@herbertgao/pi-extensions 2026.8.13 → 2026.8.14

package/node_modules/pi-web-access/xcrawl.ts

@@ -0,0 +1,264 @@
+import { existsSync, readFileSync } from "node:fs";
+import { activityMonitor } from "./activity.ts";
+import type { SearchOptions, SearchResponse } from "./perplexity.ts";
+import { hasCredentialSource, redactCredential, resolveCredential } from "./credential-source.ts";
+import { getWebSearchConfigPath } from "./utils.ts";
+
+const XCRAWL_API_URL = "https://run.xcrawl.com/v1/serp";
+const CONFIG_PATH = getWebSearchConfigPath();
+// The SERP API is usually fast (a few seconds, cached responses are quicker),
+// but leave generous headroom before treating a slow job as a provider failure.
+const SEARCH_TIMEOUT_MS = 60_000;
+
+interface WebSearchConfig {
+	xcrawlApiKey?: unknown;
+}
+
+interface XCrawlSerpResult {
+	title?: unknown;
+	link?: unknown;
+	snippet?: unknown;
+}
+
+interface XCrawlSerpResponse {
+	search_metadata?: unknown;
+	organic_results?: unknown;
+}
+
+let cachedConfig: WebSearchConfig | null = null;
+
+function loadConfig(): WebSearchConfig {
+	if (cachedConfig) return cachedConfig;
+	if (!existsSync(CONFIG_PATH)) {
+		cachedConfig = {};
+		return cachedConfig;
+	}
+
+	const raw = readFileSync(CONFIG_PATH, "utf-8");
+	let parsed: unknown;
+	try {
+		parsed = JSON.parse(raw);
+	} catch (err) {
+		const message = err instanceof Error ? err.message : String(err);
+		throw new Error(`Failed to parse ${CONFIG_PATH}: ${message}`);
+	}
+	if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
+		throw new Error(`Invalid config in ${CONFIG_PATH}: expected a JSON object`);
+	}
+	cachedConfig = parsed as WebSearchConfig;
+	return cachedConfig;
+}
+
+async function getApiKey(signal?: AbortSignal): Promise<string | null> {
+	return resolveCredential({
+		provider: "XCrawl",
+		configuredValue: loadConfig().xcrawlApiKey,
+		environmentValue: process.env.XCRAWL_API_KEY,
+		signal,
+	});
+}
+
+export function isXcrawlAvailable(): boolean {
+	return hasCredentialSource({ provider: "XCrawl", configuredValue: loadConfig().xcrawlApiKey, environmentValue: process.env.XCRAWL_API_KEY });
+}
+
+function errorMessage(err: unknown): string {
+	return err instanceof Error ? err.message : String(err);
+}
+
+function invalidResponse(message: string): Error {
+	return new Error(`XCrawl API returned invalid response: ${message}`);
+}
+
+function normalizeCount(value: number | undefined): number {
+	if (typeof value !== "number" || !Number.isFinite(value)) return 5;
+	return Math.max(1, Math.min(Math.floor(value), 20));
+}
+
+function hostnameOf(url: string): string {
+	try {
+		return new URL(url).hostname.toLowerCase();
+	} catch {
+		return "";
+	}
+}
+
+// XCrawl's Google SERP results can carry links relative to the API origin
+// (e.g. "/goto?url=..." redirect stubs) instead of the documented absolute
+// URLs. Resolve those against the API origin so callers always receive a
+// well-formed absolute URL; absolute http(s) links pass through unchanged.
+function absolutizeLink(link: string): string {
+	if (/^https?:\/\//i.test(link)) return link;
+	try {
+		return new URL(link, XCRAWL_API_URL).href;
+	} catch {
+		return link;
+	}
+}
+
+// Normalize a shared domainFilter entry the same way Valyu does before
+// matching: trim, lowercase, strip URL/paths/ports, validate the shape.
+function normalizeDomain(value: string): string | null {
+	let input = value.trim().toLowerCase();
+	if (!input) return null;
+	if (input.startsWith("-")) input = input.slice(1).trim();
+	if (!input) return null;
+	try {
+		const parsed = input.includes("://") ? new URL(input) : new URL(`https://${input}`);
+		input = parsed.hostname;
+	} catch {
+		input = input.split("/")[0]?.split(":")[0] ?? "";
+	}
+	input = input.replace(/^\.+|\.+$/g, "");
+	return /^[a-z0-9][a-z0-9.-]*\.[a-z]{2,}$/i.test(input) ? input : null;
+}
+
+// XCrawl's SERP API has no server-side domain filter, so apply the shared
+// include/exclude domainFilter locally instead of returning off-domain results.
+function applyDomainFilter(results: SearchResponse["results"], domainFilter: NonNullable<SearchOptions["domainFilter"]>): SearchResponse["results"] {
+	const includes: string[] = [];
+	const excludes: string[] = [];
+	for (const raw of domainFilter) {
+		const normalized = normalizeDomain(raw);
+		if (!normalized) continue;
+		if (raw.trim().startsWith("-")) excludes.push(normalized);
+		else includes.push(normalized);
+	}
+	if (!includes.length && !excludes.length) return results;
+	return results.filter((result) => {
+		const host = hostnameOf(result.url);
+		if (!host) return false;
+		const matches = (domain: string) => host === domain || host.endsWith(`.${domain}`);
+		if (excludes.some(matches)) return false;
+		if (includes.length && !includes.some(matches)) return false;
+		return true;
+	});
+}
+
+function parseResponse(value: unknown): SearchResponse["results"] {
+	if (!value || typeof value !== "object" || Array.isArray(value)) {
+		throw invalidResponse("expected an object envelope");
+	}
+	const envelope = value as XCrawlSerpResponse;
+	const metadata = envelope.search_metadata;
+	if (!metadata || typeof metadata !== "object" || Array.isArray(metadata)) {
+		throw invalidResponse("expected search_metadata object");
+	}
+	const status = (metadata as Record<string, unknown>).status;
+	if (status !== undefined && status !== "completed") {
+		throw invalidResponse(`expected search_metadata.status \"completed\", got ${JSON.stringify(status)}`);
+	}
+	if (!Array.isArray(envelope.organic_results)) throw invalidResponse("expected organic_results array");
+
+	const results: SearchResponse["results"] = [];
+	for (const [index, value] of (envelope.organic_results as unknown[]).entries()) {
+		if (!value || typeof value !== "object" || Array.isArray(value)) {
+			throw invalidResponse(`expected organic_results[${index}] object`);
+		}
+		const result = value as XCrawlSerpResult;
+		const { title, link, snippet } = result;
+		if (typeof link !== "string" || !link.trim()) throw invalidResponse(`expected organic_results[${index}].link to be a non-empty string`);
+		if (title !== null && title !== undefined && typeof title !== "string") {
+			throw invalidResponse(`expected organic_results[${index}].title to be a string or null`);
+		}
+		if (snippet !== undefined && snippet !== null && typeof snippet !== "string") {
+			throw invalidResponse(`expected organic_results[${index}].snippet to be a string or null`);
+		}
+		const resolved = absolutizeLink(link.trim());
+		results.push({
+			title: typeof title === "string" && title.trim().length > 0 ? title : resolved,
+			url: resolved,
+			snippet: typeof snippet === "string" ? snippet : "",
+		});
+	}
+
+	return results;
+}
+
+function buildAnswer(results: SearchResponse["results"]): string {
+	return results
+		.map((result) => result.snippet
+			? `${result.snippet}\nSource: ${result.title} (${result.url})`
+			: `Source: ${result.title} (${result.url})`)
+		.join("\n\n");
+}
+
+export async function searchWithXCrawl(query: string, options: SearchOptions = {}): Promise<SearchResponse> {
+	const apiKey = await getApiKey(options.signal);
+	const numResults = normalizeCount(options.numResults);
+	if (!apiKey) {
+		throw new Error(
+			"XCrawl search requires an API key. Set xcrawlApiKey in " + CONFIG_PATH +
+			" or export XCRAWL_API_KEY. Get one at https://dash.xcrawl.com/",
+		);
+	}
+	const body = { engine: "google_search", q: query };
+	const activityId = activityMonitor.logStart({ type: "api", query });
+	// Distinguishing caller cancellation from a provider-side timeout lets the
+	// timeout surface as a retriable failure instead of looking like an abort.
+	const timeoutSignal = AbortSignal.timeout(SEARCH_TIMEOUT_MS);
+	let response: Response;
+
+	try {
+		response = await fetch(XCRAWL_API_URL, {
+			method: "POST",
+			headers: {
+				Authorization: `Bearer ${apiKey}`,
+				"Content-Type": "application/json",
+			},
+			body: JSON.stringify(body),
+			signal: options.signal
+				? AbortSignal.any([timeoutSignal, options.signal])
+				: timeoutSignal,
+		});
+	} catch (err) {
+		const message = errorMessage(err);
+		if (options.signal?.aborted) {
+			activityMonitor.logComplete(activityId, 0);
+			throw new Error("Aborted");
+		}
+		// AbortSignal.timeout rejects with a TimeoutError; treat either that
+		// shape or our own fired timer as a retriable provider-side timeout so
+		// routing fallback still applies.
+		const providerTimeout = timeoutSignal.aborted || (err instanceof Error && err.name === "TimeoutError");
+		let outgoing: Error;
+		if (providerTimeout) {
+			outgoing = new Error(`XCrawl search request timed out after ${Math.round(SEARCH_TIMEOUT_MS / 1000)}s`);
+		} else {
+			const redactedMessage = redactCredential(message, apiKey);
+			outgoing = redactedMessage === message && err instanceof Error ? err : new Error(redactedMessage);
+			if (err instanceof Error && outgoing.name === "Error") outgoing.name = err.name;
+		}
+		activityMonitor.logError(activityId, redactCredential(errorMessage(outgoing), apiKey));
+		throw outgoing;
+	}
+
+	if (!response.ok) {
+		activityMonitor.logComplete(activityId, response.status);
+		const errorText = redactCredential(await response.text(), apiKey);
+		let detail = errorText.slice(0, 300);
+		try {
+			const parsed = JSON.parse(errorText) as { message?: unknown; error?: unknown };
+			if (typeof parsed.message === "string") detail = parsed.message.slice(0, 300);
+			else if (typeof parsed.error === "string") detail = parsed.error.slice(0, 300);
+		} catch {
+			// keep raw text slice
+		}
+		throw new Error(`XCrawl API error (${response.status}): ${detail}`);
+	}
+	activityMonitor.logComplete(activityId, response.status);
+
+	let payload: unknown;
+	try {
+		payload = await response.json();
+	} catch (err) {
+		throw invalidResponse(`response body is not valid JSON: ${errorMessage(err)}`);
+	}
+	const results = parseResponse(payload);
+	const filtered = (options.domainFilter?.length ? applyDomainFilter(results, options.domainFilter) : results).slice(0, numResults);
+
+	return {
+		answer: buildAnswer(filtered),
+		results: filtered,
+	};
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@herbertgao/pi-extensions",
-  "version": "2026.8.13",
+  "version": "2026.8.14",
   "description": "Aggregate installer for HerbertGao-maintained Pi extensions.",
   "keywords": [
     "extensions",
@@ -37,15 +37,14 @@
   "dependencies": {
     "@ast-grep/cli": "^0.45.0",
     "@ast-grep/napi": "^0.45.0",
-    "@czottmann/pi-automode": "1.13.0",
+    "@czottmann/pi-automode": "1.14.0",
     "@dietrichgebert/ponytail": "4.9.0",
     "@earendil-works/pi-coding-agent": "^0.84.2",
     "@earendil-works/pi-tui": "^0.84.2",
     "@effect/platform-node": "4.0.0-beta.103",
     "@effect/platform-node-shared": "4.0.0-beta.103",
-    "@herbertgao/pi-cc-extensions": "0.8.59",
-    "@herbertgao/pi-stash": "0.1.2",
-    "@herbertgao/pi-subagents": "0.17.0",
+    "@herbertgao/pi-cc-extensions": "0.8.60",
+    "@herbertgao/pi-subagents": "0.17.1",
     "@herbertgao/resume-from": "0.2.0",
     "@juicesharp/rpiv-ask-user-question": "2.7.1",
     "@juicesharp/rpiv-config": "^2.7.1",
@@ -66,7 +65,7 @@
     "@tifan/pi-handoff": "2.0.1",
     "@tifan/pi-inline-skills": "1.0.5",
     "@tifan/pi-mermaid-open": "0.2.0",
-    "@tifan/pi-preferred-thinking": "1.0.0",
+    "@tifan/pi-preferred-thinking": "1.0.1",
     "@tifan/pi-recap": "0.4.5",
     "@tifan/pi-rename": "0.5.1",
     "@tifan/pi-titlebar-spinner": "0.1.3",
@@ -90,8 +89,8 @@
     "p-limit": "^6.1.0",
     "pi-footer": "0.5.1",
     "pi-lens": "4.1.2",
-    "pi-mcp-adapter": "2.29.0",
-    "pi-web-access": "0.25.0",
+    "pi-mcp-adapter": "2.30.0",
+    "pi-web-access": "0.26.0",
     "pidusage": "^4.0.1",
     "promise.try": "^2.0.1",
     "qrcode-terminal": "^0.12.0",
@@ -115,7 +114,6 @@
     "@czottmann/pi-automode",
     "@dietrichgebert/ponytail",
     "@herbertgao/pi-cc-extensions",
-    "@herbertgao/pi-stash",
     "@herbertgao/pi-subagents",
     "@herbertgao/resume-from",
     "@juicesharp/rpiv-ask-user-question",
@@ -152,7 +150,6 @@
       "./node_modules/@tifan/pi-preferred-thinking/src/index.ts",
       "./node_modules/@tifan/pi-recap/src/index.ts",
       "./node_modules/@tifan/pi-rename/src/index.ts",
-      "./node_modules/@herbertgao/pi-stash/src/index.ts",
       "./node_modules/@herbertgao/pi-subagents/src/index.ts",
       "./node_modules/@tifan/pi-titlebar-spinner/src/index.ts",
       "./node_modules/@juicesharp/rpiv-ask-user-question/index.ts",

package/node_modules/@herbertgao/pi-stash/LICENSE

@@ -1,22 +0,0 @@
-MIT License
-
-Copyright (c) 2026 Tifan Dwi Avianto
-Copyright (c) 2026 Herbert Gao
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.

package/node_modules/@herbertgao/pi-stash/README.md

@@ -1,34 +0,0 @@
-# @herbertgao/pi-stash
-
-> HerbertGao-maintained fork of [@tifan/pi-stash](https://github.com/tifandotme/pi-extensions/tree/master/packages/pi-stash), distributed under MIT with the original attribution preserved.
-
-Stash one draft, send another message, and return to the draft while pi works.
-
-## Install
-
-```bash
-pi install npm:@herbertgao/pi-stash
-```
-
-## Keybinding
-
-pi-stash uses `Alt+S`, which does not conflict with Pi's built-in shortcuts.
-
-### macOS
-
-On macOS, press `Option+S` (`Option` is the `Alt` key). If that inserts `ß` instead of triggering pi-stash, configure the terminal to send `Option` as Meta/Escape:
-
-- Terminal.app: **Settings → Profiles → Keyboard → Use Option as Meta key**.
-- iTerm2: **Settings → Profiles → Keys → Left Option key** (or **Right Option key**) → **Esc+**.
-
-## Usage
-
-Press `Alt+S` with text in the editor to stash it. Pi clears the editor so you can send another message, then restores the draft as soon as that message is submitted.
-
-Press `Alt+S` again while the editor is empty to restore the draft manually. If the editor contains new text, pi-stash keeps both drafts unchanged and refuses to overwrite either one.
-
-The stash belongs to the current session and survives reloads and restarts. Slash commands and `!` shell commands do not consume it.
-
-## License
-
-[MIT](LICENSE)

package/node_modules/@herbertgao/pi-stash/package.json

@@ -1,51 +0,0 @@
-{
-  "name": "@herbertgao/pi-stash",
-  "version": "0.1.2",
-  "description": "Stash one pi draft and restore it after the next message.",
-  "keywords": [
-    "pi",
-    "pi-extension",
-    "pi-package",
-    "prompt",
-    "stash"
-  ],
-  "homepage": "https://github.com/HerbertGao/pi-extensions/tree/master/packages/pi-stash#readme",
-  "bugs": "https://github.com/HerbertGao/pi-extensions/issues",
-  "license": "MIT",
-  "author": "Herbert Gao",
-  "contributors": [
-    "Tifan Dwi Avianto <hi@tifan.me>"
-  ],
-  "repository": {
-    "type": "git",
-    "url": "git+https://github.com/HerbertGao/pi-extensions.git",
-    "directory": "packages/pi-stash"
-  },
-  "files": [
-    "src/**/*.ts",
-    "README.md",
-    "LICENSE"
-  ],
-  "type": "module",
-  "publishConfig": {
-    "access": "public",
-    "provenance": true
-  },
-  "scripts": {
-    "test": "node --experimental-strip-types --test tests/**/*.test.ts"
-  },
-  "engines": {
-    "node": ">=20"
-  },
-  "pi": {
-    "extensions": [
-      "./src/index.ts"
-    ]
-  },
-  "x-upstream": {
-    "package": "@tifan/pi-stash",
-    "version": "0.2.0",
-    "repository": "https://github.com/tifandotme/pi-extensions",
-    "commit": "b39d1e6359c78718cbf8fe5c74c82b89065e9a0b"
-  }
-}

package/node_modules/@herbertgao/pi-stash/src/index.ts

@@ -1,118 +0,0 @@
-import type {
-  ExtensionAPI,
-  ExtensionContext,
-  InputSource,
-  SessionEntry,
-} from "@earendil-works/pi-coding-agent"
-
-export const STASH_ENTRY_TYPE = "pi-stash-state"
-export const STASH_SHORTCUT = "alt+s"
-
-interface StashState {
-  draft: string | null
-}
-
-export type ShortcutAction =
-  | { type: "none" }
-  | { type: "reject" }
-  | { type: "restore"; draft: string }
-  | { type: "stash"; draft: string }
-
-export function getShortcutAction(
-  editorText: string,
-  pendingDraft: string | null,
-): ShortcutAction {
-  if (pendingDraft !== null) {
-    return editorText.trim()
-      ? { type: "reject" }
-      : { type: "restore", draft: pendingDraft }
-  }
-
-  return editorText.trim()
-    ? { type: "stash", draft: editorText }
-    : { type: "none" }
-}
-
-export function shouldAutoRestore(source: InputSource, text: string) {
-  return source === "interactive" && !text.trimStart().startsWith("/")
-}
-
-export function getPendingDraft(
-  entries: readonly SessionEntry[],
-): string | null {
-  let draft: string | null = null
-
-  for (const entry of entries) {
-    if (entry.type !== "custom" || entry.customType !== STASH_ENTRY_TYPE) {
-      continue
-    }
-
-    const data = entry.data
-    if (
-      typeof data === "object" &&
-      data !== null &&
-      "draft" in data &&
-      (typeof data.draft === "string" || data.draft === null)
-    ) {
-      draft = data.draft
-    }
-  }
-
-  return draft
-}
-
-export default function stashExtension(pi: ExtensionAPI) {
-  let pendingDraft: string | null = null
-
-  function saveDraft(draft: string | null) {
-    pendingDraft = draft
-    pi.appendEntry<StashState>(STASH_ENTRY_TYPE, { draft })
-  }
-
-  function restoreDraft(ctx: ExtensionContext) {
-    if (pendingDraft === null) return
-
-    const draft = pendingDraft
-    ctx.ui.setEditorText(draft)
-    saveDraft(null)
-    ctx.ui.notify("Draft restored", "info")
-  }
-
-  pi.on("session_start", (_event, ctx) => {
-    pendingDraft = getPendingDraft(ctx.sessionManager.getBranch())
-    if (
-      ctx.mode === "tui" &&
-      pendingDraft !== null &&
-      !ctx.ui.getEditorText().trim()
-    ) {
-      restoreDraft(ctx)
-    }
-  })
-
-  pi.on("input", (event, ctx) => {
-    if (shouldAutoRestore(event.source, event.text)) restoreDraft(ctx)
-  })
-
-  pi.registerShortcut(STASH_SHORTCUT, {
-    description: "Stash or restore the current draft",
-    handler: (ctx) => {
-      const action = getShortcutAction(ctx.ui.getEditorText(), pendingDraft)
-
-      switch (action.type) {
-        case "stash":
-          saveDraft(action.draft)
-          ctx.ui.setEditorText("")
-          ctx.ui.notify("Draft stashed", "info")
-          break
-        case "restore":
-          restoreDraft(ctx)
-          break
-        case "reject":
-          ctx.ui.notify("A draft is already stashed", "warning")
-          break
-        case "none":
-          break
-      }
-    },
-  })
-}