@herbertgao/pi-extensions 2026.10.0 → 2026.10.2

package/node_modules/pi-web-access/brave-rate-limit.ts

@@ -64,6 +64,12 @@ export function parseRetryAfterMs(headers: Headers, now = Date.now()): number |
 	return Math.max(0, date - now);
 }
 
+// Prepaid plans report a monthly bucket with limit 0: it always reads
+// remaining 0 but never gates requests.
+function exhaustedBuckets(headers: Headers): BraveRateLimitBucket[] {
+	return parseBraveRateLimitHeaders(headers).filter(bucket => bucket.limit > 0 && bucket.remaining === 0);
+}
+
 function defaultSleep(ms: number, signal?: AbortSignal): Promise<void> {
 	if (signal?.aborted) return Promise.reject(new Error("Aborted"));
 	return new Promise((resolve, reject) => {
@@ -122,7 +128,7 @@ export class BraveRateLimitCoordinator {
 	}
 
 	observe(headers: Headers): void {
-		const exhausted = parseBraveRateLimitHeaders(headers).filter(bucket => bucket.remaining === 0);
+		const exhausted = exhaustedBuckets(headers);
 		if (exhausted.length === 0) return;
 		const resetMs = Math.max(...exhausted.map(bucket => bucket.resetSeconds * 1000));
 		this.blockFor(resetMs);
@@ -130,7 +136,7 @@ export class BraveRateLimitCoordinator {
 
 	retryDelay(headers: Headers): number | null {
 		const advertised = parseRetryAfterMs(headers, this.now());
-		const exhausted = parseBraveRateLimitHeaders(headers).filter(bucket => bucket.remaining === 0);
+		const exhausted = exhaustedBuckets(headers);
 		const bucketDelay = exhausted.length > 0
 			? Math.max(...exhausted.map(bucket => bucket.resetSeconds * 1000))
 			: null;

package/node_modules/pi-web-access/curator-page.ts

@@ -33,6 +33,7 @@ function buildProviderButtons(
 		{ value: "bocha", label: "Bocha", available: available.bocha },
 		{ value: "ollama", label: "Ollama", available: available.ollama },
 		{ value: "searxng", label: "SearXNG", available: available.searxng },
+		{ value: "degoog", label: "degoog", available: available.degoog },
 		{ value: "duckduckgo", label: "DuckDuckGo", available: available.duckduckgo },
 		{ value: "perplexity", label: "Perplexity", available: available.perplexity },
 		{ value: "gemini", label: "Gemini", available: available.gemini },
@@ -50,6 +51,7 @@ function buildProviderButtons(
 		{ value: "baizhi", label: "Baizhi", available: available.baizhi },
 		{ value: "zai", label: "Z.ai", available: available.zai },
 		{ value: "valyu", label: "Valyu", available: available.valyu },
+		{ value: "keenable", label: "Keenable", available: available.keenable },
 	];
 
 	return providers
@@ -1476,7 +1478,7 @@ const SCRIPT = `(function() {
   var token = DATA.sessionToken;
   var timeoutSec = DATA.timeout;
   var queries = Array.isArray(DATA.queries) ? DATA.queries : [];
-  var providers = ["auto", "all", "openai", "exa", "brave", "parallel", "parallel-mcp", "tinyfish", "search1api", "searchinfinity", "querit", "tavily", "firecrawl", "jina", "serpdive", "kagi", "bocha", "ollama", "searxng", "duckduckgo", "perplexity", "gemini", "kimi", "anysearch", "xcrawl", "xai", "mistral", "brightdata", "serpbase", "serpapi", "serper", "serply", "valyu", "baizhi", "you", "zai"];
+  var providers = ["auto", "all", "openai", "exa", "brave", "parallel", "parallel-mcp", "tinyfish", "search1api", "searchinfinity", "querit", "tavily", "firecrawl", "jina", "serpdive", "kagi", "bocha", "ollama", "searxng", "degoog", "duckduckgo", "perplexity", "gemini", "kimi", "anysearch", "xcrawl", "xai", "mistral", "brightdata", "serpbase", "serpapi", "serper", "serply", "valyu", "baizhi", "you", "zai", "keenable"];
   var availProviders = DATA.availableProviders && typeof DATA.availableProviders === "object" ? DATA.availableProviders : {};
   var workflow = "summary-review";
   var initialDefaultProvider = typeof DATA.defaultProvider === "string" ? DATA.defaultProvider : "exa";
@@ -1697,6 +1699,7 @@ const SCRIPT = `(function() {
     if (provider === "bocha") return "Bocha";
     if (provider === "ollama") return "Ollama";
     if (provider === "searxng") return "SearXNG";
+    if (provider === "degoog") return "degoog";
     if (provider === "duckduckgo") return "DuckDuckGo";
     if (provider === "perplexity") return "Perplexity";
     if (provider === "exa") return "Exa";
@@ -1715,6 +1718,7 @@ const SCRIPT = `(function() {
     if (provider === "baizhi") return "Baizhi";
     if (provider === "zai") return "Z.ai";
     if (provider === "valyu") return "Valyu";
+    if (provider === "keenable") return "Keenable";
     return "Unknown";
   }
 

package/node_modules/pi-web-access/curator-server.ts

@@ -292,6 +292,7 @@ export function startCuratorServer(
 		if (provider === "bocha") return availableProviders.bocha;
 		if (provider === "ollama") return availableProviders.ollama;
 		if (provider === "searxng") return availableProviders.searxng;
+		if (provider === "degoog") return availableProviders.degoog;
 		if (provider === "duckduckgo") return availableProviders.duckduckgo;
 		if (provider === "perplexity") return availableProviders.perplexity;
 		if (provider === "exa") return availableProviders.exa;
@@ -309,6 +310,7 @@ export function startCuratorServer(
 		if (provider === "baizhi") return availableProviders.baizhi;
 		if (provider === "zai") return availableProviders.zai;
 		if (provider === "valyu") return availableProviders.valyu;
+		if (provider === "keenable") return availableProviders.keenable;
 		return false;
 	}
 

package/node_modules/pi-web-access/declared-web-links.ts

@@ -23,7 +23,7 @@ export interface DeclaredWebLink {
 }
 
 export function discoverDeclaredWebLinks(
-	document: Document,
+	document: Document | null,
 	linkHeader: string | null,
 	responseUrl: string,
 ): DeclaredWebLink[] {
@@ -41,7 +41,7 @@ export function discoverDeclaredWebLinks(
 		if (links.size >= MAX_DECLARED_LINKS) break;
 	}
 
-	if (links.size < MAX_DECLARED_LINKS) {
+	if (document && links.size < MAX_DECLARED_LINKS) {
 		const declaredBase = document.querySelector("base[href]")?.getAttribute("href");
 		const documentBase = resolveHttpUrl(declaredBase, responseUrl) ?? responseUrl;
 		for (const element of document.querySelectorAll("link[rel][href], a[rel][href]")) {

package/node_modules/pi-web-access/degoog.ts

@@ -0,0 +1,323 @@
+import { existsSync, readFileSync } from "node:fs";
+import { activityMonitor } from "./activity.ts";
+import { redactCredential, resolveCredential } from "./credential-source.ts";
+import { normalizeDomain } from "./domain-filter-normalization.ts";
+import { fetchRemoteUrl, loadSsrfConfig } from "./ssrf-protection.ts";
+import type { SearchOptions, SearchResult, SearchResponse } from "./perplexity.ts";
+import { formatSearchResultsAsAnswer } from "./search-answer-formatting.ts";
+import { normalizeSearchResultCount } from "./search-result-count-normalization.ts";
+import { getWebSearchConfigPath } from "./utils.ts";
+
+const CONFIG_PATH = getWebSearchConfigPath();
+// degoog is a self-hosted metasearch front end. The public instance keeps the
+// provider usable without configuration; DEGOOG_URL or degoogBaseUrl point it at
+// a private deployment, which is what SSRF-guarded internal hosts require.
+const DEFAULT_BASE_URL = "https://degoog.org";
+const SEARCH_TIMEOUT_MS = 30_000;
+
+interface WebSearchConfig {
+	degoogBaseUrl?: unknown;
+	degoogApiKey?: unknown;
+	degoogEngines?: unknown;
+	degoogHeaders?: unknown;
+}
+
+interface NormalizedDomainFilters {
+	allowed: string[];
+	blocked: string[];
+}
+
+interface DegoogResult {
+	title?: unknown;
+	url?: unknown;
+	snippet?: unknown;
+	content?: unknown;
+}
+
+let cachedConfig: WebSearchConfig | null = null;
+
+function loadConfig(): WebSearchConfig {
+	if (cachedConfig) return cachedConfig;
+	if (!existsSync(CONFIG_PATH)) {
+		cachedConfig = {};
+		return cachedConfig;
+	}
+	const raw = readFileSync(CONFIG_PATH, "utf-8");
+	let parsed: unknown;
+	try {
+		parsed = JSON.parse(raw);
+	} catch (err) {
+		const message = err instanceof Error ? err.message : String(err);
+		throw new Error(`Failed to parse ${CONFIG_PATH}: ${message}`);
+	}
+	if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
+		throw new Error(`Invalid config in ${CONFIG_PATH}: expected a JSON object`);
+	}
+	cachedConfig = parsed as WebSearchConfig;
+	return cachedConfig;
+}
+
+function normalizeBaseUrl(value: unknown): string | null {
+	if (typeof value !== "string") return null;
+	const trimmed = value.trim();
+	if (!trimmed) return null;
+	try {
+		const url = new URL(trimmed);
+		if (url.protocol !== "http:" && url.protocol !== "https:") return null;
+		if (url.username || url.password) return null;
+		url.pathname = url.pathname.replace(/\/+$/, "");
+		url.search = "";
+		url.hash = "";
+		return url.toString().replace(/\/+$/, "");
+	} catch {
+		return null;
+	}
+}
+
+function getBaseUrl(): string | null {
+	// An explicitly set variable wins, including an empty one that disables the
+	// provider, so a configured base never silently falls back to the public instance.
+	if (process.env.DEGOOG_URL !== undefined) return normalizeBaseUrl(process.env.DEGOOG_URL);
+	const configured = loadConfig().degoogBaseUrl;
+	if (configured !== undefined) return normalizeBaseUrl(configured);
+	return DEFAULT_BASE_URL;
+}
+
+function normalizeEngines(value: unknown): string[] {
+	if (!Array.isArray(value)) return [];
+	const engines: string[] = [];
+	for (const entry of value) {
+		if (typeof entry !== "string") continue;
+		const engine = entry.trim();
+		if (engine && !engines.includes(engine)) engines.push(engine);
+	}
+	return engines;
+}
+
+function isValidHeaderValue(value: string): boolean {
+	try {
+		new Headers({ "x-pi-web-access-validation": value });
+		return true;
+	} catch {
+		return false;
+	}
+}
+
+function normalizeHeaders(value: unknown): Record<string, string> {
+	if (!value || typeof value !== "object" || Array.isArray(value)) return {};
+	const headers: Record<string, string> = {};
+	for (const [key, headerValue] of Object.entries(value as Record<string, unknown>)) {
+		if (typeof headerValue !== "string") continue;
+		const name = key.trim();
+		// RFC 7230 token chars only — reject empty or malformed header names.
+		if (!name || !/^[!#$%&'*+.^_`|~0-9A-Za-z-]+$/.test(name)) continue;
+		if (!isValidHeaderValue(headerValue)) continue;
+		// Fetch strips surrounding whitespace before sending; store the sent form so
+		// redaction matches what an upstream error body can echo back.
+		headers[name] = headerValue.trim();
+	}
+	return headers;
+}
+
+function mergeHeaders(extra: Record<string, string>): Record<string, string> {
+	const headers: Record<string, string> = { Accept: "application/json", "Content-Type": "application/json" };
+	for (const [name, value] of Object.entries(extra)) {
+		for (const existing of Object.keys(headers)) {
+			if (existing.toLowerCase() === name.toLowerCase()) delete headers[existing];
+		}
+		headers[name] = value;
+	}
+	return headers;
+}
+
+function requireBaseUrl(): string {
+	const baseUrl = getBaseUrl();
+	if (!baseUrl) {
+		throw new Error(
+			"degoog base URL is invalid or missing. Either:\n" +
+			`  1. Create ${CONFIG_PATH} with { "degoogBaseUrl": "https://degoog.example.com" }\n` +
+			"  2. Set DEGOOG_URL to an HTTP(S) URL\n" +
+			"  3. Remove the override to use the public instance at https://degoog.org",
+		);
+	}
+	return baseUrl;
+}
+
+function normalizeDomainFilters(domainFilter: string[] | undefined): NormalizedDomainFilters {
+	const filters: NormalizedDomainFilters = { allowed: [], blocked: [] };
+	for (const raw of domainFilter ?? []) {
+		const domain = normalizeDomain(raw);
+		if (!domain) continue;
+		const target = raw.trim().startsWith("-") ? filters.blocked : filters.allowed;
+		if (!target.includes(domain)) target.push(domain);
+	}
+	return filters;
+}
+
+function buildQuery(query: string, filters: NormalizedDomainFilters): string {
+	const parts = [query];
+	if (filters.allowed.length === 1) {
+		parts.push(`site:${filters.allowed[0]}`);
+	} else if (filters.allowed.length > 1) {
+		// Group the alternatives so the query terms apply to every site branch.
+		parts.push(`(${filters.allowed.map(domain => `site:${domain}`).join(" OR ")})`);
+	}
+	for (const domain of filters.blocked) parts.push(`-site:${domain}`);
+	return parts.join(" ");
+}
+
+function hostMatchesDomain(hostname: string, domain: string): boolean {
+	return hostname === domain || hostname.endsWith(`.${domain}`);
+}
+
+function matchesDomainFilters(url: string, filters: NormalizedDomainFilters): boolean {
+	if (filters.allowed.length === 0 && filters.blocked.length === 0) return true;
+	let hostname: string;
+	try {
+		hostname = new URL(url).hostname.toLowerCase();
+	} catch {
+		return false;
+	}
+	if (filters.allowed.length > 0 && !filters.allowed.some(domain => hostMatchesDomain(hostname, domain))) return false;
+	return !filters.blocked.some(domain => hostMatchesDomain(hostname, domain));
+}
+
+function resultText(value: unknown): string {
+	return typeof value === "string" ? value.replace(/\s+/g, " ").trim() : "";
+}
+
+function errorMessage(err: unknown): string {
+	return err instanceof Error ? err.message : String(err);
+}
+
+function invalidResponse(message: string): Error {
+	return new Error(`degoog returned invalid response: ${message}`);
+}
+
+function parseResponse(value: unknown): DegoogResult[] {
+	if (!value || typeof value !== "object" || Array.isArray(value)) throw invalidResponse("expected an object envelope");
+	const envelope = value as Record<string, unknown>;
+	if (!Array.isArray(envelope.results)) throw invalidResponse("expected a results array");
+	return envelope.results as DegoogResult[];
+}
+
+function redactSecrets(text: string, secrets: Array<string | null | undefined>): string {
+	let redacted = text;
+	for (const secret of secrets) redacted = redactCredential(redacted, secret);
+	return redacted;
+}
+
+export function isDegoogAvailable(): boolean {
+	return getBaseUrl() !== null;
+}
+
+export async function searchWithDegoog(query: string, options: SearchOptions = {}): Promise<SearchResponse> {
+	const baseUrl = requireBaseUrl();
+	const numResults = normalizeSearchResultCount(options.numResults);
+	const filters = normalizeDomainFilters(options.domainFilter);
+	const searchQuery = buildQuery(query, filters);
+	const engines = normalizeEngines(loadConfig().degoogEngines);
+	const apiKey = await resolveCredential({
+		provider: "degoog",
+		configuredValue: loadConfig().degoogApiKey,
+		environmentValue: process.env.DEGOOG_API_KEY,
+		signal: options.signal,
+	});
+	const customHeaders = normalizeHeaders(loadConfig().degoogHeaders);
+	const headers = mergeHeaders(customHeaders);
+	if (apiKey) headers.Authorization = `Bearer ${apiKey}`;
+	// Reverse-proxy header values can be credentials, so redact them alongside the API
+	// key before an upstream error body can echo them back.
+	const secrets = [apiKey, ...Object.values(customHeaders)];
+
+	const body: Record<string, unknown> = { query: searchQuery, type: "web", page: 1 };
+	if (options.recencyFilter) body.time = options.recencyFilter;
+	if (engines.length > 0) body.engines = engines;
+
+	const url = new URL(`${baseUrl}/api/search`);
+	const activityId = activityMonitor.logStart({ type: "api", query: searchQuery });
+	const timeoutSignal = AbortSignal.timeout(SEARCH_TIMEOUT_MS);
+	let response: Response;
+	let entries: DegoogResult[];
+	try {
+		response = await fetchRemoteUrl(url, {
+			method: "POST",
+			headers,
+			body: JSON.stringify(body),
+			signal: options.signal ? AbortSignal.any([timeoutSignal, options.signal]) : timeoutSignal,
+		}, {
+			...loadSsrfConfig(),
+			// Credentials and reverse-proxy headers must not follow a cross-origin redirect.
+			onRedirect: ({ from, to, init }) => from.origin === to.origin
+				? init
+				: { ...init, headers: { Accept: "application/json", "Content-Type": "application/json" } },
+		});
+
+		if (!response.ok) {
+			const errorText = redactSecrets(await response.text(), secrets).slice(0, 300);
+			if (response.status === 401) {
+				throw new Error(
+					`degoog search error 401: this instance protects search routes. Set degoogApiKey in ${CONFIG_PATH} or DEGOOG_API_KEY. ${errorText.slice(0, 200)}`,
+				);
+			}
+			throw new Error(`degoog search error ${response.status}: ${errorText}`);
+		}
+
+		let rawData: unknown;
+		try {
+			rawData = await response.json();
+		} catch (err) {
+			if (err instanceof Error && err.name === "TimeoutError") throw err;
+			throw new Error(`degoog returned invalid JSON: ${errorMessage(err)}`);
+		}
+		entries = parseResponse(rawData);
+	} catch (err) {
+		if (options.signal?.aborted) {
+			activityMonitor.logComplete(activityId, 0);
+			throw new Error("Aborted");
+		}
+		const message = errorMessage(err);
+		// A caller cancellation is not a failure, but our own deadline is: distinguish
+		// them so a timeout is not recorded as a completed request with status 0.
+		const providerTimeout = timeoutSignal.aborted || (err instanceof Error && err.name === "TimeoutError");
+		const outgoing = providerTimeout
+			? new Error(`degoog request timed out after ${Math.round(SEARCH_TIMEOUT_MS / 1000)}s`)
+			: (() => {
+				const redactedMessage = redactSecrets(message, secrets);
+				if (redactedMessage === message && err instanceof Error) return err;
+				const redactedError = new Error(redactedMessage);
+				if (err instanceof Error) redactedError.name = err.name;
+				return redactedError;
+			})();
+		activityMonitor.logError(activityId, redactSecrets(errorMessage(outgoing), secrets));
+		throw outgoing;
+	}
+
+	activityMonitor.logComplete(activityId, response.status);
+	const results: SearchResult[] = [];
+	const seen = new Set<string>();
+	for (const item of entries) {
+		if (!item || typeof item !== "object") continue;
+		const rawUrl = resultText(item.url);
+		if (!rawUrl) continue;
+		let resultUrl: URL;
+		try {
+			resultUrl = new URL(rawUrl);
+		} catch {
+			continue;
+		}
+		// Only absolute HTTP(S) links are usable as citations or fetch targets.
+		if (resultUrl.protocol !== "http:" && resultUrl.protocol !== "https:") continue;
+		if (!matchesDomainFilters(resultUrl.href, filters)) continue;
+		if (seen.has(resultUrl.href)) continue;
+		seen.add(resultUrl.href);
+		results.push({
+			title: resultText(item.title) || resultUrl.href,
+			url: resultUrl.href,
+			snippet: resultText(item.snippet) || resultText(item.content),
+		});
+		if (results.length >= numResults) break;
+	}
+
+	return { answer: formatSearchResultsAsAnswer(results), results };
+}