@herbertgao/pi-extensions 2026.9.9 → 2026.9.11

package/node_modules/pi-web-access/bocha.ts

@@ -1,5 +1,7 @@
 import { existsSync, readFileSync } from "node:fs";
 import { activityMonitor } from "./activity.ts";
+import { normalizeDomain } from "./domain-filter-normalization.ts";
+import { formatSearchResultsAsAnswer } from "./search-answer-formatting.ts";
 import type { SearchOptions, SearchResponse } from "./perplexity.ts";
 import { hasCredentialSource, redactCredential, resolveCredential } from "./credential-source.ts";
 import { getWebSearchConfigPath } from "./utils.ts";
@@ -77,21 +79,6 @@ interface DomainFilters {
 	exclude: string[];
 }
 
-function normalizeDomain(value: string): string | null {
-	let input = value.trim().toLowerCase();
-	if (!input) return null;
-	if (input.startsWith("-")) input = input.slice(1).trim();
-	if (!input) return null;
-	try {
-		const parsed = input.includes("://") ? new URL(input) : new URL(`https://${input}`);
-		input = parsed.hostname;
-	} catch {
-		input = input.split("/")[0]?.split(":")[0] ?? "";
-	}
-	input = input.replace(/^\.+|\.+$/g, "");
-	return /^[a-z0-9][a-z0-9.-]*\.[a-z]{2,}$/i.test(input) ? input : null;
-}
-
 function parseDomainFilter(domainFilter: string[] | undefined): DomainFilters {
 	const filters: DomainFilters = { include: [], exclude: [] };
 	for (const raw of domainFilter ?? []) {
@@ -158,12 +145,6 @@ function parseSearchResponse(value: unknown): { results: SearchResponse["results
 	return { results };
 }
 
-function buildAnswer(results: SearchResponse["results"]): string {
-	return results.map((result) => result.snippet
-		? `${result.snippet}\nSource: ${result.title} (${result.url})`
-		: `Source: ${result.title} (${result.url})`).join("\n\n");
-}
-
 export function isBochaAvailable(): boolean {
 	return hasCredentialSource({ provider: "Bocha", configuredValue: loadConfig().bochaApiKey, environmentValue: process.env.BOCHA_API_KEY });
 }
@@ -217,5 +198,5 @@ export async function searchWithBocha(query: string, options: SearchOptions = {}
 	}
 	activityMonitor.logComplete(activityId, response.status);
 	const results = parsed.results.filter((result) => passesDomainFilters(result.url, filters)).slice(0, numResults);
-	return { answer: buildAnswer(results), results };
+	return { answer: formatSearchResultsAsAnswer(results), results };
 }

package/node_modules/pi-web-access/brave.ts

@@ -1,5 +1,7 @@
 import { existsSync, readFileSync } from "node:fs";
 import { activityMonitor } from "./activity.ts";
+import { normalizeDomain } from "./domain-filter-normalization.ts";
+import { normalizeSearchResultCount } from "./search-result-count-normalization.ts";
 import type { SearchOptions, SearchResult, SearchResponse } from "./perplexity.ts";
 import { hasCredentialSource, redactCredential, resolveCredential } from "./credential-source.ts";
 import { fetchWithCredentialRedirects, getWebSearchConfigPath, resolveApiBaseUrl } from "./utils.ts";
@@ -56,26 +58,6 @@ function getApiUrl(): string {
 	})}/web/search`;
 }
 
-function normalizeCount(value: number | undefined): number {
-	if (typeof value !== "number" || !Number.isFinite(value)) return 5;
-	return Math.max(1, Math.min(Math.floor(value), 20));
-}
-
-function normalizeDomain(value: string): string | null {
-	let input = value.trim().toLowerCase();
-	if (!input) return null;
-	if (input.startsWith("-")) input = input.slice(1).trim();
-	if (!input) return null;
-	try {
-		const parsed = input.includes("://") ? new URL(input) : new URL(`https://${input}`);
-		input = parsed.hostname;
-	} catch {
-		input = input.split("/")[0]?.split(":")[0] ?? "";
-	}
-	input = input.replace(/^\.+|\.+$/g, "");
-	return /^[a-z0-9][a-z0-9.-]*\.[a-z]{2,}$/i.test(input) ? input : null;
-}
-
 function normalizeDomainFilters(domainFilter: string[] | undefined): NormalizedDomainFilters {
 	const filters: NormalizedDomainFilters = { allowed: [], blocked: [] };
 	if (!domainFilter?.length) return filters;
@@ -151,7 +133,7 @@ export async function searchWithBrave(
 		);
 	}
 
-	const numResults = normalizeCount(options.numResults);
+	const numResults = normalizeSearchResultCount(options.numResults);
 	const domainFilters = normalizeDomainFilters(options.domainFilter);
 	const searchQuery = buildBraveQuery(query, options.domainFilter);
 	const activityId = activityMonitor.logStart({ type: "api", query: searchQuery });

package/node_modules/pi-web-access/brightdata.ts

@@ -1,5 +1,8 @@
 import { existsSync, readFileSync } from "node:fs";
 import { activityMonitor } from "./activity.ts";
+import { normalizeDomain } from "./domain-filter-normalization.ts";
+import { formatSearchResultsAsAnswer } from "./search-answer-formatting.ts";
+import { normalizeSearchResultCount } from "./search-result-count-normalization.ts";
 import type { SearchOptions, SearchResponse } from "./perplexity.ts";
 import { hasCredentialSource, redactCredential, resolveCredential } from "./credential-source.ts";
 import { getWebSearchConfigPath } from "./utils.ts";
@@ -182,26 +185,6 @@ function requireSerpZone(): string {
 	);
 }
 
-function normalizeCount(value: number | undefined): number {
-	if (typeof value !== "number" || !Number.isFinite(value)) return 5;
-	return Math.max(1, Math.min(Math.floor(value), 20));
-}
-
-function normalizeDomain(value: string): string | null {
-	let input = value.trim().toLowerCase();
-	if (!input) return null;
-	if (input.startsWith("-")) input = input.slice(1).trim();
-	if (!input) return null;
-	try {
-		const parsed = input.includes("://") ? new URL(input) : new URL(`https://${input}`);
-		input = parsed.hostname;
-	} catch {
-		input = input.split("/")[0]?.split(":")[0] ?? "";
-	}
-	input = input.replace(/^\.+|\.+$/g, "");
-	return /^[a-z0-9][a-z0-9.-]*\.[a-z]{2,}$/i.test(input) ? input : null;
-}
-
 interface DomainFilters {
 	include: string[];
 	exclude: string[];
@@ -436,16 +419,6 @@ function mapResults(
 	return mapped;
 }
 
-// A SERP zone returns ranked links, never a synthesized answer, so one is
-// assembled from the sources — the same shape brave.ts and searxng.ts produce.
-function buildAnswer(results: SearchResponse["results"]): string {
-	return results
-		.map((result) => result.snippet
-			? `${result.snippet}\nSource: ${result.title} (${result.url})`
-			: `Source: ${result.title} (${result.url})`)
-		.join("\n\n");
-}
-
 // Both halves are required: a token with no zone cannot make a request, and a
 // zone with no token cannot either. Availability therefore checks the config this
 // surface actually needs rather than merely a key, the way firecrawl.ts's
@@ -477,7 +450,7 @@ export async function searchWithBrightData(query: string, options: BrightDataSea
 	// resolver, and must not reach a billable endpoint.
 	const zone = requireSerpZone();
 	const apiKey = await requireApiKey(options.signal);
-	const numResults = normalizeCount(options.numResults);
+	const numResults = normalizeSearchResultCount(options.numResults);
 	const filters = parseDomainFilter(options.domainFilter);
 	const searchQuery = buildSearchQuery(query, filters);
 	const body: Record<string, unknown> = {
@@ -560,5 +533,5 @@ export async function searchWithBrightData(query: string, options: BrightDataSea
 
 	activityMonitor.logComplete(activityId, response.status);
 	const results = mapResults(data.organic, numResults, filters);
-	return { answer: buildAnswer(results), results };
+	return { answer: formatSearchResultsAsAnswer(results), results };
 }

package/node_modules/pi-web-access/content-find.ts

@@ -12,7 +12,6 @@ interface Match {
 interface Range {
 	start: number;
 	end: number;
-	matches: Match[];
 }
 
 function normalize(value: string): string {
@@ -27,9 +26,9 @@ function editDistanceWithin(left: string, right: string, maximum: number): boole
 		let rowMinimum = i;
 		for (let j = 1; j <= right.length; j++) {
 			const value = Math.min(
-				(previous[j] ?? 0) + 1,
-				(current[j - 1] ?? 0) + 1,
-				(previous[j - 1] ?? 0) + (left[i - 1] === right[j - 1] ? 0 : 1),
+				previous[j] + 1,
+				current[j - 1] + 1,
+				previous[j - 1] + (left[i - 1] === right[j - 1] ? 0 : 1),
 			);
 			current[j] = value;
 			rowMinimum = Math.min(rowMinimum, value);
@@ -37,7 +36,7 @@ function editDistanceWithin(left: string, right: string, maximum: number): boole
 		if (rowMinimum > maximum) return false;
 		previous = current;
 	}
-	return (previous[right.length] ?? maximum + 1) <= maximum;
+	return previous[right.length] <= maximum;
 }
 
 function literalMatches(text: string, query: string, caseInsensitive: boolean): Match[] {
@@ -69,27 +68,54 @@ function fuzzyMatches(text: string, query: string): Match[] {
 		const first = tokens.find(token => matched.some(queryToken => {
 			const maximum = queryToken.length >= 9 ? 2 : queryToken.length >= 5 ? 1 : 0;
 			return editDistanceWithin(queryToken, normalize(token[0]), maximum);
-		}));
-		const start = paragraph.index + (first?.index ?? 0);
-		matches.push({ query, start, end: start + (first?.[0].length ?? query.length) });
+		}))!;
+		const start = paragraph.index + first.index;
+		matches.push({ query, start, end: start + first[0].length });
 	}
 	return matches;
 }
 
-function mergeRanges(textLength: number, matches: Match[]): Range[] {
+function mergeRanges(ranges: Range[]): Range[] {
+	const merged: Range[] = [];
+	for (const range of [...ranges].sort((left, right) => left.start - right.start || left.end - right.end)) {
+		const previous = merged.at(-1);
+		if (previous && range.start <= previous.end) previous.end = Math.max(previous.end, range.end);
+		else merged.push({ ...range });
+	}
+	return merged;
+}
+
+function contextRanges(textLength: number, matches: Match[]): Range[] {
 	const ranges: Range[] = [];
-	for (const match of [...matches].sort((left, right) => left.start - right.start)) {
-		const start = Math.max(0, match.start - CONTEXT_CHARS);
-		const end = Math.min(textLength, match.end + CONTEXT_CHARS);
-		const previous = ranges.at(-1);
-		if (previous && start <= previous.end) {
-			previous.end = Math.max(previous.end, end);
-			previous.matches.push(match);
-		} else {
-			ranges.push({ start, end, matches: [match] });
-		}
+	for (const match of matches) {
+		ranges.push({
+			start: Math.max(0, match.start - CONTEXT_CHARS),
+			end: Math.min(textLength, match.end + CONTEXT_CHARS),
+		});
 	}
-	return ranges;
+	return mergeRanges(ranges);
+}
+
+function lowerBound(values: number[], target: number): number {
+	let low = 0;
+	let high = values.length;
+	while (low < high) {
+		const middle = (low + high) >>> 1;
+		if (values[middle] < target) low = middle + 1;
+		else high = middle;
+	}
+	return low;
+}
+
+function upperBound(values: number[], target: number): number {
+	let low = 0;
+	let high = values.length;
+	while (low < high) {
+		const middle = (low + high) >>> 1;
+		if (values[middle] <= target) low = middle + 1;
+		else high = middle;
+	}
+	return low;
 }
 
 export function findContent(
@@ -98,42 +124,131 @@ export function findContent(
 	mode: FindMode,
 ): { text: string; matchCount: number; returnedMatches: number; queryResults: Array<{ query: string; matchCount: number }> } {
 	const normalizedQueries = [...new Set(queries.map(query => query.trim()).filter(Boolean))];
-	const matches = normalizedQueries.flatMap(query => mode === "fuzzy"
-		? fuzzyMatches(text, query)
-		: literalMatches(text, query, mode === "case-insensitive"));
-	const queryResults = normalizedQueries.map(query => ({
-		query,
-		matchCount: matches.filter(match => match.query === query).length,
-	}));
+	const occurrences = normalizedQueries.map(query => {
+		const matches = mode === "fuzzy" ? fuzzyMatches(text, query) : literalMatches(text, query, mode === "case-insensitive");
+		return { query, matches, starts: matches.map(match => match.start), ends: matches.map(match => match.end) };
+	});
+	const matches = occurrences.flatMap(result => result.matches);
+	const queryResults = occurrences.map(result => ({ query: result.query, matchCount: result.matches.length }));
 
 	const heading = matches.length > 0 ? `Text matches (${mode})` : `Text matches (${mode}): no matches`;
-	const sections = [heading];
-	let formattedLength = heading.length;
-	let returnedMatches = 0;
-	for (const range of mergeRanges(text.length, matches)) {
-		const prefix = range.start > 0 ? "…" : "";
-		const suffix = range.end < text.length ? "…" : "";
-		const snippet = `${prefix}${text.slice(range.start, range.end).replace(/\s+/g, " ").trim()}${suffix}`;
-		const counts = [...new Set(range.matches.map(match => match.query))]
-			.map(query => `\"${query}\" ×${range.matches.filter(match => match.query === query).length}`)
-			.join(", ");
-		const section = `${sections.length}. ${counts}\n${snippet}`;
-		if (formattedLength + 2 + section.length > MAX_OUTPUT_CHARS) break;
-		sections.push(section);
-		formattedLength += 2 + section.length;
-		returnedMatches += range.matches.length;
-	}
-
 	const missing = queryResults.filter(result => result.matchCount === 0).map(result => `\"${result.query}\"`);
-	const footer = [
-		...(missing.length > 0 ? [`No matches: ${missing.join(", ")}`] : []),
-		...(returnedMatches < matches.length ? [`Showing ${returnedMatches} of ${matches.length} matches.`] : []),
-	];
-	for (const section of footer) {
-		if (formattedLength + 2 + section.length > MAX_OUTPUT_CHARS) break;
-		sections.push(section);
-		formattedLength += 2 + section.length;
+	const matchingQueries = occurrences.filter(result => result.matches.length > 0).map((result, index) => ({
+		...result,
+		id: `Q${index + 1}`,
+		order: index,
+	}));
+	const whitespaceRuns = [...text.matchAll(/\s+/g)].map(run => ({ start: run.index, end: run.index + run[0].length }));
+	const whitespaceStarts = whitespaceRuns.map(run => run.start);
+	const whitespaceEnds = whitespaceRuns.map(run => run.end);
+	const whitespaceSavings = [0];
+	for (const run of whitespaceRuns) whitespaceSavings.push(whitespaceSavings.at(-1)! + run.end - run.start - 1);
+	function normalizedLength(start: number, end: number): number {
+		const firstRun = upperBound(whitespaceStarts, start) - 1;
+		if (firstRun >= 0 && whitespaceEnds[firstRun] > start) start = whitespaceEnds[firstRun];
+		if (start >= end) return 0;
+		const lastRun = upperBound(whitespaceStarts, end - 1) - 1;
+		if (lastRun >= 0 && whitespaceEnds[lastRun] >= end) end = whitespaceStarts[lastRun];
+		if (start >= end) return 0;
+		const first = lowerBound(whitespaceStarts, start);
+		const last = upperBound(whitespaceEnds, end);
+		return end - start - (whitespaceSavings[last] - whitespaceSavings[first]);
+	}
+	function rangeCounts(range: Range) {
+		// Literal occurrences have fixed widths; fuzzy occurrences come from successive disjoint paragraphs.
+		return matchingQueries.flatMap(result => {
+			const first = lowerBound(result.starts, range.start);
+			const last = upperBound(result.ends, range.end);
+			return last > first ? [{ ...result, count: last - first, firstStart: result.starts[first] }] : [];
+		}).sort((left, right) => left.firstStart - right.firstStart || left.order - right.order);
+	}
+	const legend = matchingQueries.length > 0
+		? `Queries: ${matchingQueries.map(result => `${result.id} = \"${result.query}\"`).join(", ")}`
+		: "";
+	const missingNotice = missing.length > 0 ? `No matches: ${missing.join(", ")}` : "";
+	function measure(ranges: Range[], overflow = false, omitted: string[] = []) {
+		let length = heading.length;
+		let returnedMatches = 0;
+		if (overflow && legend) length += 2 + legend.length;
+		for (const [index, range] of ranges.entries()) {
+			const counts = rangeCounts(range);
+			if (counts.length === 0) continue;
+			const labelsLength = counts.reduce((total, result) => total
+				+ (overflow ? result.id.length : result.query.length + 2)
+				+ 2 + String(result.count).length, 2 * (counts.length - 1));
+			const snippetLength = normalizedLength(range.start, range.end)
+				+ (range.start > 0 ? 1 : 0) + (range.end < text.length ? 1 : 0);
+			length += 2 + String(index + 1).length + 2 + labelsLength + 1 + snippetLength;
+			returnedMatches += counts.reduce((total, result) => total + result.count, 0);
+		}
+		if (missingNotice) length += 2 + missingNotice.length;
+		if (omitted.length > 0) length += 2 + `No representative excerpt: ${omitted.join(", ")}.`.length;
+		if (returnedMatches < matches.length) length += 2 + `Showing ${returnedMatches} of ${matches.length} matches.`.length;
+		return { length, returnedMatches };
+	}
+	function formatRanges(ranges: Range[], overflow = false, omitted: string[] = []) {
+		const sections = [heading];
+		let returnedMatches = 0;
+		if (overflow && legend) sections.push(legend);
+		for (const [index, range] of ranges.entries()) {
+			const contained = rangeCounts(range);
+			if (contained.length === 0) continue;
+			const prefix = range.start > 0 ? "…" : "";
+			const suffix = range.end < text.length ? "…" : "";
+			const snippet = `${prefix}${text.slice(range.start, range.end).replace(/\s+/g, " ").trim()}${suffix}`;
+			const counts = contained
+				.map(result => `${overflow ? result.id : `\"${result.query}\"`} ×${result.count}`)
+				.join(", ");
+			sections.push(`${index + 1}. ${counts}\n${snippet}`);
+			returnedMatches += contained.reduce((total, result) => total + result.count, 0);
+		}
+		if (missingNotice) sections.push(missingNotice);
+		if (omitted.length > 0) sections.push(`No representative excerpt: ${omitted.join(", ")}.`);
+		if (returnedMatches < matches.length) sections.push(`Showing ${returnedMatches} of ${matches.length} matches.`);
+		return { text: sections.join("\n\n"), matchCount: matches.length, returnedMatches, queryResults };
 	}
 
-	return { text: sections.join("\n\n"), matchCount: matches.length, returnedMatches, queryResults };
+	const fullRanges = contextRanges(text.length, matches);
+	const full = measure(fullRanges);
+	if (full.returnedMatches === matches.length && full.length <= MAX_OUTPUT_CHARS) return formatRanges(fullRanges);
+
+	let ranges: Range[] = [];
+	let omitted = matchingQueries.map(result => result.id);
+	if (measure(ranges, true, omitted).length > MAX_OUTPUT_CHARS) {
+		const text = `${heading}\n\nUnable to format bounded excerpts: query metadata exceeds ${MAX_OUTPUT_CHARS} characters.\n\nShowing 0 of ${matches.length} matches.`;
+		return { text, matchCount: matches.length, returnedMatches: 0, queryResults };
+	}
+	const witnesses: Match[] = [];
+	for (const { id, matches: queryMatches } of matchingQueries) {
+		const proposedOmitted = omitted.filter(queryId => queryId !== id);
+		const currentLength = measure(ranges, true, omitted).length;
+		let selected: { witness: Match; ranges: Range[]; cost: number } | undefined;
+		for (const witness of queryMatches) {
+			const proposedRanges = mergeRanges([...ranges, { start: witness.start, end: witness.end }]);
+			const summary = measure(proposedRanges, true, proposedOmitted);
+			const candidate = {
+				witness,
+				ranges: proposedRanges,
+				cost: summary.length - currentLength,
+			};
+			if (summary.length <= MAX_OUTPUT_CHARS && (!selected || candidate.cost < selected.cost
+				|| candidate.cost === selected.cost && (witness.start < selected.witness.start
+					|| witness.start === selected.witness.start && witness.end < selected.witness.end))) selected = candidate;
+		}
+		if (selected) {
+			ranges = selected.ranges;
+			omitted = proposedOmitted;
+			witnesses.push(selected.witness);
+		}
+	}
+	for (const witness of witnesses) {
+		const proposed = mergeRanges([...ranges, {
+			start: Math.max(0, witness.start - CONTEXT_CHARS),
+			end: Math.min(text.length, witness.end + CONTEXT_CHARS),
+		}]);
+		if (measure(proposed, true, omitted).length <= MAX_OUTPUT_CHARS) ranges = proposed;
+	}
+	const allOccurrences = mergeRanges([...ranges, ...matches.map(match => ({ start: match.start, end: match.end }))]);
+	if (measure(allOccurrences, true, omitted).length <= MAX_OUTPUT_CHARS) ranges = allOccurrences;
+	return formatRanges(ranges, true, omitted);
 }

package/node_modules/pi-web-access/curator-page.ts

@@ -45,6 +45,7 @@ function buildProviderButtons(
 		{ value: "serpbase", label: "SerpBase", available: available.serpbase },
 		{ value: "serpapi", label: "SerpApi", available: available.serpapi },
 		{ value: "serper", label: "Serper", available: available.serper },
+		{ value: "serply", label: "Serply", available: available.serply },
 		{ value: "valyu", label: "Valyu", available: available.valyu },
 	];
 
@@ -1472,7 +1473,7 @@ const SCRIPT = `(function() {
   var token = DATA.sessionToken;
   var timeoutSec = DATA.timeout;
   var queries = Array.isArray(DATA.queries) ? DATA.queries : [];
-  var providers = ["all", "openai", "exa", "brave", "parallel", "parallel-mcp", "tinyfish", "search1api", "searchinfinity", "querit", "tavily", "firecrawl", "jina", "serpdive", "kagi", "bocha", "ollama", "searxng", "duckduckgo", "perplexity", "gemini", "kimi", "anysearch", "xcrawl", "xai", "mistral", "brightdata", "serpbase", "serpapi", "serper", "valyu"];
+  var providers = ["auto", "all", "openai", "exa", "brave", "parallel", "parallel-mcp", "tinyfish", "search1api", "searchinfinity", "querit", "tavily", "firecrawl", "jina", "serpdive", "kagi", "bocha", "ollama", "searxng", "duckduckgo", "perplexity", "gemini", "kimi", "anysearch", "xcrawl", "xai", "mistral", "brightdata", "serpbase", "serpapi", "serper", "serply", "valyu"];
   var availProviders = DATA.availableProviders && typeof DATA.availableProviders === "object" ? DATA.availableProviders : {};
   var workflow = "summary-review";
   var initialDefaultProvider = typeof DATA.defaultProvider === "string" ? DATA.defaultProvider : "exa";
@@ -1676,6 +1677,7 @@ const SCRIPT = `(function() {
   }
 
   function providerLabel(provider) {
+    if (provider === "auto") return "Auto";
     if (provider === "all") return "All";
     if (provider === "openai") return "OpenAI";
     if (provider === "brave") return "Brave";
@@ -1705,6 +1707,7 @@ const SCRIPT = `(function() {
     if (provider === "serpbase") return "SerpBase";
     if (provider === "serpapi") return "SerpApi";
     if (provider === "serper") return "Serper";
+    if (provider === "serply") return "Serply";
     if (provider === "valyu") return "Valyu";
     return "Unknown";
   }

package/node_modules/pi-web-access/curator-run.ts

@@ -7,7 +7,8 @@ export function resolveWebSearchWorkflow(input: unknown, hasUI: boolean): WebSea
 	if (normalized === "auto-summary") return "auto-summary";
 	if (!hasUI) return "none";
 	if (normalized === "none") return "none";
-	return "summary-review";
+	if (normalized === "summary-review") return "summary-review";
+	return "none";
 }
 
 export class CuratorRunState {

package/node_modules/pi-web-access/curator-server.ts

@@ -304,6 +304,7 @@ export function startCuratorServer(
 		if (provider === "brightdata") return availableProviders.brightdata;
 		if (provider === "serpbase") return availableProviders.serpbase;
 		if (provider === "serper") return availableProviders.serper;
+		if (provider === "serply") return availableProviders.serply;
 		if (provider === "valyu") return availableProviders.valyu;
 		return false;
 	}