@browserwright/pi 0.0.1

package/core/probe.ts

@@ -0,0 +1,182 @@
+/**
+ * Probe: learn what a provider's failures actually look like.
+ *
+ * A provider's failWhen rules are supposed to come from evidence, not from
+ * guesses, so this runs one provider against a fixed set of real URLs chosen
+ * to sit on its capability boundary (SPA shell, bot wall, login wall, PDF,
+ * huge page, localhost) and reports what came back.
+ *
+ * Two deliberate constraints:
+ *   - Probe is manual only. It hits real sites, and the browserwright rung
+ *     opens tabs in the user's own browser.
+ *   - Evidence files store summaries only, never whole pages. Real pages can
+ *     carry the user's logged-in content.
+ */
+
+import { existsSync, readFileSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { makeExecutor } from "./chain.ts";
+import { EXTENSION_DIR } from "./config.ts";
+import { failureReason, inspectText, resolveFailWhen } from "./predicates.ts";
+import type { PiConfig, Provider } from "./types.ts";
+
+export interface ProbeCase {
+	name: string;
+	url: string;
+	/** What this case is meant to expose. Kept in the evidence file. */
+	expose: string;
+}
+
+export interface ProbeRow {
+	case: string;
+	url: string;
+	expose: string;
+	/** Transport-level success, before failWhen is applied. */
+	fetched: boolean;
+	ms: number;
+	chars: number;
+	/** Why the transport failed. */
+	error?: string;
+	/** Why the current failWhen rule would reject the content, if it would. */
+	wouldReject?: string;
+	/** First 200 characters, whitespace collapsed. The signature to write rules from. */
+	head?: string;
+}
+
+const DEFAULT_CASES: ProbeCase[] = [
+	{ name: "article", url: "https://en.wikipedia.org/wiki/Markdown", expose: "normal long article" },
+	{ name: "spa-shell", url: "https://web.telegram.org/", expose: "client-rendered shell, no server HTML" },
+	{ name: "bot-wall", url: "https://www.g2.com/", expose: "bot challenge / interstitial" },
+	{ name: "login-wall", url: "https://x.com/elonmusk", expose: "needs the user's session cookies" },
+	{ name: "not-found", url: "https://example.com/definitely-not-here", expose: "404 handling" },
+	{
+		name: "huge",
+		url: "https://en.wikipedia.org/wiki/List_of_Latin_phrases_(full)",
+		expose: "page far past the 50KB truncation limit",
+	},
+	{
+		name: "pdf",
+		url: "https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf",
+		expose: "non-HTML content type",
+	},
+	{ name: "localhost", url: "http://localhost:3000/", expose: "local dev server (remote providers cannot reach it)" },
+];
+
+export function loadProbeCases(dir: string = EXTENSION_DIR): ProbeCase[] {
+	const path = join(dir, "probe-cases.json");
+	if (!existsSync(path)) return DEFAULT_CASES;
+	try {
+		const parsed = JSON.parse(readFileSync(path, "utf8")) as ProbeCase[];
+		return Array.isArray(parsed) && parsed.length > 0 ? parsed : DEFAULT_CASES;
+	} catch (error) {
+		console.error(`[webfetch] probe-cases.json unreadable (${(error as Error).message}), using defaults`);
+		return DEFAULT_CASES;
+	}
+}
+
+export async function runProbe(
+	provider: Provider,
+	cases: ProbeCase[],
+	options: {
+		config: PiConfig;
+		dir: string;
+		onCase?: (probeCase: ProbeCase, index: number) => void;
+	},
+): Promise<ProbeRow[]> {
+	// Bypass the chain on purpose: probing is about one provider's raw
+	// behaviour, including on URLs its `when` predicate would normally skip.
+	// Probe cases are URLs, so a probe run is always a fetch-role call.
+	const executor = makeExecutor<string>(options.config, { dir: options.dir, role: "fetch" });
+	const rule = resolveFailWhen(options.config.defaultFailWhen, provider.failWhen);
+	const rows: ProbeRow[] = [];
+
+	for (const [index, probeCase] of cases.entries()) {
+		// Respect a provider's declared rate limit, or its own limiter turns the
+		// probe into a page of 429s that read like "this provider cannot do it".
+		if (index > 0 && provider.probeDelayMs) {
+			await new Promise((resolve) => setTimeout(resolve, provider.probeDelayMs));
+		}
+		options.onCase?.(probeCase, index);
+		const startedAt = performance.now();
+		let outcome: Awaited<ReturnType<typeof executor>>;
+		try {
+			outcome = await executor(provider, probeCase.url);
+		} catch (error) {
+			outcome = { ok: false, reason: (error as Error).message };
+		}
+		const ms = Math.round(performance.now() - startedAt);
+
+		if (!outcome.ok || outcome.content === undefined) {
+			rows.push({
+				case: probeCase.name,
+				url: probeCase.url,
+				expose: probeCase.expose,
+				fetched: false,
+				ms,
+				chars: 0,
+				error: outcome.reason ?? "failed",
+			});
+			continue;
+		}
+
+		rows.push({
+			case: probeCase.name,
+			url: probeCase.url,
+			expose: probeCase.expose,
+			fetched: true,
+			ms,
+			chars: outcome.content.length,
+			wouldReject: failureReason(outcome.content, rule, inspectText),
+			head: outcome.content.replace(/\s+/g, " ").trim().slice(0, 200),
+		});
+	}
+
+	return rows;
+}
+
+export function formatProbeReport(providerName: string, rows: ProbeRow[]): string {
+	const lines = [`webfetch probe — ${providerName}`, ""];
+	for (const row of rows) {
+		const verdict = !row.fetched
+			? `ERROR ${row.error}`
+			: row.wouldReject
+				? `REJECTED (${row.wouldReject})`
+				: "accepted";
+		lines.push(`${row.case.padEnd(11)} ${String(row.chars).padStart(7)} chars  ${String(row.ms).padStart(5)}ms  ${verdict}`);
+		if (row.head) lines.push(`            head: ${row.head.slice(0, 140)}`);
+	}
+	lines.push(
+		"",
+		"Write failWhen rules for this provider from the signatures above:",
+		"a `matches` entry for each wall/shell phrase, and minChars only if a",
+		"legitimately short page cannot be confused with an empty one.",
+	);
+	return lines.join("\n");
+}
+
+export function saveProbeEvidence(providerName: string, rows: ProbeRow[], dir: string = EXTENSION_DIR): string {
+	const payload = {
+		provider: providerName,
+		// Stamped by the caller's clock; probe results drift as sites change,
+		// so a rule traced back to this file needs to know how old it is.
+		probedAt: new Date().toISOString(),
+		note: "Summaries only — never whole pages, which may contain the user's logged-in content.",
+		rows,
+	};
+	const body = `${JSON.stringify(payload, null, 2)}\n`;
+	const path = join(dir, "providers", `${providerName}.probe.json`);
+
+	// Evidence belongs next to the declaration it justifies — but when this
+	// package is installed from npm it lives under node_modules, which is both
+	// read-only in some setups and wiped on the next update. Fall back rather
+	// than losing a probe run that just spent a minute opening real tabs.
+	try {
+		writeFileSync(path, body, "utf8");
+		return path;
+	} catch {
+		const fallback = join(tmpdir(), `${providerName}.probe.json`);
+		writeFileSync(fallback, body, "utf8");
+		return fallback;
+	}
+}

package/core/results.ts

@@ -0,0 +1,141 @@
+/**
+ * Coercing an arbitrary JSON list into SearchResult rows.
+ *
+ * Every hosted search API returns the same three facts under different names
+ * (`link` vs `url` vs `href`, `snippet` vs `description` vs `content`). Doing
+ * the aliasing here is what keeps a new search provider a pure JSON drop-in
+ * instead of a code change — the same promise `kind: "http"` already makes for
+ * reader APIs.
+ */
+
+import type { AnswerBox, KnowledgeGraph, SearchPayload, SearchResult } from "./types.ts";
+
+const TITLE_KEYS = ["title", "name", "heading"] as const;
+const URL_KEYS = ["url", "link", "href"] as const;
+const SNIPPET_KEYS = ["snippet", "description", "content", "text", "excerpt"] as const;
+const DATE_KEYS = ["date", "published", "publishedDate", "published_date"] as const;
+
+const RESULTS_KEYS = ["results", "organic", "organic_results", "items", "webPages"] as const;
+const ANSWER_KEYS = ["answerBox", "answer_box", "answer", "featured_snippet"] as const;
+const KG_KEYS = ["knowledgeGraph", "knowledge_graph"] as const;
+const PAA_KEYS = ["peopleAlsoAsk", "people_also_ask", "relatedQuestions"] as const;
+const RELATED_KEYS = ["relatedSearches", "related_searches", "relatedQueries"] as const;
+
+function firstValue(row: Record<string, unknown>, keys: readonly string[]): unknown {
+	for (const key of keys) {
+		if (row[key] !== undefined && row[key] !== null) return row[key];
+	}
+	return undefined;
+}
+
+/** Pull a list of plain strings out of whatever shape the API used for it. */
+function stringList(value: unknown): string[] | undefined {
+	if (!Array.isArray(value)) return undefined;
+	const out: string[] = [];
+	for (const item of value) {
+		if (typeof item === "string" && item.trim()) {
+			out.push(item.trim());
+			continue;
+		}
+		if (item && typeof item === "object") {
+			// Serper-style rows: {question: "..."} / {query: "..."}
+			const row = item as Record<string, unknown>;
+			const text = firstString(row, ["question", "query", "title", "text", "name"]);
+			if (text) out.push(text);
+		}
+	}
+	return out.length > 0 ? [...new Set(out)] : undefined;
+}
+
+function firstString(row: Record<string, unknown>, keys: readonly string[]): string | undefined {
+	for (const key of keys) {
+		const value = row[key];
+		if (typeof value === "string" && value.trim()) return value.trim();
+	}
+	return undefined;
+}
+
+/**
+ * Keep only rows that carry at least a URL — a row without one is not a search
+ * result the model can act on, and silently keeping it would inflate the count
+ * that `minResults` guards.
+ */
+export function normalizeResults(value: unknown): SearchResult[] {
+	if (!Array.isArray(value)) return [];
+
+	const out: SearchResult[] = [];
+	for (const item of value) {
+		if (!item || typeof item !== "object") continue;
+		const row = item as Record<string, unknown>;
+		const url = firstString(row, URL_KEYS);
+		if (!url) continue;
+		out.push({
+			position: out.length + 1,
+			title: firstString(row, TITLE_KEYS) ?? url,
+			url,
+			snippet: firstString(row, SNIPPET_KEYS),
+			date: firstString(row, DATE_KEYS),
+		});
+	}
+	return out;
+}
+
+function normalizeAnswerBox(value: unknown): AnswerBox | undefined {
+	if (typeof value === "string") {
+		return value.trim() ? { kind: "featured-snippet", text: value.trim() } : undefined;
+	}
+	if (!value || typeof value !== "object") return undefined;
+	const row = value as Record<string, unknown>;
+	const text = firstString(row, ["text", "answer", "snippet", "description", "content"]);
+	if (!text) return undefined;
+	const kind = firstString(row, ["kind", "type"]);
+	return { kind: kind === "ai-overview" ? "ai-overview" : "featured-snippet", text };
+}
+
+function normalizeKnowledgeGraph(value: unknown): KnowledgeGraph | undefined {
+	if (!value || typeof value !== "object") return undefined;
+	const row = value as Record<string, unknown>;
+	const attributes = row.attributes;
+	const graph: KnowledgeGraph = {
+		title: firstString(row, ["title", "name"]),
+		subtitle: firstString(row, ["subtitle", "type", "category"]),
+		description: firstString(row, ["description", "snippet"]),
+	};
+	if (attributes && typeof attributes === "object" && !Array.isArray(attributes)) {
+		const flat: Record<string, string> = {};
+		for (const [key, raw] of Object.entries(attributes as Record<string, unknown>)) {
+			if (typeof raw === "string" && raw.trim()) flat[key] = raw.trim();
+		}
+		if (Object.keys(flat).length > 0) graph.attributes = flat;
+	}
+	return graph.title || graph.description || graph.attributes ? graph : undefined;
+}
+
+/**
+ * Coerce a whole search response into a SearchPayload.
+ *
+ * Accepts a bare array (just organic rows) or an object keyed the way hosted
+ * APIs key it. That aliasing is what keeps a hosted search provider a pure JSON
+ * drop-in: a Serper response, for instance, maps field-for-field with no code.
+ */
+export function normalizeSearchPayload(value: unknown): SearchPayload {
+	if (Array.isArray(value)) return { results: normalizeResults(value) };
+	if (!value || typeof value !== "object") return { results: [] };
+
+	const body = value as Record<string, unknown>;
+	const payload: SearchPayload = { results: normalizeResults(firstValue(body, RESULTS_KEYS)) };
+
+	const answerBox = normalizeAnswerBox(firstValue(body, ANSWER_KEYS));
+	if (answerBox) payload.answerBox = answerBox;
+
+	const knowledgeGraph = normalizeKnowledgeGraph(firstValue(body, KG_KEYS));
+	if (knowledgeGraph) payload.knowledgeGraph = knowledgeGraph;
+
+	const peopleAlsoAsk = stringList(firstValue(body, PAA_KEYS));
+	if (peopleAlsoAsk) payload.peopleAlsoAsk = peopleAlsoAsk;
+
+	const relatedSearches = stringList(firstValue(body, RELATED_KEYS));
+	if (relatedSearches) payload.relatedSearches = relatedSearches;
+
+	return payload;
+}

package/core/types.ts

@@ -0,0 +1,241 @@
+/**
+ * Shared types for the provider contract.
+ *
+ * A provider is a declaration, not code. Three kinds are supported:
+ *   - "http"    : issue one HTTP request, optionally pluck a field out of JSON
+ *   - "command" : run one argv, take stdout
+ *   - "module"  : hand off to a TS module in this package — the escape hatch for
+ *                 a provider that needs real logic (a session lifecycle, retry,
+ *                 progress reporting) rather than one shot at a subprocess.
+ *
+ * Everything a provider needs to say about itself lives in its JSON file: which
+ * tool it serves, what format it returns, when it is applicable, and what its
+ * output looks like when it has failed despite reporting success.
+ */
+
+/** Which tool a provider serves. A provider belongs to exactly one. */
+export type Role = "fetch" | "search";
+
+export const ROLES: readonly Role[] = ["fetch", "search"];
+
+/** What the provider hands back. The core never converts between these. */
+export type ReturnFormat = "markdown" | "html" | "text" | "results";
+
+/** One organic row of a `search` provider's payload. */
+export interface SearchResult {
+	position: number;
+	title: string;
+	url: string;
+	snippet?: string;
+	/** Publication date, when the engine states one separately from the snippet. */
+	date?: string;
+}
+
+/**
+ * The engine's own direct answer.
+ *
+ * `ai-overview` is Google's generated summary. Measured 2026-08-10: its body is
+ * **not** in the server-rendered HTML at all — it streams in afterwards — which
+ * is the single reason this extractor has to run against the live DOM rather
+ * than the document response.
+ */
+export interface AnswerBox {
+	kind: "ai-overview" | "featured-snippet";
+	text: string;
+}
+
+/** The entity panel: what the engine thinks the query is *about*. */
+export interface KnowledgeGraph {
+	title?: string;
+	subtitle?: string;
+	description?: string;
+	/** Remaining labelled facts, keyed by the engine's own attribute name. */
+	attributes?: Record<string, string>;
+}
+
+/**
+ * What a `search` provider returns. Organic rows are the contract; everything
+ * else is present only when that query happened to trigger it, so every
+ * consumer must treat the optional fields as absent by default.
+ */
+export interface SearchPayload {
+	results: SearchResult[];
+	answerBox?: AnswerBox;
+	knowledgeGraph?: KnowledgeGraph;
+	peopleAlsoAsk?: string[];
+	relatedSearches?: string[];
+}
+
+/**
+ * Subject predicate. All fields are optional; an empty match means "always".
+ * Globs are matched with `*` = any run of characters (no path semantics).
+ *
+ * The subject is a URL for `fetch` providers and the query string for `search`
+ * providers. `hostGlob` only means anything for the former — it silently never
+ * matches when the subject does not parse as a URL.
+ */
+export interface SubjectMatch {
+	/** Glob against the whole subject, e.g. "http://localhost*". */
+	urlGlob?: string[];
+	/** Glob against the hostname only, e.g. "*.local". URLs only. */
+	hostGlob?: string[];
+	/** Negation. `{ not: { hostGlob: ["localhost"] } }` = anything but localhost. */
+	not?: SubjectMatch;
+}
+
+/**
+ * When to treat a "successful" call as a failure and drop to the next rung.
+ *
+ * Per-provider values REPLACE the core defaults field by field; they do not
+ * merge. That is deliberate: `{"matches": []}` is how a provider opts out of
+ * the default match list entirely, which any provider returning raw HTML needs
+ * (a `<noscript>` block legitimately contains "enable JavaScript").
+ */
+export interface FailWhen {
+	/** Fail when the returned text is shorter than this. 0 disables. */
+	minChars?: number;
+	/** Fail when a list payload has fewer than this many items. 0 disables. */
+	minResults?: number;
+	/** Case-insensitive substrings that mark the payload as a wall/shell. */
+	matches?: string[];
+}
+
+interface ProviderCommon {
+	name: string;
+	/** Which tool this provider serves. Defaults to "fetch". */
+	role?: Role;
+	/** Human-readable label for status lines. Defaults to `name`. */
+	label?: string;
+	/** Set false to keep the declaration around without using it. */
+	enabled?: boolean;
+	returns: ReturnFormat;
+	/** Static capability declaration, e.g. "I am remote, I cannot see your LAN". */
+	when?: SubjectMatch;
+	failWhen?: FailWhen;
+	timeoutMs?: number;
+	/**
+	 * Extra attempts at this rung after a TRANSPORT failure. Default 0.
+	 *
+	 * Content rejected by `failWhen` is never retried: that verdict is
+	 * deterministic, so a second identical call only costs the user time (and,
+	 * for a browser rung, another tab).
+	 */
+	retries?: number;
+	/**
+	 * Case-insensitive substrings of the failure reason that make a retry
+	 * worthwhile. Omit to retry every transport failure. Use it to retry only
+	 * what is genuinely transient rather than, say, a 404.
+	 */
+	retryWhen?: string[];
+	/**
+	 * Milliseconds to wait between probe cases. A provider's own rate limit is a
+	 * static fact about it, like `when`. Probe-only: at runtime a 429 is just
+	 * another transport failure that drops a rung.
+	 */
+	probeDelayMs?: number;
+}
+
+export interface HttpProvider extends ProviderCommon {
+	kind: "http";
+	method?: "GET" | "POST";
+	/** Supports {url}/{query}, the {…Encoded} variants, {dir}, and $ENV_VAR. */
+	url: string;
+	headers?: Record<string, string>;
+	/** JSON body; string values support the same interpolation as `url`. */
+	body?: unknown;
+	/** Dot path into a JSON response, e.g. "result". Omit to use the raw body. */
+	pick?: string;
+}
+
+export interface CommandProvider extends ProviderCommon {
+	kind: "command";
+	/**
+	 * argv. Supports {url}/{query}, the {…Encoded} variants, and {dir}.
+	 * Exit code contract: 0 = success, 2 = not applicable (drop a rung),
+	 * anything else = hard error (also drops a rung, but is reported as an error).
+	 */
+	command: string[];
+	cwd?: string;
+	env?: Record<string, string>;
+}
+
+export interface ModuleProvider extends ProviderCommon {
+	kind: "module";
+	/** Module path relative to the package directory, e.g. "./providers/foo.ts". */
+	module: string;
+	/** Passed through to the runner verbatim. */
+	options?: Record<string, unknown>;
+}
+
+export type Provider = HttpProvider | CommandProvider | ModuleProvider;
+
+/** What a `kind: "module"` runner receives. */
+export interface ModuleContext {
+	/** The package directory — the same value `{dir}` interpolates to. */
+	dir: string;
+	timeoutMs: number;
+	signal?: AbortSignal;
+	options: Record<string, unknown>;
+	/**
+	 * Report intermediate progress. This is the capability a subprocess cannot
+	 * have, and the reason this provider kind exists.
+	 */
+	onProgress?: (text: string) => void;
+}
+
+/** A `kind: "module"` provider's default export. */
+export type ModuleRunner<T = unknown> = (subject: string, ctx: ModuleContext) => Promise<ProviderOutcome<T>>;
+
+/** Global config, from config.json. */
+export interface PiConfig {
+	/** Provider names per role, in the order they are attempted. */
+	order: Record<Role, string[]>;
+	/** Core default line of defence, overridable per provider. */
+	defaultFailWhen: FailWhen;
+	timeoutMs: number;
+	maxBytes: number;
+	maxLines: number;
+}
+
+/** One rung attempt, recorded for the chain trace. */
+export interface Attempt {
+	provider: string;
+	ok: boolean;
+	ms: number;
+	/** Why it failed or was skipped. Absent when ok. */
+	reason?: string;
+	/** True when the provider was never called (when-predicate or missing env). */
+	skipped?: boolean;
+}
+
+/** What a provider executor returns. */
+export interface ProviderOutcome<T = string> {
+	ok: boolean;
+	content?: T;
+	/** Populated on failure; becomes the Attempt reason. */
+	reason?: string;
+	/** HTTP status, when the executor knows one. */
+	status?: number;
+}
+
+export interface ChainResult<T = string> {
+	ok: boolean;
+	attempts: Attempt[];
+	provider?: string;
+	format?: ReturnFormat;
+	content?: T;
+}
+
+/**
+ * A payload reduced to the two things `failWhen` can reason about. Supplying
+ * one of these per role is what lets a single predicate guard both a Markdown
+ * blob and a list of search results.
+ */
+export interface Inspected {
+	/** The text the `matches` needles are searched in. */
+	text: string;
+	/** Item count, for list payloads. Undefined for text payloads. */
+	count?: number;
+}
+
+export type Inspector<T> = (value: T) => Inspected;