@browserwright/pi 0.18.1 → 0.18.3

package/README.md

@@ -4,12 +4,14 @@ Two tools for [pi](https://github.com/badlogic/pi-mono), backed by **declarative
 providers** that drive [browserwright](https://github.com/broven/browserwright):
 
 ```
-bw_web_fetch(url, provider?)   → the page as Markdown
+bw_web_fetch(url, provider?)   → the page as Markdown, or raw text for text endpoints
 bw_web_search(query, provider?) → ranked links + the SERP features Google showed
 ```
 
-Both run through the user's **own Chrome**, so they see what the user sees —
-including pages behind a login. Zero npm dependencies; `typebox` and the pi
+The browserwright paths run through the user's **own Chrome**, so they see what
+the user sees — including pages behind a login. Fetch also has a raw-text
+fallback for endpoints such as GitHub Raw; that fallback makes a direct request
+and returns the text body verbatim. Zero npm dependencies; `typebox` and the pi
 packages come from pi's own install.
 
 ## Install
@@ -106,17 +108,20 @@ truncated: 382 of 480 lines (49.7KB of 71.5KB) · full: /tmp/browserwright-pi-xx
 
 ## What ships, and what does not
 
-This package ships **only the browserwright rungs**. There is one per tool:
+This package ships the browserwright-backed rungs plus a text fallback for fetch.
+There is one browserwright rung per tool:
 
 | tool | provider | kind |
 |------|----------|------|
-| `bw_web_fetch` | `browserwright` | `command` — `browserwright markdown <url>` |
+| `bw_web_fetch` | `browserwright` → `raw` | `command` — browser-rendered HTML, then `module` — text body verbatim |
 | `bw_web_search` | `browserwright-search` | `module` — a session lifecycle in TS |
 
-That is a real trade-off, and it points the wrong way for casual fetches: every
-`bw_web_fetch` opens a tab in the daily browser and takes ~4-7s, where a hosted
-reader API answers in ~1s without touching Chrome. What you get for it is login
-state and full JS rendering, which no anonymous rung has.
+That is a real trade-off, and it points the wrong way for casual HTML fetches: a
+browserwright `bw_web_fetch` opens a tab in the daily browser and takes ~4-7s,
+where a hosted reader API answers in ~1s without touching Chrome. What you get
+for the browser rung is login state and full JS rendering. Text endpoints such
+as GitHub Raw skip the browser conversion failure and are returned verbatim by
+the `raw` fallback.
 
 **The chain engine is still here.** Drop your own JSON into `providers/` to add a
 cheaper or anonymous rung ahead of the browser one — nothing needs to be
@@ -204,7 +209,8 @@ extraction has to run against the live DOM rather than the document response.
 ### `returns`
 
 `markdown` | `html` | `text` | `results`. **The core never converts between
-them**; it only labels the output so the model knows what it is reading.
+them**; it only labels the output so the model knows what it is reading. The
+built-in `raw` fetch provider returns accepted text response bodies unchanged.
 
 ## failWhen: the reason the chain exists
 
@@ -222,9 +228,17 @@ legitimately inside raw HTML and inside search results *about* JavaScript.
 | field | applies to | note |
 |-------|-----------|------|
 | `minChars` | text payloads only | default 0 (off) |
-| `minResults` | list payloads only | an empty list is rejected regardless |
+| `minResults` | list payloads only | an empty list is rejected regardless, unless the engine asserted it |
 | `matches` | both | searched in the text, or in joined titles + snippets |
 
+The one exception is an **asserted** empty. When a search provider reports
+`noMatch: true` — Google says "did not match any documents" in prose, on an
+ordinary HTTP 200 page — the empty list is the engine's answer and is accepted,
+`minResults` included. Everything else empty is still rejected, because that is
+what a consent wall or a captcha looks like from here. Without the distinction a
+query the engine answered correctly reaches the model as `bw_web_search failed`,
+which reads as broken tooling rather than as a query worth rewriting.
+
 `minChars` is deliberately not applied to a list, and `minResults` not to text:
 the two floors measure different things, and applying both would reject a short
 but complete set of hits.

package/config.json

@@ -1,6 +1,6 @@
 {
   "order": {
-    "fetch": ["browserwright"],
+    "fetch": ["browserwright", "raw"],
     "search": ["browserwright-search"]
   },
   "defaultFailWhen": {

package/core/config.ts

@@ -24,13 +24,14 @@ const LOG_PREFIX = "[browserwright-pi]";
 
 const DEFAULT_CONFIG: PiConfig = {
 	order: {
-		fetch: ["browserwright"],
+		fetch: ["browserwright", "raw"],
 		search: ["browserwright-search"],
 	},
 	// The default line of defence. minChars stays 0 on purpose: a false positive
 	// escalates to a rung that opens a tab in the user's real Chrome, so
 	// over-eager rejection interrupts them. Per-provider thresholds are meant to
-	// come from `/bw probe` evidence, not from guesses.
+	// come from `/bw probe` evidence, not from guesses. The raw text rung opts out
+	// of the phrase list because source files may legitimately contain those words.
 	defaultFailWhen: {
 		minChars: 0,
 		minResults: 0,

package/core/format.ts

@@ -154,6 +154,20 @@ export function renderResults(result: ChainResult<SearchPayload>, query: string)
 		if (lines.length > 0) out.push("", "## Knowledge panel", ...lines);
 	}
 
+	if (results.length === 0) {
+		// Reaching the success path with no rows means the engine said so itself
+		// (see SearchPayload.noMatch) — everything else is rejected upstream. Say
+		// which it was, and say what to do next: the model's move here is to
+		// rewrite the query, not to conclude the tool is broken.
+		out.push(
+			"",
+			payload.noMatch
+				? "The search engine states that nothing matched this query. That is its answer, not a failure — " +
+					"widen the query before retrying: drop a `site:` path, a quoted phrase, or the least essential keywords."
+				: "No rows were extracted from the results page.",
+		);
+	}
+
 	if (results.length > 0) {
 		out.push("");
 		for (const row of results) {
@@ -171,7 +185,7 @@ export function renderResults(result: ChainResult<SearchPayload>, query: string)
 		out.push("## Related searches", payload.relatedSearches.join(" · "), "");
 	}
 
-	out.push("Use bw_web_fetch on a URL above to read it.");
+	if (results.length > 0) out.push("Use bw_web_fetch on a URL above to read it.");
 	return out.join("\n");
 }
 

package/core/predicates.ts

@@ -74,6 +74,7 @@ export const inspectText: Inspector<string> = (content) => ({ text: content });
 export const inspectSearch: Inspector<SearchPayload> = (payload) => ({
 	count: payload.results.length,
 	text: payload.results.map((r) => `${r.title} ${r.snippet ?? ""}`).join("\n"),
+	authoritativeEmpty: payload.noMatch === true,
 });
 
 /**
@@ -89,9 +90,13 @@ export function failureReason<T>(value: T, rule: FailWhen, inspect: Inspector<T>
 	const count = inspected.count;
 
 	if (count !== undefined) {
-		// A search that parsed cleanly but found nothing is a failure worth
-		// falling through on: an interstitial usually yields a valid, empty list
-		// rather than an error.
+		// An engine that said "no documents matched" has answered the question.
+		// Accepting that is what stops a correctly-answered query from reaching
+		// the model as a failed tool call; the needle scan below is skipped with
+		// it, which costs nothing because the haystack is empty either way.
+		if (count === 0 && inspected.authoritativeEmpty) return undefined;
+		// Any other empty list is a failure worth falling through on: an
+		// interstitial usually yields a valid, empty list rather than an error.
 		if (count === 0) return "no results";
 		const minResults = rule.minResults ?? 0;
 		if (minResults > 0 && count < minResults) {

package/core/results.ts

@@ -20,6 +20,7 @@ const ANSWER_KEYS = ["answerBox", "answer_box", "answer", "featured_snippet"] as
 const KG_KEYS = ["knowledgeGraph", "knowledge_graph"] as const;
 const PAA_KEYS = ["peopleAlsoAsk", "people_also_ask", "relatedQuestions"] as const;
 const RELATED_KEYS = ["relatedSearches", "related_searches", "relatedQueries"] as const;
+const NO_MATCH_KEYS = ["noMatch", "no_match", "zeroResults"] as const;
 
 function firstValue(row: Record<string, unknown>, keys: readonly string[]): unknown {
 	for (const key of keys) {
@@ -137,5 +138,9 @@ export function normalizeSearchPayload(value: unknown): SearchPayload {
 	const relatedSearches = stringList(firstValue(body, RELATED_KEYS));
 	if (relatedSearches) payload.relatedSearches = relatedSearches;
 
+	// Only meaningful next to an empty list. A source claiming both rows and
+	// "nothing matched" is contradicting itself, and the rows are the evidence.
+	if (payload.results.length === 0 && firstValue(body, NO_MATCH_KEYS) === true) payload.noMatch = true;
+
 	return payload;
 }

package/core/types.ts

@@ -60,6 +60,18 @@ export interface KnowledgeGraph {
  */
 export interface SearchPayload {
 	results: SearchResult[];
+	/**
+	 * The engine stated, in so many words, that nothing matched.
+	 *
+	 * Only ever set alongside an empty `results`, and it is the one thing that
+	 * separates an honestly empty search from the failure mode that looks
+	 * identical from the outside: a captcha or consent wall parses perfectly and
+	 * also yields zero rows. Without this flag both have to be rejected, and a
+	 * query the engine answered correctly is reported to the model as a broken
+	 * tool — which is exactly what it then works around instead of rewriting the
+	 * query.
+	 */
+	noMatch?: boolean;
 	answerBox?: AnswerBox;
 	knowledgeGraph?: KnowledgeGraph;
 	peopleAlsoAsk?: string[];
@@ -236,6 +248,12 @@ export interface Inspected {
 	text: string;
 	/** Item count, for list payloads. Undefined for text payloads. */
 	count?: number;
+	/**
+	 * A zero count that the source asserted rather than one we inferred from a
+	 * payload we could not read. Lifts the blanket rejection of an empty list —
+	 * see `SearchPayload.noMatch`.
+	 */
+	authoritativeEmpty?: boolean;
 }
 
 export type Inspector<T> = (value: T) => Inspected;

package/index.ts

@@ -58,14 +58,14 @@ export default function (pi: ExtensionAPI) {
 		name: "bw_web_fetch",
 		label: "Fetch Web Page",
 		description:
-			"Fetch a URL and return its content as Markdown. " +
+			"Fetch a URL and return its content as Markdown or text. " +
 			`Tries providers in order until one returns usable content: ${config.order.fetch.join(" → ")}. ` +
 			"The response header states which provider answered and what format the body is in. " +
 			"Output over 50KB is truncated and the full text written to a temp file whose path is given.",
-		promptSnippet: "Fetch a URL as markdown, through the user's real browser",
+		promptSnippet: "Fetch a URL as Markdown or text, through the user's real browser",
 		promptGuidelines: [
 			"Prefer `bw_web_fetch` over curl or a shell HTTP client for reading web pages — it renders JavaScript " +
-				"and carries the user's login state, so it can read pages an anonymous request cannot.",
+				"and carries the user's login state, while its text fallback also reads raw source endpoints.",
 		],
 		parameters: Type.Object({
 			url: Type.String({ description: "HTTP(S) URL to fetch" }),
@@ -180,6 +180,9 @@ export default function (pi: ExtensionAPI) {
 					query,
 					provider: result.provider,
 					count: result.content?.results.length ?? 0,
+					// Zero rows is a success now, so the trace has to record which
+					// kind of zero it was.
+					noMatch: result.content?.noMatch === true,
 					features: {
 						answerBox: Boolean(result.content?.answerBox),
 						knowledgeGraph: Boolean(result.content?.knowledgeGraph),

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@browserwright/pi",
-  "version": "0.18.1",
-  "description": "bw_web_fetch and bw_web_search for the pi coding agent, driving the user's real browser through browserwright",
+  "version": "0.18.3",
+  "description": "bw_web_fetch and bw_web_search for the pi coding agent, using browserwright plus a raw text fallback",
   "license": "AGPL-3.0-only",
   "type": "module",
   "keywords": [

package/providers/browserwright-search.json

@@ -31,7 +31,11 @@
     "them as a reason.",
     "minResults 1 is the real guard. A captcha or consent wall parses perfectly",
     "and yields an empty list, which is the failure mode that would otherwise be",
-    "indistinguishable from an honestly empty search.",
+    "indistinguishable from an honestly empty search. The extractor tells the two",
+    "apart by reading Google's own sentence ('did not match any documents') off",
+    "the page and setting noMatch, which lifts minResults for that one case: an",
+    "engine that says nothing matched has answered the query, and reporting that",
+    "as a tool failure only teaches the model to route around a working tool.",
     "searchUrl is a template so the engine can be swapped without touching code.",
     "{queryEncoded} and {limit} are the only substitutions.",
     "SERP features (measured 2026-08-10, see docs/adr/0008). The page is server-",

package/providers/browserwright-search.ts

@@ -246,6 +246,17 @@ function buildScript(query: string, limit: number, outPath: string, searchUrl: s
     return out.slice(0, 10);
   }, []);
 
+  // --- explicit "nothing matched" ---------------------------------------
+  // Google says this in prose on an otherwise ordinary page: no error, no
+  // interstitial, HTTP 200, zero rows. Reading the sentence is the only way to
+  // tell that empty list apart from the one a consent wall produces. Measured
+  // 2026-09-02 on a site:-with-path query, which is the shape that reaches zero
+  // most often.
+  const noMatch = attempt(() => {
+    const body = (document.body.innerText || '').toLowerCase();
+    return ['did not match any documents', 'no results found for'].some((n) => body.includes(n));
+  }, false);
+
   // --- related searches --------------------------------------------------
   // Scoped to #botstuff: the same href pattern at the top of the page is
   // Google's own tab bar ("Images", "News", "Past hour"), not a related query.
@@ -266,7 +277,7 @@ function buildScript(query: string, limit: number, outPath: string, searchUrl: s
     return out.slice(0, 10);
   }, []);
 
-  return { results, answerBox, knowledgeGraph, peopleAlsoAsk, relatedSearches };
+  return { results, answerBox, knowledgeGraph, peopleAlsoAsk, relatedSearches, noMatch };
 }`;
 
 	// json.dumps gives us correctly escaped Python string literals for free, so
@@ -310,6 +321,9 @@ function buildScript(query: string, limit: number, outPath: string, searchUrl: s
 		'            "knowledgeGraph": data.get("knowledgeGraph"),',
 		'            "peopleAlsoAsk": data.get("peopleAlsoAsk") or [],',
 		'            "relatedSearches": data.get("relatedSearches") or [],',
+		// Only ever consulted when rows is empty, but carried unconditionally so
+		// the shape of the payload does not depend on the outcome.
+		'            "noMatch": bool(data.get("noMatch")),',
 		"        }",
 		"        if not rows:",
 		// An interstitial parses fine and yields zero rows; say which kind it was.

package/providers/raw-text.ts

@@ -0,0 +1,78 @@
+/**
+ * The raw text fetch rung.
+ *
+ * `browserwright markdown` deliberately only converts HTML. Public source
+ * endpoints such as raw.githubusercontent.com return the source as text/plain,
+ * so they need a reader that returns the response body verbatim instead of
+ * trying to turn a browser's non-HTML document into Markdown.
+ */
+
+import type { ModuleContext, ProviderOutcome } from "../core/types.ts";
+
+/**
+ * MIME types whose response body is safe to hand back as text.
+ *
+ * `text/*` covers source files served as text/plain. The application types
+ * cover APIs and source/document formats that are commonly served without a
+ * text/* MIME type. Binary responses such as application/pdf and
+ * application/octet-stream are intentionally not decoded here.
+ */
+const TEXT_APPLICATION_TYPES = new Set([
+	"application/graphql",
+	"application/javascript",
+	"application/json",
+	"application/ld+json",
+	"application/manifest+json",
+	"application/sql",
+	"application/toml",
+	"application/typescript",
+	"application/xml",
+	"application/x-javascript",
+	"application/x-yaml",
+	"application/yaml",
+]);
+
+export function isTextContentType(raw: string | null | undefined): boolean {
+	const contentType = (raw ?? "").split(";", 1)[0].trim().toLowerCase();
+	return contentType.startsWith("text/") || TEXT_APPLICATION_TYPES.has(contentType);
+}
+
+export default async function rawText(
+	subject: string,
+	ctx: ModuleContext,
+): Promise<ProviderOutcome<string>> {
+	let response: Response;
+	try {
+		response = await fetch(subject, {
+			redirect: "follow",
+			signal: ctx.signal,
+		});
+	} catch (error) {
+		if (ctx.signal?.aborted) return { ok: false, reason: "aborted" };
+		return { ok: false, reason: `fetch failed: ${(error as Error).message}` };
+	}
+
+	const contentType = response.headers.get("content-type") ?? "";
+	if (!isTextContentType(contentType)) {
+		const type = contentType.split(";", 1)[0].trim() || "unknown";
+		return {
+			ok: false,
+			status: response.status,
+			reason: `not a text response (Content-Type: ${type})`,
+		};
+	}
+
+	let text: string;
+	try {
+		text = await response.text();
+	} catch (error) {
+		return { ok: false, status: response.status, reason: `could not read response: ${(error as Error).message}` };
+	}
+
+	if (!response.ok) {
+		const hint = text.slice(0, 200).replace(/\s+/g, " ").trim();
+		return { ok: false, status: response.status, reason: `http ${response.status}${hint ? `: ${hint}` : ""}` };
+	}
+
+	return { ok: true, content: text, status: response.status };
+}

package/providers/raw.json

@@ -0,0 +1,16 @@
+{
+  "name": "raw",
+  "role": "fetch",
+  "kind": "module",
+  "module": "./providers/raw-text.ts",
+  "returns": "text",
+  "timeoutMs": 30000,
+  "failWhen": {
+    "matches": []
+  },
+  "_note": [
+    "Returns text responses verbatim for endpoints such as raw.githubusercontent.com.",
+    "The browserwright provider remains first, so HTML gets browser rendering and the user's login state when available.",
+    "Binary responses are rejected rather than decoded as corrupt text."
+  ]
+}