@browserwright/pi 0.18.1 → 0.18.2

package/README.md

@@ -4,12 +4,14 @@ Two tools for [pi](https://github.com/badlogic/pi-mono), backed by **declarative
 providers** that drive [browserwright](https://github.com/broven/browserwright):
 
 ```
-bw_web_fetch(url, provider?)   → the page as Markdown
+bw_web_fetch(url, provider?)   → the page as Markdown, or raw text for text endpoints
 bw_web_search(query, provider?) → ranked links + the SERP features Google showed
 ```
 
-Both run through the user's **own Chrome**, so they see what the user sees —
-including pages behind a login. Zero npm dependencies; `typebox` and the pi
+The browserwright paths run through the user's **own Chrome**, so they see what
+the user sees — including pages behind a login. Fetch also has a raw-text
+fallback for endpoints such as GitHub Raw; that fallback makes a direct request
+and returns the text body verbatim. Zero npm dependencies; `typebox` and the pi
 packages come from pi's own install.
 
 ## Install
@@ -106,17 +108,20 @@ truncated: 382 of 480 lines (49.7KB of 71.5KB) · full: /tmp/browserwright-pi-xx
 
 ## What ships, and what does not
 
-This package ships **only the browserwright rungs**. There is one per tool:
+This package ships the browserwright-backed rungs plus a text fallback for fetch.
+There is one browserwright rung per tool:
 
 | tool | provider | kind |
 |------|----------|------|
-| `bw_web_fetch` | `browserwright` | `command` — `browserwright markdown <url>` |
+| `bw_web_fetch` | `browserwright` → `raw` | `command` — browser-rendered HTML, then `module` — text body verbatim |
 | `bw_web_search` | `browserwright-search` | `module` — a session lifecycle in TS |
 
-That is a real trade-off, and it points the wrong way for casual fetches: every
-`bw_web_fetch` opens a tab in the daily browser and takes ~4-7s, where a hosted
-reader API answers in ~1s without touching Chrome. What you get for it is login
-state and full JS rendering, which no anonymous rung has.
+That is a real trade-off, and it points the wrong way for casual HTML fetches: a
+browserwright `bw_web_fetch` opens a tab in the daily browser and takes ~4-7s,
+where a hosted reader API answers in ~1s without touching Chrome. What you get
+for the browser rung is login state and full JS rendering. Text endpoints such
+as GitHub Raw skip the browser conversion failure and are returned verbatim by
+the `raw` fallback.
 
 **The chain engine is still here.** Drop your own JSON into `providers/` to add a
 cheaper or anonymous rung ahead of the browser one — nothing needs to be
@@ -204,7 +209,8 @@ extraction has to run against the live DOM rather than the document response.
 ### `returns`
 
 `markdown` | `html` | `text` | `results`. **The core never converts between
-them**; it only labels the output so the model knows what it is reading.
+them**; it only labels the output so the model knows what it is reading. The
+built-in `raw` fetch provider returns accepted text response bodies unchanged.
 
 ## failWhen: the reason the chain exists
 

package/config.json

@@ -1,6 +1,6 @@
 {
   "order": {
-    "fetch": ["browserwright"],
+    "fetch": ["browserwright", "raw"],
     "search": ["browserwright-search"]
   },
   "defaultFailWhen": {

package/core/config.ts

@@ -24,13 +24,14 @@ const LOG_PREFIX = "[browserwright-pi]";
 
 const DEFAULT_CONFIG: PiConfig = {
 	order: {
-		fetch: ["browserwright"],
+		fetch: ["browserwright", "raw"],
 		search: ["browserwright-search"],
 	},
 	// The default line of defence. minChars stays 0 on purpose: a false positive
 	// escalates to a rung that opens a tab in the user's real Chrome, so
 	// over-eager rejection interrupts them. Per-provider thresholds are meant to
-	// come from `/bw probe` evidence, not from guesses.
+	// come from `/bw probe` evidence, not from guesses. The raw text rung opts out
+	// of the phrase list because source files may legitimately contain those words.
 	defaultFailWhen: {
 		minChars: 0,
 		minResults: 0,

package/index.ts

@@ -58,14 +58,14 @@ export default function (pi: ExtensionAPI) {
 		name: "bw_web_fetch",
 		label: "Fetch Web Page",
 		description:
-			"Fetch a URL and return its content as Markdown. " +
+			"Fetch a URL and return its content as Markdown or text. " +
 			`Tries providers in order until one returns usable content: ${config.order.fetch.join(" → ")}. ` +
 			"The response header states which provider answered and what format the body is in. " +
 			"Output over 50KB is truncated and the full text written to a temp file whose path is given.",
-		promptSnippet: "Fetch a URL as markdown, through the user's real browser",
+		promptSnippet: "Fetch a URL as Markdown or text, through the user's real browser",
 		promptGuidelines: [
 			"Prefer `bw_web_fetch` over curl or a shell HTTP client for reading web pages — it renders JavaScript " +
-				"and carries the user's login state, so it can read pages an anonymous request cannot.",
+				"and carries the user's login state, while its text fallback also reads raw source endpoints.",
 		],
 		parameters: Type.Object({
 			url: Type.String({ description: "HTTP(S) URL to fetch" }),

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@browserwright/pi",
-  "version": "0.18.1",
-  "description": "bw_web_fetch and bw_web_search for the pi coding agent, driving the user's real browser through browserwright",
+  "version": "0.18.2",
+  "description": "bw_web_fetch and bw_web_search for the pi coding agent, using browserwright plus a raw text fallback",
   "license": "AGPL-3.0-only",
   "type": "module",
   "keywords": [

package/providers/raw-text.ts

@@ -0,0 +1,78 @@
+/**
+ * The raw text fetch rung.
+ *
+ * `browserwright markdown` deliberately only converts HTML. Public source
+ * endpoints such as raw.githubusercontent.com return the source as text/plain,
+ * so they need a reader that returns the response body verbatim instead of
+ * trying to turn a browser's non-HTML document into Markdown.
+ */
+
+import type { ModuleContext, ProviderOutcome } from "../core/types.ts";
+
+/**
+ * MIME types whose response body is safe to hand back as text.
+ *
+ * `text/*` covers source files served as text/plain. The application types
+ * cover APIs and source/document formats that are commonly served without a
+ * text/* MIME type. Binary responses such as application/pdf and
+ * application/octet-stream are intentionally not decoded here.
+ */
+const TEXT_APPLICATION_TYPES = new Set([
+	"application/graphql",
+	"application/javascript",
+	"application/json",
+	"application/ld+json",
+	"application/manifest+json",
+	"application/sql",
+	"application/toml",
+	"application/typescript",
+	"application/xml",
+	"application/x-javascript",
+	"application/x-yaml",
+	"application/yaml",
+]);
+
+export function isTextContentType(raw: string | null | undefined): boolean {
+	const contentType = (raw ?? "").split(";", 1)[0].trim().toLowerCase();
+	return contentType.startsWith("text/") || TEXT_APPLICATION_TYPES.has(contentType);
+}
+
+export default async function rawText(
+	subject: string,
+	ctx: ModuleContext,
+): Promise<ProviderOutcome<string>> {
+	let response: Response;
+	try {
+		response = await fetch(subject, {
+			redirect: "follow",
+			signal: ctx.signal,
+		});
+	} catch (error) {
+		if (ctx.signal?.aborted) return { ok: false, reason: "aborted" };
+		return { ok: false, reason: `fetch failed: ${(error as Error).message}` };
+	}
+
+	const contentType = response.headers.get("content-type") ?? "";
+	if (!isTextContentType(contentType)) {
+		const type = contentType.split(";", 1)[0].trim() || "unknown";
+		return {
+			ok: false,
+			status: response.status,
+			reason: `not a text response (Content-Type: ${type})`,
+		};
+	}
+
+	let text: string;
+	try {
+		text = await response.text();
+	} catch (error) {
+		return { ok: false, status: response.status, reason: `could not read response: ${(error as Error).message}` };
+	}
+
+	if (!response.ok) {
+		const hint = text.slice(0, 200).replace(/\s+/g, " ").trim();
+		return { ok: false, status: response.status, reason: `http ${response.status}${hint ? `: ${hint}` : ""}` };
+	}
+
+	return { ok: true, content: text, status: response.status };
+}

package/providers/raw.json

@@ -0,0 +1,16 @@
+{
+  "name": "raw",
+  "role": "fetch",
+  "kind": "module",
+  "module": "./providers/raw-text.ts",
+  "returns": "text",
+  "timeoutMs": 30000,
+  "failWhen": {
+    "matches": []
+  },
+  "_note": [
+    "Returns text responses verbatim for endpoints such as raw.githubusercontent.com.",
+    "The browserwright provider remains first, so HTML gets browser rendering and the user's login state when available.",
+    "Binary responses are rejected rather than decoded as corrupt text."
+  ]
+}