@browserwright/pi 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +661 -0
- package/README.md +312 -0
- package/config.json +14 -0
- package/core/chain.ts +137 -0
- package/core/config.ts +90 -0
- package/core/exec-command.ts +80 -0
- package/core/exec-http.ts +138 -0
- package/core/exec-module.ts +95 -0
- package/core/format.ts +211 -0
- package/core/predicates.ts +228 -0
- package/core/probe.ts +182 -0
- package/core/results.ts +141 -0
- package/core/types.ts +241 -0
- package/index.ts +276 -0
- package/package.json +51 -0
- package/probe-cases.json +22 -0
- package/probe-run.ts +34 -0
- package/providers/browserwright-search.json +56 -0
- package/providers/browserwright-search.ts +427 -0
- package/providers/browserwright.json +54 -0
- package/verify.ts +91 -0
|
@@ -0,0 +1,427 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The `web_search` rung: drive a real search engine in the user's own Chrome.
|
|
3
|
+
*
|
|
4
|
+
* This is a `kind: "module"` provider rather than a `kind: "command"` one
|
|
5
|
+
* because a search is not one shot at a subprocess. It is: mint a session,
|
|
6
|
+
* navigate, extract from the live DOM, tear the session down — with a retry in
|
|
7
|
+
* the middle and a guaranteed teardown at the end. A shell script can express
|
|
8
|
+
* that, and one used to (`.retired/browserwright.sh`, 122 lines), which is
|
|
9
|
+
* exactly the experience this file exists to avoid repeating.
|
|
10
|
+
*
|
|
11
|
+
* Six measured behaviours of the executor are designed around here. They are
|
|
12
|
+
* not hypothetical; each one produced a silent wrong answer at least once:
|
|
13
|
+
*
|
|
14
|
+
* 1. Executor stdout is truncated at ~10KB SILENTLY, with exit 0. So the
|
|
15
|
+
* payload never travels on stdout — the script writes it to a file and
|
|
16
|
+
* prints only a byte count, which we verify. This mirrors what the
|
|
17
|
+
* `browserwright markdown` command does internally for the same reason.
|
|
18
|
+
* 2. `sys.exit()` inside the executor KILLS it, and the next call fails with
|
|
19
|
+
* ExecutorUnavailable. The script therefore never calls it; "no results" is
|
|
20
|
+
* data that comes back, not an exit status.
|
|
21
|
+
* 3. Chrome serves its own `neterror` page through a *successful* navigation,
|
|
22
|
+
* so `page.content()` returns an error document rather than raising.
|
|
23
|
+
* 4. The executor can die mid-call. One `session reset` + retry recovers it.
|
|
24
|
+
* 5. browserwright reports errors as one JSON object per line on stderr;
|
|
25
|
+
* unwrapped here so the chain trace reads as a sentence.
|
|
26
|
+
* 6. A session that is not ended leaks a Chrome tab group into the user's
|
|
27
|
+
* window, so teardown lives in a `finally` that runs on every path.
|
|
28
|
+
*/
|
|
29
|
+
|
|
30
|
+
import { spawn } from "node:child_process";
|
|
31
|
+
import { readFileSync, rmSync } from "node:fs";
|
|
32
|
+
import { tmpdir } from "node:os";
|
|
33
|
+
import { join } from "node:path";
|
|
34
|
+
import { normalizeSearchPayload } from "../core/results.ts";
|
|
35
|
+
import type { ModuleContext, ProviderOutcome, SearchPayload } from "../core/types.ts";
|
|
36
|
+
|
|
37
|
+
const BIN = "browserwright";
|
|
38
|
+
const READY_MARKER = "BW_SEARCH_OK";
|
|
39
|
+
|
|
40
|
+
/** Error types that mean "the plumbing broke", not "the page said no". */
|
|
41
|
+
const TRANSIENT = new Set(["ExecutorUnavailable", "PageBindTimeout", "DaemonUnavailable", "CDPError"]);
|
|
42
|
+
|
|
43
|
+
interface Run {
|
|
44
|
+
code: number | null;
|
|
45
|
+
stdout: string;
|
|
46
|
+
stderr: string;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
function run(args: string[], options: { input?: string; signal?: AbortSignal; timeoutMs: number }): Promise<Run> {
|
|
50
|
+
// Adding an "abort" listener to an ALREADY-aborted signal never fires it, so
|
|
51
|
+
// without this check a cancelled call still spawns a process and waits out
|
|
52
|
+
// the full timeout. Teardown deliberately passes no signal so it still runs.
|
|
53
|
+
if (options.signal?.aborted) {
|
|
54
|
+
return Promise.resolve({ code: null, stdout: "", stderr: "aborted" });
|
|
55
|
+
}
|
|
56
|
+
return new Promise((resolve) => {
|
|
57
|
+
const child = spawn(BIN, args, { stdio: ["pipe", "pipe", "pipe"] });
|
|
58
|
+
let stdout = "";
|
|
59
|
+
let stderr = "";
|
|
60
|
+
let settled = false;
|
|
61
|
+
|
|
62
|
+
const finish = (result: Run) => {
|
|
63
|
+
if (settled) return;
|
|
64
|
+
settled = true;
|
|
65
|
+
clearTimeout(timer);
|
|
66
|
+
options.signal?.removeEventListener("abort", onAbort);
|
|
67
|
+
resolve(result);
|
|
68
|
+
};
|
|
69
|
+
|
|
70
|
+
const timer = setTimeout(() => {
|
|
71
|
+
child.kill("SIGKILL");
|
|
72
|
+
finish({ code: null, stdout, stderr: `${stderr}\ntimeout after ${options.timeoutMs}ms` });
|
|
73
|
+
}, options.timeoutMs);
|
|
74
|
+
|
|
75
|
+
const onAbort = () => {
|
|
76
|
+
child.kill("SIGKILL");
|
|
77
|
+
finish({ code: null, stdout, stderr: `${stderr}\naborted` });
|
|
78
|
+
};
|
|
79
|
+
options.signal?.addEventListener("abort", onAbort, { once: true });
|
|
80
|
+
|
|
81
|
+
child.stdout.on("data", (chunk) => {
|
|
82
|
+
stdout += chunk;
|
|
83
|
+
});
|
|
84
|
+
child.stderr.on("data", (chunk) => {
|
|
85
|
+
stderr += chunk;
|
|
86
|
+
});
|
|
87
|
+
child.on("error", (error) => finish({ code: null, stdout, stderr: `spawn failed: ${error.message}` }));
|
|
88
|
+
child.on("close", (code) => finish({ code, stdout, stderr }));
|
|
89
|
+
|
|
90
|
+
if (options.input !== undefined) child.stdin.end(options.input);
|
|
91
|
+
else child.stdin.end();
|
|
92
|
+
});
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* browserwright writes one JSON object per line on stderr for its own errors,
|
|
97
|
+
* but a generic exception arrives as a bare Python traceback and a usage error
|
|
98
|
+
* as plain text. Pull out a sentence whichever shape it is.
|
|
99
|
+
*/
|
|
100
|
+
export function explain(stderr: string): { message: string; type?: string; retryable?: boolean } {
|
|
101
|
+
const lines = stderr
|
|
102
|
+
.split("\n")
|
|
103
|
+
.map((line) => line.trim())
|
|
104
|
+
.filter(Boolean);
|
|
105
|
+
|
|
106
|
+
for (const line of [...lines].reverse()) {
|
|
107
|
+
if (!line.startsWith("{")) continue;
|
|
108
|
+
try {
|
|
109
|
+
const parsed = JSON.parse(line) as { msg?: string; type?: string; retryable?: boolean };
|
|
110
|
+
if (parsed.msg) return { message: parsed.msg, type: parsed.type, retryable: parsed.retryable };
|
|
111
|
+
} catch {
|
|
112
|
+
// Not the envelope — keep scanning older lines.
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
return { message: lines.at(-1) ?? "no output" };
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
function isTransient(info: { type?: string; retryable?: boolean; message: string }): boolean {
|
|
119
|
+
if (info.retryable) return true;
|
|
120
|
+
if (info.type && TRANSIENT.has(info.type)) return true;
|
|
121
|
+
return /executor/i.test(info.message);
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/**
|
|
125
|
+
* The extraction script, run inside the session's executor.
|
|
126
|
+
*
|
|
127
|
+
* Selectors live here rather than in the Python package on purpose: a Google
|
|
128
|
+
* layout change is then a `npm publish` of this package, not a PyPI release of
|
|
129
|
+
* browserwright plus an upgrade on every machine.
|
|
130
|
+
*/
|
|
131
|
+
function buildScript(query: string, limit: number, outPath: string, searchUrl: string): string {
|
|
132
|
+
// Every extractor is independently guarded. Google restyles constantly, and
|
|
133
|
+
// the failure mode that matters is one changed container silently taking the
|
|
134
|
+
// whole search down with it — organic rows must survive a broken AI-Overview
|
|
135
|
+
// selector, and vice versa.
|
|
136
|
+
const js = `() => {
|
|
137
|
+
const QUERY = ${JSON.stringify(query)}.toLowerCase();
|
|
138
|
+
const attempt = (fn, fallback) => { try { return fn(); } catch (e) { return fallback; } };
|
|
139
|
+
const clean = (s) => (s || '')
|
|
140
|
+
.replace(/\\s*\\bRead more\\s*$/i, '')
|
|
141
|
+
.replace(/\\s*\\bShow more\\s*$/i, '')
|
|
142
|
+
.replace(/\\s+/g, ' ')
|
|
143
|
+
.trim();
|
|
144
|
+
|
|
145
|
+
// --- organic rows ------------------------------------------------------
|
|
146
|
+
const results = attempt(() => {
|
|
147
|
+
const rows = [];
|
|
148
|
+
document.querySelectorAll('#search a h3').forEach((h3) => {
|
|
149
|
+
const a = h3.closest('a');
|
|
150
|
+
if (!a || !a.href) return;
|
|
151
|
+
const block = h3.closest('div[data-hveid]');
|
|
152
|
+
let snippet = '';
|
|
153
|
+
if (block) {
|
|
154
|
+
const sn = block.querySelector('div[data-sncf], div[style*="-webkit-line-clamp"]');
|
|
155
|
+
snippet = sn ? sn.innerText : '';
|
|
156
|
+
if (!snippet) {
|
|
157
|
+
const long = (block.innerText || '').split('\\n').filter((s) => s.length > 40);
|
|
158
|
+
snippet = long[0] || '';
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
// Google prefixes dated results with "Mar 5, 2025 — ". Promote that to a
|
|
162
|
+
// field so the model can judge freshness without parsing prose.
|
|
163
|
+
let date = null;
|
|
164
|
+
const m = snippet.match(/^([A-Z][a-z]{2} \\d{1,2}, \\d{4})\\s*[—·\\-]\\s*/);
|
|
165
|
+
if (m) { date = m[1]; snippet = snippet.slice(m[0].length); }
|
|
166
|
+
rows.push({ title: clean(h3.innerText), url: a.href, snippet: clean(snippet).slice(0, 400), date });
|
|
167
|
+
});
|
|
168
|
+
return rows;
|
|
169
|
+
}, []);
|
|
170
|
+
|
|
171
|
+
// --- answer box / AI Overview -----------------------------------------
|
|
172
|
+
// Class names here are obfuscated and rotate, so anchor on the accessible
|
|
173
|
+
// heading instead and walk up until the container actually holds the body.
|
|
174
|
+
const answerBox = attempt(() => {
|
|
175
|
+
const head = [...document.querySelectorAll('[role="heading"]')]
|
|
176
|
+
.find((e) => (e.innerText || '').trim() === 'AI Overview');
|
|
177
|
+
if (!head) return null;
|
|
178
|
+
let node = head;
|
|
179
|
+
for (let i = 0; i < 8 && node; i++) {
|
|
180
|
+
const t = node.innerText || '';
|
|
181
|
+
if (t.length >= 150) {
|
|
182
|
+
const body = clean(t.replace(/^AI Overview\\s*/, ''))
|
|
183
|
+
.replace(/AI responses may include mistakes.*$/i, '')
|
|
184
|
+
// innerText splices the citation chips into the prose ("Reddit +2").
|
|
185
|
+
// The bare counters are pure noise; the source names are left alone.
|
|
186
|
+
.replace(/\\s\\+\\d+\\b/g, '')
|
|
187
|
+
.trim();
|
|
188
|
+
return body ? { kind: 'ai-overview', text: body.slice(0, 4000) } : null;
|
|
189
|
+
}
|
|
190
|
+
node = node.parentElement;
|
|
191
|
+
}
|
|
192
|
+
return null;
|
|
193
|
+
}, null);
|
|
194
|
+
|
|
195
|
+
// --- knowledge panel ---------------------------------------------------
|
|
196
|
+
// data-attrid is semantic markup rather than a styling class, which makes it
|
|
197
|
+
// the one stable hook on this page.
|
|
198
|
+
const knowledgeGraph = attempt(() => {
|
|
199
|
+
const pick = (sel) => {
|
|
200
|
+
const e = document.querySelector('[data-attrid="' + sel + '"]');
|
|
201
|
+
return e ? clean(e.innerText).slice(0, 300) : undefined;
|
|
202
|
+
};
|
|
203
|
+
const title = pick('title');
|
|
204
|
+
const subtitle = pick('subtitle');
|
|
205
|
+
const description = pick('wa:/description') || pick('description');
|
|
206
|
+
const attributes = {};
|
|
207
|
+
document.querySelectorAll('[data-attrid]').forEach((e) => {
|
|
208
|
+
const id = e.getAttribute('data-attrid') || '';
|
|
209
|
+
// Keep only labelled facts: "kc:/…:label" style ids carry real values,
|
|
210
|
+
// the rest are layout containers and image slots.
|
|
211
|
+
if (!/^kc:/.test(id)) return;
|
|
212
|
+
const label = id.split(':').pop() || id;
|
|
213
|
+
// Google renders the label above the value, so innerText yields
|
|
214
|
+
// "Programming language TypeScript" for a key already named
|
|
215
|
+
// programming_language. Drop the echoed label.
|
|
216
|
+
// Built with plain string ops rather than a constructed RegExp: the label
|
|
217
|
+
// is engine-supplied, and a metacharacter in it would either throw (which
|
|
218
|
+
// the guard above would silently swallow) or match the wrong thing.
|
|
219
|
+
const human = label.replace(/_/g, ' ').toLowerCase();
|
|
220
|
+
let text = clean(e.innerText);
|
|
221
|
+
if (text.toLowerCase().startsWith(human)) {
|
|
222
|
+
text = text.slice(human.length).replace(/^[\\s:\\uff1a]+/, '');
|
|
223
|
+
}
|
|
224
|
+
if (!text || text.length > 160) return;
|
|
225
|
+
if (!attributes[label]) attributes[label] = text;
|
|
226
|
+
});
|
|
227
|
+
if (!title && !description && Object.keys(attributes).length === 0) return null;
|
|
228
|
+
const out = { title, subtitle, description };
|
|
229
|
+
if (Object.keys(attributes).length) out.attributes = attributes;
|
|
230
|
+
return out;
|
|
231
|
+
}, null);
|
|
232
|
+
|
|
233
|
+
// --- people also ask ---------------------------------------------------
|
|
234
|
+
// data-q also carries the original query on the search box, so drop anything
|
|
235
|
+
// that just echoes what was asked.
|
|
236
|
+
const peopleAlsoAsk = attempt(() => {
|
|
237
|
+
const seen = new Set();
|
|
238
|
+
const out = [];
|
|
239
|
+
document.querySelectorAll('[data-q]').forEach((e) => {
|
|
240
|
+
const q = clean(e.getAttribute('data-q'));
|
|
241
|
+
if (!q || q.toLowerCase() === QUERY) return;
|
|
242
|
+
if (seen.has(q.toLowerCase())) return;
|
|
243
|
+
seen.add(q.toLowerCase());
|
|
244
|
+
out.push(q);
|
|
245
|
+
});
|
|
246
|
+
return out.slice(0, 10);
|
|
247
|
+
}, []);
|
|
248
|
+
|
|
249
|
+
// --- related searches --------------------------------------------------
|
|
250
|
+
// Scoped to #botstuff: the same href pattern at the top of the page is
|
|
251
|
+
// Google's own tab bar ("Images", "News", "Past hour"), not a related query.
|
|
252
|
+
const relatedSearches = attempt(() => {
|
|
253
|
+
const bot = document.querySelector('#botstuff');
|
|
254
|
+
if (!bot) return [];
|
|
255
|
+
const seen = new Set();
|
|
256
|
+
const out = [];
|
|
257
|
+
bot.querySelectorAll('a[href*="/search?"]').forEach((a) => {
|
|
258
|
+
const t = clean(a.innerText);
|
|
259
|
+
// Pagination shares this selector: bare page numbers plus the nav labels.
|
|
260
|
+
if (!t || /^\\d+$/.test(t) || t.length < 3 || t.length > 80) return;
|
|
261
|
+
if (/^(next|previous|prev|more results?)$/i.test(t)) return;
|
|
262
|
+
if (t.toLowerCase() === QUERY || seen.has(t.toLowerCase())) return;
|
|
263
|
+
seen.add(t.toLowerCase());
|
|
264
|
+
out.push(t);
|
|
265
|
+
});
|
|
266
|
+
return out.slice(0, 10);
|
|
267
|
+
}, []);
|
|
268
|
+
|
|
269
|
+
return { results, answerBox, knowledgeGraph, peopleAlsoAsk, relatedSearches };
|
|
270
|
+
}`;
|
|
271
|
+
|
|
272
|
+
// json.dumps gives us correctly escaped Python string literals for free, so
|
|
273
|
+
// a query containing quotes or newlines cannot break out of the script.
|
|
274
|
+
return [
|
|
275
|
+
"import json",
|
|
276
|
+
`QUERY = ${JSON.stringify(query)}`,
|
|
277
|
+
`LIMIT = ${limit}`,
|
|
278
|
+
`OUT = ${JSON.stringify(outPath)}`,
|
|
279
|
+
`URL = ${JSON.stringify(searchUrl)}`,
|
|
280
|
+
`JS = ${JSON.stringify(js)}`,
|
|
281
|
+
"payload = {}",
|
|
282
|
+
"try:",
|
|
283
|
+
" page.goto(URL)",
|
|
284
|
+
" html = None",
|
|
285
|
+
" data = None",
|
|
286
|
+
// A search engine can bounce the page right after load (consent, region
|
|
287
|
+
// redirect), which destroys the execution context mid-evaluate. Observed
|
|
288
|
+
// while mapping these selectors; one settle-and-retry clears it.
|
|
289
|
+
" for _ in range(2):",
|
|
290
|
+
" try:",
|
|
291
|
+
" html = page.content()",
|
|
292
|
+
" data = page.evaluate(JS)",
|
|
293
|
+
" break",
|
|
294
|
+
" except Exception as e:",
|
|
295
|
+
' if "ontext was destroyed" not in str(e):',
|
|
296
|
+
" raise",
|
|
297
|
+
' page.wait_for_load_state("domcontentloaded")',
|
|
298
|
+
" page.wait_for_timeout(800)",
|
|
299
|
+
" if data is None:",
|
|
300
|
+
' payload = {"failed": "the page kept navigating; could not read results"}',
|
|
301
|
+
// (3) Chrome's own error document arrives through a successful navigation.
|
|
302
|
+
' elif "neterror" in html and "error-code" in html:',
|
|
303
|
+
' payload = {"blocked": "the browser could not reach the search engine"}',
|
|
304
|
+
" else:",
|
|
305
|
+
" rows = data.get(\"results\") or []",
|
|
306
|
+
" payload = {",
|
|
307
|
+
' "results": rows[:LIMIT],',
|
|
308
|
+
' "url": page.url,',
|
|
309
|
+
' "answerBox": data.get("answerBox"),',
|
|
310
|
+
' "knowledgeGraph": data.get("knowledgeGraph"),',
|
|
311
|
+
' "peopleAlsoAsk": data.get("peopleAlsoAsk") or [],',
|
|
312
|
+
' "relatedSearches": data.get("relatedSearches") or [],',
|
|
313
|
+
" }",
|
|
314
|
+
" if not rows:",
|
|
315
|
+
// An interstitial parses fine and yields zero rows; say which kind it was.
|
|
316
|
+
" low = html.lower()",
|
|
317
|
+
' for needle in ("recaptcha", "unusual traffic", "detected unusual"):',
|
|
318
|
+
" if needle in low:",
|
|
319
|
+
' payload = {"blocked": "search engine returned an interstitial (%s)" % needle}',
|
|
320
|
+
" break",
|
|
321
|
+
"except Exception as e:",
|
|
322
|
+
// (2) never sys.exit() — a raise would kill the executor for the next call.
|
|
323
|
+
' payload = {"failed": "%s: %s" % (type(e).__name__, e)}',
|
|
324
|
+
"data = json.dumps(payload, ensure_ascii=False)",
|
|
325
|
+
'with open(OUT, "w", encoding="utf-8") as fh:',
|
|
326
|
+
" fh.write(data)",
|
|
327
|
+
// (1) only a byte count travels on stdout, never the payload itself.
|
|
328
|
+
`print(${JSON.stringify(READY_MARKER)}, len(data.encode("utf-8")))`,
|
|
329
|
+
].join("\n");
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
interface Attempted {
|
|
333
|
+
ok: boolean;
|
|
334
|
+
payload?: SearchPayload;
|
|
335
|
+
reason?: string;
|
|
336
|
+
transient?: boolean;
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
async function attempt(sid: string, script: string, outPath: string, ctx: ModuleContext): Promise<Attempted> {
|
|
340
|
+
const exec = await run(["-s", sid, "--code-stdin"], {
|
|
341
|
+
input: script,
|
|
342
|
+
signal: ctx.signal,
|
|
343
|
+
timeoutMs: ctx.timeoutMs,
|
|
344
|
+
});
|
|
345
|
+
|
|
346
|
+
if (exec.code !== 0) {
|
|
347
|
+
const info = explain(exec.stderr);
|
|
348
|
+
return { ok: false, reason: info.message, transient: isTransient(info) };
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
const marker = exec.stdout.match(new RegExp(`${READY_MARKER}\\s+(\\d+)`));
|
|
352
|
+
if (!marker) {
|
|
353
|
+
return { ok: false, reason: "extraction script produced no completion marker", transient: true };
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
let raw: string;
|
|
357
|
+
try {
|
|
358
|
+
raw = readFileSync(outPath, "utf8");
|
|
359
|
+
} catch (error) {
|
|
360
|
+
return { ok: false, reason: `could not read extraction output: ${(error as Error).message}` };
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
// (1) The byte count is the whole point of writing to a file: a short read
|
|
364
|
+
// here is the signature of the silent stdout truncation this avoids.
|
|
365
|
+
const expected = Number(marker[1]);
|
|
366
|
+
const actual = Buffer.byteLength(raw, "utf8");
|
|
367
|
+
if (actual !== expected) {
|
|
368
|
+
return { ok: false, reason: `truncated transfer: expected ${expected} bytes, read ${actual}` };
|
|
369
|
+
}
|
|
370
|
+
|
|
371
|
+
const body = JSON.parse(raw) as { blocked?: string; failed?: string };
|
|
372
|
+
if (body.blocked) return { ok: false, reason: body.blocked };
|
|
373
|
+
if (body.failed) return { ok: false, reason: body.failed };
|
|
374
|
+
|
|
375
|
+
// The script emits exactly the key names normalizeSearchPayload aliases, so
|
|
376
|
+
// the browser rung and a hosted JSON rung land on the same shape.
|
|
377
|
+
return { ok: true, payload: normalizeSearchPayload(body) };
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
const runner = async (query: string, ctx: ModuleContext): Promise<ProviderOutcome<SearchPayload>> => {
|
|
381
|
+
const limit = Number(ctx.options.limit ?? 10);
|
|
382
|
+
const template = String(
|
|
383
|
+
ctx.options.searchUrl ?? "https://www.google.com/search?q={queryEncoded}&hl=en&num={limit}",
|
|
384
|
+
);
|
|
385
|
+
const searchUrl = template
|
|
386
|
+
.replaceAll("{queryEncoded}", encodeURIComponent(query))
|
|
387
|
+
.replaceAll("{limit}", String(limit));
|
|
388
|
+
|
|
389
|
+
const outPath = join(tmpdir(), `bw-search-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}.json`);
|
|
390
|
+
const script = buildScript(query, limit, outPath, searchUrl);
|
|
391
|
+
|
|
392
|
+
ctx.onProgress?.("opening a search session");
|
|
393
|
+
const created = await run(
|
|
394
|
+
["session", "new", "--backend=extension", "--name=pi-websearch"],
|
|
395
|
+
{ signal: ctx.signal, timeoutMs: ctx.timeoutMs },
|
|
396
|
+
);
|
|
397
|
+
if (created.code !== 0) {
|
|
398
|
+
return { ok: false, reason: explain(created.stderr).message };
|
|
399
|
+
}
|
|
400
|
+
// stdout is the bare session id and nothing else; the human-readable
|
|
401
|
+
// "OK: session N created" goes to stderr. Reading "the last line" of the
|
|
402
|
+
// two streams merged is what the retired wrapper did, and why it was fragile.
|
|
403
|
+
const sid = created.stdout.trim();
|
|
404
|
+
if (!sid) return { ok: false, reason: "session new printed no id" };
|
|
405
|
+
|
|
406
|
+
try {
|
|
407
|
+
ctx.onProgress?.(`searching (session ${sid})`);
|
|
408
|
+
let result = await attempt(sid, script, outPath, ctx);
|
|
409
|
+
|
|
410
|
+
// (4) One recycle-and-retry. Only for plumbing failures — a page that
|
|
411
|
+
// said no will say no again, and retrying it just costs the user a tab.
|
|
412
|
+
if (!result.ok && result.transient && !ctx.signal?.aborted) {
|
|
413
|
+
ctx.onProgress?.("executor died, recycling and retrying once");
|
|
414
|
+
await run(["session", "reset", sid], { signal: ctx.signal, timeoutMs: 30_000 });
|
|
415
|
+
result = await attempt(sid, script, outPath, ctx);
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
if (!result.ok) return { ok: false, reason: result.reason ?? "search failed" };
|
|
419
|
+
return { ok: true, content: result.payload ?? { results: [] } };
|
|
420
|
+
} finally {
|
|
421
|
+
// (6) Always. A leaked session is a tab group left in the user's window.
|
|
422
|
+
await run(["session", "end", `--session=${sid}`], { timeoutMs: 30_000 });
|
|
423
|
+
rmSync(outPath, { force: true });
|
|
424
|
+
}
|
|
425
|
+
};
|
|
426
|
+
|
|
427
|
+
export default runner;
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "browserwright",
|
|
3
|
+
"label": "browserwright (your Chrome)",
|
|
4
|
+
"role": "fetch",
|
|
5
|
+
"kind": "command",
|
|
6
|
+
"command": [
|
|
7
|
+
"browserwright",
|
|
8
|
+
"markdown",
|
|
9
|
+
"{url}",
|
|
10
|
+
"--max-chars=0",
|
|
11
|
+
"--name=pi-webfetch"
|
|
12
|
+
],
|
|
13
|
+
"returns": "markdown",
|
|
14
|
+
"timeoutMs": 90000,
|
|
15
|
+
"retries": 1,
|
|
16
|
+
"retryWhen": [
|
|
17
|
+
"PageBindTimeout",
|
|
18
|
+
"retryable"
|
|
19
|
+
],
|
|
20
|
+
"failWhen": {
|
|
21
|
+
"matches": []
|
|
22
|
+
},
|
|
23
|
+
"_note": [
|
|
24
|
+
"browserwright >= 0.9.0 has a native one-shot markdown command that owns its",
|
|
25
|
+
"whole lifecycle: it creates a throwaway session, navigates, converts, and",
|
|
26
|
+
"tears the session down. Verified 2026-08-09: no leaked sessions afterwards.",
|
|
27
|
+
"--backend defaults to `extension`, the user's real Chrome, which is what we",
|
|
28
|
+
"want: the isolated cdp backend would launch a new Chrome and steal the macOS",
|
|
29
|
+
"active window. Do-not-disturb outranks isolation. It also means this rung",
|
|
30
|
+
"carries the user's login state, which is the whole reason it is worth having.",
|
|
31
|
+
"--max-chars=0 disables its 8000-char print cap. Measured: a Wikipedia article",
|
|
32
|
+
"comes through stdout at 46,033 bytes, so the ~10KB executor stdout truncation",
|
|
33
|
+
"that forced the old wrapper script does not apply on this path.",
|
|
34
|
+
"--mode defaults to `auto`: extract the main content, and fall back to the",
|
|
35
|
+
"page minus nav/aside/footer if extraction collapses. Switch to --mode=full",
|
|
36
|
+
"here if a docs or API-reference page ever comes back missing its tables or",
|
|
37
|
+
"code blocks — that is readability's known weak spot, and `full` keeps every",
|
|
38
|
+
"link.",
|
|
39
|
+
"failWhen.matches: [] switches OFF the core default list, and the probe on",
|
|
40
|
+
"2026-08-09 says keep it that way: every case that returned content returned",
|
|
41
|
+
"genuine content, including x.com, where this rung got a real logged-in",
|
|
42
|
+
"profile view (2199 chars) that an anonymous fetch is refused.",
|
|
43
|
+
"This is the ONLY fetch rung this package ships, so a false rejection here",
|
|
44
|
+
"fails the whole call rather than dropping to something else. That asymmetry",
|
|
45
|
+
"is why the bar for rejecting is deliberately high. Drop your own provider",
|
|
46
|
+
"JSON into this directory to add a cheaper or anonymous rung ahead of it.",
|
|
47
|
+
"Observed but deliberately accepted: g2.com returns a 1330-char skip-links",
|
|
48
|
+
"stub after 62 seconds — thin, and uncomfortably close to the 90s timeout.",
|
|
49
|
+
"Known upstream bug (2026-08-09): a PDF URL exits 3 with a raw",
|
|
50
|
+
"`TargetClosedError` traceback, not the documented UnsupportedContentType",
|
|
51
|
+
"refusal, so the chain trace carries a Playwright stack instead of a sentence.",
|
|
52
|
+
"retries/retryWhen: measured 2026-08-09, PageBindTimeout ('timed out binding Playwright to session target ... no replacement page was created') fires intermittently on a healthy daemon and the CLI itself marks it retryable:true. With one rung per tool there is nothing to fall through to, so one retry here is the difference between a blip and a failed call. Scoped by retryWhen so a 404 or an auth wall is not retried."
|
|
53
|
+
]
|
|
54
|
+
}
|
package/verify.ts
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Live verification harness. Runs the real chains against real inputs without
|
|
3
|
+
* pi and without an LLM, so "does each rung actually work" is answerable
|
|
4
|
+
* cheaply — including the search rung, which no unit test can exercise because
|
|
5
|
+
* it needs a browser and the user's Chrome.
|
|
6
|
+
*
|
|
7
|
+
* node verify.ts # fetch example.com + a canned search
|
|
8
|
+
* node verify.ts https://example.com # fetch a specific URL
|
|
9
|
+
* node verify.ts --search "playwright aria" # search a specific query
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
import { makeExecutor, runChain } from "./core/chain.ts";
|
|
13
|
+
import { EXTENSION_DIR, loadConfig, loadProviders } from "./core/config.ts";
|
|
14
|
+
import { renderFailure, renderResults, renderSuccess } from "./core/format.ts";
|
|
15
|
+
import { inspectSearch, inspectText, providersForRole } from "./core/predicates.ts";
|
|
16
|
+
import type { SearchPayload } from "./core/types.ts";
|
|
17
|
+
|
|
18
|
+
const config = loadConfig();
|
|
19
|
+
const providers = loadProviders();
|
|
20
|
+
|
|
21
|
+
const args = process.argv.slice(2);
|
|
22
|
+
const searchAt = args.indexOf("--search");
|
|
23
|
+
const query = searchAt >= 0 ? (args[searchAt + 1] ?? "playwright aria snapshot") : undefined;
|
|
24
|
+
const url = searchAt >= 0 ? undefined : (args[0] ?? "https://example.com");
|
|
25
|
+
|
|
26
|
+
console.log(`providers loaded: ${[...providers.keys()].join(", ")}`);
|
|
27
|
+
console.log(`fetch order: ${config.order.fetch.join(" → ")}`);
|
|
28
|
+
console.log(`search order: ${config.order.search.join(" → ")}\n`);
|
|
29
|
+
|
|
30
|
+
if (url !== undefined) {
|
|
31
|
+
console.log(`── fetch chain: ${url} ──`);
|
|
32
|
+
const chained = await runChain<string>({
|
|
33
|
+
providers,
|
|
34
|
+
config,
|
|
35
|
+
role: "fetch",
|
|
36
|
+
subject: url,
|
|
37
|
+
inspect: inspectText,
|
|
38
|
+
executor: makeExecutor<string>(config, { dir: EXTENSION_DIR, role: "fetch" }),
|
|
39
|
+
onAttempt: (provider) => console.log(` trying ${provider.name}…`),
|
|
40
|
+
});
|
|
41
|
+
console.log(
|
|
42
|
+
chained.ok
|
|
43
|
+
? `${renderSuccess(chained, { url, maxBytes: config.maxBytes, maxLines: config.maxLines }).slice(0, 600)}\n`
|
|
44
|
+
: `${renderFailure(chained, url, { tool: "web_fetch", alternatives: [...providersForRole(providers, "fetch").keys()] })}\n`,
|
|
45
|
+
);
|
|
46
|
+
|
|
47
|
+
console.log("── each fetch provider in isolation ──");
|
|
48
|
+
for (const name of providersForRole(providers, "fetch").keys()) {
|
|
49
|
+
const forced = await runChain<string>({
|
|
50
|
+
providers,
|
|
51
|
+
config,
|
|
52
|
+
role: "fetch",
|
|
53
|
+
subject: url,
|
|
54
|
+
inspect: inspectText,
|
|
55
|
+
forced: name,
|
|
56
|
+
executor: makeExecutor<string>(config, { dir: EXTENSION_DIR, role: "fetch" }),
|
|
57
|
+
});
|
|
58
|
+
const attempt = forced.attempts.at(-1);
|
|
59
|
+
const ms = attempt ? `${(attempt.ms / 1000).toFixed(1)}s` : "-";
|
|
60
|
+
console.log(
|
|
61
|
+
forced.ok
|
|
62
|
+
? `✓ ${name.padEnd(22)} ${ms.padStart(6)} ${forced.content?.length} chars (${forced.format})`
|
|
63
|
+
: `✗ ${name.padEnd(22)} ${ms.padStart(6)} ${attempt?.reason}`,
|
|
64
|
+
);
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
if (query !== undefined) {
|
|
69
|
+
console.log(`── search chain: ${JSON.stringify(query)} ──`);
|
|
70
|
+
const searched = await runChain<SearchPayload>({
|
|
71
|
+
providers,
|
|
72
|
+
config,
|
|
73
|
+
role: "search",
|
|
74
|
+
subject: query,
|
|
75
|
+
inspect: inspectSearch,
|
|
76
|
+
executor: makeExecutor<SearchPayload>(config, {
|
|
77
|
+
dir: EXTENSION_DIR,
|
|
78
|
+
role: "search",
|
|
79
|
+
onProgress: (text) => console.log(` ${text}`),
|
|
80
|
+
}),
|
|
81
|
+
onAttempt: (provider) => console.log(` trying ${provider.name}…`),
|
|
82
|
+
});
|
|
83
|
+
console.log(
|
|
84
|
+
searched.ok
|
|
85
|
+
? renderResults(searched, query)
|
|
86
|
+
: renderFailure(searched, query, {
|
|
87
|
+
tool: "web_search",
|
|
88
|
+
alternatives: [...providersForRole(providers, "search").keys()],
|
|
89
|
+
}),
|
|
90
|
+
);
|
|
91
|
+
}
|