@browserwright/pi 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +661 -0
- package/README.md +312 -0
- package/config.json +14 -0
- package/core/chain.ts +137 -0
- package/core/config.ts +90 -0
- package/core/exec-command.ts +80 -0
- package/core/exec-http.ts +138 -0
- package/core/exec-module.ts +95 -0
- package/core/format.ts +211 -0
- package/core/predicates.ts +228 -0
- package/core/probe.ts +182 -0
- package/core/results.ts +141 -0
- package/core/types.ts +241 -0
- package/index.ts +276 -0
- package/package.json +51 -0
- package/probe-cases.json +22 -0
- package/probe-run.ts +34 -0
- package/providers/browserwright-search.json +56 -0
- package/providers/browserwright-search.ts +427 -0
- package/providers/browserwright.json +54 -0
- package/verify.ts +91 -0
package/README.md
ADDED
|
@@ -0,0 +1,312 @@
|
|
|
1
|
+
# @browserwright/pi
|
|
2
|
+
|
|
3
|
+
Two tools for [pi](https://github.com/badlogic/pi-mono), backed by **declarative
|
|
4
|
+
providers** that drive [browserwright](https://github.com/broven/browserwright):
|
|
5
|
+
|
|
6
|
+
```
|
|
7
|
+
web_fetch(url, provider?) → the page as Markdown
|
|
8
|
+
web_search(query, provider?) → ranked links + the SERP features Google showed
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
Both run through the user's **own Chrome**, so they see what the user sees —
|
|
12
|
+
including pages behind a login. Zero npm dependencies; `typebox` and the pi
|
|
13
|
+
packages come from pi's own install.
|
|
14
|
+
|
|
15
|
+
## Install
|
|
16
|
+
|
|
17
|
+
```bash
|
|
18
|
+
pi install npm:@browserwright/pi
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
Requires the `browserwright` CLI (>= 0.9.0) on `PATH` and its daemon running:
|
|
22
|
+
|
|
23
|
+
```bash
|
|
24
|
+
uv tool install browserwright
|
|
25
|
+
browserwright-daemon serve
|
|
26
|
+
browserwright version check # expect drift=equal
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
The npm package and the Python package are cut from the same git tag, so their
|
|
30
|
+
versions always match. Install them together.
|
|
31
|
+
|
|
32
|
+
## The two tools
|
|
33
|
+
|
|
34
|
+
`web_search` returns **links, never page bodies**. The model then calls
|
|
35
|
+
`web_fetch` on the one or two worth reading. That split is deliberate: fetching
|
|
36
|
+
all ten hits costs ~30 seconds and 50KB+ of context to answer a question that
|
|
37
|
+
usually needs one of them.
|
|
38
|
+
|
|
39
|
+
### What `web_search` returns
|
|
40
|
+
|
|
41
|
+
| field | contents |
|
|
42
|
+
|-------|----------|
|
|
43
|
+
| `results[]` | `position`, `title`, `url`, `snippet`, `date` |
|
|
44
|
+
| `answerBox` | Google's AI Overview, labelled in the output as generated and unsourced |
|
|
45
|
+
| `knowledgeGraph` | `title`, `subtitle`, `description`, `attributes` |
|
|
46
|
+
| `peopleAlsoAsk[]` | the expandable questions (capped at 10) |
|
|
47
|
+
| `relatedSearches[]` | the queries at the foot of the page (capped at 10) |
|
|
48
|
+
|
|
49
|
+
Everything except `results` is absent on most queries and is omitted entirely
|
|
50
|
+
rather than rendered empty.
|
|
51
|
+
|
|
52
|
+
### What it does not return
|
|
53
|
+
|
|
54
|
+
Measured against what Serper and SerpApi expose, so you know when to reach for
|
|
55
|
+
one of those instead. Nothing here is blocked by the architecture — the SERP
|
|
56
|
+
carries all of it — these are simply extractors nobody has needed yet.
|
|
57
|
+
|
|
58
|
+
**Not implemented (a selector away):**
|
|
59
|
+
|
|
60
|
+
| missing | why it hasn't been done |
|
|
61
|
+
|---------|------------------------|
|
|
62
|
+
| `sitelinks` | only render for brand-shaped queries; the two probes that looked for them found none, so there was nothing to write a selector against |
|
|
63
|
+
| `topStories` / `images` / `videos` | vertical carousels. An agent that wants news or images is better served asking for them explicitly than having them folded into every search |
|
|
64
|
+
| `places` / local pack | needs a location the daemon does not have; results would silently reflect the user's IP |
|
|
65
|
+
| `shopping` | product rows, ratings, prices — a different consumer than "find me a source" |
|
|
66
|
+
| ads | deliberately not extracted. They are the one part of the page that is *paid to look like* a result |
|
|
67
|
+
| spelling correction / "Did you mean" | cheap to add, nobody has asked |
|
|
68
|
+
| `searchInformation` | total-result count and timing. Google's own totals are estimates, so the field would be precise-looking and wrong |
|
|
69
|
+
| answer-box source URL | we take the AI Overview text but not its citation links |
|
|
70
|
+
|
|
71
|
+
**Structural, not a missing extractor:**
|
|
72
|
+
|
|
73
|
+
- **No pagination.** One page per call, `num` results. There is no offset
|
|
74
|
+
parameter and adding one means another 10-20s round trip per page.
|
|
75
|
+
- **`position` is not the engine's rank.** It is the index after rows without a
|
|
76
|
+
URL are dropped, so a dropped row shifts everything below it. Fine for finding
|
|
77
|
+
sources; wrong for rank tracking — use a hosted API if you need true ranks.
|
|
78
|
+
- **Results are personalised.** They come through the user's own browser, IP and
|
|
79
|
+
login state, so region, language and search history all affect them. That is
|
|
80
|
+
the point of this rung, but it also means results are **not reproducible** and
|
|
81
|
+
are unsuitable as an objective baseline.
|
|
82
|
+
- **One engine at a time.** `searchUrl` in the provider declaration is a
|
|
83
|
+
template, so pointing it at another engine is a config edit — but the
|
|
84
|
+
extractors are written against Google's DOM and will not transfer as-is.
|
|
85
|
+
- **No usage or quota metadata**, because there is no account behind it.
|
|
86
|
+
|
|
87
|
+
If a missing field matters more than login state does, the answer is usually to
|
|
88
|
+
drop a hosted-API provider JSON into `providers/` and put it ahead of this rung,
|
|
89
|
+
not to extend the extractor. `normalizeSearchPayload` already maps Serper's
|
|
90
|
+
response field-for-field.
|
|
91
|
+
|
|
92
|
+
The response header tells the model what it got, because pi's tool `details`
|
|
93
|
+
field never reaches the LLM — it only feeds the TUI renderer:
|
|
94
|
+
|
|
95
|
+
```
|
|
96
|
+
# Example Domain
|
|
97
|
+
https://example.com
|
|
98
|
+
provider=browserwright · format=markdown · 385B
|
|
99
|
+
chain: browserwright✗1.1s browserwright-search✓2.8s ← only when >1 rung ran
|
|
100
|
+
truncated: 382 of 480 lines (49.7KB of 71.5KB) · full: /tmp/browserwright-pi-xxx.txt
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
## What ships, and what does not
|
|
104
|
+
|
|
105
|
+
This package ships **only the browserwright rungs**. There is one per tool:
|
|
106
|
+
|
|
107
|
+
| tool | provider | kind |
|
|
108
|
+
|------|----------|------|
|
|
109
|
+
| `web_fetch` | `browserwright` | `command` — `browserwright markdown <url>` |
|
|
110
|
+
| `web_search` | `browserwright-search` | `module` — a session lifecycle in TS |
|
|
111
|
+
|
|
112
|
+
That is a real trade-off, and it points the wrong way for casual fetches: every
|
|
113
|
+
`web_fetch` opens a tab in the daily browser and takes ~4-7s, where a hosted
|
|
114
|
+
reader API answers in ~1s without touching Chrome. What you get for it is login
|
|
115
|
+
state and full JS rendering, which no anonymous rung has.
|
|
116
|
+
|
|
117
|
+
**The chain engine is still here.** Drop your own JSON into `providers/` to add a
|
|
118
|
+
cheaper or anonymous rung ahead of the browser one — nothing needs to be
|
|
119
|
+
registered, and a provider missing from `config.json`'s `order` is appended
|
|
120
|
+
rather than ignored.
|
|
121
|
+
|
|
122
|
+
## Adding a provider
|
|
123
|
+
|
|
124
|
+
### kind: "http"
|
|
125
|
+
|
|
126
|
+
```json
|
|
127
|
+
{
|
|
128
|
+
"name": "example",
|
|
129
|
+
"role": "fetch",
|
|
130
|
+
"kind": "http",
|
|
131
|
+
"method": "POST",
|
|
132
|
+
"url": "https://api.example.com/read",
|
|
133
|
+
"headers": { "Authorization": "Bearer $EXAMPLE_TOKEN" },
|
|
134
|
+
"body": { "url": "{url}" },
|
|
135
|
+
"pick": "result",
|
|
136
|
+
"returns": "markdown"
|
|
137
|
+
}
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
- `role` is `fetch` (the default) or `search`. It decides which tool can reach
|
|
141
|
+
the provider, and which tokens it may use: `{url}`/`{urlEncoded}` for fetch,
|
|
142
|
+
`{query}`/`{queryEncoded}` for search. `{dir}` is available to both.
|
|
143
|
+
- Tokens are substituted first, then `$ENV_VAR`.
|
|
144
|
+
- A referenced env var that is unset makes the rung **skip** with
|
|
145
|
+
`missing env NAME` rather than sending the literal `$NAME`. A literal value
|
|
146
|
+
passes through untouched — but prefer `$ENV` for anything secret, since these
|
|
147
|
+
files are meant to be shareable.
|
|
148
|
+
- `pick` is a dot path into a JSON response. For a `search` provider it must
|
|
149
|
+
land on an **array** of organic rows; they are coerced from whatever field
|
|
150
|
+
names the API uses (`link`/`url`/`href`, `snippet`/`description`/`content`, …).
|
|
151
|
+
The SERP-feature fields are read from the top level of the same body by their
|
|
152
|
+
usual names (`answerBox`/`answer_box`, `knowledgeGraph`, `peopleAlsoAsk`,
|
|
153
|
+
`relatedSearches`, …), so a hosted search API is a pure JSON drop-in — omit
|
|
154
|
+
`pick` entirely and the whole response is mapped for you.
|
|
155
|
+
|
|
156
|
+
### kind: "command"
|
|
157
|
+
|
|
158
|
+
```json
|
|
159
|
+
{
|
|
160
|
+
"name": "example",
|
|
161
|
+
"kind": "command",
|
|
162
|
+
"command": ["{dir}/providers/example.sh", "{url}"],
|
|
163
|
+
"returns": "html"
|
|
164
|
+
}
|
|
165
|
+
```
|
|
166
|
+
|
|
167
|
+
Exit code contract — this is what lets a shell script participate without the
|
|
168
|
+
core knowing anything about the tool it wraps:
|
|
169
|
+
|
|
170
|
+
| exit | meaning |
|
|
171
|
+
|------|---------|
|
|
172
|
+
| `0` | success, stdout is the content |
|
|
173
|
+
| `2` | not applicable — drop to the next rung, not an error |
|
|
174
|
+
| other | hard error — also drops a rung, reported as an error |
|
|
175
|
+
|
|
176
|
+
The last line of stderr becomes the reason in the chain trace, so make it a
|
|
177
|
+
sentence.
|
|
178
|
+
|
|
179
|
+
### kind: "module"
|
|
180
|
+
|
|
181
|
+
The escape hatch for a provider that needs real logic — a multi-step lifecycle,
|
|
182
|
+
its own retries, progress reporting. It gets the event loop instead of one
|
|
183
|
+
process:
|
|
184
|
+
|
|
185
|
+
```json
|
|
186
|
+
{ "name": "example", "kind": "module", "module": "./providers/example.ts", "returns": "results" }
|
|
187
|
+
```
|
|
188
|
+
|
|
189
|
+
The module default-exports `(subject, ctx) => Promise<ProviderOutcome<T>>`.
|
|
190
|
+
`ctx` carries `dir`, `timeoutMs`, `signal`, `options` (verbatim from the
|
|
191
|
+
declaration) and `onProgress`. Cancellation is cooperative: there is no process
|
|
192
|
+
to kill, so the runner must unwind its own resources when `ctx.signal` fires.
|
|
193
|
+
|
|
194
|
+
`providers/browserwright-search.ts` is the worked example. Its header documents
|
|
195
|
+
the six measured executor behaviours it is built around, and its declaration
|
|
196
|
+
records why each SERP extractor anchors where it does — including the finding
|
|
197
|
+
that Google's AI Overview body is **not** in the server-rendered HTML at all, so
|
|
198
|
+
extraction has to run against the live DOM rather than the document response.
|
|
199
|
+
|
|
200
|
+
### `returns`
|
|
201
|
+
|
|
202
|
+
`markdown` | `html` | `text` | `results`. **The core never converts between
|
|
203
|
+
them**; it only labels the output so the model knows what it is reading.
|
|
204
|
+
|
|
205
|
+
## failWhen: the reason the chain exists
|
|
206
|
+
|
|
207
|
+
The common real-world failure is not an error. It is **HTTP 200 with a JS shell,
|
|
208
|
+
a cookie wall, or a login page** — and for search, **a perfectly parsed empty
|
|
209
|
+
list**. Without content-level rejection the first rung "succeeds", the model gets
|
|
210
|
+
garbage, and later rungs never run.
|
|
211
|
+
|
|
212
|
+
Two layers: the core default in `config.json` → `defaultFailWhen`, and
|
|
213
|
+
`failWhen` per provider. Per-provider values **replace** the default field by
|
|
214
|
+
field; they do not merge. That is what makes `"matches": []` a working opt-out,
|
|
215
|
+
which the browser rung relies on — phrases like "enable JavaScript" appear
|
|
216
|
+
legitimately inside raw HTML and inside search results *about* JavaScript.
|
|
217
|
+
|
|
218
|
+
| field | applies to | note |
|
|
219
|
+
|-------|-----------|------|
|
|
220
|
+
| `minChars` | text payloads only | default 0 (off) |
|
|
221
|
+
| `minResults` | list payloads only | an empty list is rejected regardless |
|
|
222
|
+
| `matches` | both | searched in the text, or in joined titles + snippets |
|
|
223
|
+
|
|
224
|
+
`minChars` is deliberately not applied to a list, and `minResults` not to text:
|
|
225
|
+
the two floors measure different things, and applying both would reject a short
|
|
226
|
+
but complete set of hits.
|
|
227
|
+
|
|
228
|
+
`minChars` defaults to **0 (off)**. A false positive here fails the whole call,
|
|
229
|
+
because there is only one rung — so the bar for rejecting is deliberately high.
|
|
230
|
+
|
|
231
|
+
## retries: for what is genuinely transient
|
|
232
|
+
|
|
233
|
+
```json
|
|
234
|
+
{ "retries": 1, "retryWhen": ["PageBindTimeout", "retryable"] }
|
|
235
|
+
```
|
|
236
|
+
|
|
237
|
+
Retries apply to **transport failures only**. Content rejected by `failWhen` is
|
|
238
|
+
never retried: that verdict is deterministic, so a second identical call only
|
|
239
|
+
costs time and, for a browser rung, another tab.
|
|
240
|
+
|
|
241
|
+
`retryWhen` scopes it. Measured 2026-08-09: browserwright intermittently fails
|
|
242
|
+
with `PageBindTimeout` on a healthy daemon and marks it `retryable: true`
|
|
243
|
+
itself. With one rung per tool there is nothing to fall through to, so that one
|
|
244
|
+
retry is the difference between a blip and a failed call — while a 404 is not
|
|
245
|
+
worth repeating.
|
|
246
|
+
|
|
247
|
+
## Probe: rules from evidence, not guesses
|
|
248
|
+
|
|
249
|
+
```
|
|
250
|
+
/browserwright list # show both chains
|
|
251
|
+
/browserwright probe # every fetch provider
|
|
252
|
+
/browserwright probe browserwright
|
|
253
|
+
```
|
|
254
|
+
|
|
255
|
+
Runs a provider against the real URLs in `probe-cases.json` — a normal article,
|
|
256
|
+
a client-rendered shell, a bot wall, a login wall, a 404, a page past the
|
|
257
|
+
truncation limit, a PDF, and localhost — then prints what came back and writes
|
|
258
|
+
`providers/<name>.probe.json`.
|
|
259
|
+
|
|
260
|
+
Constraints:
|
|
261
|
+
|
|
262
|
+
- **Manual only.** It hits real sites and opens tabs in the user's browser. It
|
|
263
|
+
asks for confirmation first.
|
|
264
|
+
- **Fetch providers only** — the cases are URLs.
|
|
265
|
+
- **Evidence files store summaries, never whole pages.** Real pages can carry
|
|
266
|
+
the user's logged-in content.
|
|
267
|
+
- Results drift as sites change, so every evidence file is timestamped.
|
|
268
|
+
- When installed from npm the package lives under `node_modules`, so evidence
|
|
269
|
+
falls back to the temp dir rather than being lost.
|
|
270
|
+
|
|
271
|
+
## Tests
|
|
272
|
+
|
|
273
|
+
```bash
|
|
274
|
+
node --test 'core/*.test.ts' # 83 cases, no network, no browser
|
|
275
|
+
node verify.ts # real fetch chain against a real URL
|
|
276
|
+
node verify.ts --search "…" # real search chain, opens a tab
|
|
277
|
+
```
|
|
278
|
+
|
|
279
|
+
The unit tests need Node >= 23.6 for unflagged TypeScript type stripping. The
|
|
280
|
+
package itself has no such floor — pi loads it through jiti.
|
|
281
|
+
|
|
282
|
+
The executor is injected throughout `core/`, which is why nothing there touches
|
|
283
|
+
the network. That is also where the tests are, because that is the code which
|
|
284
|
+
fails **silently**: a rung never tried, a JS shell accepted as success, or an
|
|
285
|
+
empty result list returned as an answer produce no error — just quietly worse
|
|
286
|
+
answers.
|
|
287
|
+
|
|
288
|
+
## Errors are thrown, not returned
|
|
289
|
+
|
|
290
|
+
`AgentToolResult` has no `isError` field. pi's agent loop hardcodes
|
|
291
|
+
`isError: false` on the normal return path and only sets it in the `catch`
|
|
292
|
+
around `execute`. A tool that returns `{isError: true}` therefore records a
|
|
293
|
+
failed call as a **successful** one: the TUI does not mark it, and observers of
|
|
294
|
+
the `tool_result` event see `isError: false`. So a chain failure here throws.
|
|
295
|
+
|
|
296
|
+
## Deliberately not built
|
|
297
|
+
|
|
298
|
+
- **No cache.** Overflow past 50KB goes to a temp file whose path the model
|
|
299
|
+
gets; it then uses `read` and `grep`, which beat any pagination parameter.
|
|
300
|
+
- **No site route table.** A route that sends a host straight to a heavy rung
|
|
301
|
+
destroys the evidence that would later invalidate it. The waste is exposed in
|
|
302
|
+
`chain:` instead — add a route when it actually annoys you.
|
|
303
|
+
- **No SSRF guard.** This is a local CLI, not a server: there is no external
|
|
304
|
+
attacker, and blocking private addresses would remove localhost fetching,
|
|
305
|
+
which is a real workflow. The requested URL is printed so internal fetches
|
|
306
|
+
stay visible in the transcript.
|
|
307
|
+
- **No body fetching inside `web_search`.** See the two-tool split above.
|
|
308
|
+
|
|
309
|
+
## License
|
|
310
|
+
|
|
311
|
+
[AGPL-3.0-only](LICENSE), the same as browserwright itself — copyleft including
|
|
312
|
+
network service use.
|
package/config.json
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
{
|
|
2
|
+
"order": {
|
|
3
|
+
"fetch": ["browserwright"],
|
|
4
|
+
"search": ["browserwright-search"]
|
|
5
|
+
},
|
|
6
|
+
"defaultFailWhen": {
|
|
7
|
+
"minChars": 0,
|
|
8
|
+
"minResults": 0,
|
|
9
|
+
"matches": ["enable javascript", "just a moment", "checking your browser", "captcha"]
|
|
10
|
+
},
|
|
11
|
+
"timeoutMs": 30000,
|
|
12
|
+
"maxBytes": 51200,
|
|
13
|
+
"maxLines": 2000
|
|
14
|
+
}
|
package/core/chain.ts
ADDED
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The fallback engine.
|
|
3
|
+
*
|
|
4
|
+
* Walks the selected providers in order, and treats a provider as failed both
|
|
5
|
+
* when it errors and when it succeeds with a payload that its own failWhen rule
|
|
6
|
+
* rejects. The executor is injected so this whole file is testable without a
|
|
7
|
+
* network or a browser.
|
|
8
|
+
*
|
|
9
|
+
* The engine is payload-agnostic: it never looks inside `content`, it only asks
|
|
10
|
+
* the caller-supplied `inspect` to reduce it to text plus an item count. That is
|
|
11
|
+
* what lets one chain serve `web_fetch` (a Markdown blob) and `web_search`
|
|
12
|
+
* (a list of results).
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
import { execCommand } from "./exec-command.ts";
|
|
16
|
+
import { execHttp } from "./exec-http.ts";
|
|
17
|
+
import { execModule } from "./exec-module.ts";
|
|
18
|
+
import { failureReason, providersForRole, resolveFailWhen, selectProviders } from "./predicates.ts";
|
|
19
|
+
import type { Attempt, ChainResult, Inspector, PiConfig, Provider, ProviderOutcome, Role } from "./types.ts";
|
|
20
|
+
|
|
21
|
+
export type Executor<T> = (provider: Provider, subject: string) => Promise<ProviderOutcome<T>>;
|
|
22
|
+
|
|
23
|
+
export interface RunChainOptions<T> {
|
|
24
|
+
providers: Map<string, Provider>;
|
|
25
|
+
config: PiConfig;
|
|
26
|
+
/** Which tool is asking. Selects both the provider set and the order. */
|
|
27
|
+
role: Role;
|
|
28
|
+
/** A URL for fetch, the raw query for search. */
|
|
29
|
+
subject: string;
|
|
30
|
+
/** Reduces a payload to what failWhen can reason about. */
|
|
31
|
+
inspect: Inspector<T>;
|
|
32
|
+
/** Explicit provider from the model. Disables fallback. */
|
|
33
|
+
forced?: string;
|
|
34
|
+
executor: Executor<T>;
|
|
35
|
+
/** Called before each attempt so the caller can show a status line. */
|
|
36
|
+
onAttempt?: (provider: Provider, index: number, total: number) => void;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
async function callOnce<T>(
|
|
40
|
+
executor: Executor<T>,
|
|
41
|
+
provider: Provider,
|
|
42
|
+
subject: string,
|
|
43
|
+
): Promise<ProviderOutcome<T>> {
|
|
44
|
+
try {
|
|
45
|
+
return await executor(provider, subject);
|
|
46
|
+
} catch (error) {
|
|
47
|
+
return { ok: false, reason: (error as Error).message };
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/** No `retryWhen` means "any transport failure is worth one more go". */
|
|
52
|
+
function shouldRetry(retryWhen: string[] | undefined, reason: string | undefined): boolean {
|
|
53
|
+
if (!retryWhen || retryWhen.length === 0) return true;
|
|
54
|
+
const haystack = (reason ?? "").toLowerCase();
|
|
55
|
+
return retryWhen.some((needle) => needle && haystack.includes(needle.toLowerCase()));
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
export async function runChain<T>(options: RunChainOptions<T>): Promise<ChainResult<T>> {
|
|
59
|
+
const { config, role, subject, forced, executor, inspect } = options;
|
|
60
|
+
const eligible = providersForRole(options.providers, role);
|
|
61
|
+
const { chain, skipped } = selectProviders(eligible, config.order[role] ?? [], subject, forced);
|
|
62
|
+
|
|
63
|
+
const attempts: Attempt[] = skipped.map((entry) => ({
|
|
64
|
+
provider: entry.name,
|
|
65
|
+
ok: false,
|
|
66
|
+
ms: 0,
|
|
67
|
+
reason: entry.reason,
|
|
68
|
+
skipped: true,
|
|
69
|
+
}));
|
|
70
|
+
|
|
71
|
+
for (const [index, provider] of chain.entries()) {
|
|
72
|
+
options.onAttempt?.(provider, index, chain.length);
|
|
73
|
+
const startedAt = performance.now();
|
|
74
|
+
let outcome = await callOnce(executor, provider, subject);
|
|
75
|
+
|
|
76
|
+
// Transport-level retry, before the content gate. browserwright reports
|
|
77
|
+
// some failures as explicitly `retryable` (a target that vanished between
|
|
78
|
+
// binding and use); with one rung per tool there is nothing to fall
|
|
79
|
+
// through to, so not retrying turns a blip into a failed call.
|
|
80
|
+
let remaining = provider.retries ?? 0;
|
|
81
|
+
while (!outcome.ok && remaining > 0 && shouldRetry(provider.retryWhen, outcome.reason)) {
|
|
82
|
+
remaining -= 1;
|
|
83
|
+
options.onAttempt?.(provider, index, chain.length);
|
|
84
|
+
outcome = await callOnce(executor, provider, subject);
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
const ms = performance.now() - startedAt;
|
|
88
|
+
|
|
89
|
+
if (!outcome.ok || outcome.content === undefined) {
|
|
90
|
+
attempts.push({ provider: provider.name, ok: false, ms, reason: outcome.reason ?? "failed" });
|
|
91
|
+
continue;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
// Succeeded at the transport level — now apply this provider's own
|
|
95
|
+
// line of defence to the payload it returned.
|
|
96
|
+
const rule = resolveFailWhen(config.defaultFailWhen, provider.failWhen);
|
|
97
|
+
const rejected = failureReason(outcome.content, rule, inspect);
|
|
98
|
+
if (rejected) {
|
|
99
|
+
attempts.push({ provider: provider.name, ok: false, ms, reason: rejected });
|
|
100
|
+
continue;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
attempts.push({ provider: provider.name, ok: true, ms });
|
|
104
|
+
return {
|
|
105
|
+
ok: true,
|
|
106
|
+
attempts,
|
|
107
|
+
provider: provider.name,
|
|
108
|
+
format: provider.returns,
|
|
109
|
+
content: outcome.content,
|
|
110
|
+
};
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
return { ok: false, attempts };
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
export interface ExecutorOptions {
|
|
117
|
+
dir: string;
|
|
118
|
+
role: Role;
|
|
119
|
+
signal?: AbortSignal;
|
|
120
|
+
/** Forwarded to `kind: "module"` runners, which are the only ones that stream. */
|
|
121
|
+
onProgress?: (text: string) => void;
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/** Wire the real executors. Kept separate so tests can skip it entirely. */
|
|
125
|
+
export function makeExecutor<T>(config: PiConfig, options: ExecutorOptions): Executor<T> {
|
|
126
|
+
const { dir, role, signal, onProgress } = options;
|
|
127
|
+
return async (provider, subject) => {
|
|
128
|
+
const shared = { dir, role, timeoutMs: config.timeoutMs, signal };
|
|
129
|
+
if (provider.kind === "http") {
|
|
130
|
+
return (await execHttp(provider, subject, shared)) as ProviderOutcome<T>;
|
|
131
|
+
}
|
|
132
|
+
if (provider.kind === "module") {
|
|
133
|
+
return await execModule<T>(provider, subject, { ...shared, onProgress });
|
|
134
|
+
}
|
|
135
|
+
return (await execCommand(provider, subject, shared)) as ProviderOutcome<T>;
|
|
136
|
+
};
|
|
137
|
+
}
|
package/core/config.ts
ADDED
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Config and provider loading.
|
|
3
|
+
*
|
|
4
|
+
* Provider declarations are plain JSON files in providers/. Adding a provider
|
|
5
|
+
* means dropping one file in there — no code change, no registration table.
|
|
6
|
+
* A declaration that names no `role` serves `web_fetch`, which is what the
|
|
7
|
+
* majority of reader APIs are.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import { existsSync, readdirSync, readFileSync } from "node:fs";
|
|
11
|
+
import { dirname, join } from "node:path";
|
|
12
|
+
import { fileURLToPath } from "node:url";
|
|
13
|
+
import { ROLES, type PiConfig, type Provider, type Role } from "./types.ts";
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* The package's own directory, used for {dir} and for resolving module
|
|
17
|
+
* providers. Two dirnames up from here, so this file must stay exactly one
|
|
18
|
+
* directory below the package root — everything else hangs off this value.
|
|
19
|
+
*/
|
|
20
|
+
export const EXTENSION_DIR = dirname(dirname(fileURLToPath(import.meta.url)));
|
|
21
|
+
|
|
22
|
+
const CONFIG_FILE = "config.json";
|
|
23
|
+
const LOG_PREFIX = "[browserwright-pi]";
|
|
24
|
+
|
|
25
|
+
const DEFAULT_CONFIG: PiConfig = {
|
|
26
|
+
order: {
|
|
27
|
+
fetch: ["browserwright"],
|
|
28
|
+
search: ["browserwright-search"],
|
|
29
|
+
},
|
|
30
|
+
// The default line of defence. minChars stays 0 on purpose: a false positive
|
|
31
|
+
// escalates to a rung that opens a tab in the user's real Chrome, so
|
|
32
|
+
// over-eager rejection interrupts them. Per-provider thresholds are meant to
|
|
33
|
+
// come from `/browserwright probe` evidence, not from guesses.
|
|
34
|
+
defaultFailWhen: {
|
|
35
|
+
minChars: 0,
|
|
36
|
+
minResults: 0,
|
|
37
|
+
matches: ["enable javascript", "just a moment", "checking your browser", "captcha"],
|
|
38
|
+
},
|
|
39
|
+
timeoutMs: 30_000,
|
|
40
|
+
maxBytes: 50 * 1024,
|
|
41
|
+
maxLines: 2000,
|
|
42
|
+
};
|
|
43
|
+
|
|
44
|
+
function readJson<T>(path: string): T | undefined {
|
|
45
|
+
try {
|
|
46
|
+
return JSON.parse(readFileSync(path, "utf8")) as T;
|
|
47
|
+
} catch (error) {
|
|
48
|
+
console.error(`${LOG_PREFIX} failed to read ${path}: ${(error as Error).message}`);
|
|
49
|
+
return undefined;
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export function loadConfig(dir: string = EXTENSION_DIR): PiConfig {
|
|
54
|
+
const path = join(dir, CONFIG_FILE);
|
|
55
|
+
if (!existsSync(path)) return DEFAULT_CONFIG;
|
|
56
|
+
const raw = readJson<Partial<PiConfig>>(path) ?? {};
|
|
57
|
+
return {
|
|
58
|
+
...DEFAULT_CONFIG,
|
|
59
|
+
...raw,
|
|
60
|
+
// Merged per role rather than replaced wholesale, so overriding one
|
|
61
|
+
// tool's order does not silently blank the other's.
|
|
62
|
+
order: { ...DEFAULT_CONFIG.order, ...(raw.order ?? {}) },
|
|
63
|
+
defaultFailWhen: { ...DEFAULT_CONFIG.defaultFailWhen, ...(raw.defaultFailWhen ?? {}) },
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
export function loadProviders(dir: string = EXTENSION_DIR): Map<string, Provider> {
|
|
68
|
+
const providerDir = join(dir, "providers");
|
|
69
|
+
const providers = new Map<string, Provider>();
|
|
70
|
+
if (!existsSync(providerDir)) return providers;
|
|
71
|
+
|
|
72
|
+
for (const file of readdirSync(providerDir).sort()) {
|
|
73
|
+
// *.probe.json holds probe evidence, not declarations. Runners for
|
|
74
|
+
// module providers live here too and are imported, never loaded as JSON.
|
|
75
|
+
if (!file.endsWith(".json") || file.endsWith(".probe.json")) continue;
|
|
76
|
+
const declared = readJson<Provider>(join(providerDir, file));
|
|
77
|
+
if (!declared) continue;
|
|
78
|
+
if (!declared.name || !declared.kind || !declared.returns) {
|
|
79
|
+
console.error(`${LOG_PREFIX} ${file}: needs name, kind and returns — skipped`);
|
|
80
|
+
continue;
|
|
81
|
+
}
|
|
82
|
+
if (declared.role && !ROLES.includes(declared.role as Role)) {
|
|
83
|
+
console.error(`${LOG_PREFIX} ${file}: unknown role ${JSON.stringify(declared.role)} — skipped`);
|
|
84
|
+
continue;
|
|
85
|
+
}
|
|
86
|
+
providers.set(declared.name, declared);
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
return providers;
|
|
90
|
+
}
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The "command" provider kind: run one argv, take stdout.
|
|
3
|
+
*
|
|
4
|
+
* Exit code contract (the whole reason a command can participate in the chain
|
|
5
|
+
* without the core knowing anything about it):
|
|
6
|
+
* 0 success, stdout is the content
|
|
7
|
+
* 2 not applicable — drop to the next rung, this is not an error
|
|
8
|
+
* anything else hard error — also drops a rung, but is reported as an error
|
|
9
|
+
*
|
|
10
|
+
* The provider script is the thing that understands its own tool, so it owns
|
|
11
|
+
* the "is this page actually empty" judgement and signals it with exit 2.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import { spawn } from "node:child_process";
|
|
15
|
+
import { interpolate, subjectTokens } from "./predicates.ts";
|
|
16
|
+
import type { CommandProvider, ProviderOutcome, Role } from "./types.ts";
|
|
17
|
+
|
|
18
|
+
export async function execCommand(
|
|
19
|
+
provider: CommandProvider,
|
|
20
|
+
subject: string,
|
|
21
|
+
options: { dir: string; role: Role; timeoutMs: number; signal?: AbortSignal },
|
|
22
|
+
): Promise<ProviderOutcome<string>> {
|
|
23
|
+
const tokens = subjectTokens(options.role, subject, options.dir);
|
|
24
|
+
const argv = provider.command.map((part) => interpolate(part, tokens));
|
|
25
|
+
if (argv.length === 0) return { ok: false, reason: "empty command" };
|
|
26
|
+
|
|
27
|
+
const [bin, ...args] = argv;
|
|
28
|
+
const timeoutMs = provider.timeoutMs ?? options.timeoutMs;
|
|
29
|
+
|
|
30
|
+
return await new Promise<ProviderOutcome<string>>((resolve) => {
|
|
31
|
+
let settled = false;
|
|
32
|
+
const finish = (outcome: ProviderOutcome<string>) => {
|
|
33
|
+
if (settled) return;
|
|
34
|
+
settled = true;
|
|
35
|
+
clearTimeout(timer);
|
|
36
|
+
options.signal?.removeEventListener("abort", onAbort);
|
|
37
|
+
resolve(outcome);
|
|
38
|
+
};
|
|
39
|
+
|
|
40
|
+
const child = spawn(bin, args, {
|
|
41
|
+
cwd: provider.cwd ? interpolate(provider.cwd, tokens) : options.dir,
|
|
42
|
+
env: { ...process.env, ...(provider.env ?? {}) },
|
|
43
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
const timer = setTimeout(() => {
|
|
47
|
+
child.kill("SIGKILL");
|
|
48
|
+
finish({ ok: false, reason: `timeout after ${timeoutMs}ms` });
|
|
49
|
+
}, timeoutMs);
|
|
50
|
+
|
|
51
|
+
const onAbort = () => {
|
|
52
|
+
child.kill("SIGKILL");
|
|
53
|
+
finish({ ok: false, reason: "aborted" });
|
|
54
|
+
};
|
|
55
|
+
options.signal?.addEventListener("abort", onAbort, { once: true });
|
|
56
|
+
|
|
57
|
+
let stdout = "";
|
|
58
|
+
let stderr = "";
|
|
59
|
+
child.stdout.on("data", (chunk) => {
|
|
60
|
+
stdout += chunk;
|
|
61
|
+
});
|
|
62
|
+
child.stderr.on("data", (chunk) => {
|
|
63
|
+
stderr += chunk;
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
child.on("error", (error) => {
|
|
67
|
+
finish({ ok: false, reason: `spawn failed: ${error.message}` });
|
|
68
|
+
});
|
|
69
|
+
|
|
70
|
+
child.on("close", (code) => {
|
|
71
|
+
if (code === 0) return finish({ ok: true, content: stdout });
|
|
72
|
+
if (code === 2) {
|
|
73
|
+
const why = stderr.trim().split("\n").pop() ?? "";
|
|
74
|
+
return finish({ ok: false, reason: `not applicable${why ? `: ${why}` : ""}` });
|
|
75
|
+
}
|
|
76
|
+
const detail = (stderr.trim() || stdout.trim()).split("\n").pop() ?? "";
|
|
77
|
+
finish({ ok: false, reason: `exit ${code}${detail ? `: ${detail.slice(0, 200)}` : ""}` });
|
|
78
|
+
});
|
|
79
|
+
});
|
|
80
|
+
}
|