@parcha/pi-browser 0.1.0-beta.17 → 0.1.0-beta.19

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. package/CHANGELOG.md +5 -0
  2. package/README.md +177 -39
  3. package/dist/coding-agent-local.js +2 -1
  4. package/dist/coding-agent.d.ts +42 -2
  5. package/dist/coding-agent.js +159 -21
  6. package/dist/core/clip.d.ts +31 -0
  7. package/dist/core/clip.js +49 -0
  8. package/dist/core/contract.d.ts +4 -1
  9. package/dist/core/contract.js +40 -7
  10. package/dist/core/custody.d.ts +79 -5
  11. package/dist/core/custody.js +66 -5
  12. package/dist/core/decisions.d.ts +4 -0
  13. package/dist/core/dial.d.ts +7 -0
  14. package/dist/core/dial.js +41 -0
  15. package/dist/core/downloads.d.ts +1 -0
  16. package/dist/core/downloads.js +17 -5
  17. package/dist/core/driving-guard.d.ts +13 -0
  18. package/dist/core/driving-guard.js +61 -0
  19. package/dist/core/driving.d.ts +61 -0
  20. package/dist/core/driving.js +156 -0
  21. package/dist/core/effects.js +33 -13
  22. package/dist/core/evidence.d.ts +3 -1
  23. package/dist/core/failures.d.ts +5 -1
  24. package/dist/core/failures.js +20 -4
  25. package/dist/core/files.d.ts +8 -0
  26. package/dist/core/files.js +74 -0
  27. package/dist/core/find.d.ts +3 -1
  28. package/dist/core/find.js +2 -1
  29. package/dist/core/guard.d.ts +9 -1
  30. package/dist/core/guard.js +7 -2
  31. package/dist/core/host.d.ts +49 -1
  32. package/dist/core/images.d.ts +10 -2
  33. package/dist/core/images.js +4 -4
  34. package/dist/core/lease.d.ts +1 -1
  35. package/dist/core/lease.js +2 -1
  36. package/dist/core/session-control.d.ts +2 -0
  37. package/dist/core/session-control.js +5 -0
  38. package/dist/core/tools.d.ts +70 -4
  39. package/dist/core/tools.js +288 -45
  40. package/dist/core/unanswered.d.ts +3 -0
  41. package/dist/core/unanswered.js +17 -0
  42. package/dist/core/usage.d.ts +2 -0
  43. package/dist/core/usage.js +6 -0
  44. package/dist/core/web.d.ts +1 -1
  45. package/dist/core/web.js +9 -7
  46. package/dist/driver/extension-worker.d.ts +4 -2
  47. package/dist/driver/extension-worker.js +16 -10
  48. package/dist/driver/page-driver.d.ts +8 -0
  49. package/dist/driver/page-driver.js +128 -0
  50. package/dist/driver/stagehand.d.ts +3 -1
  51. package/dist/driver/stagehand.js +40 -66
  52. package/dist/durable.d.ts +16 -4
  53. package/dist/durable.js +415 -82
  54. package/dist/index.d.ts +2 -0
  55. package/dist/index.js +2 -0
  56. package/dist/live-view/index.d.ts +4 -0
  57. package/dist/live-view/index.js +4 -0
  58. package/dist/live-view/protocol.d.ts +152 -0
  59. package/dist/live-view/protocol.js +84 -0
  60. package/dist/live-view/provider.d.ts +6 -0
  61. package/dist/live-view/provider.js +23 -0
  62. package/dist/live-view/server.d.ts +50 -0
  63. package/dist/live-view/server.js +342 -0
  64. package/dist/live-view/stream.d.ts +7 -0
  65. package/dist/live-view/stream.js +388 -0
  66. package/dist/live-view/token.d.ts +6 -0
  67. package/dist/live-view/token.js +13 -0
  68. package/dist/live-view/viewer.d.ts +6 -0
  69. package/dist/live-view/viewer.js +350 -0
  70. package/dist/live-view/ws.d.ts +19 -0
  71. package/dist/live-view/ws.js +176 -0
  72. package/dist/providers/assigned.d.ts +18 -0
  73. package/dist/providers/assigned.js +76 -0
  74. package/dist/providers/cdp-call.js +12 -2
  75. package/dist/providers/cdp.js +37 -11
  76. package/dist/recorder/follow.d.ts +10 -0
  77. package/dist/recorder/follow.js +27 -2
  78. package/dist/recorder/frames.d.ts +7 -0
  79. package/dist/recorder/frames.js +73 -5
  80. package/dist/store.d.ts +22 -3
  81. package/dist/store.js +75 -4
  82. package/dist/testing/backend.d.ts +26 -0
  83. package/dist/testing/backend.js +21 -3
  84. package/dist/testing/driver.d.ts +18 -1
  85. package/dist/testing/driver.js +4 -0
  86. package/dist/web.js +2 -2
  87. package/package.json +10 -2
@@ -1,21 +1,113 @@
1
- import { BrowserFailureError } from "./custody.js";
1
+ import { BrowserFailureError, FenceRefusal, loadedNoPage, passedThrough } from "./custody.js";
2
2
  import { defaultInline } from "./evidence.js";
3
- import { browserFailure, classifyBrowserError } from "./failures.js";
3
+ import { fetchDownloads } from "./downloads.js";
4
+ import { browserFailure, classifyBrowserError, transportFactOf } from "./failures.js";
4
5
  import { createRepeatGuard } from "./guard.js";
6
+ import { regionOf } from "./clip.js";
5
7
  import { captureWithinBudget } from "./images.js";
6
8
  import { orderFacts, receiptFacts, sha256Hex } from "./receipts.js";
7
9
  import { scrubPageUrl } from "./redact.js";
10
+ import { UNANSWERED_CALL } from "./unanswered.js";
8
11
  import { pageNotice, wallFailure, wallOf } from "./web.js";
9
12
  const WHATS = ["markdown", "text", "html", "url", "title"];
10
13
  const DROPPED_RUN = "The connection to your browser session dropped; the session is still open, and your next browser call reconnects to it with its pages and cookies. This run may have acted before the drop: take a snapshot to see the page before running again.";
11
14
  const DROPPED_FOR_GOOD = "The connection to your browser session dropped and could not be re-established, though the session is still open. Call browser_relaunch for a fresh session (its pages and cookies are gone), or use web_fetch.";
15
+ const HUNG_PAGE = "After a fresh connection to your browser session, the page still did not answer this call: the page itself is stuck, not the browser. Call browser_relaunch for a fresh session (its pages and cookies are gone), then open the page again.";
16
+ const DROPPED_RELEASED = "The connection to your browser session dropped and could not be re-established, so that session was released: a session that stops answering keeps running and billing. Your next browser call opens a fresh one, without the old pages or cookies; browser_relaunch opens it now. Or use web_fetch.";
17
+ const KEPT_FOR_PERSON = "The browser is held for a person, and its connection could not be re-established; it is kept for them. Wait for them to finish, or use web_fetch.";
18
+ const RELAUNCH_NEXT = { action: "browser_relaunch", note: "Call browser_relaunch for a fresh session now, or make your next browser call, which opens one." };
19
+ /** The same remedy for a session custody did not release (a stuck page, or a release that failed): the next ordinary call would use it. */
20
+ const RELAUNCH_ONLY = { action: "browser_relaunch", note: "Call browser_relaunch for a fresh session; this one is still attached, and your next browser call would use it." };
21
+ const RECONNECT = { budgetMs: 90_000, tryMs: 61_000, backoffMs: [1_000, 3_000] };
22
+ const pause = (ms, signal) => new Promise((resolve) => {
23
+ const done = () => { clearTimeout(timer); signal?.removeEventListener("abort", done); resolve(); };
24
+ const timer = setTimeout(done, ms);
25
+ signal?.addEventListener("abort", done, { once: true });
26
+ });
12
27
  const text = (value, isError = false) => ({ content: [{ type: "text", text: typeof value === "string" ? value : JSON.stringify(value) }], ...(isError ? { isError: true } : {}) });
13
28
  const withNews = (news, body) => (news ? `${news}\n\n${body}` : body);
29
+ const refused = (message) => ({ ok: false, code: "refused", retryable: false, effect: "none", message });
30
+ /** A read of a browser that has loaded no page yet: it is fresh, and the next step is to navigate, to the replaced
31
+ * session's last page when there is one. */
32
+ function freshBrowser(replaced) {
33
+ const why = replaced ? `it replaced the session released${replaced.at ? ` at ${replaced.at}` : ""} (${replaced.reason})` : "it was just opened";
34
+ const go = replaced?.lastUrl ? `Navigate to the last page, ${replaced.lastUrl}, with run, then read again.` : "Navigate to the page you need with run, then read again.";
35
+ return { ok: false, code: "not_content", retryable: true, effect: "none", message: `This browser is fresh and has loaded no page yet: ${why}. ${go}`, detail: "", next: { action: "drive_page", note: go } };
36
+ }
14
37
  const pageUrl = (call, raw) => scrubPageUrl(raw, call.redact);
15
- /** A cut run that was observed and sent nothing that left the browser is safe to run again. */
16
- const harmless = (p) => p.observed === true && !(p.sent ?? []).some((r) => r.held !== "denied");
38
+ const MAX_FRAMES = 5;
39
+ const FRAME_TEXT_CHARS = 200;
40
+ const FRAME_TEXT_READ_CHARS = 20_000;
41
+ const FRAME_SCAN_CHARS = 2_000_000;
42
+ /** What the judge is shown of the page's frames, in document order: a frame's content is not in the page's text, so a page whose content sits in a challenge frame reads as empty without it. `frames` are the distinct origins (origin only: a
43
+ * frame's path and query can carry a token; http(s) only; a relative source resolves against the page's `<base href>` when it has one, as the browser does) and `frameText` what the page puts inside the frame elements (the fallback a browser
44
+ * does not render, where a block page keeps its message), collapsed to one line and cut at 200 characters. A frame is an element: markup inside a script, a comment, a noscript or a template is not one, so it neither counts nor takes one of the five.
45
+ * A page that cannot be parsed gives none. */
46
+ async function framesOf(html, pageUrl, redact) {
47
+ try {
48
+ const { parse } = await import("node-html-parser");
49
+ // HTML's own void elements (the parser's default list lacks `frame`, so an unclosed <frame> would swallow its sibling).
50
+ const root = parse(html.slice(0, FRAME_SCAN_CHARS), { voidTag: { tags: ["area", "base", "br", "col", "embed", "frame", "hr", "img", "input", "keygen", "link", "meta", "param", "source", "track", "wbr"] } });
51
+ let base = pageUrl;
52
+ const baseHref = root.querySelector("base[href]")?.getAttribute("href");
53
+ if (baseHref) {
54
+ try {
55
+ base = new URL(baseHref, pageUrl).href;
56
+ }
57
+ catch { /* a base that does not parse: the page's own URL stands */ }
58
+ }
59
+ const origins = new Set();
60
+ const texts = new Set();
61
+ for (const frame of root.querySelectorAll("iframe, frame")) {
62
+ if (origins.size >= MAX_FRAMES && texts.size >= MAX_FRAMES)
63
+ break;
64
+ let inert = false;
65
+ for (let up = frame.parentNode; up; up = up.parentNode)
66
+ if (up.rawTagName?.toLowerCase() === "template") {
67
+ inert = true;
68
+ break;
69
+ }
70
+ if (inert)
71
+ continue;
72
+ const source = frame.getAttribute("src")?.trim();
73
+ if (source && origins.size < MAX_FRAMES) {
74
+ try {
75
+ const url = new URL(source, base);
76
+ if (url.protocol === "http:" || url.protocol === "https:")
77
+ origins.add(redact(url.origin));
78
+ }
79
+ catch { /* a source that does not parse names no origin */ }
80
+ }
81
+ if (texts.size < MAX_FRAMES) {
82
+ // Redacted WHOLE, then cut: a registered secret the cut split would no longer match, and half of it would reach the judge. The read bound is far past the cut, so nothing the cut keeps can straddle it.
83
+ const text = redact(frame.text.replace(/\s+/g, " ").trim().slice(0, FRAME_TEXT_READ_CHARS)).slice(0, FRAME_TEXT_CHARS);
84
+ if (text)
85
+ texts.add(text);
86
+ }
87
+ }
88
+ return { frames: [...origins], frameText: [...texts] };
89
+ }
90
+ catch {
91
+ return { frames: [], frameText: [] };
92
+ }
93
+ }
94
+ /** An effectful call the page never answered within its deadline: it may have acted, and the identical call is refused until a look. */
95
+ const UNANSWERED_EFFECT = {
96
+ browser_upload: "The page did not answer your browser_upload in time, so the file may already be set. Look at the page with snapshot or browser_read before you upload it again.",
97
+ };
98
+ const unansweredEffect = (tool) => UNANSWERED_EFFECT[tool] ?? `The page did not answer your ${tool} call in time, so it may already have taken effect. Look at the page with snapshot or browser_read before you repeat it.`;
99
+ /** The refusal of an identical replay-unsafe action, by tool. */
100
+ const SAME = {
101
+ run: "This is the same code, and it may already have taken effect, so it is not run again now: look at the page with snapshot or browser_read before you run this again.",
102
+ browser_upload: "This is the same upload, and it may already have taken effect, so it is not repeated now: look at the page with snapshot or browser_read before you upload it again.",
103
+ };
104
+ /** A cut `run` that was observed and sent nothing that left the browser is safe to run again. */
105
+ const harmlessRun = (p) => p.observed === true && !(p.sent ?? []).some((r) => r.held !== "denied");
106
+ /** A cut upload is never harmless: it sets a file into an input, which changes the page without any request, so an observer that saw none says nothing about whether the file is set. */
107
+ const harmlessUpload = (_p) => false;
108
+ const harmless = (p) => (p.tool === "browser_upload" ? harmlessUpload : harmlessRun)(p);
17
109
  function interruptedNotice(p, now) {
18
- const where = `Your run call ${p.callId}${p.url ? ` on ${p.url}` : ""}`;
110
+ const where = `Your ${p.tool ?? "run"} call ${p.callId}${p.url ? ` on ${p.url}` : ""}`;
19
111
  const sent = (p.sent ?? []).filter((r) => r.held !== "denied").map((r) => `${r.method} ${r.origin}${r.path}`);
20
112
  if (p.finished) {
21
113
  const head = `${where} ran, but the process stopped before its answer reached you`;
@@ -49,32 +141,89 @@ async function markdown(html, fallback) {
49
141
  return { body: fallback, extractor: "inner-text", ext: "txt" };
50
142
  }
51
143
  /** `guards` holds each conversation's repeat guard; a host that owns the conversation's lifecycle passes it to evict one. */
52
- export function pageTools({ guards = new Map() } = {}) {
53
- const guardOf = (conversationId) => guards.get(conversationId) ?? guards.set(conversationId, createRepeatGuard()).get(conversationId);
54
- async function failureOf(error, effectful, port, call) {
144
+ /** `reconnect`: the reconnect timings, for tests; hosts keep the defaults. */
145
+ export function pageTools({ guards = new Map(), reconnect: timings = RECONNECT } = {}) {
146
+ const guardOf = (conversationId, redact) => guards.get(conversationId) ?? guards.set(conversationId, createRepeatGuard(redact)).get(conversationId);
147
+ async function failureOf(error, effectful, port, call, action) {
55
148
  if (error instanceof BrowserFailureError)
56
149
  return error.failure;
57
- const facts = { aborted: call.signal?.aborted === true };
150
+ const facts = { aborted: call.signal?.aborted === true, action, redact: call.redact };
58
151
  const first = classifyBrowserError(error, effectful, facts);
59
152
  if (first.code === "aborted" || first.code === "auth" || !(await port.ended().catch(() => false)))
60
153
  return first;
61
154
  return classifyBrowserError(error, effectful, { ...facts, sessionEnded: true });
62
155
  }
63
156
  /** A replay-unsafe action is journaled before it acts and marked finished after; the entry stays until a later call
64
- * settles it, so a process that dies before pi commits the answer leaves a cut for the resumed run. */
65
- async function dispatched(port, call, s, key, action) {
66
- await port.dispatching({ callId: call.callId, codeSha: key, url: s.record.lastUrl, observed: s.effects?.holding === true, watched: s.effects != null });
157
+ * settles it, so a process that dies before pi commits the answer leaves a cut for the resumed run. One that a hold
158
+ * fenced before any of its driver calls got through never reached the page: its entry is settled at once, as not sent,
159
+ * so no look is owed for it in this process or a resumed one. */
160
+ async function dispatched(tool, port, call, s, key, action) {
161
+ await port.dispatching({ callId: call.callId, ...(tool === "browser_upload" ? { tool } : {}), codeSha: key, url: s.record.lastUrl, observed: s.effects?.holding === true, watched: s.effects != null });
162
+ const before = passedThrough(s.driver);
163
+ let notSent = false;
67
164
  try {
68
165
  return await action();
69
166
  }
167
+ catch (error) {
168
+ notSent = error instanceof FenceRefusal && before !== null && passedThrough(s.driver) === before;
169
+ throw error;
170
+ }
70
171
  finally {
71
- await port.mark("finished", call.callId).catch(() => undefined);
172
+ await (notSent ? port.settle(call.callId) : port.mark("finished", call.callId)).catch(() => undefined);
173
+ }
174
+ }
175
+ /** Reconnects a dropped session: the first try at once, then each backoff step while a whole try still fits in the budget
176
+ * and the call lasts. A refusal that is not a dropped browser (an abort, credentials) ends the tries. */
177
+ async function reconnect(port, call) {
178
+ const until = Date.now() + timings.budgetMs;
179
+ let error = null;
180
+ for (const wait of [0, ...timings.backoffMs]) {
181
+ if (wait > 0) {
182
+ // The next try must fit whole after its pause: a try is not bounded on our side, so one that could not finish in
183
+ // the budget never starts.
184
+ if (call.signal?.aborted || until - (Date.now() + wait) < timings.tryMs)
185
+ break;
186
+ await pause(wait, call.signal);
187
+ if (call.signal?.aborted)
188
+ break;
189
+ }
190
+ // The try's attach is bounded by what is left of the budget: one that waits on the browser (an earlier init still
191
+ // finishing in it) stops there instead of running past it. The timer is a ref'd one, cleared when the try ends, so a
192
+ // reconnect in progress keeps the process alive (AbortSignal.timeout's timer does not, and Node 22 then drains).
193
+ const deadline = new AbortController();
194
+ const timer = setTimeout(() => deadline.abort(new DOMException("the reconnect's budget ran out", "TimeoutError")), Math.max(0, until - Date.now()));
195
+ try {
196
+ return { session: await port.session(call.signal ? AbortSignal.any([call.signal, deadline.signal]) : deadline.signal) };
197
+ }
198
+ catch (failed) {
199
+ error = failed;
200
+ }
201
+ finally {
202
+ clearTimeout(timer);
203
+ }
204
+ // The budget ran out, not the call: that is a reconnect that failed, and the session goes on to be released as lost.
205
+ if (deadline.signal.aborted && !call.signal?.aborted) {
206
+ error = new BrowserFailureError(browserFailure("browser_unavailable", "The browser did not answer a reconnect within its budget."));
207
+ break;
208
+ }
209
+ if (notDropped(error))
210
+ break;
72
211
  }
212
+ // A call cancelled while it reconnects ends aborted, whatever the last try failed with: the session is the run's, not
213
+ // the cancelled call's, so it is never released for that.
214
+ if (call.signal?.aborted)
215
+ return { error: Object.assign(new Error("the call was cancelled while it reconnected"), { name: "AbortError" }) };
216
+ return { error };
73
217
  }
218
+ /** A failure no reconnect can help: an abort, credentials, a refusal, a replaced session, one held for a person. A
219
+ * provider that fails the attach (command_failed, timeout, rate_limited, browser_unavailable) is tried again. */
220
+ const FINAL = new Set(["aborted", "auth", "refused", "session_replaced", "waiting_for_human"]);
221
+ const notDropped = (error) => FINAL.has(error instanceof BrowserFailureError ? error.failure.code : classifyBrowserError(error, false).code);
74
222
  async function perform(tool, args, port, call, kind, body) {
75
223
  const effect = typeof kind === "object" ? kind.effect : null;
76
224
  const effectful = effect !== null;
77
- const guard = guardOf(call.conversationId);
225
+ const action = effectful && tool !== "run" ? `Your ${tool} call` : undefined;
226
+ const guard = guardOf(call.conversationId, call.redact);
78
227
  const refused = guard.check(tool, args);
79
228
  if (refused)
80
229
  return text(refused, true);
@@ -83,57 +232,106 @@ export function pageTools({ guards = new Map() } = {}) {
83
232
  let news = "";
84
233
  let cut = null;
85
234
  let facts = null;
86
- const tell = (out) => (facts ? { ...out, details: { ...out.details, pendingEffect: facts } } : out);
235
+ // The first call after a release the model did not ask for: by what and when, beside the notice that says it.
236
+ let previous = null;
237
+ const tell = (out) => (facts || previous ? { ...out, details: { ...out.details, ...(facts ? { pendingEffect: facts } : {}), ...(previous ? { previous_session: previous } : {}) } } : out);
87
238
  const done = (out) => { guard.note(tool, args, null); return tell(out); };
88
- const fail = (failure) => { guard.note(tool, args, failure); return tell(text(news ? { ...failure, notice: news } : failure, true)); };
89
- const looked = async (out) => { if (kind === "look" && cut?.refused)
90
- await port.settle(cut.callId); return done(out); };
239
+ // The notice leads a failure, never after its own keys: a call told nothing else first reads it.
240
+ const fail = (failure) => { guard.note(tool, args, failure); return tell(text(news ? { notice: news, ...(previous ? { previous_session: previous } : {}), ...failure } : failure, true)); };
241
+ let cuts = [];
242
+ // A look after a refusal settles every refused cut in the chain whose warning a saved answer had already told; one this call is
243
+ // telling stays until a later look, so a process that dies before this answer is saved tells it again.
244
+ const looked = async (out) => {
245
+ if (kind === "look") {
246
+ guard.looked();
247
+ for (const each of cuts)
248
+ if (each.refused && each.told)
249
+ await port.settle(each.callId);
250
+ }
251
+ return done(out);
252
+ };
91
253
  try {
92
254
  const s = await port.session();
93
255
  news = s.notice ?? "";
256
+ if (s.notice && s.replaced)
257
+ previous = { released: true, reason: s.replaced.reason, at: s.replaced.at };
94
258
  cut = s.interrupted;
95
- const same = cut !== null && effect === cut.codeSha && !harmless(cut);
259
+ cuts = s.cuts ?? (cut ? [cut] : []);
260
+ // A cut that sent nothing, told by an answer that was saved, is over.
261
+ if (cut?.told && harmless(cut)) {
262
+ await port.settle(cut.callId);
263
+ cuts = cuts.filter((each) => each !== cut);
264
+ cut = null;
265
+ }
266
+ // The cut this action would repeat: any in the chain. A repeat of a newer one while an older is still to be told is
267
+ // refused about itself, with its own notice and facts; the older one is told by the next call.
268
+ const repeated = effect === null ? null : cuts.find((each) => each.codeSha === effect && !harmless(each)) ?? null;
269
+ if (repeated && repeated !== cut) {
270
+ facts = factsOf(repeated, pageUrl(call, await s.driver.url().catch(() => "gone")));
271
+ if (!repeated.told) {
272
+ news = [news, interruptedNotice(repeated, facts.urlNow)].filter(Boolean).join(" ");
273
+ await port.mark("told", repeated.callId, call.callId);
274
+ }
275
+ await port.mark("refused", repeated.callId);
276
+ return done(text(browserFailure("effect_unknown", `${news ? `${news} ` : ""}${SAME[tool] ?? SAME.run}`, { retryable: true, effect: "possibly_effected" }), true));
277
+ }
278
+ const same = repeated !== null;
279
+ // The same action as one the page never answered in this process: refused like a cut, until a look.
280
+ const held = effect !== null && guard.held(tool, effect);
96
281
  if (cut && (!cut.told || same))
97
282
  facts = factsOf(cut, pageUrl(call, await s.driver.url().catch(() => "gone")));
98
283
  if (cut && !cut.told) {
99
284
  news = [news, interruptedNotice(cut, facts.urlNow)].filter(Boolean).join(" ");
100
- await (harmless(cut) ? port.settle(cut.callId) : port.mark("told", cut.callId));
285
+ // Told by this call: it counts once this call's answer, carrying the facts, is saved, so a process that dies first
286
+ // leaves the cut to be told again.
287
+ await port.mark("told", cut.callId, call.callId);
101
288
  }
102
- if (cut && same) {
103
- await port.mark("refused", cut.callId);
104
- return done(text(browserFailure("effect_unknown", `${news ? `${news} ` : ""}This is the same code, and it may already have taken effect, so it is not run again now: look at the page with snapshot or browser_read before you run this again.`, { retryable: true, effect: "possibly_effected" }), true));
289
+ if ((cut && same) || held) {
290
+ if (cut && same)
291
+ await port.mark("refused", cut.callId);
292
+ return done(text(browserFailure("effect_unknown", `${news ? `${news} ` : ""}${SAME[tool] ?? SAME.run}`, { retryable: true, effect: "possibly_effected" }), true));
105
293
  }
106
- return await looked(effect === null ? await body(s, news) : await dispatched(port, call, s, effect, () => body(s, news)));
294
+ return await looked(effect === null ? await body(s, news) : await dispatched(tool, port, call, s, effect, () => body(s, news)));
107
295
  }
108
296
  catch (error) {
109
- const failure = await failureOf(error, effectful, port, call);
297
+ const failure = await failureOf(error, effectful, port, call, action);
110
298
  if (failure.code !== "browser_unavailable" || error instanceof BrowserFailureError)
111
299
  return fail(failure);
112
300
  // A dropped CDP connection to a session the provider still runs: a read reconnects and runs once more; a run, which
113
301
  // may have acted, is not repeated.
114
302
  await port.dropped().catch(() => undefined);
303
+ if (effectful && transportFactOf(error).errorName === UNANSWERED_CALL) {
304
+ guard.hold(tool, effect);
305
+ return fail({ ...failure, effect: "possibly_effected", retryable: true, message: unansweredEffect(tool) });
306
+ }
115
307
  if (effectful)
116
308
  return fail({ ...failure, retryable: true, message: DROPPED_RUN });
117
- let again;
118
- try {
119
- again = await port.session();
120
- }
121
- catch (reconnect) {
122
- const refused = await failureOf(reconnect, effectful, port, call);
123
- return fail(refused.code === "aborted" || refused.code === "auth" ? refused : { ...refused, code: "browser_unavailable", retryable: false, message: DROPPED_FOR_GOOD });
124
- }
125
- try {
126
- return await looked(await body(again, news));
127
- }
128
- catch (retried) {
129
- return fail(await failureOf(retried, effectful, port, call));
309
+ const reconnected = await reconnect(port, call);
310
+ if ("session" in reconnected) {
311
+ try {
312
+ return await looked(await body(reconnected.session, news));
313
+ }
314
+ catch (retried) {
315
+ const failure = await failureOf(retried, effectful, port, call, action);
316
+ // The browser answered the reconnect, so a call it leaves unanswered again is the page's, not an unreachable browser.
317
+ return fail(failure.code === "browser_unavailable" && transportFactOf(retried).errorName === UNANSWERED_CALL ? { ...failure, retryable: false, message: HUNG_PAGE, next: RELAUNCH_ONLY } : failure);
318
+ }
130
319
  }
320
+ const refused = await failureOf(reconnected.error, effectful, port, call, action);
321
+ if (refused.code === "aborted" || refused.code === "auth")
322
+ return fail(refused);
323
+ // Every try failed while the provider still runs it: that session is released as lost (it would keep billing), unless a
324
+ // person holds or drives it, and the next call opens a fresh one.
325
+ const left = await port.unreachable().catch(() => null);
326
+ if (left?.kept)
327
+ return fail({ ...refused, code: "browser_unavailable", retryable: false, message: KEPT_FOR_PERSON });
328
+ return fail({ ...refused, code: "browser_unavailable", retryable: false, message: left?.released ? DROPPED_RELEASED : DROPPED_FOR_GOOD, next: left?.released ? RELAUNCH_NEXT : RELAUNCH_ONLY });
131
329
  }
132
330
  }
133
331
  async function file(s, call, record) {
134
332
  const facts = orderFacts({ ...record.facts, ...receiptFacts({ body: record.body, session: s.record, recorded: s.recorded }) });
135
333
  const filed = await call.evidence.file(call.label, { ...record, facts });
136
- return { path: filed?.path ?? null, bodyLine: filed?.bodyLine ?? null, sha256: facts.sha256 };
334
+ return { path: filed?.path ?? null, bodyLine: filed?.bodyLine ?? null, bodyPath: filed?.bodyPath ?? null, sha256: facts.sha256 };
137
335
  }
138
336
  return {
139
337
  snapshot: (args, port, call) => perform("snapshot", args ?? {}, port, call, "look", async (s, news) => text(withNews(news, await s.driver.snapshot(args ?? {})))),
@@ -163,29 +361,37 @@ export function pageTools({ guards = new Map() } = {}) {
163
361
  });
164
362
  },
165
363
  screenshot: (args, port, call) => {
166
- const { ask: asked, ...requested } = args ?? {};
364
+ const { ask: asked, clip: wantedClip, id: wantedId, selector: wantedSelector, ...requested } = args ?? {};
167
365
  const ask = typeof asked === "string" && asked.trim() ? asked.trim() : undefined;
168
366
  return perform("screenshot", args ?? {}, port, call, "read", async (s, news) => {
169
- const shot = await captureWithinBudget((options) => s.driver.screenshot(options), requested);
367
+ // A region goes through the same capture as the whole page, so whatever guards a capture guards a clipped one.
368
+ const region = await regionOf({ clip: wantedClip, id: wantedId, selector: wantedSelector, fullPage: requested.fullPage }, s.driver.locate?.bind(s.driver));
369
+ const shot = await captureWithinBudget((options) => s.driver.screenshot(options), region ? { ...requested, fullPage: false, clip: region.clip } : requested);
170
370
  const url = pageUrl(call, await s.driver.url());
171
- const receipt = await file(s, call, { tool: "screenshot", args: requested, status: "ok", facts: { final_url: url }, body: Buffer.from(shot.data, "base64"), ext: shot.size.type === "png" ? "png" : "jpg" });
371
+ const receipt = await file(s, call, { tool: "screenshot", args: region ? { ...requested, clip: region.clip, ...region.of } : requested, status: "ok", facts: { final_url: url }, body: Buffer.from(shot.data, "base64"), ext: shot.size.type === "png" ? "png" : "jpg" });
172
372
  const first = shot.adjusted?.first;
173
- const adjusted = shot.adjusted && `adjusted: the first capture${first ? ` (${first.width}x${first.height})` : ""} was over the ${shot.adjusted.why === "long_edge" ? "2,000 px long edge" : "size"} budget; this is the viewport at ${shot.size.width}x${shot.size.height}`;
373
+ const adjusted = shot.adjusted && `adjusted: the first capture${first ? ` (${first.width}x${first.height})` : ""} was over the ${shot.adjusted.why === "long_edge" ? "2,000 px long edge" : "size"} budget; this is ${region ? "the clip" : "the viewport"} at ${shot.size.width}x${shot.size.height}`;
174
374
  const lines = [news, `url: ${url}`, `evidence: ${receipt.path ?? "(not filed)"}`, adjusted].filter(Boolean).join("\n");
175
- return { content: [{ type: "text", text: lines }, { type: "image", data: shot.data, mimeType: shot.mimeType }], details: { evidence: receipt.path, sha256: receipt.sha256, ...(ask ? { media: { ask } } : {}) } };
375
+ const media = { kind: "screenshot", path: receipt.bodyPath, mime: shot.mimeType, bytes: Buffer.from(shot.data, "base64").byteLength, width: shot.size.width, height: shot.size.height, ...(region ? { clip: region.clip, ...region.of } : {}), ...(ask ? { ask } : {}) };
376
+ return { content: [{ type: "text", text: lines }, { type: "image", data: shot.data, mimeType: shot.mimeType }], details: { evidence: receipt.path, sha256: receipt.sha256, media } };
176
377
  });
177
378
  },
178
379
  browser_read: (args, port, call) => {
179
380
  const what = args?.what && WHATS.includes(args.what) ? args.what : "markdown";
180
381
  return perform("browser_read", { what }, port, call, "look", async (s, news) => {
181
382
  const page = await s.driver.page();
383
+ // A browser that has loaded no page yet, on an about:blank nothing has written into, is fresh, not walled: it says
384
+ // so, and where to go. A script that wrote a title or text into about:blank made a page, which is read.
385
+ if (page.url === "about:blank" && loadedNoPage(s.record) && !page.title.trim() && !page.text.trim())
386
+ throw new BrowserFailureError(freshBrowser(s.replaced ?? null));
182
387
  const read = what === "markdown" ? await markdown(page.html, page.text)
183
388
  : what === "html" ? { body: page.html, extractor: "raw", ext: "html" }
184
389
  : what === "text" ? { body: page.text, extractor: "inner-text", ext: "txt" }
185
390
  : { body: what === "url" ? pageUrl(call, page.url) : page.title, ext: "txt" };
186
391
  const body = call.redact(read.body);
187
392
  const [url, title] = [pageUrl(call, page.url), call.redact(page.title)];
188
- const verdict = call.classifyPage ? await call.classifyPage({ url, status: null, contentType: "text/html", text: page.text }).catch(() => ({ wall: "unjudged", injection: null, confidence: null })) : null;
393
+ const { frames, frameText } = await framesOf(page.html, page.url, call.redact);
394
+ const verdict = call.classifyPage ? await call.classifyPage({ url, status: null, contentType: "text/html", text: call.redact(page.text), ...(frames.length ? { frames } : {}), ...(frameText.length ? { frameText } : {}) }).catch(() => ({ wall: "unjudged", injection: null, confidence: null })) : null;
189
395
  const wall = wallOf(verdict);
190
396
  const facts = { final_url: url, title, ...(read.extractor ? { extractor: read.extractor } : {}), via: "browser", ...(verdict?.guard ? { page_guard: call.redact(verdict.guard) } : {}) };
191
397
  const receipt = await file(s, call, { tool: "browser_read", args: { what }, status: wall ? "failed" : "ok", facts, body, ext: read.ext });
@@ -196,5 +402,42 @@ export function pageTools({ guards = new Map() } = {}) {
196
402
  return { content: [{ type: "text", text: shown }], details: { evidence: receipt.path, sha256: receipt.sha256, url } };
197
403
  });
198
404
  },
405
+ browser_upload: async (args, port, call) => {
406
+ const asked = typeof args?.path === "string" ? args.path : "";
407
+ const selector = typeof args?.selector === "string" ? args.selector.trim() : "";
408
+ if (!call.files)
409
+ return text(refused("This host gives the browser no workspace to upload from."), true);
410
+ if (!selector)
411
+ return text(refused("selector must name the page's file input."), true);
412
+ // The host resolves the path before any browser call, so a refused file never reaches the page.
413
+ const file = await call.files.resolve(asked);
414
+ if ("refused" in file)
415
+ return text(refused(`browser_upload refused the file: ${file.refused}.`), true);
416
+ const uploaded = { name: file.name, bytes: file.bytes.byteLength, sha256: sha256Hex(file.bytes) };
417
+ const codeSha = sha256Hex(JSON.stringify({ upload: selector, sha256: uploaded.sha256 }));
418
+ // A page can send the file once its input changes: an upload is journaled like a run and never repeated.
419
+ return perform("browser_upload", { path: asked, selector }, port, call, { effect: codeSha }, async (s, news) => {
420
+ if (!s.driver.setInputFiles)
421
+ throw new BrowserFailureError(refused("This browser driver cannot set a file input."));
422
+ const sent = [];
423
+ const stop = s.effects?.record((row) => { sent.push({ method: row.method, url: call.redact(`${row.origin}${row.path}`), held: row.held }); return port.journal(row); });
424
+ try {
425
+ await s.driver.setInputFiles(selector, { name: file.name, mimeType: file.mime, buffer: file.bytes });
426
+ }
427
+ finally {
428
+ await s.effects?.flush(call.signal).catch(() => undefined);
429
+ stop?.();
430
+ }
431
+ return { content: [{ type: "text", text: JSON.stringify({ ok: true, uploaded, selector, ...(news ? { notice: news } : {}) }) }], details: { uploaded, ...(sent.length ? { sent } : {}) } };
432
+ });
433
+ },
434
+ browser_downloads: (args, port, call) => perform("browser_downloads", {}, port, call, "read", async (s, news) => {
435
+ if (!call.workspace)
436
+ throw new BrowserFailureError(refused("This host gives the browser no workspace to copy downloads into."));
437
+ if (!s.downloads)
438
+ throw new BrowserFailureError(refused("This browser provider cannot list the files its sessions download."));
439
+ const result = await fetchDownloads({ provider: { downloads: s.downloads }, ref: { id: s.sessionId, tag: s.tag }, session: s.sessionId, workspace: call.workspace, redact: call.redact });
440
+ return { content: [{ type: "text", text: JSON.stringify({ ok: true, ...result, ...(news ? { notice: news } : {}) }) }], details: { downloads: result } };
441
+ }),
199
442
  };
200
443
  }
@@ -0,0 +1,3 @@
1
+ /** The name a host call that passed its deadline unanswered fails with. */
2
+ export declare const UNANSWERED_CALL = "UnansweredHostCallError";
3
+ export declare function withinDeadline<T>(call: Promise<T>, ms: number, what: string): Promise<T>;
@@ -0,0 +1,17 @@
1
+ // A host call the browser does not answer within its deadline counts as a dropped connection (browser_unavailable): a
2
+ // read reconnects and runs once more on a fresh driver. Nothing cancels the call itself, so whatever the browser answers
3
+ // later settles a promise nobody reads.
4
+ /** The name a host call that passed its deadline unanswered fails with. */
5
+ export const UNANSWERED_CALL = "UnansweredHostCallError";
6
+ export async function withinDeadline(call, ms, what) {
7
+ let timer;
8
+ const deadline = new Promise((_, reject) => {
9
+ timer = setTimeout(() => reject(Object.assign(new Error(`${what} had no answer from the browser within ${ms} ms`), { name: UNANSWERED_CALL })), ms);
10
+ });
11
+ try {
12
+ return await Promise.race([call, deadline]);
13
+ }
14
+ finally {
15
+ clearTimeout(timer);
16
+ }
17
+ }
@@ -11,6 +11,8 @@ export declare const fetchUsage: (prices: PriceTable, proxied: boolean) => ToolU
11
11
  export declare const searchUsage: (prices: PriceTable) => ToolUsage;
12
12
  type Spent = SessionRecord["spent"];
13
13
  export declare const startSpent: (createdAt: string) => Spent;
14
+ /** The record after a charge, with its list-priced dollars: the ones it already held plus this charge's when the price was the list's. `after` is what `sessionCharge` returned. */
15
+ export declare function withEstimate(before: Spent, after: Spent, listPriced: boolean): Spent;
14
16
  /** `final` (the release) tops up to the provider's minimum and marks the record, so a retried release adds nothing. */
15
17
  export declare function sessionCharge(spent: Spent, at: string, prices: PriceTable, state?: PriceState, final?: boolean): {
16
18
  spent: Spent;
@@ -23,6 +23,12 @@ export const searchUsage = (prices) => toolUsage(prices.searchUsd);
23
23
  const PICO = 1e12;
24
24
  const HOUR_MS = 3600000n;
25
25
  export const startSpent = (createdAt) => ({ seconds: 0, usd: 0, chargedThrough: createdAt });
26
+ /** The record after a charge, with its list-priced dollars: the ones it already held plus this charge's when the price was the list's. `after` is what `sessionCharge` returned. */
27
+ export function withEstimate(before, after, listPriced) {
28
+ const added = Math.max(0, Math.round((after.usd - before.usd) * PICO)) / PICO;
29
+ const estimated = Math.round(((before.estimatedUsd ?? 0) + (listPriced ? added : 0)) * PICO) / PICO;
30
+ return estimated > 0 ? { ...after, estimatedUsd: estimated } : after;
31
+ }
26
32
  /** `final` (the release) tops up to the provider's minimum and marks the record, so a retried release adds nothing. */
27
33
  export function sessionCharge(spent, at, prices, state = "priced", final = false) {
28
34
  if (spent.final)
@@ -63,7 +63,7 @@ export type WebToolsOptions = {
63
63
  };
64
64
  };
65
65
  export type Next = {
66
- action: "drive_page" | "other_source";
66
+ action: "drive_page" | "other_source" | "browser_relaunch";
67
67
  note: string;
68
68
  };
69
69
  export type WebFailure = BrowserFailure & {
package/dist/core/web.js CHANGED
@@ -34,6 +34,8 @@ function pdfOf(raw) {
34
34
  }
35
35
  return { size: Math.floor(((digits + padding) * 3) / 4) - padding, bytes: () => Buffer.from(text, "base64") };
36
36
  }
37
+ /** What a call the run cancelled answers, wherever it was cancelled. */
38
+ const ABORTED = { ok: false, code: "aborted", retryable: false, effect: "none", message: "The call was cancelled by the run." };
37
39
  export const WALLS = {
38
40
  captcha_or_bot_check: { code: "blocked", what: "a CAPTCHA or bot check", next: { action: "drive_page", note: "Relaunch the browser Verified with browser_relaunch({verified:true}), then drive the page with run and snapshot." } },
39
41
  region_or_geo_block: { code: "blocked", what: "a region block", next: { action: "drive_page", note: "Relaunch the browser from the page's own country with browser_relaunch({geolocation}), then drive the page with run and snapshot." } },
@@ -72,13 +74,13 @@ export function createWebTools(options) {
72
74
  const prices = { ...LIST_PRICES, ...options.prices };
73
75
  const redact = createRedactor(options.redact?.values);
74
76
  const guards = new Map();
75
- const guardOf = (conversation) => guards.get(conversation) ?? guards.set(conversation, createRepeatGuard()).get(conversation);
77
+ const guardOf = (conversation) => guards.get(conversation) ?? guards.set(conversation, createRepeatGuard(redact)).get(conversation);
76
78
  const fail = (failure) => ({ text: JSON.stringify(failure, (_k, v) => (typeof v === "string" ? redact(v) : v)), isError: true, details: { code: failure.code }, usd: 0 });
77
79
  const failureOf = (error, tool, signal) => {
78
- const base = classifyBrowserError(error, false, { aborted: signal?.aborted === true });
80
+ const base = classifyBrowserError(error, false, { aborted: signal?.aborted === true, redact });
79
81
  const status = transportFactOf(error).status;
80
82
  if (base.code === "aborted")
81
- return { ...base, message: "The call was cancelled by the run." };
83
+ return { ...base, message: ABORTED.message };
82
84
  if (status === 429)
83
85
  return { ...base, code: "rate_limited", retryable: true, message: `The ${tool} provider is rate limiting this run. Wait a moment and try again, or use another source.` };
84
86
  if (status === 401 || status === 403)
@@ -88,7 +90,7 @@ export function createWebTools(options) {
88
90
  return { ...base, message: `The ${tool} call failed (${base.code}). Try once more, then use another source.` };
89
91
  };
90
92
  const httpFailure = (url, status, content) => {
91
- const detail = content.slice(0, 500);
93
+ const detail = redact(content).slice(0, 500);
92
94
  if (status === 429)
93
95
  return { ok: false, code: "rate_limited", retryable: true, effect: "none", message: `${url} answered 429: the site is rate limiting the fetch. Wait a moment and try again, or use another source.`, detail };
94
96
  if (REFUSED_STATUSES.has(status))
@@ -230,7 +232,7 @@ export function createWebTools(options) {
230
232
  // The model and the receipt keep the query the model sent; the judge never sees a query value.
231
233
  const pageUrl = scrubFetchedUrl(page.url, url, redact);
232
234
  const verdict = options.classifyPage
233
- ? await options.classifyPage({ url: scrubPageUrl(page.url, redact), status: page.status, contentType: page.contentType, text: page.content }).catch(() => ({ wall: "unjudged", injection: null, confidence: null }))
235
+ ? await options.classifyPage({ url: scrubPageUrl(page.url, redact), status: page.status, contentType: page.contentType, text: redact(page.content) }).catch(() => ({ wall: "unjudged", injection: null, confidence: null }))
234
236
  : null;
235
237
  // Each page's receipt records its own judgment: a later page never inherits an earlier one's.
236
238
  guardLine = verdict?.guard;
@@ -262,7 +264,7 @@ export function createWebTools(options) {
262
264
  break;
263
265
  }
264
266
  if (signal?.aborted || primary?.code === "aborted")
265
- return { ...fail({ ok: false, code: "aborted", retryable: false, effect: "none", message: "The call was cancelled by the run." }), usd };
267
+ return { ...fail(ABORTED), usd };
266
268
  const base = primary ?? { ok: false, code: "refused", retryable: false, effect: "none", message: "No source can fetch pages in this run." };
267
269
  const others = tried.filter((name) => name !== primaryVia);
268
270
  const final = { ...base, message: `${base.message}${others.length ? ` Also tried: ${others.join(", ")}.` : ""}${base.next ? "" : " Drive the page with run and snapshot in a live session, or try a different URL."}` };
@@ -296,7 +298,7 @@ export function createWebTools(options) {
296
298
  // Identical calls in flight share one provider call, so parallel copies of a failing fetch cannot all pass the repeat
297
299
  // guard. A cancelled follower answers aborted alone; a cancelled leader leaves its followers to call themselves.
298
300
  const inflight = new Map();
299
- const cancelled = { text: JSON.stringify({ ok: false, code: "aborted", retryable: false, effect: "none", message: "The call was cancelled by the run." }), isError: true, details: { code: "aborted" }, usd: 0 };
301
+ const cancelled = fail(ABORTED);
300
302
  const coalesce = (key, signal, run) => {
301
303
  const running = inflight.get(key);
302
304
  if (!running) {
@@ -3,5 +3,7 @@ export declare const STAGEHAND_EXTENSION = "Stagehand Runtime";
3
3
  * so a new attach would be refused. The close is sent here through the worker's own RPC, and the worker keeps running:
4
4
  * a worker closed under load is sometimes never restarted, and its attach waits out Stagehand's 60 s init deadline.
5
5
  * With no worker running nothing is held, and the worker is woken until it is listed. A target list the browser does
6
- * not give by DISCOVERY_MS is not a list without a worker: the attach fails. */
7
- export declare function installedExtension(cdpUrl: string, signal?: AbortSignal, known?: string): Promise<string | null>;
6
+ * not give by DISCOVERY_MS is not a list without a worker: the attach fails. `replyMs`: how long the close waits for the
7
+ * runtime's reply. An init still running in the extension answers it only once that init ends, since the runtime runs
8
+ * both one after the other; the socket outlives the deadline by that much more. */
9
+ export declare function installedExtension(cdpUrl: string, signal?: AbortSignal, known?: string, replyMs?: number): Promise<string | null>;