@parcha/pi-browser 0.1.0-beta.18 → 0.1.0-beta.20

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. package/CHANGELOG.md +5 -0
  2. package/README.md +177 -39
  3. package/dist/coding-agent-local.js +2 -1
  4. package/dist/coding-agent.d.ts +42 -2
  5. package/dist/coding-agent.js +159 -21
  6. package/dist/core/clip.d.ts +31 -0
  7. package/dist/core/clip.js +49 -0
  8. package/dist/core/contract.d.ts +4 -1
  9. package/dist/core/contract.js +40 -7
  10. package/dist/core/custody.d.ts +79 -5
  11. package/dist/core/custody.js +66 -5
  12. package/dist/core/decisions.d.ts +4 -0
  13. package/dist/core/dial.d.ts +7 -0
  14. package/dist/core/dial.js +41 -0
  15. package/dist/core/downloads.d.ts +1 -0
  16. package/dist/core/downloads.js +17 -5
  17. package/dist/core/driving-guard.d.ts +13 -0
  18. package/dist/core/driving-guard.js +61 -0
  19. package/dist/core/driving.d.ts +61 -0
  20. package/dist/core/driving.js +156 -0
  21. package/dist/core/effects.js +33 -13
  22. package/dist/core/evidence.d.ts +3 -1
  23. package/dist/core/failures.d.ts +5 -1
  24. package/dist/core/failures.js +20 -4
  25. package/dist/core/files.d.ts +8 -0
  26. package/dist/core/files.js +74 -0
  27. package/dist/core/find.d.ts +3 -1
  28. package/dist/core/find.js +2 -1
  29. package/dist/core/guard.d.ts +9 -1
  30. package/dist/core/guard.js +7 -2
  31. package/dist/core/host.d.ts +49 -1
  32. package/dist/core/images.d.ts +10 -2
  33. package/dist/core/images.js +4 -4
  34. package/dist/core/lease.d.ts +1 -1
  35. package/dist/core/lease.js +2 -1
  36. package/dist/core/session-control.d.ts +2 -0
  37. package/dist/core/session-control.js +5 -0
  38. package/dist/core/tools.d.ts +70 -4
  39. package/dist/core/tools.js +288 -45
  40. package/dist/core/unanswered.d.ts +3 -0
  41. package/dist/core/unanswered.js +17 -0
  42. package/dist/core/usage.d.ts +2 -0
  43. package/dist/core/usage.js +6 -0
  44. package/dist/core/web.d.ts +1 -1
  45. package/dist/core/web.js +9 -7
  46. package/dist/driver/extension-worker.d.ts +4 -2
  47. package/dist/driver/extension-worker.js +16 -10
  48. package/dist/driver/page-driver.d.ts +8 -0
  49. package/dist/driver/page-driver.js +128 -0
  50. package/dist/driver/stagehand.d.ts +3 -1
  51. package/dist/driver/stagehand.js +40 -66
  52. package/dist/durable.d.ts +16 -4
  53. package/dist/durable.js +415 -82
  54. package/dist/index.d.ts +2 -0
  55. package/dist/index.js +2 -0
  56. package/dist/live-view/index.d.ts +4 -0
  57. package/dist/live-view/index.js +4 -0
  58. package/dist/live-view/protocol.d.ts +152 -0
  59. package/dist/live-view/protocol.js +84 -0
  60. package/dist/live-view/provider.d.ts +6 -0
  61. package/dist/live-view/provider.js +23 -0
  62. package/dist/live-view/server.d.ts +50 -0
  63. package/dist/live-view/server.js +342 -0
  64. package/dist/live-view/stream.d.ts +7 -0
  65. package/dist/live-view/stream.js +388 -0
  66. package/dist/live-view/token.d.ts +6 -0
  67. package/dist/live-view/token.js +13 -0
  68. package/dist/live-view/viewer.d.ts +6 -0
  69. package/dist/live-view/viewer.js +350 -0
  70. package/dist/live-view/ws.d.ts +19 -0
  71. package/dist/live-view/ws.js +176 -0
  72. package/dist/providers/assigned.d.ts +18 -0
  73. package/dist/providers/assigned.js +76 -0
  74. package/dist/providers/cdp-call.js +12 -2
  75. package/dist/providers/cdp.js +37 -11
  76. package/dist/recorder/follow.d.ts +10 -0
  77. package/dist/recorder/follow.js +27 -2
  78. package/dist/recorder/frames.d.ts +7 -0
  79. package/dist/recorder/frames.js +73 -5
  80. package/dist/store.d.ts +22 -3
  81. package/dist/store.js +75 -4
  82. package/dist/testing/backend.d.ts +26 -0
  83. package/dist/testing/backend.js +21 -3
  84. package/dist/testing/driver.d.ts +18 -1
  85. package/dist/testing/driver.js +4 -0
  86. package/dist/web.js +2 -2
  87. package/package.json +10 -2
@@ -3,14 +3,18 @@ import { existsSync } from "node:fs";
3
3
  import { mkdtemp, rm } from "node:fs/promises";
4
4
  import os from "node:os";
5
5
  import path from "node:path";
6
+ import { setTimeout as sleep } from "node:timers/promises";
6
7
  import { BROWSER_TOOLS, WEB_TOOLS, browserSection } from "./core/contract.js";
7
- import { BrowserFailureError, closeDriver, custodyTools, newSessionRecord, notices, profileRefusal, sessionSpec, } from "./core/custody.js";
8
+ import { BrowserFailureError, closeDriver, custodyTools, FIXED_SESSION_RELAUNCH, fenced, HANDED_BACK, loadedNoPage, moveRecord, newSessionRecord, notices, NOTHING_TO_HOLD, profileRefusal, sessionSpec, waitingForHuman, backoffMs, NO_HANDOFF, } from "./core/custody.js";
8
9
  import { browserFailure, classifyBrowserError } from "./core/failures.js";
10
+ import { workspaceFiles } from "./core/files.js";
9
11
  import { leaseTag } from "./core/lease.js";
10
12
  import { createRedactor, redactDeep, scrubPageUrl } from "./core/redact.js";
11
13
  import { useWebsite, useWebsiteContract } from "./core/sites.js";
14
+ import { drivingGuarded } from "./core/driving-guard.js";
15
+ import { drivingGates, keptFor, personOn, rearmWhenLeft } from "./core/driving.js";
12
16
  import { pageTools } from "./core/tools.js";
13
- import { CREATE_DEADLINE_MS, DEFAULT_BROWSER_POLICY, idleTracker, keyedSerializer, providerKeyValues } from "./core/session-control.js";
17
+ import { CREATE_DEADLINE_MS, DEFAULT_BROWSER_POLICY, idleTracker, keyedSerializer, oneAtATime, providerKeyValues } from "./core/session-control.js";
14
18
  import { createWebTools } from "./core/web.js";
15
19
  import { fileEvidenceSink, findChrome, httpBackup } from "./coding-agent-local.js";
16
20
  export { findChrome } from "./coding-agent-local.js";
@@ -30,13 +34,33 @@ export class LocalCustody {
30
34
  lastReleased = null;
31
35
  seq = 0;
32
36
  notice = null;
37
+ /** The session an idle release ended, told with its notice and named while its replacement has loaded no page; any
38
+ * other release ends it, and a failed launch never touches it. */
39
+ replacedBy = null;
33
40
  queue = keyedSerializer();
34
41
  idle;
42
+ /** The poll of a session the idle timer kept for a person (#1098): stopped when the session is released or closed. */
43
+ leaving = null;
35
44
  constructor(d) {
36
45
  this.d = d;
37
46
  this.idle = idleTracker({ idleMs: d.config.policy.idleReleaseS * 1000, onIdle: (key, fired) => void this.serial(async () => {
38
- if (this.idle.isCurrent(key, fired) && this.live)
39
- await this.finish(this.live, "idle").catch(() => undefined);
47
+ const live = this.live;
48
+ if (!this.idle.isCurrent(key, fired) || !live || live.record.state === "held")
49
+ return;
50
+ // A person driving or watching the session through the live view keeps it (#1098); it is released one idle window
51
+ // after they leave.
52
+ const gate = personOn(d.gates ?? drivingGates, live.record.tag);
53
+ if (gate) {
54
+ this.leaving?.();
55
+ this.leaving = rearmWhenLeft(gate, () => this.live === live, () => { this.leaving = null; this.idle.activity(key, 0); }, d.config.policy.idleReleaseS * 1000);
56
+ return;
57
+ }
58
+ await this.finish(live, "idle").catch(() => undefined);
59
+ // Whatever the release answered, a session that is over is told to the next call, which opens a fresh browser.
60
+ if (this.live !== live) {
61
+ this.notice = notices.released("idle", live.record.lastUrl, d.config.policy.idleReleaseS);
62
+ this.replacedBy = { reason: "idle", at: live.record.endedAt, lastUrl: live.record.lastUrl };
63
+ }
40
64
  }) });
41
65
  }
42
66
  serial(run) {
@@ -46,7 +70,7 @@ export class LocalCustody {
46
70
  this.d.keep(SESSION_ENTRY, { tag: record.tag, resourceId: record.resourceId, state: record.state, reason: record.releaseReason, ...(record.endedAt ? { endedAt: record.endedAt } : {}) });
47
71
  }
48
72
  failure(error, signal) {
49
- return error instanceof BrowserFailureError ? error : new BrowserFailureError(classifyBrowserError(error, false, { aborted: signal?.aborted === true }));
73
+ return error instanceof BrowserFailureError ? error : new BrowserFailureError(classifyBrowserError(error, false, { aborted: signal?.aborted === true, redact: this.d.redact }));
50
74
  }
51
75
  /** Release every session the branch's entries leave open (a browser a crashed process left running); never adopted. */
52
76
  async releaseOrphans(entries) {
@@ -69,9 +93,9 @@ export class LocalCustody {
69
93
  this.d.keep(SESSION_ENTRY, { tag, resourceId, state: "released", reason: "lost" });
70
94
  }
71
95
  }
72
- async attached(live) {
96
+ async attached(live, bound) {
73
97
  const provider = await this.d.provider();
74
- const signal = AbortSignal.any([this.d.signal, AbortSignal.timeout(CREATE_DEADLINE_MS)]);
98
+ const signal = AbortSignal.any([this.d.signal, AbortSignal.timeout(CREATE_DEADLINE_MS), ...(bound ? [bound] : [])]);
75
99
  try {
76
100
  live.driver ??= await this.d.driver(await provider.attach(live.ref, signal), signal, { batchTimeoutMs: this.d.config.policy.batchTimeoutMs });
77
101
  }
@@ -80,7 +104,9 @@ export class LocalCustody {
80
104
  }
81
105
  const notice = this.notice;
82
106
  this.notice = null;
83
- return { driver: live.driver, tag: live.record.tag, sessionId: live.ref.id, record: live.record, notice, interrupted: null, recorded: !!provider.recordings, effects: null };
107
+ const fence = live.record.fence ?? 0;
108
+ const replaced = this.replacedBy && loadedNoPage(live.record) ? this.replacedBy : null;
109
+ return { driver: fenced(live.driver, fence, () => live.mark), tag: live.record.tag, sessionId: live.ref.id, record: live.record, notice, interrupted: null, recorded: !!provider.recordings, effects: null, fence, downloads: provider.downloads ?? null, replaced };
84
110
  }
85
111
  async launch(overrides) {
86
112
  const provider = await this.d.provider();
@@ -103,6 +129,11 @@ export class LocalCustody {
103
129
  return attached;
104
130
  }
105
131
  catch (error) {
132
+ // A session the host assigned is the task's only one: an attach that failed keeps it live, and the next call attaches it again.
133
+ if (live && provider.caps.fixedSession) {
134
+ this.live = live;
135
+ throw this.failure(error, signal);
136
+ }
106
137
  // A create that threw may still have made a browser: with no ref, look it up by its tag. The record ends released
107
138
  // only when every release is confirmed; otherwise it stays open for the next start to retry.
108
139
  const refs = live ? [live.ref] : await provider.findByTag(tag).catch(() => null);
@@ -123,6 +154,11 @@ export class LocalCustody {
123
154
  * one again), its record stays open for the directory, which is released again by the next release, the shutdown or the next start; the
124
155
  * caller is still told what is left. */
125
156
  async finish(live, reason) {
157
+ // A released session's poll (#1098) ends with it, whatever released it.
158
+ this.leaving?.();
159
+ this.leaving = null;
160
+ if (reason !== "idle")
161
+ this.replacedBy = null;
126
162
  await this.sweepLeftovers();
127
163
  // A close that never comes back (Stagehand after a reconnect) never holds up the provider's release.
128
164
  if (live.driver)
@@ -148,6 +184,10 @@ export class LocalCustody {
148
184
  this.lastReleased = live.record;
149
185
  this.save(live.record);
150
186
  }
187
+ async refuseHeld() {
188
+ if (this.live?.record.state === "held" && !(await this.retireEnded()))
189
+ throw new BrowserFailureError(waitingForHuman(this.live.record.handoff));
190
+ }
151
191
  /** Releases again each browser that is gone and whose directory was left; one that still cannot be removed stays for the next time, and the
152
192
  * last such failure is returned for a caller that was asked to release. */
153
193
  async sweepLeftovers() {
@@ -175,14 +215,20 @@ export class LocalCustody {
175
215
  await this.finish(live, "ended").catch(() => undefined);
176
216
  return true;
177
217
  }
178
- session() {
218
+ session(signal) {
179
219
  return this.serial(async () => {
220
+ await this.refuseHeld();
180
221
  await this.retireEnded();
181
- return this.live ? this.attached(this.live) : this.launch({});
222
+ return this.live ? this.attached(this.live, signal) : this.launch({});
182
223
  });
183
224
  }
225
+ async current() { return this.live?.record.tag ?? null; }
184
226
  relaunch(overrides) {
185
227
  return this.serial(async () => {
228
+ await this.refuseHeld();
229
+ // A session the host assigned is the only one: refused before anything is released.
230
+ if ((await this.d.provider()).caps.fixedSession)
231
+ throw new BrowserFailureError(FIXED_SESSION_RELAUNCH);
186
232
  // A browser that is gone with only its directory left is over: the new one opens, and the directory is retried later.
187
233
  if (this.live)
188
234
  await this.finish(this.live, "relaunch").catch((error) => { if (this.live)
@@ -208,11 +254,44 @@ export class LocalCustody {
208
254
  close(reason) {
209
255
  return this.serial(async () => {
210
256
  this.idle.cancel(this.d.config.label);
257
+ this.leaving?.();
258
+ this.leaving = null;
211
259
  if (this.live)
212
260
  await this.finish(this.live, reason).catch(() => undefined);
213
261
  await this.sweepLeftovers();
214
262
  });
215
263
  }
264
+ hold(requestId) {
265
+ return this.serial(async () => {
266
+ const live = this.live;
267
+ if (live?.record.state === "held" && live.record.handoff !== requestId)
268
+ throw new BrowserFailureError(waitingForHuman(live.record.handoff));
269
+ if (!live)
270
+ throw new BrowserFailureError(NOTHING_TO_HOLD);
271
+ if (live.record.state === "live") {
272
+ moveRecord(live.record, { state: "held", handoff: requestId, at: now() });
273
+ live.mark = { fence: live.record.fence, failure: waitingForHuman(requestId) };
274
+ this.save(live.record);
275
+ const driver = live.driver;
276
+ live.driver = null;
277
+ if (driver)
278
+ await closeDriver(driver);
279
+ }
280
+ return { tag: live.record.tag, sessionId: live.ref.id };
281
+ });
282
+ }
283
+ resume(notice) {
284
+ return this.serial(async () => {
285
+ const live = this.live;
286
+ if (live?.record.state !== "held")
287
+ return;
288
+ moveRecord(live.record, { state: "live", at: now() });
289
+ live.mark = { fence: live.record.fence, failure: HANDED_BACK };
290
+ this.save(live.record);
291
+ if (notice)
292
+ this.notice = [this.notice, notice].filter(Boolean).join(" ");
293
+ });
294
+ }
216
295
  async dispatching() { }
217
296
  async journal() { }
218
297
  async mark() { }
@@ -222,6 +301,56 @@ export class LocalCustody {
222
301
  ended() {
223
302
  return this.serial(() => this.retireEnded());
224
303
  }
304
+ unreachable() {
305
+ return this.serial(async () => {
306
+ const live = this.live;
307
+ if (!live)
308
+ return { released: false, lastUrl: null };
309
+ const lastUrl = live.record.lastUrl;
310
+ // A session held for a person is theirs: custody never releases it on its own.
311
+ if (live.record.state === "held")
312
+ return { released: false, lastUrl, kept: "held" };
313
+ // A session a person drives or watches through the live view is theirs: custody never releases it on its own.
314
+ const person = personOn(this.d.gates ?? drivingGates, live.record.tag);
315
+ const kept = person && keptFor(person);
316
+ if (kept)
317
+ return { released: false, lastUrl, kept };
318
+ await this.finish(live, "lost").catch(() => undefined);
319
+ if (this.live === live)
320
+ return { released: false, lastUrl }; // the release failed, so the session is kept to retry it
321
+ this.notice = notices.unreachable(lastUrl);
322
+ this.replacedBy = { reason: "lost", at: live.record.endedAt, lastUrl };
323
+ return { released: true, lastUrl };
324
+ });
325
+ }
326
+ /** Blocks on the port until the person answers or the call is aborted; the session comes back either way. */
327
+ async handoff(ask, signal) {
328
+ const port = this.d.handoff;
329
+ if (!port)
330
+ throw new BrowserFailureError(NO_HANDOFF);
331
+ const { tag, sessionId } = await this.hold(ask.requestId);
332
+ const stop = signal ? AbortSignal.any([this.d.signal, signal]) : this.d.signal;
333
+ try {
334
+ const provider = await this.d.provider();
335
+ const liveView = provider.liveView ? await provider.liveView({ id: sessionId, tag }).catch(() => null) : null;
336
+ const asked = await port.request({ requestId: ask.requestId, sessionTag: tag, sessionId, liveView, instruction: ask.instruction, completionCondition: ask.completionCondition }, stop);
337
+ if ("refused" in asked)
338
+ return asked;
339
+ for (let attempt = 0;; attempt += 1) {
340
+ const answer = await port.outcome(asked.interactionId, stop).catch((error) => { if (stop.aborted)
341
+ throw error; return "pending"; });
342
+ if (answer !== "pending")
343
+ return { outcome: answer };
344
+ await sleep(backoffMs(attempt), undefined, { signal: stop });
345
+ }
346
+ }
347
+ catch (error) {
348
+ throw this.failure(error, stop);
349
+ }
350
+ finally {
351
+ await this.resume(null);
352
+ }
353
+ }
225
354
  dropped() {
226
355
  return this.serial(async () => {
227
356
  const driver = this.live?.driver;
@@ -285,7 +414,7 @@ async function openRuntime(pi, settings, ctx, reason, listSites) {
285
414
  const sessionId = ctx.sessionManager.getSessionId?.() ?? randomUUID();
286
415
  const config = { label: "browser", run: sessionId, policy };
287
416
  const driver = options.driver ?? (async (target, signal, driverOptions) => (await import("./driver/stagehand.js")).stagehandDriver()(target, signal, driverOptions));
288
- const custody = new LocalCustody({ config, provider, driver, redact, signal: closing.signal, keep: (type, data) => pi.appendEntry(type, data) });
417
+ const custody = new LocalCustody({ config, gates: options.gates, provider, driver, redact, signal: closing.signal, keep: (type, data) => pi.appendEntry(type, data), handoff: options.handoff });
289
418
  // A fork's branch holds its parent's entries: the parent released its own browser when it ended.
290
419
  if (reason !== "fork")
291
420
  await custody.releaseOrphans(ctx.sessionManager.getBranch()).catch(() => undefined);
@@ -301,14 +430,21 @@ async function openRuntime(pi, settings, ctx, reason, listSites) {
301
430
  const sites = options.sites && listSites ? await options.sites.list(closing.signal).catch(() => []) : [];
302
431
  let closed = null;
303
432
  return {
304
- redact, web, searches: !!chosenProvider?.search, sites,
433
+ redact, web, searches: !!chosenProvider?.search, sharedBrowser: chosenProvider?.caps.sharedBrowser === true, sites,
305
434
  async call(tool, args, callId, signal) {
306
435
  const impl = impls[tool];
307
436
  custody.activity(1);
308
437
  try {
309
- const call = { callId, conversationId: CONVERSATION, signal, label: config.label, redact, evidence };
438
+ const call = { callId, conversationId: CONVERSATION, signal, label: config.label, redact, evidence, workspace: ctx.cwd, files: options.files ?? workspaceFiles(ctx.cwd) };
310
439
  return await impl(args, custody, call);
311
440
  }
441
+ catch (error) {
442
+ // A typed failure thrown before the tool's own handling (the driving guard refuses a release or relaunch for a person who is driving before the tool runs) comes
443
+ // back as the envelope the durable adapter gives, so the model reads its code and `retryable`; other errors keep their path.
444
+ if (error instanceof BrowserFailureError)
445
+ return { content: [{ type: "text", text: JSON.stringify(error.failure) }], isError: true };
446
+ throw error;
447
+ }
312
448
  finally {
313
449
  custody.activity(-1);
314
450
  }
@@ -345,15 +481,16 @@ export function createBrowserExtension(options = {}) {
345
481
  const env = options.env ?? process.env;
346
482
  const policy = { ...DEFAULT_BROWSER_POLICY, ...(env.PI_BROWSER_PROXIES === "1" ? { proxies: true } : {}), ...options.policy };
347
483
  // pi checks a call's arguments against the tool's schema before execute, so each implementation receives its own shape.
348
- const impls = { ...custodyTools, ...pageTools() };
484
+ // Every browser tool is a model call on its session's driving gate (see durable.ts).
485
+ const impls = drivingGuarded({ ...custodyTools, ...pageTools() }, options.gates);
349
486
  return function piBrowser(pi) {
350
487
  let runtime = null;
351
488
  let opening = null;
352
489
  let webSearchRegistered = false;
353
490
  let sitesRegistered = false;
354
- // Children on one existing browser (--browser-endpoint) would drive the same pages, so there they take turns.
355
- let turn = Promise.resolve();
356
- const inTurn = (run) => { const mine = turn.then(run, run); turn = mine.catch(() => undefined); return mine; };
491
+ // Children on a provider whose sessions share one browser (an endpoint, named by the flag or in the host's own provider)
492
+ // would drive the same pages, so there they take turns.
493
+ const inTurn = oneAtATime();
357
494
  pi.registerFlag("browser-provider", { type: "string", description: "Browser provider: cdp (a local Chrome, the default), browserbase (needs BROWSERBASE_API_KEY) or kernel (needs KERNEL_API_KEY)" });
358
495
  pi.registerFlag("browser-endpoint", { type: "string", description: "Attach to a Chrome DevTools endpoint (http://host:port or ws://...) instead of starting Chrome" });
359
496
  pi.registerFlag("browser-chrome", { type: "string", description: "Path to the Chrome or Chromium to start" });
@@ -378,7 +515,8 @@ export function createBrowserExtension(options = {}) {
378
515
  };
379
516
  pi.on("session_start", async (event, ctx) => { await end(); await runtimeOf(ctx, event.reason); });
380
517
  pi.on("session_shutdown", end);
381
- pi.on("before_agent_start", (event) => { event.systemPromptOptions.sections.browser = browserSection(policy); });
518
+ const fileTools = options.fileTools !== false;
519
+ pi.on("before_agent_start", (event) => { event.systemPromptOptions.sections.browser = browserSection(policy, { downloads: fileTools, upload: fileTools }); });
382
520
  async function childSession(ctx, prompt, signal) {
383
521
  const sdk = await import("@earendil-works/pi-coding-agent");
384
522
  const model = ctx.model;
@@ -398,7 +536,7 @@ export function createBrowserExtension(options = {}) {
398
536
  if (typeof value === "string" || typeof value === "boolean")
399
537
  flags.set(flag, value);
400
538
  }
401
- const extension = createBrowserExtension({ ...options, sites: undefined });
539
+ const extension = createBrowserExtension({ ...options, sites: undefined, fileTools: false });
402
540
  const runtime = await sdk.createAgentSessionRuntime(async ({ cwd, sessionManager, sessionStartEvent }) => {
403
541
  const services = await sdk.createAgentSessionServices({
404
542
  cwd, agentDir, modelRuntime, extensionFlagValues: flags,
@@ -446,14 +584,14 @@ export function createBrowserExtension(options = {}) {
446
584
  const rt = await runtimeOf(ctx);
447
585
  const out = await useWebsite(params, port, listing, { signal: signal ?? new AbortController().signal, runChild: (task, childSignal) => {
448
586
  const run = () => childSession(ctx, task.prompt, childSignal);
449
- return flagOf(pi, "browser-endpoint", env.PI_BROWSER_ENDPOINT) ? inTurn(run) : run();
587
+ return rt.sharedBrowser ? inTurn(run) : run();
450
588
  } });
451
589
  return { content: out.content.map((item) => item.type === "text" ? { ...item, text: rt.redact(item.text) } : item), details: undefined, ...(out.isError ? { isError: true } : {}) };
452
590
  },
453
591
  });
454
592
  }
455
593
  for (const contract of BROWSER_TOOLS)
456
- if (contract.listedWhen === "always")
594
+ if (contract.listedWhen === "always" || (fileTools && ["downloads", "upload"].includes(contract.listedWhen)) || (contract.listedWhen === "handoff" && options.handoff))
457
595
  registerTool(pi, contract, runtimeOf);
458
596
  registerTool(pi, WEB_TOOLS.find((tool) => tool.name === "web_fetch"), runtimeOf);
459
597
  };
@@ -0,0 +1,31 @@
1
+ import type { Clip } from "./images.js";
2
+ export type Target = {
3
+ id?: string;
4
+ selector?: string;
5
+ };
6
+ /** What the page says about a target now: the viewport, and for an element how many match and, when one does, its box in CSS pixels
7
+ * from the viewport's top-left corner after it was scrolled into view. */
8
+ export type Located = {
9
+ viewport: {
10
+ width: number;
11
+ height: number;
12
+ };
13
+ matches?: number;
14
+ box?: Clip | null;
15
+ };
16
+ export type Locate = (target?: Target) => Promise<Located>;
17
+ export type Region = {
18
+ clip: Clip;
19
+ of: {
20
+ id: string;
21
+ } | {
22
+ selector: string;
23
+ } | null;
24
+ };
25
+ /** The region the arguments ask for, or null for the whole page; throws a typed `refused` for one that cannot be captured. */
26
+ export declare function regionOf(args: {
27
+ clip?: unknown;
28
+ id?: unknown;
29
+ selector?: unknown;
30
+ fullPage?: boolean;
31
+ }, locate: Locate | undefined): Promise<Region | null>;
@@ -0,0 +1,49 @@
1
+ // A screenshot of a region: the page's own rectangle for a clip the model names, or for an element it names by snapshot ID or CSS
2
+ // selector. Every refusal is typed and says why, before a capture is made.
3
+ import { BrowserFailureError } from "./custody.js";
4
+ const refuse = (message) => { throw new BrowserFailureError({ ok: false, code: "refused", retryable: false, effect: "none", message: `screenshot: ${message} No capture was made.` }); };
5
+ const fits = (box, viewport) => box.x >= 0 && box.y >= 0 && box.x + box.width <= viewport.width + 0.5 && box.y + box.height <= viewport.height + 0.5;
6
+ const shown = (box) => `x ${round(box.x)}, y ${round(box.y)}, ${round(box.width)} wide, ${round(box.height)} high`;
7
+ const round = (n) => Math.round(n * 10) / 10;
8
+ /** The model this harness runs sends every property of a tool's schema, an empty one for each it has no use for: "" for text, 0 for
9
+ * numbers. So a blank id or selector, and a clip with no width and no height, are no request at all, and `fullPage` beside a region
10
+ * gives way to the region. */
11
+ const blank = (v) => v === undefined || v === null || (typeof v === "string" && v.trim() === "");
12
+ const unsetClip = (c) => c === undefined || c === null || (typeof c === "object" && Number(c.width) === 0 && Number(c.height) === 0);
13
+ /** The region the arguments ask for, or null for the whole page; throws a typed `refused` for one that cannot be captured. */
14
+ export async function regionOf(args, locate) {
15
+ const given = [!unsetClip(args.clip), !blank(args.id), !blank(args.selector)].filter(Boolean).length;
16
+ if (given === 0)
17
+ return null;
18
+ if (given > 1)
19
+ refuse("give one of clip, id or selector, not several.");
20
+ if (!locate)
21
+ refuse("this browser driver cannot place a region on the page.");
22
+ const place = locate;
23
+ if (!unsetClip(args.clip)) {
24
+ const c = args.clip;
25
+ const clip = { x: Number(c?.x), y: Number(c?.y), width: Number(c?.width), height: Number(c?.height) };
26
+ if (![clip.x, clip.y, clip.width, clip.height].every(Number.isFinite))
27
+ refuse("clip needs numbers for x, y, width and height.");
28
+ if (clip.width <= 0 || clip.height <= 0)
29
+ refuse(`the clip is empty (${clip.width} wide, ${clip.height} high).`);
30
+ const { viewport } = await place();
31
+ if (!fits(clip, viewport))
32
+ refuse(`the clip (${shown(clip)}) leaves the viewport (${viewport.width} wide, ${viewport.height} high); scroll to it, or ask for a smaller one.`);
33
+ return { clip, of: null };
34
+ }
35
+ const id = blank(args.id) ? undefined : String(args.id).trim();
36
+ const selector = blank(args.selector) ? undefined : String(args.selector).trim();
37
+ const what = id !== undefined ? `id ${id}` : `selector ${selector}`;
38
+ const found = await place(id !== undefined ? { id } : { selector });
39
+ if (!found.matches)
40
+ refuse(`${what} matches nothing on the page now.`);
41
+ if (found.matches > 1)
42
+ refuse(`${what} matches ${found.matches} elements; it must match exactly one.`);
43
+ const box = found.box;
44
+ if (!box || !(box.width > 0) || !(box.height > 0))
45
+ refuse(`the element ${what} has no area (it is empty or hidden).`);
46
+ if (!fits(box, found.viewport))
47
+ refuse(`the element ${what} (${shown(box)}) does not fit the viewport (${found.viewport.width} wide, ${found.viewport.height} high) once scrolled into view; use a clip inside it.`);
48
+ return { clip: box, of: id !== undefined ? { id } : { selector: selector } };
49
+ }
@@ -14,7 +14,7 @@ export type ToolContract = {
14
14
  /** One sequential call makes its whole round sequential. */
15
15
  readonly executionMode: "sequential" | "parallel";
16
16
  readonly evidence: "none" | "receipt" | "image" | "files" | "urls";
17
- readonly listedWhen: "always" | "downloads" | "fetch" | "search" | "sites";
17
+ readonly listedWhen: "always" | "downloads" | "upload" | "fetch" | "search" | "sites" | "handoff";
18
18
  };
19
19
  export declare const BROWSER_TOOLS: readonly ToolContract[];
20
20
  export declare const WEB_TOOLS: readonly ToolContract[];
@@ -23,4 +23,7 @@ export declare const WEB_TOOL_NAMES: readonly string[];
23
23
  /** Depends only on configuration, never live state, so every request renders the same text and keeps the prompt cache. */
24
24
  export declare function browserSection(policy: {
25
25
  readonly idleReleaseS: number;
26
+ }, offered?: {
27
+ readonly downloads?: boolean;
28
+ readonly upload?: boolean;
26
29
  }): string;
@@ -50,6 +50,9 @@ const screenshot = {
50
50
  type: { type: "string", enum: ["png", "jpeg"] },
51
51
  quality: { type: "number", minimum: 0, maximum: 100 },
52
52
  ask: { type: "string", description: "What you want to know from this capture. It travels with the image, so a host can answer it from the pixels when your model cannot see them." },
53
+ clip: { ...object({ x: { type: "number" }, y: { type: "number" }, width: { type: "number" }, height: { type: "number" } }, ["x", "y", "width", "height"]), description: "Capture only this region, in CSS pixels from the viewport's top-left corner (what the page shows now). Width and height 0 mean no clip: leave it out, or send zeros, to capture the whole viewport. A region that leaves the viewport is refused. Use one of clip, id or selector; an empty id or selector means none." },
54
+ id: { type: "string", description: "Capture only this element: a bracketed ID from the latest snapshot, as a string. It is scrolled into view first. It must match one element." },
55
+ selector: { type: "string", description: "Capture only the element this CSS selector matches (when there is no snapshot ID for it). It is scrolled into view first. It must match exactly one element." },
53
56
  }),
54
57
  effect: "read", replay: "safe", executionMode: "sequential", evidence: "image", listedWhen: "always",
55
58
  };
@@ -58,7 +61,8 @@ const browserRead = {
58
61
  label: "Browser: read",
59
62
  description: "Read the CURRENT page of the browser as evidence and persist it under evidence/ with the page URL: " +
60
63
  "`what` = markdown (the whole page as markdown, the default for citing), text, html, url, or title. " +
61
- "This is the citable door for anything you saw in the browser: snapshot, screenshot and run return values are not evidence. Cite the returned URL.",
64
+ "This is the citable door for anything you saw in the browser: snapshot, screenshot and run return values are not evidence. Cite the returned URL. " +
65
+ "A browser page URL in a receipt keeps its query names but shows every value replaced by a redaction marker, the word redacted in square brackets (the site chose where the browser went, and a value in it can be a credential); web_fetch keeps a query value that is in the URL you asked for.",
62
66
  parameters: object({ what: { type: "string", enum: ["markdown", "text", "html", "url", "title"], description: "Default markdown." } }),
63
67
  effect: "read", replay: "safe", executionMode: "sequential", evidence: "receipt", listedWhen: "always",
64
68
  };
@@ -73,11 +77,11 @@ const browserRelease = {
73
77
  const browserRelaunch = {
74
78
  name: "browser_relaunch",
75
79
  label: "Browser: relaunch",
76
- description: "Replace the browser session with a fresh one, after a site blocked it (403/429, a CAPTCHA wall, a bot-detection page, an empty page that should have content). The new session keeps the run's Verified setting unless `verified` names one: `verified: true` where the run's sessions are not Verified (a Verified browser carries a stronger identity and costs more). " +
80
+ description: "Replace the browser session with a fresh one, after a site blocked it (403/429, a CAPTCHA wall, a bot-detection page, an empty page that should have content). The new session is Verified whenever the run's sessions are, and a relaunch never lowers that: `verified: true` asks for a Verified session where the run's sessions are not (a Verified browser carries a stronger identity and costs more). " +
77
81
  "Use `geolocation` to browse as a visitor from a given place: the proxy egresses from that country (ISO alpha-2), optionally a US state and a city, so geo-aware sites (storefront region, prices, payment methods, content availability) show what a local sees. " +
78
82
  "Page state is lost; navigate again with run.",
79
83
  parameters: object({
80
- verified: { type: "boolean", description: "true launches the new session as a Verified browser, false as one that is not. Omitted, the new session keeps the run's setting." },
84
+ verified: { type: "boolean", description: "true launches the new session as a Verified browser. false, or leaving it out, keeps the run's setting: a Verified run's sessions stay Verified." },
81
85
  geolocation: {
82
86
  type: "object",
83
87
  additionalProperties: false,
@@ -92,6 +96,19 @@ const browserRelaunch = {
92
96
  }),
93
97
  effect: "custody", replay: "safe", executionMode: "sequential", evidence: "none", listedWhen: "always",
94
98
  };
99
+ const browserRequestHandoff = {
100
+ name: "browser_request_handoff",
101
+ label: "Browser: hand to a person",
102
+ description: "Hand the browser to a person for a step only a human can do: a CAPTCHA you cannot pass, a sign-in that needs their device, a confirmation only they can give. " +
103
+ "Say in instruction what they should do and in completion_condition what the page shows once it is done. The call waits until the person finishes, cancels or the request expires; while they have the browser, every other browser call fails with waiting_for_human. " +
104
+ "The same request_id retries the same request. Afterwards take a snapshot and check the completion condition on the page before acting.",
105
+ parameters: object({
106
+ request_id: { type: "string", pattern: "^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$", description: "A UUID you choose for this request; the same one retries the same request." },
107
+ instruction: { type: "string", minLength: 1, maxLength: 2000, description: "What the person should do, in plain words." },
108
+ completion_condition: { type: "string", minLength: 1, maxLength: 1000, description: "What the page shows once the step is done, for you to verify afterwards." },
109
+ }, ["request_id", "instruction", "completion_condition"]),
110
+ effect: "custody", replay: "safe", executionMode: "sequential", evidence: "none", listedWhen: "handoff",
111
+ };
95
112
  const browserDownloads = {
96
113
  name: "browser_downloads",
97
114
  label: "Browser: downloads",
@@ -100,12 +117,23 @@ const browserDownloads = {
100
117
  parameters: object({}),
101
118
  effect: "read", replay: "safe", executionMode: "sequential", evidence: "files", listedWhen: "downloads",
102
119
  };
120
+ const browserUpload = {
121
+ name: "browser_upload",
122
+ label: "Browser: upload",
123
+ description: "Choose one workspace file in a file input on the page, as a person picking it would; nothing is submitted. " +
124
+ "Returns the file's name, size and SHA-256; submit the form with run afterwards.",
125
+ parameters: object({
126
+ path: { type: "string", minLength: 1, maxLength: 1000, description: "The file's path relative to the workspace, for example reports/q3.pdf." },
127
+ selector: { type: "string", minLength: 1, maxLength: 500, description: "CSS selector of the <input type=\"file\">, for example input[type=file]." },
128
+ }, ["path", "selector"]),
129
+ effect: "effect", replay: "unsafe", executionMode: "sequential", evidence: "none", listedWhen: "upload",
130
+ };
103
131
  const webFetch = {
104
132
  name: "web_fetch",
105
133
  label: "Web fetch",
106
134
  description: "Fetch one URL as clean markdown without a browser session: fast and cheap, and it handles most static and many dynamic pages. " +
107
135
  "Use it for reading articles, filings, registries, PDFs-as-pages. If it returns blocked or empty content, drive the page with run/snapshot instead. " +
108
- "The result is persisted as evidence with its URL.",
136
+ "The result is persisted as evidence with its URL: a query value that is in the URL you asked for stays in it, any other value of the final URL is replaced by a redaction marker (the word redacted in square brackets).",
109
137
  parameters: object({
110
138
  url: { type: "string", description: "Absolute http(s) URL." },
111
139
  format: { type: "string", enum: ["markdown", "raw"], description: "Default markdown." },
@@ -119,13 +147,16 @@ const webSearch = {
119
147
  parameters: object({ query: { type: "string" }, n: { type: "number", description: "Results to return, 1-25 (default 10)." } }, ["query"]),
120
148
  effect: "read", replay: "safe", executionMode: "parallel", evidence: "urls", listedWhen: "search",
121
149
  };
122
- export const BROWSER_TOOLS = Object.freeze([snapshot, run, screenshot, browserRead, browserRelease, browserRelaunch, browserDownloads]);
150
+ export const BROWSER_TOOLS = Object.freeze([snapshot, run, screenshot, browserRead, browserRelease, browserRelaunch, browserRequestHandoff, browserDownloads, browserUpload]);
123
151
  export const WEB_TOOLS = Object.freeze([webFetch, webSearch]);
124
152
  export const BROWSER_TOOL_NAMES = Object.freeze(BROWSER_TOOLS.map((tool) => tool.name));
125
153
  export const WEB_TOOL_NAMES = Object.freeze(WEB_TOOLS.map((tool) => tool.name));
126
154
  /** Depends only on configuration, never live state, so every request renders the same text and keeps the prompt cache. */
127
- export function browserSection(policy) {
155
+ export function browserSection(policy, offered = {}) {
128
156
  const idle = Math.round(policy.idleReleaseS);
157
+ const download = offered.downloads
158
+ ? " A file the page offers as a download: click its link, then call browser_downloads, which copies it into the workspace under downloads/ (listed in downloads/manifest.json); open it with read."
159
+ : " A file the page offers as a download cannot be collected from the browser: take the link's URL from the page and fetch it with `web_fetch` when that tool is offered, or cite the page that links it.";
129
160
  return [
130
161
  "You control one remote browser:",
131
162
  "- snapshot: the active page's accessibility tree, with bracketed element IDs for run actions.",
@@ -134,10 +165,12 @@ export function browserSection(policy) {
134
165
  "- browser_read: the current page's content, filed as evidence with its URL.",
135
166
  "- browser_relaunch: a fresh session, Verified or egressing from a given place.",
136
167
  "- browser_release: release the session when you are done browsing.",
168
+ ...(offered.downloads ? ["- browser_downloads: copy the files this session downloaded into the workspace, each with its SHA-256."] : []),
169
+ ...(offered.upload ? ["- browser_upload: choose a workspace file in a file input on the page."] : []),
137
170
  "",
138
171
  "Snapshot IDs are valid only for the latest snapshot of the active page; snapshot again after navigation or a stale ID. Use snapshot actions for single clicks and fills, and one run code call for a multi-step workflow that returns only the values you need. Do not launch another browser; open more tabs with context.newPage() inside run code.",
139
172
  "",
140
- "The `run` page is Playwright-shaped, not Playwright. On this browser these throw: `page.waitForEvent(\"download\")`, `page.route`, `page.waitForResponse`, `context.newCDPSession`, and `page.on` for `pageerror`, `framenavigated` or `response`. Wait for what the page shows instead (`page.waitForSelector`, `page.waitForLoadState`, or poll `page.url()` or a locator's text in a short loop), read results from the page itself (a locator's text, then `browser_read` to cite it), and never intercept or rewrite requests. A file the page offers as a download cannot be collected from the browser: take the link's URL from the page and fetch it with `web_fetch` when that tool is offered, or cite the page that links it.",
173
+ "The `run` page is Playwright-shaped, not Playwright. On this browser these throw: `page.waitForEvent(\"download\")`, `page.route`, `page.waitForResponse`, `context.newCDPSession`, and `page.on` for `pageerror`, `framenavigated` or `response`. Wait for what the page shows instead (`page.waitForSelector`, `page.waitForLoadState`, or poll `page.url()` or a locator's text in a short loop), read results from the page itself (a locator's text, then `browser_read` to cite it), and never intercept or rewrite requests." + download,
141
174
  "",
142
175
  // A page call made while a submit's navigation commits throws -32000 or hangs; a bounded wait and one retry avoid
143
176
  // it.