@parcha/pi-browser 0.1.0-beta.17 → 0.1.0-beta.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +5 -0
- package/README.md +177 -39
- package/dist/coding-agent-local.js +2 -1
- package/dist/coding-agent.d.ts +42 -2
- package/dist/coding-agent.js +159 -21
- package/dist/core/clip.d.ts +31 -0
- package/dist/core/clip.js +49 -0
- package/dist/core/contract.d.ts +4 -1
- package/dist/core/contract.js +40 -7
- package/dist/core/custody.d.ts +79 -5
- package/dist/core/custody.js +66 -5
- package/dist/core/decisions.d.ts +4 -0
- package/dist/core/dial.d.ts +7 -0
- package/dist/core/dial.js +41 -0
- package/dist/core/downloads.d.ts +1 -0
- package/dist/core/downloads.js +17 -5
- package/dist/core/driving-guard.d.ts +13 -0
- package/dist/core/driving-guard.js +61 -0
- package/dist/core/driving.d.ts +61 -0
- package/dist/core/driving.js +156 -0
- package/dist/core/effects.js +33 -13
- package/dist/core/evidence.d.ts +3 -1
- package/dist/core/failures.d.ts +5 -1
- package/dist/core/failures.js +20 -4
- package/dist/core/files.d.ts +8 -0
- package/dist/core/files.js +74 -0
- package/dist/core/find.d.ts +3 -1
- package/dist/core/find.js +2 -1
- package/dist/core/guard.d.ts +9 -1
- package/dist/core/guard.js +7 -2
- package/dist/core/host.d.ts +49 -1
- package/dist/core/images.d.ts +10 -2
- package/dist/core/images.js +4 -4
- package/dist/core/lease.d.ts +1 -1
- package/dist/core/lease.js +2 -1
- package/dist/core/session-control.d.ts +2 -0
- package/dist/core/session-control.js +5 -0
- package/dist/core/tools.d.ts +70 -4
- package/dist/core/tools.js +288 -45
- package/dist/core/unanswered.d.ts +3 -0
- package/dist/core/unanswered.js +17 -0
- package/dist/core/usage.d.ts +2 -0
- package/dist/core/usage.js +6 -0
- package/dist/core/web.d.ts +1 -1
- package/dist/core/web.js +9 -7
- package/dist/driver/extension-worker.d.ts +4 -2
- package/dist/driver/extension-worker.js +16 -10
- package/dist/driver/page-driver.d.ts +8 -0
- package/dist/driver/page-driver.js +128 -0
- package/dist/driver/stagehand.d.ts +3 -1
- package/dist/driver/stagehand.js +40 -66
- package/dist/durable.d.ts +16 -4
- package/dist/durable.js +415 -82
- package/dist/index.d.ts +2 -0
- package/dist/index.js +2 -0
- package/dist/live-view/index.d.ts +4 -0
- package/dist/live-view/index.js +4 -0
- package/dist/live-view/protocol.d.ts +152 -0
- package/dist/live-view/protocol.js +84 -0
- package/dist/live-view/provider.d.ts +6 -0
- package/dist/live-view/provider.js +23 -0
- package/dist/live-view/server.d.ts +50 -0
- package/dist/live-view/server.js +342 -0
- package/dist/live-view/stream.d.ts +7 -0
- package/dist/live-view/stream.js +388 -0
- package/dist/live-view/token.d.ts +6 -0
- package/dist/live-view/token.js +13 -0
- package/dist/live-view/viewer.d.ts +6 -0
- package/dist/live-view/viewer.js +350 -0
- package/dist/live-view/ws.d.ts +19 -0
- package/dist/live-view/ws.js +176 -0
- package/dist/providers/assigned.d.ts +18 -0
- package/dist/providers/assigned.js +76 -0
- package/dist/providers/cdp-call.js +12 -2
- package/dist/providers/cdp.js +37 -11
- package/dist/recorder/follow.d.ts +10 -0
- package/dist/recorder/follow.js +27 -2
- package/dist/recorder/frames.d.ts +7 -0
- package/dist/recorder/frames.js +73 -5
- package/dist/store.d.ts +22 -3
- package/dist/store.js +75 -4
- package/dist/testing/backend.d.ts +26 -0
- package/dist/testing/backend.js +21 -3
- package/dist/testing/driver.d.ts +18 -1
- package/dist/testing/driver.js +4 -0
- package/dist/web.js +2 -2
- package/package.json +10 -2
package/dist/coding-agent.js
CHANGED
|
@@ -3,14 +3,18 @@ import { existsSync } from "node:fs";
|
|
|
3
3
|
import { mkdtemp, rm } from "node:fs/promises";
|
|
4
4
|
import os from "node:os";
|
|
5
5
|
import path from "node:path";
|
|
6
|
+
import { setTimeout as sleep } from "node:timers/promises";
|
|
6
7
|
import { BROWSER_TOOLS, WEB_TOOLS, browserSection } from "./core/contract.js";
|
|
7
|
-
import { BrowserFailureError, closeDriver, custodyTools, newSessionRecord, notices, profileRefusal, sessionSpec, } from "./core/custody.js";
|
|
8
|
+
import { BrowserFailureError, closeDriver, custodyTools, FIXED_SESSION_RELAUNCH, fenced, HANDED_BACK, loadedNoPage, moveRecord, newSessionRecord, notices, NOTHING_TO_HOLD, profileRefusal, sessionSpec, waitingForHuman, backoffMs, NO_HANDOFF, } from "./core/custody.js";
|
|
8
9
|
import { browserFailure, classifyBrowserError } from "./core/failures.js";
|
|
10
|
+
import { workspaceFiles } from "./core/files.js";
|
|
9
11
|
import { leaseTag } from "./core/lease.js";
|
|
10
12
|
import { createRedactor, redactDeep, scrubPageUrl } from "./core/redact.js";
|
|
11
13
|
import { useWebsite, useWebsiteContract } from "./core/sites.js";
|
|
14
|
+
import { drivingGuarded } from "./core/driving-guard.js";
|
|
15
|
+
import { drivingGates, keptFor, personOn, rearmWhenLeft } from "./core/driving.js";
|
|
12
16
|
import { pageTools } from "./core/tools.js";
|
|
13
|
-
import { CREATE_DEADLINE_MS, DEFAULT_BROWSER_POLICY, idleTracker, keyedSerializer, providerKeyValues } from "./core/session-control.js";
|
|
17
|
+
import { CREATE_DEADLINE_MS, DEFAULT_BROWSER_POLICY, idleTracker, keyedSerializer, oneAtATime, providerKeyValues } from "./core/session-control.js";
|
|
14
18
|
import { createWebTools } from "./core/web.js";
|
|
15
19
|
import { fileEvidenceSink, findChrome, httpBackup } from "./coding-agent-local.js";
|
|
16
20
|
export { findChrome } from "./coding-agent-local.js";
|
|
@@ -30,13 +34,33 @@ export class LocalCustody {
|
|
|
30
34
|
lastReleased = null;
|
|
31
35
|
seq = 0;
|
|
32
36
|
notice = null;
|
|
37
|
+
/** The session an idle release ended, told with its notice and named while its replacement has loaded no page; any
|
|
38
|
+
* other release ends it, and a failed launch never touches it. */
|
|
39
|
+
replacedBy = null;
|
|
33
40
|
queue = keyedSerializer();
|
|
34
41
|
idle;
|
|
42
|
+
/** The poll of a session the idle timer kept for a person (#1098): stopped when the session is released or closed. */
|
|
43
|
+
leaving = null;
|
|
35
44
|
constructor(d) {
|
|
36
45
|
this.d = d;
|
|
37
46
|
this.idle = idleTracker({ idleMs: d.config.policy.idleReleaseS * 1000, onIdle: (key, fired) => void this.serial(async () => {
|
|
38
|
-
|
|
39
|
-
|
|
47
|
+
const live = this.live;
|
|
48
|
+
if (!this.idle.isCurrent(key, fired) || !live || live.record.state === "held")
|
|
49
|
+
return;
|
|
50
|
+
// A person driving or watching the session through the live view keeps it (#1098); it is released one idle window
|
|
51
|
+
// after they leave.
|
|
52
|
+
const gate = personOn(d.gates ?? drivingGates, live.record.tag);
|
|
53
|
+
if (gate) {
|
|
54
|
+
this.leaving?.();
|
|
55
|
+
this.leaving = rearmWhenLeft(gate, () => this.live === live, () => { this.leaving = null; this.idle.activity(key, 0); }, d.config.policy.idleReleaseS * 1000);
|
|
56
|
+
return;
|
|
57
|
+
}
|
|
58
|
+
await this.finish(live, "idle").catch(() => undefined);
|
|
59
|
+
// Whatever the release answered, a session that is over is told to the next call, which opens a fresh browser.
|
|
60
|
+
if (this.live !== live) {
|
|
61
|
+
this.notice = notices.released("idle", live.record.lastUrl, d.config.policy.idleReleaseS);
|
|
62
|
+
this.replacedBy = { reason: "idle", at: live.record.endedAt, lastUrl: live.record.lastUrl };
|
|
63
|
+
}
|
|
40
64
|
}) });
|
|
41
65
|
}
|
|
42
66
|
serial(run) {
|
|
@@ -46,7 +70,7 @@ export class LocalCustody {
|
|
|
46
70
|
this.d.keep(SESSION_ENTRY, { tag: record.tag, resourceId: record.resourceId, state: record.state, reason: record.releaseReason, ...(record.endedAt ? { endedAt: record.endedAt } : {}) });
|
|
47
71
|
}
|
|
48
72
|
failure(error, signal) {
|
|
49
|
-
return error instanceof BrowserFailureError ? error : new BrowserFailureError(classifyBrowserError(error, false, { aborted: signal?.aborted === true }));
|
|
73
|
+
return error instanceof BrowserFailureError ? error : new BrowserFailureError(classifyBrowserError(error, false, { aborted: signal?.aborted === true, redact: this.d.redact }));
|
|
50
74
|
}
|
|
51
75
|
/** Release every session the branch's entries leave open (a browser a crashed process left running); never adopted. */
|
|
52
76
|
async releaseOrphans(entries) {
|
|
@@ -69,9 +93,9 @@ export class LocalCustody {
|
|
|
69
93
|
this.d.keep(SESSION_ENTRY, { tag, resourceId, state: "released", reason: "lost" });
|
|
70
94
|
}
|
|
71
95
|
}
|
|
72
|
-
async attached(live) {
|
|
96
|
+
async attached(live, bound) {
|
|
73
97
|
const provider = await this.d.provider();
|
|
74
|
-
const signal = AbortSignal.any([this.d.signal, AbortSignal.timeout(CREATE_DEADLINE_MS)]);
|
|
98
|
+
const signal = AbortSignal.any([this.d.signal, AbortSignal.timeout(CREATE_DEADLINE_MS), ...(bound ? [bound] : [])]);
|
|
75
99
|
try {
|
|
76
100
|
live.driver ??= await this.d.driver(await provider.attach(live.ref, signal), signal, { batchTimeoutMs: this.d.config.policy.batchTimeoutMs });
|
|
77
101
|
}
|
|
@@ -80,7 +104,9 @@ export class LocalCustody {
|
|
|
80
104
|
}
|
|
81
105
|
const notice = this.notice;
|
|
82
106
|
this.notice = null;
|
|
83
|
-
|
|
107
|
+
const fence = live.record.fence ?? 0;
|
|
108
|
+
const replaced = this.replacedBy && loadedNoPage(live.record) ? this.replacedBy : null;
|
|
109
|
+
return { driver: fenced(live.driver, fence, () => live.mark), tag: live.record.tag, sessionId: live.ref.id, record: live.record, notice, interrupted: null, recorded: !!provider.recordings, effects: null, fence, downloads: provider.downloads ?? null, replaced };
|
|
84
110
|
}
|
|
85
111
|
async launch(overrides) {
|
|
86
112
|
const provider = await this.d.provider();
|
|
@@ -103,6 +129,11 @@ export class LocalCustody {
|
|
|
103
129
|
return attached;
|
|
104
130
|
}
|
|
105
131
|
catch (error) {
|
|
132
|
+
// A session the host assigned is the task's only one: an attach that failed keeps it live, and the next call attaches it again.
|
|
133
|
+
if (live && provider.caps.fixedSession) {
|
|
134
|
+
this.live = live;
|
|
135
|
+
throw this.failure(error, signal);
|
|
136
|
+
}
|
|
106
137
|
// A create that threw may still have made a browser: with no ref, look it up by its tag. The record ends released
|
|
107
138
|
// only when every release is confirmed; otherwise it stays open for the next start to retry.
|
|
108
139
|
const refs = live ? [live.ref] : await provider.findByTag(tag).catch(() => null);
|
|
@@ -123,6 +154,11 @@ export class LocalCustody {
|
|
|
123
154
|
* one again), its record stays open for the directory, which is released again by the next release, the shutdown or the next start; the
|
|
124
155
|
* caller is still told what is left. */
|
|
125
156
|
async finish(live, reason) {
|
|
157
|
+
// A released session's poll (#1098) ends with it, whatever released it.
|
|
158
|
+
this.leaving?.();
|
|
159
|
+
this.leaving = null;
|
|
160
|
+
if (reason !== "idle")
|
|
161
|
+
this.replacedBy = null;
|
|
126
162
|
await this.sweepLeftovers();
|
|
127
163
|
// A close that never comes back (Stagehand after a reconnect) never holds up the provider's release.
|
|
128
164
|
if (live.driver)
|
|
@@ -148,6 +184,10 @@ export class LocalCustody {
|
|
|
148
184
|
this.lastReleased = live.record;
|
|
149
185
|
this.save(live.record);
|
|
150
186
|
}
|
|
187
|
+
async refuseHeld() {
|
|
188
|
+
if (this.live?.record.state === "held" && !(await this.retireEnded()))
|
|
189
|
+
throw new BrowserFailureError(waitingForHuman(this.live.record.handoff));
|
|
190
|
+
}
|
|
151
191
|
/** Releases again each browser that is gone and whose directory was left; one that still cannot be removed stays for the next time, and the
|
|
152
192
|
* last such failure is returned for a caller that was asked to release. */
|
|
153
193
|
async sweepLeftovers() {
|
|
@@ -175,14 +215,20 @@ export class LocalCustody {
|
|
|
175
215
|
await this.finish(live, "ended").catch(() => undefined);
|
|
176
216
|
return true;
|
|
177
217
|
}
|
|
178
|
-
session() {
|
|
218
|
+
session(signal) {
|
|
179
219
|
return this.serial(async () => {
|
|
220
|
+
await this.refuseHeld();
|
|
180
221
|
await this.retireEnded();
|
|
181
|
-
return this.live ? this.attached(this.live) : this.launch({});
|
|
222
|
+
return this.live ? this.attached(this.live, signal) : this.launch({});
|
|
182
223
|
});
|
|
183
224
|
}
|
|
225
|
+
async current() { return this.live?.record.tag ?? null; }
|
|
184
226
|
relaunch(overrides) {
|
|
185
227
|
return this.serial(async () => {
|
|
228
|
+
await this.refuseHeld();
|
|
229
|
+
// A session the host assigned is the only one: refused before anything is released.
|
|
230
|
+
if ((await this.d.provider()).caps.fixedSession)
|
|
231
|
+
throw new BrowserFailureError(FIXED_SESSION_RELAUNCH);
|
|
186
232
|
// A browser that is gone with only its directory left is over: the new one opens, and the directory is retried later.
|
|
187
233
|
if (this.live)
|
|
188
234
|
await this.finish(this.live, "relaunch").catch((error) => { if (this.live)
|
|
@@ -208,11 +254,44 @@ export class LocalCustody {
|
|
|
208
254
|
close(reason) {
|
|
209
255
|
return this.serial(async () => {
|
|
210
256
|
this.idle.cancel(this.d.config.label);
|
|
257
|
+
this.leaving?.();
|
|
258
|
+
this.leaving = null;
|
|
211
259
|
if (this.live)
|
|
212
260
|
await this.finish(this.live, reason).catch(() => undefined);
|
|
213
261
|
await this.sweepLeftovers();
|
|
214
262
|
});
|
|
215
263
|
}
|
|
264
|
+
hold(requestId) {
|
|
265
|
+
return this.serial(async () => {
|
|
266
|
+
const live = this.live;
|
|
267
|
+
if (live?.record.state === "held" && live.record.handoff !== requestId)
|
|
268
|
+
throw new BrowserFailureError(waitingForHuman(live.record.handoff));
|
|
269
|
+
if (!live)
|
|
270
|
+
throw new BrowserFailureError(NOTHING_TO_HOLD);
|
|
271
|
+
if (live.record.state === "live") {
|
|
272
|
+
moveRecord(live.record, { state: "held", handoff: requestId, at: now() });
|
|
273
|
+
live.mark = { fence: live.record.fence, failure: waitingForHuman(requestId) };
|
|
274
|
+
this.save(live.record);
|
|
275
|
+
const driver = live.driver;
|
|
276
|
+
live.driver = null;
|
|
277
|
+
if (driver)
|
|
278
|
+
await closeDriver(driver);
|
|
279
|
+
}
|
|
280
|
+
return { tag: live.record.tag, sessionId: live.ref.id };
|
|
281
|
+
});
|
|
282
|
+
}
|
|
283
|
+
resume(notice) {
|
|
284
|
+
return this.serial(async () => {
|
|
285
|
+
const live = this.live;
|
|
286
|
+
if (live?.record.state !== "held")
|
|
287
|
+
return;
|
|
288
|
+
moveRecord(live.record, { state: "live", at: now() });
|
|
289
|
+
live.mark = { fence: live.record.fence, failure: HANDED_BACK };
|
|
290
|
+
this.save(live.record);
|
|
291
|
+
if (notice)
|
|
292
|
+
this.notice = [this.notice, notice].filter(Boolean).join(" ");
|
|
293
|
+
});
|
|
294
|
+
}
|
|
216
295
|
async dispatching() { }
|
|
217
296
|
async journal() { }
|
|
218
297
|
async mark() { }
|
|
@@ -222,6 +301,56 @@ export class LocalCustody {
|
|
|
222
301
|
ended() {
|
|
223
302
|
return this.serial(() => this.retireEnded());
|
|
224
303
|
}
|
|
304
|
+
unreachable() {
|
|
305
|
+
return this.serial(async () => {
|
|
306
|
+
const live = this.live;
|
|
307
|
+
if (!live)
|
|
308
|
+
return { released: false, lastUrl: null };
|
|
309
|
+
const lastUrl = live.record.lastUrl;
|
|
310
|
+
// A session held for a person is theirs: custody never releases it on its own.
|
|
311
|
+
if (live.record.state === "held")
|
|
312
|
+
return { released: false, lastUrl, kept: "held" };
|
|
313
|
+
// A session a person drives or watches through the live view is theirs: custody never releases it on its own.
|
|
314
|
+
const person = personOn(this.d.gates ?? drivingGates, live.record.tag);
|
|
315
|
+
const kept = person && keptFor(person);
|
|
316
|
+
if (kept)
|
|
317
|
+
return { released: false, lastUrl, kept };
|
|
318
|
+
await this.finish(live, "lost").catch(() => undefined);
|
|
319
|
+
if (this.live === live)
|
|
320
|
+
return { released: false, lastUrl }; // the release failed, so the session is kept to retry it
|
|
321
|
+
this.notice = notices.unreachable(lastUrl);
|
|
322
|
+
this.replacedBy = { reason: "lost", at: live.record.endedAt, lastUrl };
|
|
323
|
+
return { released: true, lastUrl };
|
|
324
|
+
});
|
|
325
|
+
}
|
|
326
|
+
/** Blocks on the port until the person answers or the call is aborted; the session comes back either way. */
|
|
327
|
+
async handoff(ask, signal) {
|
|
328
|
+
const port = this.d.handoff;
|
|
329
|
+
if (!port)
|
|
330
|
+
throw new BrowserFailureError(NO_HANDOFF);
|
|
331
|
+
const { tag, sessionId } = await this.hold(ask.requestId);
|
|
332
|
+
const stop = signal ? AbortSignal.any([this.d.signal, signal]) : this.d.signal;
|
|
333
|
+
try {
|
|
334
|
+
const provider = await this.d.provider();
|
|
335
|
+
const liveView = provider.liveView ? await provider.liveView({ id: sessionId, tag }).catch(() => null) : null;
|
|
336
|
+
const asked = await port.request({ requestId: ask.requestId, sessionTag: tag, sessionId, liveView, instruction: ask.instruction, completionCondition: ask.completionCondition }, stop);
|
|
337
|
+
if ("refused" in asked)
|
|
338
|
+
return asked;
|
|
339
|
+
for (let attempt = 0;; attempt += 1) {
|
|
340
|
+
const answer = await port.outcome(asked.interactionId, stop).catch((error) => { if (stop.aborted)
|
|
341
|
+
throw error; return "pending"; });
|
|
342
|
+
if (answer !== "pending")
|
|
343
|
+
return { outcome: answer };
|
|
344
|
+
await sleep(backoffMs(attempt), undefined, { signal: stop });
|
|
345
|
+
}
|
|
346
|
+
}
|
|
347
|
+
catch (error) {
|
|
348
|
+
throw this.failure(error, stop);
|
|
349
|
+
}
|
|
350
|
+
finally {
|
|
351
|
+
await this.resume(null);
|
|
352
|
+
}
|
|
353
|
+
}
|
|
225
354
|
dropped() {
|
|
226
355
|
return this.serial(async () => {
|
|
227
356
|
const driver = this.live?.driver;
|
|
@@ -285,7 +414,7 @@ async function openRuntime(pi, settings, ctx, reason, listSites) {
|
|
|
285
414
|
const sessionId = ctx.sessionManager.getSessionId?.() ?? randomUUID();
|
|
286
415
|
const config = { label: "browser", run: sessionId, policy };
|
|
287
416
|
const driver = options.driver ?? (async (target, signal, driverOptions) => (await import("./driver/stagehand.js")).stagehandDriver()(target, signal, driverOptions));
|
|
288
|
-
const custody = new LocalCustody({ config, provider, driver, redact, signal: closing.signal, keep: (type, data) => pi.appendEntry(type, data) });
|
|
417
|
+
const custody = new LocalCustody({ config, gates: options.gates, provider, driver, redact, signal: closing.signal, keep: (type, data) => pi.appendEntry(type, data), handoff: options.handoff });
|
|
289
418
|
// A fork's branch holds its parent's entries: the parent released its own browser when it ended.
|
|
290
419
|
if (reason !== "fork")
|
|
291
420
|
await custody.releaseOrphans(ctx.sessionManager.getBranch()).catch(() => undefined);
|
|
@@ -301,14 +430,21 @@ async function openRuntime(pi, settings, ctx, reason, listSites) {
|
|
|
301
430
|
const sites = options.sites && listSites ? await options.sites.list(closing.signal).catch(() => []) : [];
|
|
302
431
|
let closed = null;
|
|
303
432
|
return {
|
|
304
|
-
redact, web, searches: !!chosenProvider?.search, sites,
|
|
433
|
+
redact, web, searches: !!chosenProvider?.search, sharedBrowser: chosenProvider?.caps.sharedBrowser === true, sites,
|
|
305
434
|
async call(tool, args, callId, signal) {
|
|
306
435
|
const impl = impls[tool];
|
|
307
436
|
custody.activity(1);
|
|
308
437
|
try {
|
|
309
|
-
const call = { callId, conversationId: CONVERSATION, signal, label: config.label, redact, evidence };
|
|
438
|
+
const call = { callId, conversationId: CONVERSATION, signal, label: config.label, redact, evidence, workspace: ctx.cwd, files: options.files ?? workspaceFiles(ctx.cwd) };
|
|
310
439
|
return await impl(args, custody, call);
|
|
311
440
|
}
|
|
441
|
+
catch (error) {
|
|
442
|
+
// A typed failure thrown before the tool's own handling (the driving guard refuses a release or relaunch for a person who is driving before the tool runs) comes
|
|
443
|
+
// back as the envelope the durable adapter gives, so the model reads its code and `retryable`; other errors keep their path.
|
|
444
|
+
if (error instanceof BrowserFailureError)
|
|
445
|
+
return { content: [{ type: "text", text: JSON.stringify(error.failure) }], isError: true };
|
|
446
|
+
throw error;
|
|
447
|
+
}
|
|
312
448
|
finally {
|
|
313
449
|
custody.activity(-1);
|
|
314
450
|
}
|
|
@@ -345,15 +481,16 @@ export function createBrowserExtension(options = {}) {
|
|
|
345
481
|
const env = options.env ?? process.env;
|
|
346
482
|
const policy = { ...DEFAULT_BROWSER_POLICY, ...(env.PI_BROWSER_PROXIES === "1" ? { proxies: true } : {}), ...options.policy };
|
|
347
483
|
// pi checks a call's arguments against the tool's schema before execute, so each implementation receives its own shape.
|
|
348
|
-
|
|
484
|
+
// Every browser tool is a model call on its session's driving gate (see durable.ts).
|
|
485
|
+
const impls = drivingGuarded({ ...custodyTools, ...pageTools() }, options.gates);
|
|
349
486
|
return function piBrowser(pi) {
|
|
350
487
|
let runtime = null;
|
|
351
488
|
let opening = null;
|
|
352
489
|
let webSearchRegistered = false;
|
|
353
490
|
let sitesRegistered = false;
|
|
354
|
-
// Children on one
|
|
355
|
-
|
|
356
|
-
const inTurn = (
|
|
491
|
+
// Children on a provider whose sessions share one browser (an endpoint, named by the flag or in the host's own provider)
|
|
492
|
+
// would drive the same pages, so there they take turns.
|
|
493
|
+
const inTurn = oneAtATime();
|
|
357
494
|
pi.registerFlag("browser-provider", { type: "string", description: "Browser provider: cdp (a local Chrome, the default), browserbase (needs BROWSERBASE_API_KEY) or kernel (needs KERNEL_API_KEY)" });
|
|
358
495
|
pi.registerFlag("browser-endpoint", { type: "string", description: "Attach to a Chrome DevTools endpoint (http://host:port or ws://...) instead of starting Chrome" });
|
|
359
496
|
pi.registerFlag("browser-chrome", { type: "string", description: "Path to the Chrome or Chromium to start" });
|
|
@@ -378,7 +515,8 @@ export function createBrowserExtension(options = {}) {
|
|
|
378
515
|
};
|
|
379
516
|
pi.on("session_start", async (event, ctx) => { await end(); await runtimeOf(ctx, event.reason); });
|
|
380
517
|
pi.on("session_shutdown", end);
|
|
381
|
-
|
|
518
|
+
const fileTools = options.fileTools !== false;
|
|
519
|
+
pi.on("before_agent_start", (event) => { event.systemPromptOptions.sections.browser = browserSection(policy, { downloads: fileTools, upload: fileTools }); });
|
|
382
520
|
async function childSession(ctx, prompt, signal) {
|
|
383
521
|
const sdk = await import("@earendil-works/pi-coding-agent");
|
|
384
522
|
const model = ctx.model;
|
|
@@ -398,7 +536,7 @@ export function createBrowserExtension(options = {}) {
|
|
|
398
536
|
if (typeof value === "string" || typeof value === "boolean")
|
|
399
537
|
flags.set(flag, value);
|
|
400
538
|
}
|
|
401
|
-
const extension = createBrowserExtension({ ...options, sites: undefined });
|
|
539
|
+
const extension = createBrowserExtension({ ...options, sites: undefined, fileTools: false });
|
|
402
540
|
const runtime = await sdk.createAgentSessionRuntime(async ({ cwd, sessionManager, sessionStartEvent }) => {
|
|
403
541
|
const services = await sdk.createAgentSessionServices({
|
|
404
542
|
cwd, agentDir, modelRuntime, extensionFlagValues: flags,
|
|
@@ -446,14 +584,14 @@ export function createBrowserExtension(options = {}) {
|
|
|
446
584
|
const rt = await runtimeOf(ctx);
|
|
447
585
|
const out = await useWebsite(params, port, listing, { signal: signal ?? new AbortController().signal, runChild: (task, childSignal) => {
|
|
448
586
|
const run = () => childSession(ctx, task.prompt, childSignal);
|
|
449
|
-
return
|
|
587
|
+
return rt.sharedBrowser ? inTurn(run) : run();
|
|
450
588
|
} });
|
|
451
589
|
return { content: out.content.map((item) => item.type === "text" ? { ...item, text: rt.redact(item.text) } : item), details: undefined, ...(out.isError ? { isError: true } : {}) };
|
|
452
590
|
},
|
|
453
591
|
});
|
|
454
592
|
}
|
|
455
593
|
for (const contract of BROWSER_TOOLS)
|
|
456
|
-
if (contract.listedWhen === "always")
|
|
594
|
+
if (contract.listedWhen === "always" || (fileTools && ["downloads", "upload"].includes(contract.listedWhen)) || (contract.listedWhen === "handoff" && options.handoff))
|
|
457
595
|
registerTool(pi, contract, runtimeOf);
|
|
458
596
|
registerTool(pi, WEB_TOOLS.find((tool) => tool.name === "web_fetch"), runtimeOf);
|
|
459
597
|
};
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import type { Clip } from "./images.js";
|
|
2
|
+
export type Target = {
|
|
3
|
+
id?: string;
|
|
4
|
+
selector?: string;
|
|
5
|
+
};
|
|
6
|
+
/** What the page says about a target now: the viewport, and for an element how many match and, when one does, its box in CSS pixels
|
|
7
|
+
* from the viewport's top-left corner after it was scrolled into view. */
|
|
8
|
+
export type Located = {
|
|
9
|
+
viewport: {
|
|
10
|
+
width: number;
|
|
11
|
+
height: number;
|
|
12
|
+
};
|
|
13
|
+
matches?: number;
|
|
14
|
+
box?: Clip | null;
|
|
15
|
+
};
|
|
16
|
+
export type Locate = (target?: Target) => Promise<Located>;
|
|
17
|
+
export type Region = {
|
|
18
|
+
clip: Clip;
|
|
19
|
+
of: {
|
|
20
|
+
id: string;
|
|
21
|
+
} | {
|
|
22
|
+
selector: string;
|
|
23
|
+
} | null;
|
|
24
|
+
};
|
|
25
|
+
/** The region the arguments ask for, or null for the whole page; throws a typed `refused` for one that cannot be captured. */
|
|
26
|
+
export declare function regionOf(args: {
|
|
27
|
+
clip?: unknown;
|
|
28
|
+
id?: unknown;
|
|
29
|
+
selector?: unknown;
|
|
30
|
+
fullPage?: boolean;
|
|
31
|
+
}, locate: Locate | undefined): Promise<Region | null>;
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
// A screenshot of a region: the page's own rectangle for a clip the model names, or for an element it names by snapshot ID or CSS
|
|
2
|
+
// selector. Every refusal is typed and says why, before a capture is made.
|
|
3
|
+
import { BrowserFailureError } from "./custody.js";
|
|
4
|
+
const refuse = (message) => { throw new BrowserFailureError({ ok: false, code: "refused", retryable: false, effect: "none", message: `screenshot: ${message} No capture was made.` }); };
|
|
5
|
+
const fits = (box, viewport) => box.x >= 0 && box.y >= 0 && box.x + box.width <= viewport.width + 0.5 && box.y + box.height <= viewport.height + 0.5;
|
|
6
|
+
const shown = (box) => `x ${round(box.x)}, y ${round(box.y)}, ${round(box.width)} wide, ${round(box.height)} high`;
|
|
7
|
+
const round = (n) => Math.round(n * 10) / 10;
|
|
8
|
+
/** The model this harness runs sends every property of a tool's schema, an empty one for each it has no use for: "" for text, 0 for
|
|
9
|
+
* numbers. So a blank id or selector, and a clip with no width and no height, are no request at all, and `fullPage` beside a region
|
|
10
|
+
* gives way to the region. */
|
|
11
|
+
const blank = (v) => v === undefined || v === null || (typeof v === "string" && v.trim() === "");
|
|
12
|
+
const unsetClip = (c) => c === undefined || c === null || (typeof c === "object" && Number(c.width) === 0 && Number(c.height) === 0);
|
|
13
|
+
/** The region the arguments ask for, or null for the whole page; throws a typed `refused` for one that cannot be captured. */
|
|
14
|
+
export async function regionOf(args, locate) {
|
|
15
|
+
const given = [!unsetClip(args.clip), !blank(args.id), !blank(args.selector)].filter(Boolean).length;
|
|
16
|
+
if (given === 0)
|
|
17
|
+
return null;
|
|
18
|
+
if (given > 1)
|
|
19
|
+
refuse("give one of clip, id or selector, not several.");
|
|
20
|
+
if (!locate)
|
|
21
|
+
refuse("this browser driver cannot place a region on the page.");
|
|
22
|
+
const place = locate;
|
|
23
|
+
if (!unsetClip(args.clip)) {
|
|
24
|
+
const c = args.clip;
|
|
25
|
+
const clip = { x: Number(c?.x), y: Number(c?.y), width: Number(c?.width), height: Number(c?.height) };
|
|
26
|
+
if (![clip.x, clip.y, clip.width, clip.height].every(Number.isFinite))
|
|
27
|
+
refuse("clip needs numbers for x, y, width and height.");
|
|
28
|
+
if (clip.width <= 0 || clip.height <= 0)
|
|
29
|
+
refuse(`the clip is empty (${clip.width} wide, ${clip.height} high).`);
|
|
30
|
+
const { viewport } = await place();
|
|
31
|
+
if (!fits(clip, viewport))
|
|
32
|
+
refuse(`the clip (${shown(clip)}) leaves the viewport (${viewport.width} wide, ${viewport.height} high); scroll to it, or ask for a smaller one.`);
|
|
33
|
+
return { clip, of: null };
|
|
34
|
+
}
|
|
35
|
+
const id = blank(args.id) ? undefined : String(args.id).trim();
|
|
36
|
+
const selector = blank(args.selector) ? undefined : String(args.selector).trim();
|
|
37
|
+
const what = id !== undefined ? `id ${id}` : `selector ${selector}`;
|
|
38
|
+
const found = await place(id !== undefined ? { id } : { selector });
|
|
39
|
+
if (!found.matches)
|
|
40
|
+
refuse(`${what} matches nothing on the page now.`);
|
|
41
|
+
if (found.matches > 1)
|
|
42
|
+
refuse(`${what} matches ${found.matches} elements; it must match exactly one.`);
|
|
43
|
+
const box = found.box;
|
|
44
|
+
if (!box || !(box.width > 0) || !(box.height > 0))
|
|
45
|
+
refuse(`the element ${what} has no area (it is empty or hidden).`);
|
|
46
|
+
if (!fits(box, found.viewport))
|
|
47
|
+
refuse(`the element ${what} (${shown(box)}) does not fit the viewport (${found.viewport.width} wide, ${found.viewport.height} high) once scrolled into view; use a clip inside it.`);
|
|
48
|
+
return { clip: box, of: id !== undefined ? { id } : { selector: selector } };
|
|
49
|
+
}
|
package/dist/core/contract.d.ts
CHANGED
|
@@ -14,7 +14,7 @@ export type ToolContract = {
|
|
|
14
14
|
/** One sequential call makes its whole round sequential. */
|
|
15
15
|
readonly executionMode: "sequential" | "parallel";
|
|
16
16
|
readonly evidence: "none" | "receipt" | "image" | "files" | "urls";
|
|
17
|
-
readonly listedWhen: "always" | "downloads" | "fetch" | "search" | "sites";
|
|
17
|
+
readonly listedWhen: "always" | "downloads" | "upload" | "fetch" | "search" | "sites" | "handoff";
|
|
18
18
|
};
|
|
19
19
|
export declare const BROWSER_TOOLS: readonly ToolContract[];
|
|
20
20
|
export declare const WEB_TOOLS: readonly ToolContract[];
|
|
@@ -23,4 +23,7 @@ export declare const WEB_TOOL_NAMES: readonly string[];
|
|
|
23
23
|
/** Depends only on configuration, never live state, so every request renders the same text and keeps the prompt cache. */
|
|
24
24
|
export declare function browserSection(policy: {
|
|
25
25
|
readonly idleReleaseS: number;
|
|
26
|
+
}, offered?: {
|
|
27
|
+
readonly downloads?: boolean;
|
|
28
|
+
readonly upload?: boolean;
|
|
26
29
|
}): string;
|
package/dist/core/contract.js
CHANGED
|
@@ -50,6 +50,9 @@ const screenshot = {
|
|
|
50
50
|
type: { type: "string", enum: ["png", "jpeg"] },
|
|
51
51
|
quality: { type: "number", minimum: 0, maximum: 100 },
|
|
52
52
|
ask: { type: "string", description: "What you want to know from this capture. It travels with the image, so a host can answer it from the pixels when your model cannot see them." },
|
|
53
|
+
clip: { ...object({ x: { type: "number" }, y: { type: "number" }, width: { type: "number" }, height: { type: "number" } }, ["x", "y", "width", "height"]), description: "Capture only this region, in CSS pixels from the viewport's top-left corner (what the page shows now). Width and height 0 mean no clip: leave it out, or send zeros, to capture the whole viewport. A region that leaves the viewport is refused. Use one of clip, id or selector; an empty id or selector means none." },
|
|
54
|
+
id: { type: "string", description: "Capture only this element: a bracketed ID from the latest snapshot, as a string. It is scrolled into view first. It must match one element." },
|
|
55
|
+
selector: { type: "string", description: "Capture only the element this CSS selector matches (when there is no snapshot ID for it). It is scrolled into view first. It must match exactly one element." },
|
|
53
56
|
}),
|
|
54
57
|
effect: "read", replay: "safe", executionMode: "sequential", evidence: "image", listedWhen: "always",
|
|
55
58
|
};
|
|
@@ -58,7 +61,8 @@ const browserRead = {
|
|
|
58
61
|
label: "Browser: read",
|
|
59
62
|
description: "Read the CURRENT page of the browser as evidence and persist it under evidence/ with the page URL: " +
|
|
60
63
|
"`what` = markdown (the whole page as markdown, the default for citing), text, html, url, or title. " +
|
|
61
|
-
"This is the citable door for anything you saw in the browser: snapshot, screenshot and run return values are not evidence. Cite the returned URL."
|
|
64
|
+
"This is the citable door for anything you saw in the browser: snapshot, screenshot and run return values are not evidence. Cite the returned URL. " +
|
|
65
|
+
"A browser page URL in a receipt keeps its query names but shows every value replaced by a redaction marker, the word redacted in square brackets (the site chose where the browser went, and a value in it can be a credential); web_fetch keeps a query value that is in the URL you asked for.",
|
|
62
66
|
parameters: object({ what: { type: "string", enum: ["markdown", "text", "html", "url", "title"], description: "Default markdown." } }),
|
|
63
67
|
effect: "read", replay: "safe", executionMode: "sequential", evidence: "receipt", listedWhen: "always",
|
|
64
68
|
};
|
|
@@ -73,11 +77,11 @@ const browserRelease = {
|
|
|
73
77
|
const browserRelaunch = {
|
|
74
78
|
name: "browser_relaunch",
|
|
75
79
|
label: "Browser: relaunch",
|
|
76
|
-
description: "Replace the browser session with a fresh one, after a site blocked it (403/429, a CAPTCHA wall, a bot-detection page, an empty page that should have content). The new session
|
|
80
|
+
description: "Replace the browser session with a fresh one, after a site blocked it (403/429, a CAPTCHA wall, a bot-detection page, an empty page that should have content). The new session is Verified whenever the run's sessions are, and a relaunch never lowers that: `verified: true` asks for a Verified session where the run's sessions are not (a Verified browser carries a stronger identity and costs more). " +
|
|
77
81
|
"Use `geolocation` to browse as a visitor from a given place: the proxy egresses from that country (ISO alpha-2), optionally a US state and a city, so geo-aware sites (storefront region, prices, payment methods, content availability) show what a local sees. " +
|
|
78
82
|
"Page state is lost; navigate again with run.",
|
|
79
83
|
parameters: object({
|
|
80
|
-
verified: { type: "boolean", description: "true launches the new session as a Verified browser
|
|
84
|
+
verified: { type: "boolean", description: "true launches the new session as a Verified browser. false, or leaving it out, keeps the run's setting: a Verified run's sessions stay Verified." },
|
|
81
85
|
geolocation: {
|
|
82
86
|
type: "object",
|
|
83
87
|
additionalProperties: false,
|
|
@@ -92,6 +96,19 @@ const browserRelaunch = {
|
|
|
92
96
|
}),
|
|
93
97
|
effect: "custody", replay: "safe", executionMode: "sequential", evidence: "none", listedWhen: "always",
|
|
94
98
|
};
|
|
99
|
+
const browserRequestHandoff = {
|
|
100
|
+
name: "browser_request_handoff",
|
|
101
|
+
label: "Browser: hand to a person",
|
|
102
|
+
description: "Hand the browser to a person for a step only a human can do: a CAPTCHA you cannot pass, a sign-in that needs their device, a confirmation only they can give. " +
|
|
103
|
+
"Say in instruction what they should do and in completion_condition what the page shows once it is done. The call waits until the person finishes, cancels or the request expires; while they have the browser, every other browser call fails with waiting_for_human. " +
|
|
104
|
+
"The same request_id retries the same request. Afterwards take a snapshot and check the completion condition on the page before acting.",
|
|
105
|
+
parameters: object({
|
|
106
|
+
request_id: { type: "string", pattern: "^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$", description: "A UUID you choose for this request; the same one retries the same request." },
|
|
107
|
+
instruction: { type: "string", minLength: 1, maxLength: 2000, description: "What the person should do, in plain words." },
|
|
108
|
+
completion_condition: { type: "string", minLength: 1, maxLength: 1000, description: "What the page shows once the step is done, for you to verify afterwards." },
|
|
109
|
+
}, ["request_id", "instruction", "completion_condition"]),
|
|
110
|
+
effect: "custody", replay: "safe", executionMode: "sequential", evidence: "none", listedWhen: "handoff",
|
|
111
|
+
};
|
|
95
112
|
const browserDownloads = {
|
|
96
113
|
name: "browser_downloads",
|
|
97
114
|
label: "Browser: downloads",
|
|
@@ -100,12 +117,23 @@ const browserDownloads = {
|
|
|
100
117
|
parameters: object({}),
|
|
101
118
|
effect: "read", replay: "safe", executionMode: "sequential", evidence: "files", listedWhen: "downloads",
|
|
102
119
|
};
|
|
120
|
+
const browserUpload = {
|
|
121
|
+
name: "browser_upload",
|
|
122
|
+
label: "Browser: upload",
|
|
123
|
+
description: "Choose one workspace file in a file input on the page, as a person picking it would; nothing is submitted. " +
|
|
124
|
+
"Returns the file's name, size and SHA-256; submit the form with run afterwards.",
|
|
125
|
+
parameters: object({
|
|
126
|
+
path: { type: "string", minLength: 1, maxLength: 1000, description: "The file's path relative to the workspace, for example reports/q3.pdf." },
|
|
127
|
+
selector: { type: "string", minLength: 1, maxLength: 500, description: "CSS selector of the <input type=\"file\">, for example input[type=file]." },
|
|
128
|
+
}, ["path", "selector"]),
|
|
129
|
+
effect: "effect", replay: "unsafe", executionMode: "sequential", evidence: "none", listedWhen: "upload",
|
|
130
|
+
};
|
|
103
131
|
const webFetch = {
|
|
104
132
|
name: "web_fetch",
|
|
105
133
|
label: "Web fetch",
|
|
106
134
|
description: "Fetch one URL as clean markdown without a browser session: fast and cheap, and it handles most static and many dynamic pages. " +
|
|
107
135
|
"Use it for reading articles, filings, registries, PDFs-as-pages. If it returns blocked or empty content, drive the page with run/snapshot instead. " +
|
|
108
|
-
"The result is persisted as evidence with its URL.",
|
|
136
|
+
"The result is persisted as evidence with its URL: a query value that is in the URL you asked for stays in it, any other value of the final URL is replaced by a redaction marker (the word redacted in square brackets).",
|
|
109
137
|
parameters: object({
|
|
110
138
|
url: { type: "string", description: "Absolute http(s) URL." },
|
|
111
139
|
format: { type: "string", enum: ["markdown", "raw"], description: "Default markdown." },
|
|
@@ -119,13 +147,16 @@ const webSearch = {
|
|
|
119
147
|
parameters: object({ query: { type: "string" }, n: { type: "number", description: "Results to return, 1-25 (default 10)." } }, ["query"]),
|
|
120
148
|
effect: "read", replay: "safe", executionMode: "parallel", evidence: "urls", listedWhen: "search",
|
|
121
149
|
};
|
|
122
|
-
export const BROWSER_TOOLS = Object.freeze([snapshot, run, screenshot, browserRead, browserRelease, browserRelaunch, browserDownloads]);
|
|
150
|
+
export const BROWSER_TOOLS = Object.freeze([snapshot, run, screenshot, browserRead, browserRelease, browserRelaunch, browserRequestHandoff, browserDownloads, browserUpload]);
|
|
123
151
|
export const WEB_TOOLS = Object.freeze([webFetch, webSearch]);
|
|
124
152
|
export const BROWSER_TOOL_NAMES = Object.freeze(BROWSER_TOOLS.map((tool) => tool.name));
|
|
125
153
|
export const WEB_TOOL_NAMES = Object.freeze(WEB_TOOLS.map((tool) => tool.name));
|
|
126
154
|
/** Depends only on configuration, never live state, so every request renders the same text and keeps the prompt cache. */
|
|
127
|
-
export function browserSection(policy) {
|
|
155
|
+
export function browserSection(policy, offered = {}) {
|
|
128
156
|
const idle = Math.round(policy.idleReleaseS);
|
|
157
|
+
const download = offered.downloads
|
|
158
|
+
? " A file the page offers as a download: click its link, then call browser_downloads, which copies it into the workspace under downloads/ (listed in downloads/manifest.json); open it with read."
|
|
159
|
+
: " A file the page offers as a download cannot be collected from the browser: take the link's URL from the page and fetch it with `web_fetch` when that tool is offered, or cite the page that links it.";
|
|
129
160
|
return [
|
|
130
161
|
"You control one remote browser:",
|
|
131
162
|
"- snapshot: the active page's accessibility tree, with bracketed element IDs for run actions.",
|
|
@@ -134,10 +165,12 @@ export function browserSection(policy) {
|
|
|
134
165
|
"- browser_read: the current page's content, filed as evidence with its URL.",
|
|
135
166
|
"- browser_relaunch: a fresh session, Verified or egressing from a given place.",
|
|
136
167
|
"- browser_release: release the session when you are done browsing.",
|
|
168
|
+
...(offered.downloads ? ["- browser_downloads: copy the files this session downloaded into the workspace, each with its SHA-256."] : []),
|
|
169
|
+
...(offered.upload ? ["- browser_upload: choose a workspace file in a file input on the page."] : []),
|
|
137
170
|
"",
|
|
138
171
|
"Snapshot IDs are valid only for the latest snapshot of the active page; snapshot again after navigation or a stale ID. Use snapshot actions for single clicks and fills, and one run code call for a multi-step workflow that returns only the values you need. Do not launch another browser; open more tabs with context.newPage() inside run code.",
|
|
139
172
|
"",
|
|
140
|
-
"The `run` page is Playwright-shaped, not Playwright. On this browser these throw: `page.waitForEvent(\"download\")`, `page.route`, `page.waitForResponse`, `context.newCDPSession`, and `page.on` for `pageerror`, `framenavigated` or `response`. Wait for what the page shows instead (`page.waitForSelector`, `page.waitForLoadState`, or poll `page.url()` or a locator's text in a short loop), read results from the page itself (a locator's text, then `browser_read` to cite it), and never intercept or rewrite requests.
|
|
173
|
+
"The `run` page is Playwright-shaped, not Playwright. On this browser these throw: `page.waitForEvent(\"download\")`, `page.route`, `page.waitForResponse`, `context.newCDPSession`, and `page.on` for `pageerror`, `framenavigated` or `response`. Wait for what the page shows instead (`page.waitForSelector`, `page.waitForLoadState`, or poll `page.url()` or a locator's text in a short loop), read results from the page itself (a locator's text, then `browser_read` to cite it), and never intercept or rewrite requests." + download,
|
|
141
174
|
"",
|
|
142
175
|
// A page call made while a submit's navigation commits throws -32000 or hangs; a bounded wait and one retry avoid
|
|
143
176
|
// it.
|