@parcha/pi-browser 0.1.0-beta.33 → 0.1.0-beta.34
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +17 -0
- package/NOTICE +30 -0
- package/README.md +28 -178
- package/dist/core/contract.js +14 -5
- package/dist/core/custody.d.ts +6 -0
- package/dist/core/custody.js +6 -0
- package/dist/core/driving-guard.js +17 -2
- package/dist/core/host.d.ts +2 -2
- package/dist/core/tools.d.ts +9 -0
- package/dist/driver/cdp.d.ts +6 -0
- package/dist/driver/cdp.js +245 -0
- package/dist/driver/page-driver.js +29 -3
- package/dist/durable/store.js +13 -4
- package/dist/durable.d.ts +1 -0
- package/dist/durable.js +1 -0
- package/dist/live-view/provider.d.ts +2 -2
- package/dist/live-view/provider.js +2 -2
- package/dist/live-view/server.d.ts +1 -1
- package/dist/providers/chrome-args.d.ts +3 -0
- package/dist/providers/chrome-args.js +13 -2
- package/dist/vendor/browser-use-pi/ax.d.ts +106 -0
- package/dist/vendor/browser-use-pi/ax.js +642 -0
- package/dist/vendor/browser-use-pi/cdp.d.ts +19 -0
- package/dist/vendor/browser-use-pi/cdp.js +106 -0
- package/dist/vendor/browser-use-pi/page.d.ts +38 -0
- package/dist/vendor/browser-use-pi/page.js +112 -0
- package/package.json +6 -24
- package/src/vendor/browser-use-pi/UPSTREAM.json +25 -0
- package/src/vendor/stagehand-facade/UPSTREAM.json +0 -4
- package/dist/coding-agent-local.d.ts +0 -12
- package/dist/coding-agent-local.js +0 -72
- package/dist/coding-agent.d.ts +0 -135
- package/dist/coding-agent.js +0 -641
- package/dist/providers/assigned.d.ts +0 -18
- package/dist/providers/assigned.js +0 -75
- package/dist/recorder/credential.d.ts +0 -9
- package/dist/recorder/credential.js +0 -49
- package/dist/recorder/follow.d.ts +0 -36
- package/dist/recorder/follow.js +0 -163
- package/dist/recorder/frames.d.ts +0 -87
- package/dist/recorder/frames.js +0 -377
- package/dist/recorder/index.d.ts +0 -4
- package/dist/recorder/index.js +0 -7
- package/dist/recorder/jpeg.d.ts +0 -8
- package/dist/recorder/jpeg.js +0 -70
- package/dist/recorder/track.d.ts +0 -107
- package/dist/recorder/track.js +0 -280
- package/dist/vendor/stagehand-facade/screenshot-transport.d.ts +0 -15
- package/dist/vendor/stagehand-facade/screenshot-transport.js +0 -51
|
@@ -0,0 +1,245 @@
|
|
|
1
|
+
import { refusedError } from "../core/custody.js";
|
|
2
|
+
import { openDial } from "../core/dial.js";
|
|
3
|
+
import { AxHelpers, UnknownIdError } from "../vendor/browser-use-pi/ax.js";
|
|
4
|
+
import { CDP, CDPConnectionClosedError } from "../vendor/browser-use-pi/cdp.js";
|
|
5
|
+
import { Page } from "../vendor/browser-use-pi/page.js";
|
|
6
|
+
import { NAVIGATED_SNAPSHOT_ERROR, NO_HYDRATED_SNAPSHOT_ERROR, STALE_SNAPSHOT_ID_ERROR } from "../core/failures.js";
|
|
7
|
+
/** The package's stale-snapshot words, as the Stagehand driver says them: plain errors, so a failure reads them as a stale snapshot. */
|
|
8
|
+
const stale = (message) => new Error(message);
|
|
9
|
+
const staleSnapshotIdError = (id) => STALE_SNAPSHOT_ID_ERROR.replace("${id}", id);
|
|
10
|
+
/** What `run` with code is told on this browser. */
|
|
11
|
+
export const CDP_CODE_REFUSED = 'run code is not available on this browser (the agent\'s browser computer): send actions instead, each an op with a snapshot id and a value, for example {"actions":[{"op":"click","id":"0-42"}]}, and navigate with {"actions":[{"op":"goto","url":"https://example.com"}]}. Read the page with browser_read.';
|
|
12
|
+
/** How long one CDP command may take before the action is said to be unanswered. */
|
|
13
|
+
const COMMAND_MS = 30_000;
|
|
14
|
+
/**
|
|
15
|
+
* One element's box once it is scrolled into view, in CSS pixels from the top page's viewport corner, or how many a
|
|
16
|
+
* selector matches. An element in a same-origin iframe (an id from the snapshot) gets each enclosing frame's
|
|
17
|
+
* content-box offset added, and the viewport is the top page's, as the screenshot's clip is.
|
|
18
|
+
*/
|
|
19
|
+
const LOCATE_JS = `function(selector){
|
|
20
|
+
let top = (this && this.ownerDocument ? this.ownerDocument : document).defaultView;
|
|
21
|
+
while (top.frameElement) top = top.parent;
|
|
22
|
+
const viewport = { width: top.innerWidth, height: top.innerHeight };
|
|
23
|
+
const found = selector === null ? null : selector === undefined ? this : document.querySelectorAll(selector);
|
|
24
|
+
if (found === null) return { viewport };
|
|
25
|
+
const list = found instanceof NodeList ? [...found] : [found.nodeType === 1 ? found : found.parentElement];
|
|
26
|
+
if (list.length !== 1 || !list[0]) return { viewport, matches: list.filter(Boolean).length };
|
|
27
|
+
list[0].scrollIntoView({ block: 'center', inline: 'center' });
|
|
28
|
+
const r = list[0].getBoundingClientRect();
|
|
29
|
+
let x = r.x, y = r.y;
|
|
30
|
+
for (let w = list[0].ownerDocument.defaultView; w && w.frameElement; w = w.parent) {
|
|
31
|
+
const f = w.frameElement, fr = f.getBoundingClientRect(), cs = getComputedStyle(f);
|
|
32
|
+
x += fr.x + f.clientLeft + parseFloat(cs.paddingLeft);
|
|
33
|
+
y += fr.y + f.clientTop + parseFloat(cs.paddingTop);
|
|
34
|
+
}
|
|
35
|
+
return { viewport, matches: 1, box: { x, y, width: r.width, height: r.height } };
|
|
36
|
+
}`;
|
|
37
|
+
/** The browser's tabs, in the browser's order (DevTools' own pages left out). */
|
|
38
|
+
async function tabsOf(cdp) {
|
|
39
|
+
const { targetInfos } = await cdp.send("Target.getTargets");
|
|
40
|
+
return targetInfos.filter((t) => t.type === "page" && !t.url.startsWith("devtools://"));
|
|
41
|
+
}
|
|
42
|
+
/** A blank tab opened for the driver, when the browser has none. */
|
|
43
|
+
async function blankTab(cdp) {
|
|
44
|
+
return (await cdp.send("Target.createTarget", { url: "about:blank" })).targetId;
|
|
45
|
+
}
|
|
46
|
+
/** The raw-CDP page driver: snapshot ids and actions only. */
|
|
47
|
+
export function cdpDriver() {
|
|
48
|
+
return async (target, signal) => {
|
|
49
|
+
signal?.throwIfAborted();
|
|
50
|
+
const cdp = await CDP.open(openDial(target), COMMAND_MS);
|
|
51
|
+
let page;
|
|
52
|
+
/** Every tab the driver has seen, so one that appears later is known as new. */
|
|
53
|
+
const seen = new Set();
|
|
54
|
+
/** The tab that opened each tab the driver followed, to go back to when that one closes. */
|
|
55
|
+
const openers = new Map();
|
|
56
|
+
/** Whether the last run moved to another tab (movedTab). */
|
|
57
|
+
let movedTab = false;
|
|
58
|
+
try {
|
|
59
|
+
const tabs = await tabsOf(cdp);
|
|
60
|
+
for (const t of tabs)
|
|
61
|
+
seen.add(t.targetId);
|
|
62
|
+
// The browser's first tab, or a blank one opened for the driver when it has none.
|
|
63
|
+
page = await Page.attach(cdp, tabs[0]?.targetId ?? (await blankTab(cdp)));
|
|
64
|
+
seen.add(page.targetId);
|
|
65
|
+
}
|
|
66
|
+
catch (error) {
|
|
67
|
+
cdp.close();
|
|
68
|
+
throw error;
|
|
69
|
+
}
|
|
70
|
+
const ax = new AxHelpers(() => page, () => cdp);
|
|
71
|
+
/**
|
|
72
|
+
* Drives the tab the driven tab opened (a popup, or a link that opens a new tab), or, when the driven tab has closed,
|
|
73
|
+
* the tab that opened it (else the browser's last, else a blank one). The old tab's snapshot ids are dropped, so an
|
|
74
|
+
* action by id fails as a stale snapshot until the next snapshot. Answers the line that says so, or nothing when
|
|
75
|
+
* the driven tab stays.
|
|
76
|
+
*/
|
|
77
|
+
const follow = async () => {
|
|
78
|
+
const tabs = await tabsOf(cdp);
|
|
79
|
+
const opened = tabs.filter((t) => !seen.has(t.targetId) && t.openerId === page.targetId).at(-1);
|
|
80
|
+
for (const t of tabs)
|
|
81
|
+
seen.add(t.targetId);
|
|
82
|
+
let next = opened?.targetId;
|
|
83
|
+
if (!next && !tabs.some((t) => t.targetId === page.targetId)) {
|
|
84
|
+
const back = openers.get(page.targetId);
|
|
85
|
+
next = tabs.find((t) => t.targetId === back)?.targetId ?? tabs.at(-1)?.targetId ?? (await blankTab(cdp));
|
|
86
|
+
seen.add(next);
|
|
87
|
+
}
|
|
88
|
+
if (!next)
|
|
89
|
+
return undefined;
|
|
90
|
+
const old = page;
|
|
91
|
+
page = await Page.attach(cdp, next);
|
|
92
|
+
if (opened)
|
|
93
|
+
openers.set(next, old.targetId);
|
|
94
|
+
ax.forget();
|
|
95
|
+
await cdp.send("Target.detachFromTarget", { sessionId: old.sessionId }).catch(() => undefined);
|
|
96
|
+
await ax.settle({ capMs: 3000 }).catch(() => undefined);
|
|
97
|
+
const { targetInfo } = await cdp.send("Target.getTargetInfo", { targetId: next });
|
|
98
|
+
const now = opened ? "the page opened a new tab; now driving it" : "the tab closed; now driving another";
|
|
99
|
+
return `[tab] ${now}: ${targetInfo.url}.`;
|
|
100
|
+
};
|
|
101
|
+
/** The latest snapshot's ids hold: there is one, the page still shows its document, and every id is in it. */
|
|
102
|
+
const check = async (actions, checked = false) => {
|
|
103
|
+
if (actions.every((a) => a.op === "goto"))
|
|
104
|
+
return;
|
|
105
|
+
if (!ax.hydrated || !ax.taken)
|
|
106
|
+
throw stale(NO_HYDRATED_SNAPSHOT_ERROR);
|
|
107
|
+
if (!checked) {
|
|
108
|
+
// A probe the page can't answer is a navigated page; a dropped connection is said as one, not as a stale snapshot.
|
|
109
|
+
const now = await ax.probe().catch((error) => {
|
|
110
|
+
if (error instanceof CDPConnectionClosedError)
|
|
111
|
+
throw error;
|
|
112
|
+
return undefined;
|
|
113
|
+
});
|
|
114
|
+
if (!now || now.doc !== ax.taken.doc)
|
|
115
|
+
throw stale(NAVIGATED_SNAPSHOT_ERROR);
|
|
116
|
+
}
|
|
117
|
+
for (const action of actions) {
|
|
118
|
+
if (action.op === "goto")
|
|
119
|
+
break; // what follows a goto is on the next page: its ids are checked when it runs
|
|
120
|
+
try {
|
|
121
|
+
ax.nodeOf(action.id);
|
|
122
|
+
}
|
|
123
|
+
catch {
|
|
124
|
+
throw stale(staleSnapshotIdError(action.id));
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
};
|
|
128
|
+
const runOne = async (action) => {
|
|
129
|
+
switch (action.op) {
|
|
130
|
+
case "goto": {
|
|
131
|
+
const before = await ax.probe().catch(() => undefined);
|
|
132
|
+
await page.goto(action.url);
|
|
133
|
+
// The page's ids were the old document's: the next action by id fails as a stale snapshot.
|
|
134
|
+
ax.forget();
|
|
135
|
+
const settled = await ax.settle({ capMs: 3000 });
|
|
136
|
+
return `[ok] goto ${action.url} -> ${await ax.change(before, settled.probe)}`;
|
|
137
|
+
}
|
|
138
|
+
case "click":
|
|
139
|
+
return ax.click(action.id);
|
|
140
|
+
case "hover":
|
|
141
|
+
return ax.hover(action.id);
|
|
142
|
+
case "fill":
|
|
143
|
+
return ax.fill(action.id, action.value);
|
|
144
|
+
case "type":
|
|
145
|
+
return ax.type(action.id, action.text, action.delay);
|
|
146
|
+
case "press":
|
|
147
|
+
return ax.press(action.id, action.key);
|
|
148
|
+
case "select":
|
|
149
|
+
return ax.select(action.id, action.values);
|
|
150
|
+
default:
|
|
151
|
+
throw refusedError(`run takes the ops goto, click, hover, fill, type, press and select, not ${JSON.stringify(action.op)}`);
|
|
152
|
+
}
|
|
153
|
+
};
|
|
154
|
+
return {
|
|
155
|
+
// A snapshot is of the tab the agent is on now: a tab the page opened since, or the one left when it closed.
|
|
156
|
+
snapshot: async (options = {}) => {
|
|
157
|
+
const moved = await follow();
|
|
158
|
+
const shot = await ax.snapshot(options.includeIframes === undefined ? {} : { includeIframes: options.includeIframes });
|
|
159
|
+
return moved ? `${moved}\n${shot}` : shot;
|
|
160
|
+
},
|
|
161
|
+
run: async ({ code, actions, checked, rest: after = 0 }) => {
|
|
162
|
+
movedTab = false;
|
|
163
|
+
if (code !== undefined)
|
|
164
|
+
throw refusedError(CDP_CODE_REFUSED);
|
|
165
|
+
if (!actions?.length)
|
|
166
|
+
throw refusedError("run requires exactly one of code or actions");
|
|
167
|
+
await check(actions, checked === true);
|
|
168
|
+
const lines = [];
|
|
169
|
+
for (const [i, action] of actions.entries()) {
|
|
170
|
+
try {
|
|
171
|
+
lines.push(await runOne(action));
|
|
172
|
+
}
|
|
173
|
+
catch (error) {
|
|
174
|
+
// An id after a goto in the same run names nothing of the new page: a stale snapshot.
|
|
175
|
+
if (error instanceof UnknownIdError)
|
|
176
|
+
throw stale(staleSnapshotIdError(error.id));
|
|
177
|
+
if (!lines.length)
|
|
178
|
+
throw error;
|
|
179
|
+
// What ran is said with what failed: the actions before it took effect.
|
|
180
|
+
throw new Error(`${lines.join("\n")}\n${error instanceof Error ? error.message : String(error)}`, { cause: error });
|
|
181
|
+
}
|
|
182
|
+
// An action that opened a tab (or closed this one) moves the driver there; the rest name the old tab's ids.
|
|
183
|
+
// The action already took effect, so a failure to look is not its failure.
|
|
184
|
+
const moved = await follow().catch(() => undefined);
|
|
185
|
+
if (moved) {
|
|
186
|
+
movedTab = true;
|
|
187
|
+
lines.push(`${moved} Take a snapshot before acting by id.`);
|
|
188
|
+
// This run's actions after this one, and the batch's that the guard would send after them.
|
|
189
|
+
const rest = actions.length - i - 1 + after;
|
|
190
|
+
if (rest)
|
|
191
|
+
lines.push(`[stopped] ${rest} more action${rest === 1 ? "" : "s"} not run: they name the old tab's snapshot ids.`);
|
|
192
|
+
break;
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
return lines.join("\n");
|
|
196
|
+
},
|
|
197
|
+
movedTab: () => movedTab,
|
|
198
|
+
checkActions: (actions) => check(actions),
|
|
199
|
+
url: async () => {
|
|
200
|
+
const { targetInfo } = await cdp.send("Target.getTargetInfo", { targetId: page.targetId });
|
|
201
|
+
return targetInfo.url;
|
|
202
|
+
},
|
|
203
|
+
forgetSnapshots: () => ax.forget(),
|
|
204
|
+
locate: async (place) => {
|
|
205
|
+
if (place?.id !== undefined) {
|
|
206
|
+
let node;
|
|
207
|
+
try {
|
|
208
|
+
node = ax.nodeOf(place.id);
|
|
209
|
+
}
|
|
210
|
+
catch {
|
|
211
|
+
throw stale(ax.hydrated ? staleSnapshotIdError(place.id) : NO_HYDRATED_SNAPSHOT_ERROR);
|
|
212
|
+
}
|
|
213
|
+
const { object } = await page.cdp("DOM.resolveNode", { backendNodeId: node });
|
|
214
|
+
if (!object.objectId)
|
|
215
|
+
return { viewport: await page.evaluate(() => ({ width: innerWidth, height: innerHeight })), matches: 0 };
|
|
216
|
+
try {
|
|
217
|
+
const answer = await page.cdp("Runtime.callFunctionOn", { objectId: object.objectId, functionDeclaration: LOCATE_JS, arguments: [{ value: undefined }], returnByValue: true });
|
|
218
|
+
return answer.result.value;
|
|
219
|
+
}
|
|
220
|
+
finally {
|
|
221
|
+
await page.cdp("Runtime.releaseObject", { objectId: object.objectId }).catch(() => { });
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
const selector = place?.selector ?? null;
|
|
225
|
+
return page.evaluate(`(${LOCATE_JS}).call(document, ${JSON.stringify(selector)})`);
|
|
226
|
+
},
|
|
227
|
+
screenshot: async ({ type = "png", quality, fullPage, clip }) => {
|
|
228
|
+
const data = await page.screenshot({ type, ...(quality === undefined ? {} : { quality }), ...(fullPage === undefined ? {} : { fullPage }), ...(clip ? { clip } : {}) });
|
|
229
|
+
return { data, mimeType: type === "jpeg" ? "image/jpeg" : "image/png" };
|
|
230
|
+
},
|
|
231
|
+
// One evaluate reads all four, so they are one document's.
|
|
232
|
+
page: () => page.evaluate(() => ({
|
|
233
|
+
url: location.href,
|
|
234
|
+
title: document.title,
|
|
235
|
+
text: document.body ? document.body.innerText : "",
|
|
236
|
+
html: document.documentElement ? document.documentElement.outerHTML : "",
|
|
237
|
+
})),
|
|
238
|
+
// The browser keeps running: custody releases it through the provider.
|
|
239
|
+
close: async () => {
|
|
240
|
+
await cdp.send("Target.detachFromTarget", { sessionId: page.sessionId }).catch(() => undefined);
|
|
241
|
+
cdp.close();
|
|
242
|
+
},
|
|
243
|
+
};
|
|
244
|
+
};
|
|
245
|
+
}
|
|
@@ -133,6 +133,25 @@ export function pageDriver(stagehand, deadlines) {
|
|
|
133
133
|
kept().set(page.pageId, { ...current, url: snapshot.url });
|
|
134
134
|
}
|
|
135
135
|
};
|
|
136
|
+
/** A batch with a goto: the actions one at a time, a goto through the facade's page (http, https or about:blank only), each
|
|
137
|
+
* other action by its id as the facade runs it, so an id after a goto meets the new page and fails as a stale snapshot. */
|
|
138
|
+
const withGoto = async (actions, checked) => {
|
|
139
|
+
if (!actions.some((action) => action.op === "goto"))
|
|
140
|
+
return checked ? afterChecked(actions) : facade.runActions(actions);
|
|
141
|
+
let completed = 0;
|
|
142
|
+
let url = "";
|
|
143
|
+
for (const action of actions) {
|
|
144
|
+
if (action.op === "goto") {
|
|
145
|
+
if (!/^(https?:\/\/|about:blank$)/i.test(action.url))
|
|
146
|
+
throw new Error(`goto takes an http or https URL (or about:blank), not ${JSON.stringify(action.url.slice(0, 100))}`);
|
|
147
|
+
url = await facade.run(`await page.goto(${JSON.stringify(action.url)}); return page.url();`);
|
|
148
|
+
}
|
|
149
|
+
else
|
|
150
|
+
url = (await facade.runActions([action])).url;
|
|
151
|
+
completed++;
|
|
152
|
+
}
|
|
153
|
+
return { completed, url };
|
|
154
|
+
};
|
|
136
155
|
return {
|
|
137
156
|
snapshot: async (snapshotOptions) => {
|
|
138
157
|
const tree = await reads.snapshot(snapshotOptions);
|
|
@@ -142,19 +161,26 @@ export function pageDriver(stagehand, deadlines) {
|
|
|
142
161
|
snapshotDocuments.set(page.pageId, { xpathById: taken.xpathById, document: documentOf(page) });
|
|
143
162
|
return tree;
|
|
144
163
|
},
|
|
145
|
-
run: (input) => input.actions !== undefined ? (input.
|
|
164
|
+
run: (input) => input.actions !== undefined ? withGoto(input.actions, input.checked === true) : facade.run(input.code ?? ""),
|
|
146
165
|
// As the facade's own batch checks a run's actions before it sends any (runActionsNow): the active page's snapshot,
|
|
147
|
-
// the page still at its address, and every id in it. Read from the facade's snapshots, as forgetSnapshots does.
|
|
166
|
+
// the page still at its address, and every id in it. Read from the facade's snapshots, as forgetSnapshots does. A
|
|
167
|
+
// batch of gotos alone names no id, so it needs no snapshot, as the raw-CDP driver's check holds too.
|
|
148
168
|
checkActions: async (actions) => {
|
|
169
|
+
if (actions.every((action) => action.op === "goto"))
|
|
170
|
+
return;
|
|
149
171
|
const page = await active();
|
|
150
172
|
const snapshot = facade.snapshotsByPage.get(page.pageId);
|
|
151
173
|
if (!snapshot)
|
|
152
174
|
throw new Error(NO_HYDRATED_SNAPSHOT_ERROR);
|
|
153
175
|
if ((await page.url()) !== snapshot.url)
|
|
154
176
|
throw new Error(NAVIGATED_SNAPSHOT_ERROR);
|
|
155
|
-
for (const action of actions)
|
|
177
|
+
for (const action of actions) {
|
|
178
|
+
// What follows a goto is on the next page: its ids are checked when it runs.
|
|
179
|
+
if (action.op === "goto")
|
|
180
|
+
break;
|
|
156
181
|
if (!snapshot.xpathById[action.id])
|
|
157
182
|
throw new Error(staleSnapshotIdError(action.id));
|
|
183
|
+
}
|
|
158
184
|
},
|
|
159
185
|
url: async () => documentUrl(stagehand, await active()),
|
|
160
186
|
// The facade keeps each page's last snapshot's ids for its own actions; with none, an action by id fails NO_HYDRATED (a stale snapshot).
|
package/dist/durable/store.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { defineDoc, defineTask } from "@earendil-works/pi-durable";
|
|
2
|
-
import { backoffMs, moveRecord, notices } from "../core/custody.js";
|
|
2
|
+
import { backoffMs, HANDOFF_POLL_MS, moveRecord, notices } from "../core/custody.js";
|
|
3
3
|
export const now = () => new Date().toISOString();
|
|
4
4
|
const iso = (ms) => new Date(ms).toISOString();
|
|
5
5
|
const cidOf = (row) => row.conversationId;
|
|
@@ -183,8 +183,9 @@ export function custodyStore({ name, Config, Sessions, providerOf, emit, meter,
|
|
|
183
183
|
}, context);
|
|
184
184
|
},
|
|
185
185
|
});
|
|
186
|
-
/** Polls the port until it answers,
|
|
187
|
-
* this request.
|
|
186
|
+
/** Polls the port until it answers, every HANDOFF_POLL_MS while the answer is pending, then gives the session back with
|
|
187
|
+
* the outcome's notice, if it is still held for this request. `attempt` counts the reads in a row that failed, which
|
|
188
|
+
* back off (backoffMs); a read that answers "pending" starts it again. */
|
|
188
189
|
const HandoffTask = defineTask({
|
|
189
190
|
name: `${name}.handoff`,
|
|
190
191
|
version: 1,
|
|
@@ -206,6 +207,7 @@ export function custodyStore({ name, Config, Sessions, providerOf, emit, meter,
|
|
|
206
207
|
if (over)
|
|
207
208
|
return;
|
|
208
209
|
let answer = "pending";
|
|
210
|
+
let failed = false;
|
|
209
211
|
try {
|
|
210
212
|
if (handoff)
|
|
211
213
|
answer = await handoff.outcome(interactionId, runtime.signal);
|
|
@@ -214,11 +216,18 @@ export function custodyStore({ name, Config, Sessions, providerOf, emit, meter,
|
|
|
214
216
|
if (runtime.signal.aborted)
|
|
215
217
|
throw error;
|
|
216
218
|
runtime.report(scrubbed(error));
|
|
219
|
+
failed = true;
|
|
217
220
|
}
|
|
218
|
-
|
|
221
|
+
// A person's answer can come at any moment: while it is pending, it is asked again at a short fixed interval, so
|
|
222
|
+
// a hand-back resumes the agent within one poll. A read that failed backs off instead.
|
|
223
|
+
if (failed) {
|
|
219
224
|
await runtime.sleepUntil(runtime.now() + backoffMs(attempt), { phase: "poll", attempt: attempt + 1 }, context);
|
|
220
225
|
return;
|
|
221
226
|
}
|
|
227
|
+
if (answer === "pending") {
|
|
228
|
+
await runtime.sleepUntil(runtime.now() + HANDOFF_POLL_MS, { phase: "poll", attempt: 0 }, context);
|
|
229
|
+
return;
|
|
230
|
+
}
|
|
222
231
|
const outcome = typeof answer === "string" ? answer : answer.outcome;
|
|
223
232
|
const reason = typeof answer === "string" ? undefined : answer.reason;
|
|
224
233
|
await runtime.commit(async (tx) => {
|
package/dist/durable.d.ts
CHANGED
|
@@ -120,4 +120,5 @@ export type BrowserExtensionHandle = {
|
|
|
120
120
|
};
|
|
121
121
|
export { createWebExtension, type WebExtensionOptions } from "./durable/web.js";
|
|
122
122
|
export { CONFIRM_POLLS, RELEASE_ATTEMPTS } from "./durable/store.js";
|
|
123
|
+
export { HANDOFF_POLL_MS } from "./core/custody.js";
|
|
123
124
|
export declare function createBrowserExtension<D extends PageDriver>(options: BrowserExtensionOptions<D>): BrowserExtensionHandle;
|
package/dist/durable.js
CHANGED
|
@@ -20,6 +20,7 @@ import { withinDeadline } from "./core/unanswered.js";
|
|
|
20
20
|
import { resolvePrices, sessionCharge, withEstimate } from "./core/usage.js";
|
|
21
21
|
export { createWebExtension } from "./durable/web.js";
|
|
22
22
|
export { CONFIRM_POLLS, RELEASE_ATTEMPTS } from "./durable/store.js";
|
|
23
|
+
export { HANDOFF_POLL_MS } from "./core/custody.js";
|
|
23
24
|
const PLANE = "default";
|
|
24
25
|
const UNCONFIGURED = { label: "", run: "", policy: DEFAULT_BROWSER_POLICY };
|
|
25
26
|
const HANDOFF_ENDED = new BrowserFailureError(browserFailure("session_ended", "The browser session left the person's hands before their answer could be waited on. Your next browser call opens a fresh session; take a snapshot there before acting.", { retryable: true }));
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { BrowserProvider } from "../core/host.js";
|
|
2
2
|
import type { LiveViewHost } from "./server.js";
|
|
3
3
|
/** The provider with a live view: `liveView(ref)` reports the viewer's URL (to the host only: its `onSession` and a handoff's `request`), and a release kills the
|
|
4
|
-
* token before the provider's own release runs, so a session that is going away cannot be driven. Wrap the cdp provider with it
|
|
5
|
-
*
|
|
4
|
+
* token before the provider's own release runs, so a session that is going away cannot be driven. Wrap the cdp provider with it; the provider's own
|
|
5
|
+
* methods are untouched. */
|
|
6
6
|
export declare function liveViewProvider(provider: BrowserProvider, host: LiveViewHost): BrowserProvider;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/** The provider with a live view: `liveView(ref)` reports the viewer's URL (to the host only: its `onSession` and a handoff's `request`), and a release kills the
|
|
2
|
-
* token before the provider's own release runs, so a session that is going away cannot be driven. Wrap the cdp provider with it
|
|
3
|
-
*
|
|
2
|
+
* token before the provider's own release runs, so a session that is going away cannot be driven. Wrap the cdp provider with it; the provider's own
|
|
3
|
+
* methods are untouched. */
|
|
4
4
|
export function liveViewProvider(provider, host) {
|
|
5
5
|
const wrapped = Object.create(provider);
|
|
6
6
|
wrapped.liveView = async (ref) => {
|
|
@@ -58,7 +58,7 @@ export type LiveViewHost = {
|
|
|
58
58
|
open(tag: string, cdpUrl: string): Promise<LiveView>;
|
|
59
59
|
/** The view `open` gave for a tag, or null: for a caller that must not start anything (`liveViewProvider` asks before it attaches). */
|
|
60
60
|
known(tag: string): LiveView | null;
|
|
61
|
-
/** True while a person is driving this session (viewer input within the gate's idle window)
|
|
61
|
+
/** True while a person is driving this session (viewer input within the gate's idle window). */
|
|
62
62
|
driving(tag: string): boolean;
|
|
63
63
|
/** The token is dead, the viewer's socket closed, the stream and the gate ended. Idempotent. */
|
|
64
64
|
release(tag: string): Promise<void>;
|
|
@@ -1,3 +1,6 @@
|
|
|
1
|
+
/** The local Chrome to start: the one `PI_BROWSER_CHROME` or `CHROME_PATH` names when it exists, else the first install found
|
|
2
|
+
* on PATH or at a fixed place; null when there is none. */
|
|
3
|
+
export declare function findChrome(env?: NodeJS.ProcessEnv, exists?: (file: string) => boolean): string | null;
|
|
1
4
|
/** The directory Stagehand loads its extension from in a local Chrome (`Extensions.loadUnpacked`), resolved as Stagehand does. */
|
|
2
5
|
export declare function extensionDirectory(): string;
|
|
3
6
|
/** The Origin the extension's own DevTools socket carries. Chrome names an unpacked extension by the SHA-256 of its real path: the
|
|
@@ -1,12 +1,23 @@
|
|
|
1
1
|
// The arguments the cdp provider starts a local Chrome with, in one place, so a host that starts its own Chrome for Stagehand can
|
|
2
|
-
// take the same list rather than copy it.
|
|
2
|
+
// take the same list rather than copy it; and where that Chrome is installed (findChrome).
|
|
3
3
|
import { createHash } from "node:crypto";
|
|
4
|
-
import { realpathSync } from "node:fs";
|
|
4
|
+
import { existsSync, realpathSync } from "node:fs";
|
|
5
5
|
import path from "node:path";
|
|
6
6
|
import { fileURLToPath } from "node:url";
|
|
7
7
|
import { BrowserFailureError } from "../core/custody.js";
|
|
8
8
|
import { browserFailure } from "../core/failures.js";
|
|
9
9
|
import { nonEmpty } from "./shared.js";
|
|
10
|
+
const LINUX_NAMES = ["google-chrome", "google-chrome-stable", "chromium", "chromium-browser"];
|
|
11
|
+
/** The local Chrome to start: the one `PI_BROWSER_CHROME` or `CHROME_PATH` names when it exists, else the first install found
|
|
12
|
+
* on PATH or at a fixed place; null when there is none. */
|
|
13
|
+
export function findChrome(env = process.env, exists = existsSync) {
|
|
14
|
+
for (const named of [env.PI_BROWSER_CHROME, env.CHROME_PATH])
|
|
15
|
+
if (named && exists(named))
|
|
16
|
+
return named;
|
|
17
|
+
const onPath = (env.PATH ?? "").split(path.delimiter).filter(Boolean).flatMap((dir) => LINUX_NAMES.map((name) => path.join(dir, name)));
|
|
18
|
+
const fixed = ["/opt/google/chrome/chrome", "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", "C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe"];
|
|
19
|
+
return [...onPath, ...fixed].find((file) => exists(file)) ?? null;
|
|
20
|
+
}
|
|
10
21
|
/** Stagehand 4.1.0's DEFAULT_CHROME_FLAGS (packages/sdk-ts/src/browser/localBrowser.ts, MIT, see NOTICE), less its
|
|
11
22
|
* `--remote-allow-origins=*`, which the launch below narrows to the extension's own Origin. Without them
|
|
12
23
|
* Stagehand.create takes about 24 s on a local Chrome instead of under 100 ms. */
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
import type { CDP } from "./cdp.js";
|
|
2
|
+
import type { Page } from "./page.js";
|
|
3
|
+
type Probe = {
|
|
4
|
+
idle: number;
|
|
5
|
+
ready: string;
|
|
6
|
+
url: string;
|
|
7
|
+
title: string;
|
|
8
|
+
n: number;
|
|
9
|
+
doc: number;
|
|
10
|
+
};
|
|
11
|
+
/** A snapshot id is stale or never was: the driver turns it into the package's own words. */
|
|
12
|
+
export declare class UnknownIdError extends Error {
|
|
13
|
+
readonly id: string;
|
|
14
|
+
constructor(id: string);
|
|
15
|
+
}
|
|
16
|
+
/** Snapshot and id-based actions on Chrome's accessibility tree. */
|
|
17
|
+
export declare class AxHelpers {
|
|
18
|
+
private page;
|
|
19
|
+
private browser;
|
|
20
|
+
constructor(page: () => Page, browser: () => CDP);
|
|
21
|
+
private inflight;
|
|
22
|
+
private lastNet;
|
|
23
|
+
private tracked;
|
|
24
|
+
private dialogs;
|
|
25
|
+
private observing;
|
|
26
|
+
/** Role and name of every backend id in the last snapshot: marks new ids and names targets in action lines. */
|
|
27
|
+
private last;
|
|
28
|
+
/** The last snapshot's displayed ids, each to its backend node id; none before the first snapshot or after forget(). */
|
|
29
|
+
private ids;
|
|
30
|
+
/** The document and address the last snapshot was taken on. */
|
|
31
|
+
taken: {
|
|
32
|
+
doc: number;
|
|
33
|
+
url: string;
|
|
34
|
+
} | undefined;
|
|
35
|
+
/** Track in-flight requests per page session from CDP Network events (no page patching), and accept dialogs. */
|
|
36
|
+
private trackNetwork;
|
|
37
|
+
/** URL, title, document identity, a counter of DOM mutations (style aside) and form input/change events. */
|
|
38
|
+
probe(page?: Page): Promise<Probe>;
|
|
39
|
+
/**
|
|
40
|
+
* Event-based wait, never a fixed sleep: document parsed, no fresh in-flight requests (older than 1.5 s are treated as
|
|
41
|
+
* long-poll/analytics), and no DOM mutation for quietMs. Bounded by capMs.
|
|
42
|
+
*/
|
|
43
|
+
settle(options?: {
|
|
44
|
+
capMs?: number;
|
|
45
|
+
quietMs?: number;
|
|
46
|
+
page?: Page;
|
|
47
|
+
since?: number;
|
|
48
|
+
}): Promise<{
|
|
49
|
+
why: string;
|
|
50
|
+
pending: number;
|
|
51
|
+
ms: number;
|
|
52
|
+
probe: Probe;
|
|
53
|
+
} | {
|
|
54
|
+
why: string;
|
|
55
|
+
pending: number;
|
|
56
|
+
ms: number;
|
|
57
|
+
probe?: undefined;
|
|
58
|
+
}>;
|
|
59
|
+
private flushDialogs;
|
|
60
|
+
/**
|
|
61
|
+
* Text actually shown, lowercased with whitespace collapsed. The AX tree keeps opacity:0 text (pre-rendered success
|
|
62
|
+
* banners), which agents then report as success; checkVisibility with checkOpacity drops it.
|
|
63
|
+
*/
|
|
64
|
+
private visibleText;
|
|
65
|
+
/**
|
|
66
|
+
* AX trees of the main frame and, with `frames`, of every same-process child frame. A child frame whose tree cannot be
|
|
67
|
+
* read (a cross-origin frame Chrome runs in another process) is returned as `unread`, never dropped unsaid.
|
|
68
|
+
*/
|
|
69
|
+
private capture;
|
|
70
|
+
/** The page as one list in page order: `[frame-node] role "name" = value states`, ## headings and visible text. */
|
|
71
|
+
snapshot(options?: {
|
|
72
|
+
includeIframes?: boolean;
|
|
73
|
+
max?: number;
|
|
74
|
+
}): Promise<string>;
|
|
75
|
+
/** Drops the snapshot's ids: an action by any of them then fails as a stale snapshot. */
|
|
76
|
+
forget(): void;
|
|
77
|
+
/** Whether there is a snapshot to act on. */
|
|
78
|
+
get hydrated(): boolean;
|
|
79
|
+
/** The backend node id a displayed id names in the latest snapshot; throws UnknownIdError for any other. */
|
|
80
|
+
nodeOf(id: string): number;
|
|
81
|
+
private onNode;
|
|
82
|
+
private describe;
|
|
83
|
+
/** Scroll into view and return a clickable, unobstructed center point, or throw a precise reason. */
|
|
84
|
+
point(page: Page, id: number, editableCover?: boolean): Promise<{
|
|
85
|
+
x: number;
|
|
86
|
+
y: number;
|
|
87
|
+
covered: boolean;
|
|
88
|
+
}>;
|
|
89
|
+
/** Runs one action on a displayed id and returns one line: navigated to <url> / page changed / no change. */
|
|
90
|
+
private act;
|
|
91
|
+
change(before: Probe | undefined, after?: Probe): Promise<string>;
|
|
92
|
+
click(display: string): Promise<string>;
|
|
93
|
+
hover(display: string): Promise<string>;
|
|
94
|
+
/** Picks a native select's option by its label or value, or each of several in a multiple select. */
|
|
95
|
+
select(display: string, values: string | string[]): Promise<string>;
|
|
96
|
+
/** Replaces a field's text (Pi's type): a native date or time input by its value, any other by click, select-all and keys. */
|
|
97
|
+
fill(display: string, text: string): Promise<string>;
|
|
98
|
+
/** Types at the end of a field's whole text, as keys would; `delayMs` between characters, as paced typing asks. */
|
|
99
|
+
type(display: string, text: string, delayMs?: number): Promise<string>;
|
|
100
|
+
/** Clicks the element, then presses a key on it: a name (Enter, Tab, Escape, ArrowDown, ...) or one character, each
|
|
101
|
+
* with any of the modifiers Shift, Control (Ctrl), Alt and Meta (Command, Cmd) before it: Shift+Tab, Control+A. */
|
|
102
|
+
press(display: string, key: string): Promise<string>;
|
|
103
|
+
private enter;
|
|
104
|
+
private key;
|
|
105
|
+
}
|
|
106
|
+
export {};
|