@parcha/pi-browser 0.1.0-beta.13 → 0.1.0-beta.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/README.md +29 -8
- package/dist/coding-agent.d.ts +7 -0
- package/dist/coding-agent.js +33 -16
- package/dist/core/contract.js +1 -1
- package/dist/core/custody.d.ts +16 -3
- package/dist/core/custody.js +28 -11
- package/dist/core/downloads.d.ts +1 -0
- package/dist/core/downloads.js +2 -2
- package/dist/core/failures.d.ts +8 -0
- package/dist/core/failures.js +18 -9
- package/dist/core/guard.d.ts +1 -1
- package/dist/core/guard.js +2 -1
- package/dist/core/host.d.ts +60 -2
- package/dist/core/sites.js +2 -1
- package/dist/core/tools.d.ts +37 -2
- package/dist/core/tools.js +84 -54
- package/dist/core/usage.d.ts +1 -20
- package/dist/driver/extension-worker.d.ts +7 -0
- package/dist/driver/extension-worker.js +148 -0
- package/dist/driver/new-document.d.ts +26 -0
- package/dist/driver/new-document.js +40 -0
- package/dist/driver/stagehand.js +9 -61
- package/dist/durable.d.ts +14 -3
- package/dist/durable.js +507 -369
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/providers/browserbase.d.ts +17 -0
- package/dist/providers/browserbase.js +49 -8
- package/dist/providers/cdp.d.ts +14 -0
- package/dist/providers/cdp.js +270 -66
- package/dist/providers/kernel.js +40 -6
- package/dist/providers/profiles.d.ts +19 -0
- package/dist/providers/profiles.js +124 -0
- package/dist/providers/shared.d.ts +1 -1
- package/dist/recorder/index.d.ts +1 -0
- package/dist/recorder/index.js +1 -0
- package/dist/recorder/jpeg.d.ts +3 -0
- package/dist/recorder/jpeg.js +36 -0
- package/dist/recorder/track.d.ts +107 -0
- package/dist/recorder/track.js +280 -0
- package/dist/testing/backend.js +2 -2
- package/package.json +3 -3
package/dist/core/host.d.ts
CHANGED
|
@@ -1,5 +1,23 @@
|
|
|
1
1
|
import type { LeaseRef, LeaseState, ResourceStatus } from "./lease.js";
|
|
2
|
-
|
|
2
|
+
export type PriceState = "priced" | "unpriced" | "invalid";
|
|
3
|
+
export type PiUsage = {
|
|
4
|
+
input: 0;
|
|
5
|
+
output: 0;
|
|
6
|
+
cacheRead: 0;
|
|
7
|
+
cacheWrite: 0;
|
|
8
|
+
totalTokens: 0;
|
|
9
|
+
cost: {
|
|
10
|
+
input: 0;
|
|
11
|
+
output: 0;
|
|
12
|
+
cacheRead: 0;
|
|
13
|
+
cacheWrite: 0;
|
|
14
|
+
total: number;
|
|
15
|
+
};
|
|
16
|
+
};
|
|
17
|
+
export type ToolUsage = {
|
|
18
|
+
usage: PiUsage;
|
|
19
|
+
state: PriceState;
|
|
20
|
+
};
|
|
3
21
|
export type Geo = {
|
|
4
22
|
country: string;
|
|
5
23
|
state?: string;
|
|
@@ -11,6 +29,11 @@ export type BrowserConfig = {
|
|
|
11
29
|
run: string;
|
|
12
30
|
policy: BrowserPolicy;
|
|
13
31
|
};
|
|
32
|
+
/** A named saved sign-in; `save` keeps the session's changes. One saver per profile within a process; across processes the last to end wins. */
|
|
33
|
+
export type SavedProfile = {
|
|
34
|
+
name: string;
|
|
35
|
+
save: boolean;
|
|
36
|
+
};
|
|
14
37
|
export type BrowserPolicy = {
|
|
15
38
|
proxies: boolean;
|
|
16
39
|
verified: boolean;
|
|
@@ -18,6 +41,7 @@ export type BrowserPolicy = {
|
|
|
18
41
|
geolocation: Geo | null;
|
|
19
42
|
region: string | null;
|
|
20
43
|
contextId: string | null;
|
|
44
|
+
profile?: SavedProfile;
|
|
21
45
|
sessionTimeoutS: number;
|
|
22
46
|
idleReleaseS: number;
|
|
23
47
|
batchTimeoutMs: number;
|
|
@@ -43,6 +67,7 @@ export type SessionSpec = {
|
|
|
43
67
|
height: number;
|
|
44
68
|
};
|
|
45
69
|
contextId?: string;
|
|
70
|
+
profile?: SavedProfile;
|
|
46
71
|
navigation?: {
|
|
47
72
|
allow?: string[];
|
|
48
73
|
block?: string[];
|
|
@@ -64,7 +89,17 @@ export type ProviderCaps = {
|
|
|
64
89
|
maxLifetimeS: number;
|
|
65
90
|
survivesDisconnect: boolean;
|
|
66
91
|
releaseIsAsync: boolean;
|
|
92
|
+
profiles: boolean;
|
|
67
93
|
extension: "uploaded-per-launch" | "preinstalled" | "loaded-locally";
|
|
94
|
+
/** The session settings this provider applies; a setting left out is applied. */
|
|
95
|
+
settings?: Partial<SessionSettingsSupport>;
|
|
96
|
+
};
|
|
97
|
+
export type SessionSettingsSupport = {
|
|
98
|
+
proxies: boolean;
|
|
99
|
+
verified: boolean;
|
|
100
|
+
geolocation: boolean;
|
|
101
|
+
region: boolean;
|
|
102
|
+
context: boolean;
|
|
68
103
|
};
|
|
69
104
|
export type LiveView = {
|
|
70
105
|
fullscreen: string | null;
|
|
@@ -185,6 +220,7 @@ export type SessionRecord = {
|
|
|
185
220
|
geolocation: Geo | null;
|
|
186
221
|
region: string | null;
|
|
187
222
|
contextId: string | null;
|
|
223
|
+
profile: string | null;
|
|
188
224
|
};
|
|
189
225
|
/** Every page the session moved to, in order; the last 500 kept. */
|
|
190
226
|
navigation: Array<{
|
|
@@ -216,7 +252,10 @@ export type SentEffect = {
|
|
|
216
252
|
held: "pending" | "allowed" | "denied" | null;
|
|
217
253
|
irreversible: number | null;
|
|
218
254
|
};
|
|
219
|
-
/** `observed`: a holding observer committed each request before it left the
|
|
255
|
+
/** A replay-unsafe page action's journal entry. `observed`: a holding observer committed each request before it left the
|
|
256
|
+
* browser; `watched`: an observer recorded the requests it sent, holding them or not. `incarnation`: the custody process that wrote it; an entry from an earlier one belongs to a resumed run.
|
|
257
|
+
* `after`: the conversation's newest entry when it was journaled, where a resumed run's look for its answer starts.
|
|
258
|
+
* `finished`: the action returned. `told` and `refused`: how far a cut has been answered. */
|
|
220
259
|
export type PendingEffect = {
|
|
221
260
|
callId: string;
|
|
222
261
|
codeSha: string;
|
|
@@ -224,6 +263,25 @@ export type PendingEffect = {
|
|
|
224
263
|
at: string;
|
|
225
264
|
observed?: boolean;
|
|
226
265
|
sent?: SentEffect[];
|
|
266
|
+
watched?: boolean;
|
|
267
|
+
incarnation?: string;
|
|
268
|
+
after?: number;
|
|
269
|
+
finished?: boolean;
|
|
270
|
+
told?: boolean;
|
|
271
|
+
refused?: boolean;
|
|
272
|
+
};
|
|
273
|
+
/** A cut action's facts, beside the notice in the call's `details.pendingEffect`. `sent` is null when no observer ran, so
|
|
274
|
+
* an unknown never reads as none. */
|
|
275
|
+
export type PendingEffectFacts = {
|
|
276
|
+
callId: string;
|
|
277
|
+
urlAtDispatch: string | null;
|
|
278
|
+
urlNow: string;
|
|
279
|
+
observed: boolean;
|
|
280
|
+
sent: Array<{
|
|
281
|
+
method: string;
|
|
282
|
+
url: string;
|
|
283
|
+
held: SentEffect["held"];
|
|
284
|
+
}> | null;
|
|
227
285
|
};
|
|
228
286
|
export type SessionsState = {
|
|
229
287
|
/** Index into `sessions` of each plane's current session. */
|
package/dist/core/sites.js
CHANGED
|
@@ -12,6 +12,7 @@ export function useWebsiteContract(sites) {
|
|
|
12
12
|
"Hand a task to an agent that works on one of these websites in its own browser, and get back what it found or did:",
|
|
13
13
|
...sites.map((site) => `- ${site.site}: ${site.title}. ${site.description}`),
|
|
14
14
|
"Give a new request_id (a UUID) for each new task. Calling again with the request_id of a finished task answers its result without running it again; a task that failed runs again, and may repeat what it already did on the site.",
|
|
15
|
+
"Several calls in one turn run at once, each agent in its own browser.",
|
|
15
16
|
].join("\n"),
|
|
16
17
|
parameters: {
|
|
17
18
|
type: "object", additionalProperties: false, required: ["site", "request_id", "instructions"],
|
|
@@ -21,7 +22,7 @@ export function useWebsiteContract(sites) {
|
|
|
21
22
|
instructions: { type: "string", maxLength: MAX_SITE_INSTRUCTIONS, description: "What to do on the website and what to report back." },
|
|
22
23
|
},
|
|
23
24
|
},
|
|
24
|
-
effect: "effect", replay: "safe", executionMode: "
|
|
25
|
+
effect: "effect", replay: "safe", executionMode: "parallel", evidence: "none", listedWhen: "sites",
|
|
25
26
|
};
|
|
26
27
|
}
|
|
27
28
|
/** Only the declared shape crosses back to the model: a port's extra fields, such as capabilities or URLs, are dropped
|
package/dist/core/tools.d.ts
CHANGED
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
import { type BrowserDriver, type CustodyPort, type ToolOutput } from "./custody.js";
|
|
2
2
|
import { type EvidenceSink } from "./evidence.js";
|
|
3
3
|
import type { Decisions } from "./decisions.js";
|
|
4
|
+
import { type RepeatGuard } from "./guard.js";
|
|
4
5
|
import { type ScreenshotOptions } from "./images.js";
|
|
5
6
|
import { type Redact } from "./redact.js";
|
|
7
|
+
import type { Geo } from "./host.js";
|
|
6
8
|
export type PageDriver = BrowserDriver & {
|
|
7
9
|
snapshot(options?: {
|
|
8
10
|
includeIframes?: boolean;
|
|
@@ -33,5 +35,38 @@ export type ToolCall = {
|
|
|
33
35
|
evidence: EvidenceSink;
|
|
34
36
|
classifyPage?: Decisions["classifyPage"];
|
|
35
37
|
};
|
|
36
|
-
export type ToolImpl<D extends BrowserDriver = BrowserDriver> = (args:
|
|
37
|
-
|
|
38
|
+
export type ToolImpl<D extends BrowserDriver = BrowserDriver, A = unknown> = (args: A, port: CustodyPort<D>, call: ToolCall) => Promise<ToolOutput>;
|
|
39
|
+
/** Each browser tool's arguments, as its contract's schema admits them. */
|
|
40
|
+
export type BrowserToolArgs = {
|
|
41
|
+
snapshot: {
|
|
42
|
+
includeIframes?: boolean;
|
|
43
|
+
};
|
|
44
|
+
run: {
|
|
45
|
+
code?: string;
|
|
46
|
+
actions?: unknown[];
|
|
47
|
+
};
|
|
48
|
+
screenshot: {
|
|
49
|
+
fullPage?: boolean;
|
|
50
|
+
type?: "png" | "jpeg";
|
|
51
|
+
quality?: number;
|
|
52
|
+
ask?: string;
|
|
53
|
+
};
|
|
54
|
+
browser_read: {
|
|
55
|
+
what?: (typeof WHATS)[number];
|
|
56
|
+
};
|
|
57
|
+
browser_release: Record<string, never>;
|
|
58
|
+
browser_relaunch: {
|
|
59
|
+
verified?: boolean;
|
|
60
|
+
geolocation?: Geo;
|
|
61
|
+
};
|
|
62
|
+
browser_downloads: Record<string, never>;
|
|
63
|
+
};
|
|
64
|
+
export type BrowserToolImpls<D extends BrowserDriver = BrowserDriver> = {
|
|
65
|
+
[K in keyof BrowserToolArgs]: ToolImpl<D, BrowserToolArgs[K]>;
|
|
66
|
+
};
|
|
67
|
+
declare const WHATS: readonly ["markdown", "text", "html", "url", "title"];
|
|
68
|
+
/** `guards` holds each conversation's repeat guard; a host that owns the conversation's lifecycle passes it to evict one. */
|
|
69
|
+
export declare function pageTools({ guards }?: {
|
|
70
|
+
guards?: Map<number, RepeatGuard>;
|
|
71
|
+
}): Pick<BrowserToolImpls<PageDriver>, "snapshot" | "run" | "screenshot" | "browser_read">;
|
|
72
|
+
export {};
|
package/dist/core/tools.js
CHANGED
|
@@ -1,26 +1,41 @@
|
|
|
1
1
|
import { BrowserFailureError } from "./custody.js";
|
|
2
2
|
import { defaultInline } from "./evidence.js";
|
|
3
|
-
import { classifyBrowserError } from "./failures.js";
|
|
3
|
+
import { browserFailure, classifyBrowserError } from "./failures.js";
|
|
4
4
|
import { createRepeatGuard } from "./guard.js";
|
|
5
5
|
import { captureWithinBudget } from "./images.js";
|
|
6
6
|
import { orderFacts, receiptFacts, sha256Hex } from "./receipts.js";
|
|
7
7
|
import { scrubPageUrl } from "./redact.js";
|
|
8
8
|
import { pageNotice, wallFailure, wallOf } from "./web.js";
|
|
9
9
|
const WHATS = ["markdown", "text", "html", "url", "title"];
|
|
10
|
+
const DROPPED_RUN = "The connection to your browser session dropped; the session is still open, and your next browser call reconnects to it with its pages and cookies. This run may have acted before the drop: take a snapshot to see the page before running again.";
|
|
11
|
+
const DROPPED_FOR_GOOD = "The connection to your browser session dropped and could not be re-established, though the session is still open. Call browser_relaunch for a fresh session (its pages and cookies are gone), or use web_fetch.";
|
|
10
12
|
const text = (value, isError = false) => ({ content: [{ type: "text", text: typeof value === "string" ? value : JSON.stringify(value) }], ...(isError ? { isError: true } : {}) });
|
|
11
13
|
const withNews = (news, body) => (news ? `${news}\n\n${body}` : body);
|
|
12
14
|
const pageUrl = (call, raw) => scrubPageUrl(raw, call.redact);
|
|
13
15
|
/** A cut run that was observed and sent nothing that left the browser is safe to run again. */
|
|
14
16
|
const harmless = (p) => p.observed === true && !(p.sent ?? []).some((r) => r.held !== "denied");
|
|
15
17
|
function interruptedNotice(p, now) {
|
|
16
|
-
const
|
|
18
|
+
const where = `Your run call ${p.callId}${p.url ? ` on ${p.url}` : ""}`;
|
|
19
|
+
const sent = (p.sent ?? []).filter((r) => r.held !== "denied").map((r) => `${r.method} ${r.origin}${r.path}`);
|
|
20
|
+
if (p.finished) {
|
|
21
|
+
const head = `${where} ran, but the process stopped before its answer reached you`;
|
|
22
|
+
if (harmless(p))
|
|
23
|
+
return `${head}; it sent no request that changes anything, so it is safe to run again. The page is now ${now}.`;
|
|
24
|
+
if (sent.length)
|
|
25
|
+
return `${head}, after it sent ${sent.join(", ")}; the page is now ${now}. Check on the page whether that landed before acting again, and do not send it again if it did.`;
|
|
26
|
+
return `${head}, so what it did has taken effect; the page is now ${now}. Look at it with snapshot before acting again.`;
|
|
27
|
+
}
|
|
28
|
+
const head = `${where} was interrupted`;
|
|
17
29
|
if (harmless(p))
|
|
18
30
|
return `${head} before it sent any request that changes anything; it is safe to run again. The page is now ${now}.`;
|
|
19
|
-
const sent = (p.sent ?? []).filter((r) => r.held !== "denied").map((r) => `${r.method} ${r.origin}${r.path}`);
|
|
20
31
|
if (sent.length)
|
|
21
32
|
return `${head} after it sent ${sent.join(", ")}; the page is now ${now}. Check on the page whether that landed before acting again, and do not send it again if it did.`;
|
|
22
33
|
return `${head} and may have taken effect; the page is now ${now}. Check it with snapshot before acting again.`;
|
|
23
34
|
}
|
|
35
|
+
const factsOf = (p, now) => ({
|
|
36
|
+
callId: p.callId, urlAtDispatch: p.url, urlNow: now, observed: p.observed === true,
|
|
37
|
+
sent: p.watched ? (p.sent ?? []).map((r) => ({ method: r.method, url: `${r.origin}${r.path}`, held: r.held })) : null,
|
|
38
|
+
});
|
|
24
39
|
async function markdown(html, fallback) {
|
|
25
40
|
try {
|
|
26
41
|
const [{ parse }, { NodeHtmlMarkdown }] = await Promise.all([import("node-html-parser"), import("node-html-markdown")]);
|
|
@@ -33,8 +48,8 @@ async function markdown(html, fallback) {
|
|
|
33
48
|
catch { }
|
|
34
49
|
return { body: fallback, extractor: "inner-text", ext: "txt" };
|
|
35
50
|
}
|
|
36
|
-
|
|
37
|
-
|
|
51
|
+
/** `guards` holds each conversation's repeat guard; a host that owns the conversation's lifecycle passes it to evict one. */
|
|
52
|
+
export function pageTools({ guards = new Map() } = {}) {
|
|
38
53
|
const guardOf = (conversationId) => guards.get(conversationId) ?? guards.set(conversationId, createRepeatGuard()).get(conversationId);
|
|
39
54
|
async function failureOf(error, effectful, port, call) {
|
|
40
55
|
if (error instanceof BrowserFailureError)
|
|
@@ -45,55 +60,74 @@ export function pageTools() {
|
|
|
45
60
|
return first;
|
|
46
61
|
return classifyBrowserError(error, effectful, { ...facts, sessionEnded: true });
|
|
47
62
|
}
|
|
48
|
-
|
|
63
|
+
/** A replay-unsafe action is journaled before it acts and marked finished after; the entry stays until a later call
|
|
64
|
+
* settles it, so a process that dies before pi commits the answer leaves a cut for the resumed run. */
|
|
65
|
+
async function dispatched(port, call, s, key, action) {
|
|
66
|
+
await port.dispatching({ callId: call.callId, codeSha: key, url: s.record.lastUrl, observed: s.effects?.holding === true, watched: s.effects != null });
|
|
67
|
+
try {
|
|
68
|
+
return await action();
|
|
69
|
+
}
|
|
70
|
+
finally {
|
|
71
|
+
await port.mark("finished", call.callId).catch(() => undefined);
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
async function perform(tool, args, port, call, kind, body) {
|
|
75
|
+
const effect = typeof kind === "object" ? kind.effect : null;
|
|
76
|
+
const effectful = effect !== null;
|
|
49
77
|
const guard = guardOf(call.conversationId);
|
|
50
78
|
const refused = guard.check(tool, args);
|
|
51
79
|
if (refused)
|
|
52
80
|
return text(refused, true);
|
|
53
|
-
//
|
|
81
|
+
// A cut is told once, on the call after it, whatever that call ends in. An identical action is refused until a look
|
|
82
|
+
// that follows the refusal settles the cut.
|
|
54
83
|
let news = "";
|
|
84
|
+
let cut = null;
|
|
85
|
+
let facts = null;
|
|
86
|
+
const tell = (out) => (facts ? { ...out, details: { ...out.details, pendingEffect: facts } } : out);
|
|
87
|
+
const done = (out) => { guard.note(tool, args, null); return tell(out); };
|
|
88
|
+
const fail = (failure) => { guard.note(tool, args, failure); return tell(text(news ? { ...failure, notice: news } : failure, true)); };
|
|
89
|
+
const looked = async (out) => { if (kind === "look" && cut?.refused)
|
|
90
|
+
await port.settle(cut.callId); return done(out); };
|
|
55
91
|
try {
|
|
56
92
|
const s = await port.session();
|
|
57
93
|
news = s.notice ?? "";
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
await
|
|
94
|
+
cut = s.interrupted;
|
|
95
|
+
const same = cut !== null && effect === cut.codeSha && !harmless(cut);
|
|
96
|
+
if (cut && (!cut.told || same))
|
|
97
|
+
facts = factsOf(cut, pageUrl(call, await s.driver.url().catch(() => "gone")));
|
|
98
|
+
if (cut && !cut.told) {
|
|
99
|
+
news = [news, interruptedNotice(cut, facts.urlNow)].filter(Boolean).join(" ");
|
|
100
|
+
await (harmless(cut) ? port.settle(cut.callId) : port.mark("told", cut.callId));
|
|
62
101
|
}
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
102
|
+
if (cut && same) {
|
|
103
|
+
await port.mark("refused", cut.callId);
|
|
104
|
+
return done(text(browserFailure("effect_unknown", `${news ? `${news} ` : ""}This is the same code, and it may already have taken effect, so it is not run again now: look at the page with snapshot or browser_read before you run this again.`, { retryable: true, effect: "possibly_effected" }), true));
|
|
105
|
+
}
|
|
106
|
+
return await looked(effect === null ? await body(s, news) : await dispatched(port, call, s, effect, () => body(s, news)));
|
|
66
107
|
}
|
|
67
108
|
catch (error) {
|
|
68
|
-
|
|
69
|
-
|
|
109
|
+
const failure = await failureOf(error, effectful, port, call);
|
|
110
|
+
if (failure.code !== "browser_unavailable" || error instanceof BrowserFailureError)
|
|
111
|
+
return fail(failure);
|
|
112
|
+
// A dropped CDP connection to a session the provider still runs: a read reconnects and runs once more; a run, which
|
|
70
113
|
// may have acted, is not repeated.
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
guard.note(tool, args, null);
|
|
88
|
-
return out;
|
|
89
|
-
}
|
|
90
|
-
catch (retried) {
|
|
91
|
-
failure = await failureOf(retried, effectful, port, call);
|
|
92
|
-
}
|
|
93
|
-
}
|
|
114
|
+
await port.dropped().catch(() => undefined);
|
|
115
|
+
if (effectful)
|
|
116
|
+
return fail({ ...failure, retryable: true, message: DROPPED_RUN });
|
|
117
|
+
let again;
|
|
118
|
+
try {
|
|
119
|
+
again = await port.session();
|
|
120
|
+
}
|
|
121
|
+
catch (reconnect) {
|
|
122
|
+
const refused = await failureOf(reconnect, effectful, port, call);
|
|
123
|
+
return fail(refused.code === "aborted" || refused.code === "auth" ? refused : { ...refused, code: "browser_unavailable", retryable: false, message: DROPPED_FOR_GOOD });
|
|
124
|
+
}
|
|
125
|
+
try {
|
|
126
|
+
return await looked(await body(again, news));
|
|
127
|
+
}
|
|
128
|
+
catch (retried) {
|
|
129
|
+
return fail(await failureOf(retried, effectful, port, call));
|
|
94
130
|
}
|
|
95
|
-
guard.note(tool, args, failure);
|
|
96
|
-
return text(news ? { ...failure, notice: news } : failure, true);
|
|
97
131
|
}
|
|
98
132
|
}
|
|
99
133
|
async function file(s, call, record) {
|
|
@@ -102,17 +136,13 @@ export function pageTools() {
|
|
|
102
136
|
return { path: filed?.path ?? null, bodyLine: filed?.bodyLine ?? null, sha256: facts.sha256 };
|
|
103
137
|
}
|
|
104
138
|
return {
|
|
105
|
-
snapshot: (args, port, call) => perform("snapshot", args ?? {}, port, call,
|
|
139
|
+
snapshot: (args, port, call) => perform("snapshot", args ?? {}, port, call, "look", async (s, news) => text(withNews(news, await s.driver.snapshot(args ?? {})))),
|
|
106
140
|
run: async (args, port, call) => {
|
|
107
141
|
if ((args?.code === undefined) === (args?.actions === undefined)) {
|
|
108
|
-
return text(
|
|
142
|
+
return text(browserFailure("refused", "run requires exactly one of code or actions"), true);
|
|
109
143
|
}
|
|
110
144
|
const codeSha = sha256Hex(JSON.stringify(args.code ?? args.actions));
|
|
111
|
-
return perform("run", args, port, call,
|
|
112
|
-
if (s.interrupted?.codeSha === codeSha && !harmless(s.interrupted)) {
|
|
113
|
-
return text({ ok: false, code: "effect_unknown", retryable: true, effect: "possibly_effected", message: `${news} This is the same code, so it is not run again now: check whether it landed, then call run again only if it did not.` }, true);
|
|
114
|
-
}
|
|
115
|
-
await port.dispatching({ callId: call.callId, codeSha, url: s.record.lastUrl, observed: s.effects?.holding === true });
|
|
145
|
+
return perform("run", args, port, call, { effect: codeSha }, async (s, news) => {
|
|
116
146
|
const sent = [];
|
|
117
147
|
const stop = s.effects?.record((row) => { sent.push({ method: row.method, url: call.redact(`${row.origin}${row.path}`), held: row.held }); return port.journal(row); });
|
|
118
148
|
let value;
|
|
@@ -122,7 +152,6 @@ export function pageTools() {
|
|
|
122
152
|
finally {
|
|
123
153
|
await s.effects?.flush(call.signal).catch(() => undefined);
|
|
124
154
|
stop?.();
|
|
125
|
-
await port.settle();
|
|
126
155
|
}
|
|
127
156
|
const url = await s.driver.url().catch(() => null);
|
|
128
157
|
// Compared as custody keeps it, so a page with a query value never looks new on every run.
|
|
@@ -134,20 +163,21 @@ export function pageTools() {
|
|
|
134
163
|
});
|
|
135
164
|
},
|
|
136
165
|
screenshot: (args, port, call) => {
|
|
137
|
-
const { ask:
|
|
138
|
-
|
|
166
|
+
const { ask: asked, ...requested } = args ?? {};
|
|
167
|
+
const ask = typeof asked === "string" && asked.trim() ? asked.trim() : undefined;
|
|
168
|
+
return perform("screenshot", args ?? {}, port, call, "read", async (s, news) => {
|
|
139
169
|
const shot = await captureWithinBudget((options) => s.driver.screenshot(options), requested);
|
|
140
170
|
const url = pageUrl(call, await s.driver.url());
|
|
141
171
|
const receipt = await file(s, call, { tool: "screenshot", args: requested, status: "ok", facts: { final_url: url }, body: Buffer.from(shot.data, "base64"), ext: shot.size.type === "png" ? "png" : "jpg" });
|
|
142
172
|
const first = shot.adjusted?.first;
|
|
143
173
|
const adjusted = shot.adjusted && `adjusted: the first capture${first ? ` (${first.width}x${first.height})` : ""} was over the ${shot.adjusted.why === "long_edge" ? "2,000 px long edge" : "size"} budget; this is the viewport at ${shot.size.width}x${shot.size.height}`;
|
|
144
174
|
const lines = [news, `url: ${url}`, `evidence: ${receipt.path ?? "(not filed)"}`, adjusted].filter(Boolean).join("\n");
|
|
145
|
-
return { content: [{ type: "text", text: lines }, { type: "image", data: shot.data, mimeType: shot.mimeType }], details: { evidence: receipt.path, sha256: receipt.sha256 } };
|
|
175
|
+
return { content: [{ type: "text", text: lines }, { type: "image", data: shot.data, mimeType: shot.mimeType }], details: { evidence: receipt.path, sha256: receipt.sha256, ...(ask ? { media: { ask } } : {}) } };
|
|
146
176
|
});
|
|
147
177
|
},
|
|
148
178
|
browser_read: (args, port, call) => {
|
|
149
|
-
const what = WHATS.includes(args
|
|
150
|
-
return perform("browser_read", { what }, port, call,
|
|
179
|
+
const what = args?.what && WHATS.includes(args.what) ? args.what : "markdown";
|
|
180
|
+
return perform("browser_read", { what }, port, call, "look", async (s, news) => {
|
|
151
181
|
const page = await s.driver.page();
|
|
152
182
|
const read = what === "markdown" ? await markdown(page.html, page.text)
|
|
153
183
|
: what === "html" ? { body: page.html, extractor: "raw", ext: "html" }
|
package/dist/core/usage.d.ts
CHANGED
|
@@ -1,23 +1,4 @@
|
|
|
1
|
-
import { type PriceTable, type SessionRecord } from "./host.js";
|
|
2
|
-
export type PriceState = "priced" | "unpriced" | "invalid";
|
|
3
|
-
export type PiUsage = {
|
|
4
|
-
input: 0;
|
|
5
|
-
output: 0;
|
|
6
|
-
cacheRead: 0;
|
|
7
|
-
cacheWrite: 0;
|
|
8
|
-
totalTokens: 0;
|
|
9
|
-
cost: {
|
|
10
|
-
input: 0;
|
|
11
|
-
output: 0;
|
|
12
|
-
cacheRead: 0;
|
|
13
|
-
cacheWrite: 0;
|
|
14
|
-
total: number;
|
|
15
|
-
};
|
|
16
|
-
};
|
|
17
|
-
export type ToolUsage = {
|
|
18
|
-
usage: PiUsage;
|
|
19
|
-
state: PriceState;
|
|
20
|
-
};
|
|
1
|
+
import { type PriceState, type PriceTable, type SessionRecord, type ToolUsage } from "./host.js";
|
|
21
2
|
/** Dollars per minute: unset or blank is unpriced; anything but a non-negative decimal is invalid. Zero is a price. */
|
|
22
3
|
export declare function sessionPrice(raw: string | null | undefined): {
|
|
23
4
|
prices: Pick<PriceTable, "sessionUsdPerHour">;
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
export declare const STAGEHAND_EXTENSION = "Stagehand Runtime";
|
|
2
|
+
/** A held extension runtime keeps one Stagehand instance until that instance's close, which a dead process never sent,
|
|
3
|
+
* so a new attach would be refused. The close is sent here through the worker's own RPC, and the worker keeps running:
|
|
4
|
+
* a worker closed under load is sometimes never restarted, and its attach waits out Stagehand's 60 s init deadline.
|
|
5
|
+
* With no worker running nothing is held, and the worker is woken until it is listed. A target list the browser does
|
|
6
|
+
* not give by DISCOVERY_MS is not a list without a worker: the attach fails. */
|
|
7
|
+
export declare function installedExtension(cdpUrl: string, signal?: AbortSignal, known?: string): Promise<string | null>;
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
export const STAGEHAND_EXTENSION = "Stagehand Runtime";
|
|
2
|
+
const DISCOVERY_MS = 10_000;
|
|
3
|
+
/** Kept past the deadline so the last wake page is closed on the same socket. */
|
|
4
|
+
const CLEANUP_MS = 1_000;
|
|
5
|
+
const WAKE_TRIES = 60;
|
|
6
|
+
const CLOSE_REPLY_MS = 3_000;
|
|
7
|
+
const SEND_TO_HOST = "__stagehandSendToHost";
|
|
8
|
+
const RESET_ID = 2_147_000_000;
|
|
9
|
+
/** The browser did not list its targets by the deadline, so a runtime it may hold could not be freed. */
|
|
10
|
+
class TargetsUnlistedError extends Error {
|
|
11
|
+
constructor() { super(`the browser did not list its targets within ${DISCOVERY_MS} ms, so a Stagehand runtime it may hold could not be freed`); }
|
|
12
|
+
}
|
|
13
|
+
/** A held extension runtime keeps one Stagehand instance until that instance's close, which a dead process never sent,
|
|
14
|
+
* so a new attach would be refused. The close is sent here through the worker's own RPC, and the worker keeps running:
|
|
15
|
+
* a worker closed under load is sometimes never restarted, and its attach waits out Stagehand's 60 s init deadline.
|
|
16
|
+
* With no worker running nothing is held, and the worker is woken until it is listed. A target list the browser does
|
|
17
|
+
* not give by DISCOVERY_MS is not a list without a worker: the attach fails. */
|
|
18
|
+
export async function installedExtension(cdpUrl, signal, known) {
|
|
19
|
+
const socket = new WebSocket(cdpUrl);
|
|
20
|
+
const deadline = Date.now() + DISCOVERY_MS;
|
|
21
|
+
const timer = setTimeout(() => socket.close(), DISCOVERY_MS + CLEANUP_MS);
|
|
22
|
+
const stop = () => socket.close();
|
|
23
|
+
signal?.addEventListener("abort", stop, { once: true });
|
|
24
|
+
// A call the browser does not answer within `ms` (the deadline, unless shorter) settles empty, as does every call once
|
|
25
|
+
// the socket closes (an abort, the browser).
|
|
26
|
+
const pending = new Map();
|
|
27
|
+
const listeners = new Set();
|
|
28
|
+
let next = 0;
|
|
29
|
+
let closed = false;
|
|
30
|
+
// A send on a closed socket neither throws nor is ever answered (Node 24): once closed, every call settles empty.
|
|
31
|
+
const call = (method, params = {}, sessionId, ms = deadline - Date.now()) => new Promise((resolve) => {
|
|
32
|
+
if (closed || socket.readyState !== WebSocket.OPEN || ms <= 0)
|
|
33
|
+
return resolve(undefined);
|
|
34
|
+
const id = ++next;
|
|
35
|
+
const settle = (result) => { clearTimeout(wait); pending.delete(id); resolve(result); };
|
|
36
|
+
const wait = setTimeout(() => settle(undefined), ms);
|
|
37
|
+
pending.set(id, settle);
|
|
38
|
+
socket.send(JSON.stringify({ id, method, params, ...(sessionId ? { sessionId } : {}) }));
|
|
39
|
+
});
|
|
40
|
+
try {
|
|
41
|
+
await new Promise((resolve, reject) => {
|
|
42
|
+
socket.onopen = () => resolve();
|
|
43
|
+
// A refused connection raises `error` and, on Node 22, never `close`: waiting for `close` alone would hang the
|
|
44
|
+
// attach.
|
|
45
|
+
socket.onerror = () => reject(new Error("Failed to open CDP WebSocket"));
|
|
46
|
+
socket.onclose = () => {
|
|
47
|
+
closed = true;
|
|
48
|
+
for (const settle of [...pending.values()])
|
|
49
|
+
settle(undefined);
|
|
50
|
+
for (const listener of listeners)
|
|
51
|
+
listener({ method: "closed" });
|
|
52
|
+
reject(new Error("the CDP socket closed"));
|
|
53
|
+
};
|
|
54
|
+
socket.onmessage = (event) => {
|
|
55
|
+
const message = JSON.parse(String(event.data));
|
|
56
|
+
if (message.id !== undefined)
|
|
57
|
+
pending.get(message.id)?.(message.result);
|
|
58
|
+
else
|
|
59
|
+
for (const listener of listeners)
|
|
60
|
+
listener(message);
|
|
61
|
+
};
|
|
62
|
+
});
|
|
63
|
+
let id = known ?? null;
|
|
64
|
+
if (!id) {
|
|
65
|
+
const extensions = (await call("Extensions.getExtensions"))?.extensions ?? [];
|
|
66
|
+
const enabled = extensions.filter((extension) => extension.name === STAGEHAND_EXTENSION && extension.enabled);
|
|
67
|
+
if (enabled.length !== 1)
|
|
68
|
+
return null;
|
|
69
|
+
id = enabled[0].id;
|
|
70
|
+
}
|
|
71
|
+
/** The extension's workers; null when the browser gave no list, which is not a list without one. */
|
|
72
|
+
const workers = async () => {
|
|
73
|
+
const infos = (await call("Target.getTargets"))?.targetInfos;
|
|
74
|
+
return infos?.filter((target) => target.type === "service_worker" && target.url.startsWith(`chrome-extension://${id}/`)).map((target) => target.targetId) ?? null;
|
|
75
|
+
};
|
|
76
|
+
const running = await workers();
|
|
77
|
+
if (running === null) {
|
|
78
|
+
if (signal?.aborted || closed)
|
|
79
|
+
return id;
|
|
80
|
+
throw new TargetsUnlistedError();
|
|
81
|
+
}
|
|
82
|
+
if (running.length === 0) {
|
|
83
|
+
// Every wake page opened stays listed here until a close of it is answered. None is opened within CLEANUP_MS of the
|
|
84
|
+
// deadline, an opening's reply is awaited into the first half of that second, and the socket outlives the deadline
|
|
85
|
+
// by CLEANUP_MS, so the last pages are closed on it.
|
|
86
|
+
const wakes = new Set();
|
|
87
|
+
const closeWakes = async (ms) => { for (const page of [...wakes])
|
|
88
|
+
if ((await call("Target.closeTarget", { targetId: page }, undefined, ms)) !== undefined)
|
|
89
|
+
wakes.delete(page); };
|
|
90
|
+
for (let tries = 0; !closed && Date.now() < deadline && tries < WAKE_TRIES && ((await workers())?.length ?? 0) === 0; tries += 1) {
|
|
91
|
+
if (tries % 10 === 0 && deadline - Date.now() > CLEANUP_MS) {
|
|
92
|
+
await closeWakes();
|
|
93
|
+
const page = (await call("Target.createTarget", { url: `chrome-extension://${id}/wake-service-worker.html` }, undefined, deadline + CLEANUP_MS / 2 - Date.now()))?.targetId;
|
|
94
|
+
if (page)
|
|
95
|
+
wakes.add(page);
|
|
96
|
+
}
|
|
97
|
+
await new Promise((r) => setTimeout(r, 100));
|
|
98
|
+
}
|
|
99
|
+
await closeWakes(deadline + CLEANUP_MS - Date.now());
|
|
100
|
+
return id;
|
|
101
|
+
}
|
|
102
|
+
const sessionId = (await call("Target.attachToTarget", { targetId: running[0], flatten: true }))?.sessionId;
|
|
103
|
+
if (!sessionId)
|
|
104
|
+
return id;
|
|
105
|
+
try {
|
|
106
|
+
await call("Runtime.enable", {}, sessionId);
|
|
107
|
+
await call("Runtime.addBinding", { name: SEND_TO_HOST }, sessionId);
|
|
108
|
+
const replied = new Promise((resolve) => {
|
|
109
|
+
const wait = setTimeout(resolve, CLOSE_REPLY_MS);
|
|
110
|
+
listeners.add((event) => {
|
|
111
|
+
if (event.method === "closed") {
|
|
112
|
+
clearTimeout(wait);
|
|
113
|
+
return resolve();
|
|
114
|
+
}
|
|
115
|
+
if (event.method !== "Runtime.bindingCalled" || event.sessionId !== sessionId || event.params?.name !== SEND_TO_HOST)
|
|
116
|
+
return;
|
|
117
|
+
let reply = null;
|
|
118
|
+
try {
|
|
119
|
+
reply = JSON.parse(event.params.payload);
|
|
120
|
+
}
|
|
121
|
+
catch { }
|
|
122
|
+
if (reply?.id === RESET_ID) {
|
|
123
|
+
clearTimeout(wait);
|
|
124
|
+
resolve();
|
|
125
|
+
}
|
|
126
|
+
});
|
|
127
|
+
});
|
|
128
|
+
const close = JSON.stringify({ jsonrpc: "2.0", id: RESET_ID, method: "stagehand.close", params: {} });
|
|
129
|
+
const sent = await call("Runtime.evaluate", { expression: `void globalThis.__stagehandReceiveFromHost(${JSON.stringify(close)}); true`, returnByValue: true }, sessionId);
|
|
130
|
+
if (sent && !sent.exceptionDetails)
|
|
131
|
+
await replied;
|
|
132
|
+
}
|
|
133
|
+
finally {
|
|
134
|
+
await call("Target.detachFromTarget", { sessionId });
|
|
135
|
+
}
|
|
136
|
+
return id;
|
|
137
|
+
}
|
|
138
|
+
catch (error) {
|
|
139
|
+
if (error instanceof TargetsUnlistedError)
|
|
140
|
+
throw error;
|
|
141
|
+
return known ?? null;
|
|
142
|
+
}
|
|
143
|
+
finally {
|
|
144
|
+
clearTimeout(timer);
|
|
145
|
+
signal?.removeEventListener("abort", stop);
|
|
146
|
+
socket.close();
|
|
147
|
+
}
|
|
148
|
+
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
/** Chrome's answer to an evaluate sent with a document's context id after a navigation replaced it. Stagehand's extension
|
|
2
|
+
* relays it as a JSON-RPC error with this text, and the facade's code runs rebuild it as a plain Error with the same
|
|
3
|
+
* text and no cause. Matched by identity. */
|
|
4
|
+
export declare const CONTEXT_GONE = "-32000 Cannot find context with specified id";
|
|
5
|
+
/** A read that lands in a document a navigation has just replaced waits for the new one, then reads once more. The
|
|
6
|
+
* wait's own failure is not the read's: the second read's answer, or its error, is what comes back. */
|
|
7
|
+
export declare function readOnNewDocument<T>(read: () => Promise<T>, waitForDocument: () => Promise<unknown>): Promise<T>;
|
|
8
|
+
type Facade = {
|
|
9
|
+
snapshot(options?: {
|
|
10
|
+
includeIframes?: boolean;
|
|
11
|
+
}): Promise<string>;
|
|
12
|
+
run(code: string): Promise<unknown>;
|
|
13
|
+
};
|
|
14
|
+
/** The driver's snapshot and page reads, each read once more on the new document when a navigation replaced its own. */
|
|
15
|
+
export declare function documentReads(facade: Facade, waitForDocument: () => Promise<unknown>): {
|
|
16
|
+
snapshot: (options?: {
|
|
17
|
+
includeIframes?: boolean;
|
|
18
|
+
}) => Promise<string>;
|
|
19
|
+
page: () => Promise<{
|
|
20
|
+
url: string;
|
|
21
|
+
title: string;
|
|
22
|
+
text: string;
|
|
23
|
+
html: string;
|
|
24
|
+
}>;
|
|
25
|
+
};
|
|
26
|
+
export {};
|