@parcha/pi-browser 0.1.0-beta.33 → 0.1.0-beta.34

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. package/CHANGELOG.md +17 -0
  2. package/NOTICE +30 -0
  3. package/README.md +28 -178
  4. package/dist/core/contract.js +14 -5
  5. package/dist/core/custody.d.ts +6 -0
  6. package/dist/core/custody.js +6 -0
  7. package/dist/core/driving-guard.js +17 -2
  8. package/dist/core/host.d.ts +2 -2
  9. package/dist/core/tools.d.ts +9 -0
  10. package/dist/driver/cdp.d.ts +6 -0
  11. package/dist/driver/cdp.js +245 -0
  12. package/dist/driver/page-driver.js +29 -3
  13. package/dist/durable/store.js +13 -4
  14. package/dist/durable.d.ts +1 -0
  15. package/dist/durable.js +1 -0
  16. package/dist/live-view/provider.d.ts +2 -2
  17. package/dist/live-view/provider.js +2 -2
  18. package/dist/live-view/server.d.ts +1 -1
  19. package/dist/providers/chrome-args.d.ts +3 -0
  20. package/dist/providers/chrome-args.js +13 -2
  21. package/dist/vendor/browser-use-pi/ax.d.ts +106 -0
  22. package/dist/vendor/browser-use-pi/ax.js +642 -0
  23. package/dist/vendor/browser-use-pi/cdp.d.ts +19 -0
  24. package/dist/vendor/browser-use-pi/cdp.js +106 -0
  25. package/dist/vendor/browser-use-pi/page.d.ts +38 -0
  26. package/dist/vendor/browser-use-pi/page.js +112 -0
  27. package/package.json +6 -24
  28. package/src/vendor/browser-use-pi/UPSTREAM.json +25 -0
  29. package/src/vendor/stagehand-facade/UPSTREAM.json +0 -4
  30. package/dist/coding-agent-local.d.ts +0 -12
  31. package/dist/coding-agent-local.js +0 -72
  32. package/dist/coding-agent.d.ts +0 -135
  33. package/dist/coding-agent.js +0 -641
  34. package/dist/providers/assigned.d.ts +0 -18
  35. package/dist/providers/assigned.js +0 -75
  36. package/dist/recorder/credential.d.ts +0 -9
  37. package/dist/recorder/credential.js +0 -49
  38. package/dist/recorder/follow.d.ts +0 -36
  39. package/dist/recorder/follow.js +0 -163
  40. package/dist/recorder/frames.d.ts +0 -87
  41. package/dist/recorder/frames.js +0 -377
  42. package/dist/recorder/index.d.ts +0 -4
  43. package/dist/recorder/index.js +0 -7
  44. package/dist/recorder/jpeg.d.ts +0 -8
  45. package/dist/recorder/jpeg.js +0 -70
  46. package/dist/recorder/track.d.ts +0 -107
  47. package/dist/recorder/track.js +0 -280
  48. package/dist/vendor/stagehand-facade/screenshot-transport.d.ts +0 -15
  49. package/dist/vendor/stagehand-facade/screenshot-transport.js +0 -51
@@ -1,280 +0,0 @@
1
- import { spawn } from "node:child_process";
2
- import { createHash } from "node:crypto";
3
- import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
4
- import os from "node:os";
5
- import path from "node:path";
6
- import { jpegSampling, jpegSize } from "./jpeg.js";
7
- export class TrackExportError extends Error {
8
- code;
9
- constructor(code, message) {
10
- super(message);
11
- this.name = "TrackExportError";
12
- this.code = code;
13
- }
14
- }
15
- const even = (n) => (n % 2 === 0 ? n : n + 1);
16
- /** The timeline of one session's lines. Each frame is shown until the next one; a gap puts the slate up (from the moment it was written, except
17
- * `process_killed`, which starts at the last frame the earlier process saved: nothing is known after it) until the next frame or the end.
18
- * Null when the session has no frame. `skip` names frames whose picture could not be read: they show the slate and are not listed. */
19
- export function planTrack(lines, options = {}) {
20
- const fps = options.fps ?? 10;
21
- const one = Math.max(1, Math.round(1000 / fps));
22
- const rows = [...lines].map((row, order) => ({ row: row, order })).filter(({ row }) => (row.kind === "frame" || row.kind === "frame.gap") && typeof row.t === "number" && row.data && typeof row.data.session === "string");
23
- const byTime = [...rows].sort((a, b) => a.row.t - b.row.t || a.order - b.order);
24
- const first = byTime.find(({ row }) => row.kind === "frame");
25
- if (!first)
26
- return null;
27
- const session = first.row.data.session;
28
- const events = byTime.filter(({ row }) => row.data.session === session);
29
- const from = first.row.t;
30
- const segments = [];
31
- const frames = [];
32
- const gaps = [];
33
- const state = { current: null, gap: null };
34
- const close = (at) => {
35
- const cur = state.current;
36
- if (cur && at > cur.start)
37
- segments.push({ start: cur.start, end: at, sha256: cur.sha256, ...(cur.reason ? { reason: cur.reason } : {}) });
38
- state.current = null;
39
- };
40
- const closeGap = (at) => { if (state.gap)
41
- gaps.push({ ...state.gap, to: at }); state.gap = null; };
42
- let last = from;
43
- for (const { row } of events) {
44
- const t = row.t;
45
- last = Math.max(last, t);
46
- if (t < from)
47
- continue; // a gap written before the first frame has no place in the track
48
- if (row.kind === "frame") {
49
- const sha = row.blobs?.[0] ?? "";
50
- close(t);
51
- closeGap(t);
52
- if (options.skip?.has(sha)) {
53
- state.current = { start: t, sha256: null, reason: "frame_unreadable" };
54
- state.gap = { from: t, reason: "frame_unreadable" };
55
- }
56
- else {
57
- state.current = { start: t, sha256: sha };
58
- frames.push({ t, ms: t - from, sha256: sha, w: row.data.w, h: row.data.h });
59
- }
60
- continue;
61
- }
62
- const data = row.data;
63
- const cur = state.current;
64
- if (!cur)
65
- continue;
66
- const opened = (at) => { state.gap = { from: at, reason: data.reason, ...(data.detail ? { detail: data.detail } : {}) }; };
67
- if (cur.sha256 === null) {
68
- closeGap(t);
69
- opened(t);
70
- continue;
71
- } // the slate is already up
72
- const at = data.reason === "process_killed" ? Math.min(t, Math.max(cur.start, data.from)) : t;
73
- close(at);
74
- state.current = { start: at, sha256: null, reason: data.reason };
75
- opened(at);
76
- }
77
- // The track ends where the recording does; only a track that would have no length at all is given one frame's time.
78
- let to = Math.max(options.endAt ?? 0, last);
79
- if (to - from < one)
80
- to = from + one;
81
- close(to);
82
- closeGap(to);
83
- const width = even(frames.reduce((max, f) => Math.max(max, f.w), 2));
84
- const height = even(frames.reduce((max, f) => Math.max(max, f.h), 2));
85
- return { session, conv: first.row.conv, fps, from, to, width, height, segments, frames, gaps };
86
- }
87
- const SLATE_COLOR = "0x1f2933";
88
- const sha256 = (bytes) => createHash("sha256").update(bytes).digest("hex");
89
- /** One WebM (VP9) and one index per session that has a frame. A session's pictures are read one at a time and written straight to a private temporary
90
- * directory made for the session and removed after, so memory holds one picture at a time (not every picture of the recording), and ffmpeg reads
91
- * only files there: it is started with a short environment, not given a URL, and nothing it writes reaches the recording but the video. `readBlob`
92
- * may be asked for the same blob again by another session. */
93
- export async function exportFrameTracks(options) {
94
- const fps = options.fps ?? 10;
95
- const lines = [...options.lines];
96
- // The end of the recording is the last line of any kind, the same for every session.
97
- const recordingEnd = lines.reduce((end, row) => (typeof row?.t === "number" && row.t > end ? row.t : end), 0);
98
- const endAt = options.endAt ?? recordingEnd;
99
- const sessions = new Map();
100
- for (const row of lines) {
101
- const session = row?.data?.session;
102
- if ((row?.kind === "frame" || row?.kind === "frame.gap") && typeof session === "string")
103
- (sessions.get(session) ?? sessions.set(session, []).get(session)).push(row);
104
- }
105
- const tracks = [];
106
- for (const [, rows] of sessions) {
107
- throwIfAborted(options.signal);
108
- // One deadline for the session's whole export (every ffmpeg call of it): each call gets the time that is left.
109
- const deadline = Date.now() + (options.timeoutMs ?? 120_000);
110
- const dir = await mkdtemp(path.join(os.tmpdir(), "pib-track-"));
111
- try {
112
- const names = new Map();
113
- const skip = new Set();
114
- const skipped = [];
115
- let n = 0;
116
- for (const row of rows) {
117
- if (row.kind !== "frame")
118
- continue;
119
- const hash = row.blobs?.[0];
120
- if (!hash || names.has(hash) || skip.has(hash))
121
- continue;
122
- let bytes;
123
- try {
124
- bytes = await bounded(async () => options.readBlob(hash), options, deadline);
125
- }
126
- catch (error) {
127
- if (error instanceof TrackExportError)
128
- throw error;
129
- bytes = null;
130
- }
131
- throwIfAborted(options.signal);
132
- const reason = !bytes ? "missing" : sha256(bytes) !== hash ? "hash_mismatch" : jpegSize(bytes) === null ? "not_a_jpeg" : null;
133
- if (reason || !bytes) {
134
- skip.add(hash);
135
- skipped.push({ sha256: hash, reason: reason ?? "missing" });
136
- continue;
137
- }
138
- const file = path.join(dir, `f${n++}.jpg`);
139
- await writeFile(file, bytes);
140
- const size = jpegSize(bytes);
141
- names.set(hash, { file, w: size.w, h: size.h, sampling: jpegSampling(bytes) });
142
- }
143
- let plan = null;
144
- let webm = null;
145
- for (let attempt = 0;; attempt += 1) {
146
- plan = planTrack(rows, { fps, endAt, skip });
147
- if (!plan)
148
- break;
149
- throwIfAborted(options.signal);
150
- try {
151
- webm = await encode(plan, names, dir, options, deadline);
152
- break;
153
- }
154
- catch (error) {
155
- // A picture whose header reads but whose data does not decode fails the whole encode: find it, show the slate for it, try once more.
156
- if (attempt > 0 || !(error instanceof TrackExportError) || error.code !== "ffmpeg_failed")
157
- throw error;
158
- const undecodable = await findUndecodable(names, dir, options, deadline);
159
- if (undecodable.length === 0)
160
- throw error;
161
- for (const hash of undecodable) {
162
- names.delete(hash);
163
- skip.add(hash);
164
- skipped.push({ sha256: hash, reason: "not_a_jpeg" });
165
- }
166
- }
167
- }
168
- if (!plan || !webm)
169
- continue;
170
- const index = { version: 1, session: plan.session, fps: plan.fps, from: plan.from, to: plan.to, width: plan.width, height: plan.height, codec: "vp9", frames: plan.frames, gaps: plan.gaps };
171
- const indexBytes = Buffer.from(`${JSON.stringify(index)}\n`, "utf8");
172
- const files = { webm: sha256(webm), index: sha256(indexBytes) };
173
- tracks.push({
174
- line: { channel: "live", source: "browser", kind: "frame.track", t: plan.to, conv: plan.conv, data: { session: plan.session, fps: plan.fps, from: plan.from, to: plan.to }, blobs: [files.webm, files.index] },
175
- blobs: { [files.webm]: webm, [files.index]: indexBytes },
176
- files, index, skipped,
177
- });
178
- }
179
- finally {
180
- await rm(dir, { recursive: true, force: true });
181
- }
182
- }
183
- return tracks.sort((a, b) => a.line.data.from - b.line.data.from);
184
- }
185
- const throwIfAborted = (signal) => { if (signal?.aborted)
186
- throw new TrackExportError("aborted", "the export was aborted"); };
187
- /** A read of a blob settles with the read, or fails at the session's deadline (`ffmpeg_timeout`) or at the signal (`aborted`), whichever is first: a store
188
- * that never answers cannot hold the export past either. The read itself is not cancelled; its answer, if one ever comes, is dropped. */
189
- function bounded(read, options, deadline) {
190
- const { signal } = options;
191
- return new Promise((resolve, reject) => {
192
- if (signal?.aborted)
193
- return reject(new TrackExportError("aborted", "the export was aborted"));
194
- const left = deadline - Date.now();
195
- if (left <= 0)
196
- return reject(new TrackExportError("ffmpeg_timeout", `the export did not finish within ${options.timeoutMs ?? 120_000} ms`));
197
- const timer = setTimeout(() => settle(() => reject(new TrackExportError("ffmpeg_timeout", `the export did not finish within ${options.timeoutMs ?? 120_000} ms`))), left);
198
- const onAbort = () => settle(() => reject(new TrackExportError("aborted", "the export was aborted")));
199
- let done = false;
200
- const settle = (finish) => { if (done)
201
- return; done = true; clearTimeout(timer); signal?.removeEventListener("abort", onAbort); finish(); };
202
- signal?.addEventListener("abort", onAbort, { once: true });
203
- read().then((value) => settle(() => resolve(value)), (error) => settle(() => reject(error)));
204
- });
205
- }
206
- const quote = (file) => `'${file.replaceAll("'", "'\\''")}'`;
207
- /** The layout a browser's JPEG has, and the one every picture is brought to: 4:2:0. */
208
- const COMMON_SAMPLING = "22,11,11";
209
- /** The pictures ffmpeg cannot decode, found one at a time (only after an encode has failed). */
210
- async function findUndecodable(names, dir, options, deadline) {
211
- const bad = [];
212
- for (const [hash, picture] of names) {
213
- throwIfAborted(options.signal);
214
- try {
215
- await run(options.ffmpeg ?? "ffmpeg", ["-hide_banner", "-loglevel", "error", "-nostdin", "-xerror", "-i", path.basename(picture.file), "-f", "null", "-"], dir, options, deadline);
216
- }
217
- catch (error) {
218
- if (error instanceof TrackExportError && error.code === "ffmpeg_failed")
219
- bad.push(hash);
220
- else
221
- throw error;
222
- }
223
- }
224
- return bad;
225
- }
226
- async function encode(plan, names, dir, options, deadline) {
227
- // The slate is made at the canvas's own size, so no frame is ever scaled: a smaller frame is centred on the canvas at the size it was recorded.
228
- const slate = path.join(dir, "slate.jpg");
229
- await run(options.ffmpeg ?? "ffmpeg", ["-hide_banner", "-loglevel", "error", "-nostdin", "-y", "-f", "lavfi", "-i", `color=c=${SLATE_COLOR}:s=${plan.width}x${plan.height}`, "-frames:v", "1", "slate.jpg"], dir, options, deadline);
230
- // ffmpeg reconfigures its filter graph when a picture's size or pixel layout differs from the one before, and the picture it was holding is lost (the
231
- // video then shows the next picture for its time). So every picture reaches the concat as the canvas's size and 4:2:0: one that is not is padded to
232
- // the canvas, at the size it was recorded and centred, and converted, first.
233
- const ready = new Map();
234
- let fixed = 0;
235
- for (const [hash, picture] of names) {
236
- if (picture.w === plan.width && picture.h === plan.height && picture.sampling === COMMON_SAMPLING) {
237
- ready.set(hash, picture.file);
238
- continue;
239
- }
240
- throwIfAborted(options.signal);
241
- const out = `n${fixed++}.jpg`;
242
- await run(options.ffmpeg ?? "ffmpeg", ["-hide_banner", "-loglevel", "error", "-nostdin", "-y", "-i", path.basename(picture.file), "-vf", `pad=${plan.width}:${plan.height}:(ow-iw)/2:(oh-ih)/2:color=${SLATE_COLOR},format=yuvj420p`, "-frames:v", "1", "-q:v", "2", out], dir, options, deadline);
243
- ready.set(hash, path.join(dir, out));
244
- }
245
- const list = [];
246
- const file = (sha) => (sha === null ? slate : ready.get(sha));
247
- for (const segment of plan.segments)
248
- list.push(`file ${quote(file(segment.sha256))}`, `duration ${((segment.end - segment.start) / 1000).toFixed(3)}`);
249
- list.push(`file ${quote(file(plan.segments.at(-1).sha256))}`); // the concat demuxer ignores the last duration unless its file is named again
250
- await writeFile(path.join(dir, "list.txt"), `${list.join("\n")}\n`);
251
- const filter = `fps=${plan.fps},format=yuv420p`;
252
- const args = ["-hide_banner", "-loglevel", "error", "-nostdin", "-y", "-f", "concat", "-safe", "0", "-i", "list.txt", "-vf", filter, "-fps_mode", "cfr", "-c:v", "libvpx-vp9", "-b:v", "0", "-crf", "40", "-row-mt", "1", "-deadline", "good", "-cpu-used", "5", "-an", "out.webm"];
253
- await run(options.ffmpeg ?? "ffmpeg", args, dir, options, deadline);
254
- return new Uint8Array(await readFile(path.join(dir, "out.webm")));
255
- }
256
- function run(program, args, cwd, options, deadline) {
257
- return new Promise((resolve, reject) => {
258
- if (options.signal?.aborted)
259
- return reject(new TrackExportError("aborted", "the export was aborted"));
260
- const left = deadline - Date.now();
261
- if (left <= 0)
262
- return reject(new TrackExportError("ffmpeg_timeout", `the export did not finish within ${options.timeoutMs ?? 120_000} ms`));
263
- // Only a short list of names reaches the program: not the host's keys.
264
- const child = spawn(program, args, { cwd, env: { PATH: process.env.PATH ?? "/usr/bin:/bin", LANG: "C", LC_ALL: "C", HOME: cwd }, stdio: ["ignore", "ignore", "pipe"] });
265
- let stderr = "";
266
- let settled = false;
267
- const finish = (error) => { if (settled)
268
- return; settled = true; clearTimeout(timer); options.signal?.removeEventListener("abort", onAbort); if (error)
269
- reject(error);
270
- else
271
- resolve(); };
272
- const timer = setTimeout(() => { child.kill("SIGKILL"); finish(new TrackExportError("ffmpeg_timeout", `ffmpeg did not finish within ${options.timeoutMs ?? 120_000} ms of the session's export`)); }, left);
273
- const onAbort = () => { child.kill("SIGKILL"); finish(new TrackExportError("aborted", "the export was aborted")); };
274
- options.signal?.addEventListener("abort", onAbort, { once: true });
275
- child.stderr.on("data", (chunk) => { if (stderr.length < 4_000)
276
- stderr += chunk.toString("utf8"); });
277
- child.once("error", (error) => finish(new TrackExportError("ffmpeg_missing", `could not start ffmpeg (${program}): ${error.code ?? error.message}`)));
278
- child.once("close", (code) => finish(code === 0 ? undefined : new TrackExportError("ffmpeg_failed", `ffmpeg exited with ${code}: ${stderr.trim().slice(0, 1_000)}`)));
279
- });
280
- }
@@ -1,15 +0,0 @@
1
- import type { StagehandFacadeScreenshot } from "./tools.js";
2
- export type ScreenshotOptions = {
3
- fullPage?: boolean;
4
- type?: "png" | "jpeg";
5
- quality?: number;
6
- };
7
- type CaptureScreenshot = (options: ScreenshotOptions) => Promise<StagehandFacadeScreenshot>;
8
- export type TransportSafeScreenshot = {
9
- image: StagehandFacadeScreenshot;
10
- options: ScreenshotOptions;
11
- adjusted: boolean;
12
- };
13
- export declare function screenshotBase64BudgetFromArgs(args: string[]): number | undefined;
14
- export declare function captureScreenshotWithinBase64Budget(capture: CaptureScreenshot, requested: ScreenshotOptions, maxBase64Bytes: number): Promise<TransportSafeScreenshot>;
15
- export {};
@@ -1,51 +0,0 @@
1
- const SCREENSHOT_BUDGET_FLAG = "--max-screenshot-base64-bytes=";
2
- export function screenshotBase64BudgetFromArgs(args) {
3
- const value = args.find((arg) => arg.startsWith(SCREENSHOT_BUDGET_FLAG));
4
- if (value === undefined)
5
- return undefined;
6
- const budget = Number(value.slice(SCREENSHOT_BUDGET_FLAG.length));
7
- if (!Number.isSafeInteger(budget) || budget < 1_024) {
8
- throw new Error(`${SCREENSHOT_BUDGET_FLAG} must be an integer of at least 1024.`);
9
- }
10
- return budget;
11
- }
12
- export async function captureScreenshotWithinBase64Budget(capture, requested, maxBase64Bytes) {
13
- const attempts = screenshotAttempts(requested);
14
- for (const [index, options] of attempts.entries()) {
15
- const image = await capture(options);
16
- if (Buffer.byteLength(image.data, "utf8") <= maxBase64Bytes) {
17
- return { image, options, adjusted: index > 0 || !sameOptions(options, requested) };
18
- }
19
- }
20
- throw new Error(`Screenshot exceeds the ${maxBase64Bytes}-byte MCP transport budget after compressed viewport retries.`);
21
- }
22
- function screenshotAttempts(requested) {
23
- const initial = {
24
- fullPage: requested.fullPage ?? false,
25
- type: requested.type ?? "jpeg",
26
- ...(requested.type === "png" ? {} : { quality: requested.quality ?? 40 }),
27
- };
28
- const maxFallbackQuality = requested.type === "png" ? 40 : (requested.quality ?? 40);
29
- const attempts = [
30
- initial,
31
- {
32
- fullPage: false,
33
- type: "jpeg",
34
- quality: Math.min(maxFallbackQuality, 40),
35
- },
36
- {
37
- fullPage: false,
38
- type: "jpeg",
39
- quality: Math.min(maxFallbackQuality, 25),
40
- },
41
- {
42
- fullPage: false,
43
- type: "jpeg",
44
- quality: Math.min(maxFallbackQuality, 10),
45
- },
46
- ];
47
- return attempts.filter((candidate, index) => attempts.findIndex((other) => sameOptions(candidate, other)) === index);
48
- }
49
- function sameOptions(left, right) {
50
- return (left.fullPage === right.fullPage && left.type === right.type && left.quality === right.quality);
51
- }