@parcha/pi-browser 0.0.0-stage → 0.1.0-beta.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (108) hide show
  1. package/CHANGELOG.md +23 -0
  2. package/LICENSE +202 -0
  3. package/NOTICE +33 -0
  4. package/README.md +348 -2
  5. package/dist/coding-agent-local.d.ts +16 -0
  6. package/dist/coding-agent-local.js +138 -0
  7. package/dist/coding-agent.d.ts +79 -0
  8. package/dist/coding-agent.js +448 -0
  9. package/dist/core/contract.d.ts +26 -0
  10. package/dist/core/contract.js +154 -0
  11. package/dist/core/custody.d.ts +117 -0
  12. package/dist/core/custody.js +126 -0
  13. package/dist/core/decisions.d.ts +43 -0
  14. package/dist/core/decisions.js +4 -0
  15. package/dist/core/downloads.d.ts +35 -0
  16. package/dist/core/downloads.js +131 -0
  17. package/dist/core/effects.d.ts +63 -0
  18. package/dist/core/effects.js +256 -0
  19. package/dist/core/evidence.d.ts +43 -0
  20. package/dist/core/evidence.js +6 -0
  21. package/dist/core/failures.d.ts +36 -0
  22. package/dist/core/failures.js +68 -0
  23. package/dist/core/find.d.ts +32 -0
  24. package/dist/core/find.js +179 -0
  25. package/dist/core/guard.d.ts +6 -0
  26. package/dist/core/guard.js +25 -0
  27. package/dist/core/host.d.ts +328 -0
  28. package/dist/core/host.js +2 -0
  29. package/dist/core/images.d.ts +37 -0
  30. package/dist/core/images.js +60 -0
  31. package/dist/core/lease.d.ts +16 -0
  32. package/dist/core/lease.js +30 -0
  33. package/dist/core/receipts.d.ts +19 -0
  34. package/dist/core/receipts.js +22 -0
  35. package/dist/core/redact.d.ts +13 -0
  36. package/dist/core/redact.js +83 -0
  37. package/dist/core/session-control.d.ts +25 -0
  38. package/dist/core/session-control.js +75 -0
  39. package/dist/core/sites.d.ts +33 -0
  40. package/dist/core/sites.js +102 -0
  41. package/dist/core/tools.d.ts +72 -0
  42. package/dist/core/tools.js +200 -0
  43. package/dist/core/uri.d.ts +11 -0
  44. package/dist/core/uri.js +37 -0
  45. package/dist/core/usage.d.ts +19 -0
  46. package/dist/core/usage.js +44 -0
  47. package/dist/core/web.d.ts +98 -0
  48. package/dist/core/web.js +327 -0
  49. package/dist/driver/extension-worker.d.ts +6 -0
  50. package/dist/driver/extension-worker.js +121 -0
  51. package/dist/driver/stagehand.d.ts +25 -0
  52. package/dist/driver/stagehand.js +91 -0
  53. package/dist/durable-result.d.ts +6 -0
  54. package/dist/durable-result.js +2 -0
  55. package/dist/durable.d.ts +79 -0
  56. package/dist/durable.js +582 -0
  57. package/dist/index.d.ts +19 -0
  58. package/dist/index.js +20 -0
  59. package/dist/providers/browserbase-net.d.ts +13 -0
  60. package/dist/providers/browserbase-net.js +96 -0
  61. package/dist/providers/browserbase.d.ts +152 -0
  62. package/dist/providers/browserbase.js +262 -0
  63. package/dist/providers/cdp-call.d.ts +6 -0
  64. package/dist/providers/cdp-call.js +25 -0
  65. package/dist/providers/cdp.d.ts +55 -0
  66. package/dist/providers/cdp.js +423 -0
  67. package/dist/providers/http.d.ts +2 -0
  68. package/dist/providers/http.js +21 -0
  69. package/dist/providers/kernel.d.ts +49 -0
  70. package/dist/providers/kernel.js +246 -0
  71. package/dist/providers/shared.d.ts +47 -0
  72. package/dist/providers/shared.js +82 -0
  73. package/dist/recorder/credential.d.ts +9 -0
  74. package/dist/recorder/credential.js +49 -0
  75. package/dist/recorder/follow.d.ts +26 -0
  76. package/dist/recorder/follow.js +138 -0
  77. package/dist/recorder/frames.d.ts +80 -0
  78. package/dist/recorder/frames.js +309 -0
  79. package/dist/recorder/index.d.ts +4 -0
  80. package/dist/recorder/index.js +7 -0
  81. package/dist/recorder/jpeg.d.ts +8 -0
  82. package/dist/recorder/jpeg.js +70 -0
  83. package/dist/recorder/track.d.ts +107 -0
  84. package/dist/recorder/track.js +259 -0
  85. package/dist/store.d.ts +56 -0
  86. package/dist/store.js +196 -0
  87. package/dist/testing/backend.d.ts +231 -0
  88. package/dist/testing/backend.js +414 -0
  89. package/dist/testing/driver.d.ts +28 -0
  90. package/dist/testing/driver.js +32 -0
  91. package/dist/testing/index.d.ts +4 -0
  92. package/dist/testing/index.js +4 -0
  93. package/dist/testing/pages.d.ts +3 -0
  94. package/dist/testing/pages.js +10 -0
  95. package/dist/testing/sentinels.d.ts +29 -0
  96. package/dist/testing/sentinels.js +38 -0
  97. package/dist/vendor/stagehand-facade/contract.d.ts +348 -0
  98. package/dist/vendor/stagehand-facade/contract.js +155 -0
  99. package/dist/vendor/stagehand-facade/runtime.d.ts +90 -0
  100. package/dist/vendor/stagehand-facade/runtime.js +1640 -0
  101. package/dist/vendor/stagehand-facade/screenshot-transport.d.ts +15 -0
  102. package/dist/vendor/stagehand-facade/screenshot-transport.js +51 -0
  103. package/dist/vendor/stagehand-facade/tools.d.ts +31 -0
  104. package/dist/vendor/stagehand-facade/tools.js +139 -0
  105. package/dist/web.d.ts +41 -0
  106. package/dist/web.js +38 -0
  107. package/package.json +123 -4
  108. package/src/vendor/stagehand-facade/UPSTREAM.json +24 -0
@@ -0,0 +1,80 @@
1
+ import type { BrowserProvider } from "../core/host.js";
2
+ export type FrameData = {
3
+ session: string;
4
+ provider: string;
5
+ w: number;
6
+ h: number;
7
+ mime: "image/jpeg";
8
+ };
9
+ /** `detail`, when present, says why a `credential_on_page` gap was written: `field` (a credential or code field holds a value), `typed` (one held a value
10
+ * since the last check), `too_deep` (the page nests deeper than the check reads), `unreadable_frame` (a visible frame whose content cannot be read), `host` (the host paused
11
+ * the session). Anything the check cannot read counts as a credential being present. */
12
+ export type FrameGapData = {
13
+ session: string;
14
+ from: number;
15
+ reason: "process_killed" | "credential_on_page" | "session_released";
16
+ detail?: string;
17
+ };
18
+ /** One live line of agentrun.recording/1. `seq`, the line's place among the process's live lines, is the writer's to add. */
19
+ export type FrameLine = {
20
+ channel: "live";
21
+ source: "browser";
22
+ kind: "frame";
23
+ t: number;
24
+ gen?: number;
25
+ conv: number;
26
+ data: FrameData;
27
+ blobs: [string];
28
+ } | {
29
+ channel: "live";
30
+ source: "browser";
31
+ kind: "frame.gap";
32
+ t: number;
33
+ gen?: number;
34
+ conv: number;
35
+ data: FrameGapData;
36
+ };
37
+ /** `blobs` maps each sha256 a line names to its bytes; the sink keeps them as `blobs/<sha256>.jpg`. A sink that throws or rejects is
38
+ * ignored: a recorder never fails a browser call. */
39
+ export type FrameSink = (line: FrameLine, blobs?: Record<string, Uint8Array>) => void | Promise<void>;
40
+ export type FrameRecorderOptions = {
41
+ sink: FrameSink;
42
+ /** Milliseconds since the recording started: `t` of every line, and the `from` of a gap, are read from it. */
43
+ clock: () => number;
44
+ /** This process's generation, set on every line when given. */
45
+ gen?: () => number;
46
+ /** When a session this process did not create is taken up, the `t` of that session's last frame in an earlier process (the host reads
47
+ * it from the recording), so a `process_killed` gap can start where the frames stopped; null: it has none. Absent: no such gap. */
48
+ lastFrameAt?: (session: string) => number | null;
49
+ maxFps?: number;
50
+ maxWidth?: number;
51
+ quality?: number;
52
+ /** How long one line may wait for the sink before it counts as dropped; default 5 s. */
53
+ sinkTimeoutMs?: number;
54
+ /** The clock and the waits the page checks are paced by (default: performance.now and timers). A test passes a virtual one. */
55
+ probeClock?: ProbeClock;
56
+ /** Called with the probe clock's time as each check of the page is sent. */
57
+ onProbe?: (at: number) => void;
58
+ };
59
+ export type ProbeClock = {
60
+ now(): number;
61
+ sleep(ms: number): Promise<void>;
62
+ };
63
+ export type FrameRecorder = {
64
+ /** The provider, with every session it creates or is reattached to captured, and every release ending that capture with a gap. Its
65
+ * own methods are untouched. `conv` is the pi conversation that owns the sessions; without it, it is read from the lease tag
66
+ * (`ar-<run>-<conversation>-<n>`), and a session whose conversation cannot be told is not recorded. */
67
+ watch(provider: BrowserProvider, conv?: number): BrowserProvider;
68
+ /** Pauses a session's capture as if a credential were on its page (a code entered, a setup key shown). It lifts at the next
69
+ * navigation of the page, or at `resume`. */
70
+ pause(session: string): void;
71
+ resume(session: string): void;
72
+ stats(): {
73
+ frames: number;
74
+ gaps: number;
75
+ dropped: number;
76
+ sessions: number;
77
+ };
78
+ close(): Promise<void>;
79
+ };
80
+ export declare function frameRecorder(options: FrameRecorderOptions): FrameRecorder;
@@ -0,0 +1,309 @@
1
+ import { createHash } from "node:crypto";
2
+ import { CREDENTIAL_PROBE } from "./credential.js";
3
+ import { followPage } from "./follow.js";
4
+ import { jpegSize } from "./jpeg.js";
5
+ const MAX_QUEUED_LINES = 32;
6
+ /** The conversation id in a lease tag: `ar-<run>-<conversation>-<n>`. */
7
+ const tagConversation = (tag) => { const m = /-(\d+)-\d+$/.exec(tag); return m ? Number(m[1]) : null; };
8
+ /** One session asks its page at most 1000 / PROBE_GAP_MS = 20 times a second: arrival checks and delivery checks share one pace, each sent at least this far after the last. */
9
+ const PROBE_GAP_MS = 50;
10
+ export function frameRecorder(options) {
11
+ const maxFps = Math.max(1, Math.min(10, options.maxFps ?? 10));
12
+ const maxWidth = Math.min(800, options.maxWidth ?? 800);
13
+ const quality = options.quality ?? 60;
14
+ const spacingMs = Math.ceil(1000 / maxFps);
15
+ const captures = new Map();
16
+ const made = new Set();
17
+ const counts = { frames: 0, gaps: 0, dropped: 0 };
18
+ const sinkTimeoutMs = options.sinkTimeoutMs ?? 5_000;
19
+ const pclock = options.probeClock ?? { now: () => performance.now(), sleep: (ms) => new Promise((resolve) => setTimeout(resolve, ms)) };
20
+ let closed = false;
21
+ // Lines reach the sink one at a time, in the order they were made, and a write is never left behind: the next one starts only when the
22
+ // one before has ended. A line the sink has not stored within sinkTimeoutMs of being made counts as dropped (its write, if it had started,
23
+ // may still land, but nothing is written after it until it does); one whose turn comes after that is not written at all. A sink that never
24
+ // answers leaves one unfinished write and at most MAX_QUEUED_LINES lines waiting; the rest are dropped at once.
25
+ let queue = Promise.resolve();
26
+ let queued = 0;
27
+ /** The timers of lines waiting or being written: unreferenced while the process lives on its own, referenced once `close` waits on them. */
28
+ const sinkTimers = new Set();
29
+ /** `make` builds the line when its turn comes (a gap reads the last frame that was saved by then). */
30
+ const emit = (make) => {
31
+ if (queued >= MAX_QUEUED_LINES) {
32
+ counts.dropped += 1;
33
+ return Promise.resolve(false);
34
+ }
35
+ queued += 1;
36
+ let abandoned = false;
37
+ let answered = false;
38
+ let answer = () => { };
39
+ const result = new Promise((resolve) => { answer = (ok) => { if (answered)
40
+ return; answered = true; if (!ok)
41
+ counts.dropped += 1; resolve(ok); }; });
42
+ const timer = setTimeout(() => { abandoned = true; answer(false); }, sinkTimeoutMs);
43
+ sinkTimers.add(timer);
44
+ if (closed)
45
+ timer.ref?.();
46
+ else
47
+ timer.unref?.();
48
+ queue = queue.then(async () => {
49
+ try {
50
+ if (abandoned)
51
+ return;
52
+ const [line, blobs] = make();
53
+ await options.sink(line, blobs);
54
+ answer(true); // too late to count when the timeout already answered
55
+ }
56
+ catch {
57
+ answer(false);
58
+ }
59
+ finally {
60
+ clearTimeout(timer);
61
+ sinkTimers.delete(timer);
62
+ queued -= 1;
63
+ }
64
+ });
65
+ return result;
66
+ };
67
+ const base = (c, t) => ({ channel: "live", source: "browser", t, ...(options.gen ? { gen: options.gen() } : {}), conv: c.conv });
68
+ const gap = (c, reason, from, detail) => {
69
+ const t = options.clock();
70
+ return emit(() => [{ ...base(c, t), kind: "frame.gap", data: { session: c.ref.id, from: from ?? c.lastFrameT ?? t, reason, ...(detail ? { detail } : {}) } }]).then((ok) => { if (ok)
71
+ counts.gaps += 1; });
72
+ };
73
+ /** A credential may be on the page: the capture pauses, once, until the page navigates and the check is clean. */
74
+ const hold = (c, why) => {
75
+ c.ready = null;
76
+ c.waiting = null;
77
+ if (c.paused)
78
+ return;
79
+ c.paused = true;
80
+ c.navigated = false;
81
+ void gap(c, "credential_on_page", undefined, why ?? "field");
82
+ };
83
+ const check = async (c) => {
84
+ try {
85
+ options.onProbe?.(pclock.now());
86
+ const answer = (await c.follower?.evaluate(CREDENTIAL_PROBE));
87
+ return answer && typeof answer.hold === "boolean" ? { hold: answer.hold, why: typeof answer.why === "string" ? answer.why : null } : "unchecked";
88
+ }
89
+ catch {
90
+ return "unchecked";
91
+ } // a page that cannot be checked is not recorded
92
+ };
93
+ /** Every check of the page goes through here, one at a time: it is sent no earlier than PROBE_GAP_MS after the last one was sent, measured at the
94
+ * send, so a wait that ended late cannot be followed by one that ends on time and lands closer. Any half-open second then holds at most 20. */
95
+ const paced = (c, send) => {
96
+ let result;
97
+ c.pacing = c.pacing.then(async () => {
98
+ for (;;) {
99
+ const wait = c.lastProbeAt + PROBE_GAP_MS - pclock.now();
100
+ if (wait <= 0)
101
+ break;
102
+ await pclock.sleep(wait);
103
+ }
104
+ c.lastProbeAt = pclock.now();
105
+ result = send();
106
+ }).catch(() => { result ??= Promise.reject(new Error("the check was not sent")); });
107
+ return c.pacing.then(() => result);
108
+ };
109
+ const arm = (c) => {
110
+ if (c.timer || c.busy || c.stopped || !c.ready)
111
+ return;
112
+ c.timer = setTimeout(() => { c.timer = null; void deliver(c); }, Math.max(0, spacingMs - (Date.now() - c.lastEmitMs)));
113
+ c.timer.unref?.();
114
+ };
115
+ /** A frame is judged against the page the moment it arrives: the check that follows it stands for the page at its capture. A frame that
116
+ * arrives while a check runs waits for the next one; the one judged last is the one kept. */
117
+ function judge(c) {
118
+ if (c.probing || c.stopped || !c.waiting || !c.follower)
119
+ return;
120
+ c.probing = true;
121
+ let frame = null;
122
+ // The frame is the newest one when the check is sent: one that came while it waited for its turn replaces the older.
123
+ void paced(c, async () => { frame = c.waiting; c.waiting = null; return c.stopped || !frame ? null : check(c); })
124
+ .then((verdict) => { if (verdict === null || !frame)
125
+ c.probing = false;
126
+ else
127
+ settle(c, frame, verdict); }, () => { c.probing = false; });
128
+ }
129
+ function settle(c, frame, verdict) {
130
+ c.probing = false;
131
+ if (c.stopped)
132
+ return;
133
+ if (c.follower?.epoch() === frame.page.epoch) {
134
+ if (verdict !== "unchecked" && verdict.hold)
135
+ hold(c, verdict.why);
136
+ else if (verdict !== "unchecked") {
137
+ frame.verdict = verdict;
138
+ c.ready = frame;
139
+ }
140
+ }
141
+ judge(c);
142
+ arm(c);
143
+ }
144
+ /** The judged frame whose turn has come: checked again now (a credential seen between its capture and now drops it), then kept. */
145
+ async function deliver(c) {
146
+ if (c.busy || c.stopped || !c.ready || !c.follower)
147
+ return;
148
+ c.busy = true;
149
+ // The slot is taken when the check starts, so the next frame is a full spacing after this one whatever the check costs.
150
+ c.lastEmitMs = Date.now();
151
+ const frame = c.ready;
152
+ c.ready = null;
153
+ try {
154
+ const verdict = await paced(c, async () => (c.stopped ? "unchecked" : check(c)));
155
+ if (c.stopped)
156
+ return;
157
+ // A frame belongs to the page it showed: one held across a navigation or a move to another page is not kept.
158
+ if (c.follower?.epoch() !== frame.page.epoch)
159
+ return;
160
+ if (verdict === "unchecked")
161
+ return;
162
+ if (verdict.hold) {
163
+ hold(c, verdict.why);
164
+ return;
165
+ }
166
+ // A pause lifts at a navigation, once the page holds no credential.
167
+ if (c.paused) {
168
+ if (!c.navigated)
169
+ return;
170
+ c.paused = false;
171
+ }
172
+ const size = jpegSize(frame.bytes) ?? (frame.page.deviceWidth > 0 ? { w: Math.min(frame.page.deviceWidth, maxWidth), h: Math.round(Math.min(frame.page.deviceWidth, maxWidth) * (frame.page.deviceHeight / frame.page.deviceWidth)) } : null);
173
+ if (!size)
174
+ return;
175
+ const hash = createHash("sha256").update(frame.bytes).digest("hex");
176
+ const t = options.clock();
177
+ const saved = await emit(() => [{ ...base(c, t), kind: "frame", data: { session: c.ref.id, provider: c.provider, w: size.w, h: size.h, mime: "image/jpeg" }, blobs: [hash] }, { [hash]: new Uint8Array(frame.bytes) }]);
178
+ if (saved) {
179
+ counts.frames += 1;
180
+ c.lastFrameT = t;
181
+ }
182
+ c.lastEmitMs = Date.now();
183
+ }
184
+ finally {
185
+ c.busy = false;
186
+ arm(c);
187
+ }
188
+ }
189
+ function connect(c, url) {
190
+ c.follower = followPage(url, {
191
+ frame(bytes, page) {
192
+ if (c.stopped)
193
+ return;
194
+ // The newest frame wins: an older one still held is not worth keeping.
195
+ c.waiting = { bytes, page };
196
+ judge(c);
197
+ },
198
+ navigated() { c.navigated = true; },
199
+ closed() {
200
+ c.follower = null;
201
+ if (c.stopped || closed)
202
+ return;
203
+ c.reconnect = setTimeout(() => { c.reconnect = null; if (!c.stopped && !closed)
204
+ connect(c, url); }, 1_000);
205
+ c.reconnect.unref?.();
206
+ },
207
+ }, { maxWidth, maxHeight: 2_000, quality });
208
+ }
209
+ function start(ref, url, provider, conv) {
210
+ if (closed)
211
+ return;
212
+ const owner = conv ?? tagConversation(ref.tag);
213
+ if (owner === null || owner === undefined)
214
+ return;
215
+ if (captures.get(ref.id))
216
+ return;
217
+ const c = { ref, conv: owner, provider: provider.name, follower: null, lastFrameT: null, paused: false, navigated: false, ready: null, waiting: null, probing: false, lastProbeAt: -Infinity, pacing: Promise.resolve(), busy: false, timer: null, lastEmitMs: 0, stopped: false, reconnect: null };
218
+ captures.set(ref.id, c);
219
+ // A session first seen in this process that an earlier process already framed: the frames stopped when that process was killed.
220
+ if (!made.has(ref.id)) {
221
+ const from = options.lastFrameAt?.(ref.id) ?? null;
222
+ if (from !== null) {
223
+ c.lastFrameT = from;
224
+ void gap(c, "process_killed", from);
225
+ }
226
+ }
227
+ connect(c, url);
228
+ }
229
+ function stop(c) {
230
+ c.stopped = true;
231
+ if (c.timer)
232
+ clearTimeout(c.timer);
233
+ if (c.reconnect)
234
+ clearTimeout(c.reconnect);
235
+ c.timer = null;
236
+ c.reconnect = null;
237
+ c.ready = null;
238
+ c.waiting = null;
239
+ c.follower?.close();
240
+ c.follower = null;
241
+ captures.delete(c.ref.id);
242
+ }
243
+ return {
244
+ watch(provider, conv) {
245
+ const watched = Object.create(provider);
246
+ watched.create = async (spec, signal) => {
247
+ const ref = await provider.create(spec, signal);
248
+ made.add(ref.id);
249
+ return ref;
250
+ };
251
+ watched.attach = async (ref, signal) => {
252
+ const target = await provider.attach(ref, signal);
253
+ start(ref, target.sdkCdpUrl, provider, conv);
254
+ return target;
255
+ };
256
+ watched.release = async (ref, signal) => {
257
+ // The capture ends when the provider says the session is released: one it refuses to release is still running, and still recorded.
258
+ await provider.release(ref, signal);
259
+ const c = captures.get(ref.id);
260
+ if (c) {
261
+ stop(c);
262
+ void gap(c, "session_released");
263
+ } // storage is not the browser call's to wait for: close() flushes
264
+ else if (!closed) {
265
+ // A release made by a process that never captured the session (a later one finishing an earlier one's release): the tile
266
+ // ended where an earlier process's frames stopped.
267
+ const from = options.lastFrameAt?.(ref.id) ?? null;
268
+ const owner = conv ?? tagConversation(ref.tag);
269
+ if (from !== null && owner !== null)
270
+ void gap({ ref, conv: owner, lastFrameT: from }, "session_released");
271
+ }
272
+ };
273
+ return watched;
274
+ },
275
+ pause(session) {
276
+ const c = captures.get(session);
277
+ if (!c || c.paused)
278
+ return;
279
+ c.paused = true;
280
+ c.navigated = false;
281
+ void gap(c, "credential_on_page", undefined, "host");
282
+ },
283
+ resume(session) {
284
+ const c = captures.get(session);
285
+ if (c) {
286
+ c.paused = false;
287
+ judge(c);
288
+ arm(c);
289
+ }
290
+ },
291
+ stats: () => ({ ...counts, sessions: captures.size }),
292
+ async close() {
293
+ closed = true;
294
+ for (const t of sinkTimers)
295
+ t.ref?.();
296
+ for (const c of [...captures.values()])
297
+ stop(c);
298
+ // What was already handed over is delivered first, for as long as the sink answers: every line has its own timeout, and a sink that
299
+ // never answers is not waited on past one.
300
+ let giveUp;
301
+ try {
302
+ await Promise.race([queue, new Promise((resolve) => { giveUp = setTimeout(resolve, sinkTimeoutMs + 50); })]);
303
+ }
304
+ finally {
305
+ clearTimeout(giveUp);
306
+ }
307
+ },
308
+ };
309
+ }
@@ -0,0 +1,4 @@
1
+ export { frameRecorder, type FrameData, type FrameGapData, type FrameLine, type FrameRecorder, type FrameRecorderOptions, type FrameSink } from "./frames.js";
2
+ export { CREDENTIAL_PROBE } from "./credential.js";
3
+ export { jpegSize } from "./jpeg.js";
4
+ export { exportFrameTracks, planTrack, TrackExportError, type FrameTrackData, type FrameTrackLine, type Track, type TrackFrame, type TrackGap, type TrackIndex, type TrackOptions, type TrackPlan, type TrackSegment } from "./track.js";
@@ -0,0 +1,7 @@
1
+ // The host-side frame recorder: it screencasts each browser session over DevTools and hands agentrun.recording/1's `frame` and
2
+ // `frame.gap` lines, with the JPEG blobs, to a sink the host passes. It is read-only beside the session's own connections and writes
3
+ // nothing of its own.
4
+ export { frameRecorder } from "./frames.js";
5
+ export { CREDENTIAL_PROBE } from "./credential.js";
6
+ export { jpegSize } from "./jpeg.js";
7
+ export { exportFrameTracks, planTrack, TrackExportError } from "./track.js";
@@ -0,0 +1,8 @@
1
+ /** The pixel size a JPEG states in its first start-of-frame segment, or null when it states none. */
2
+ export declare function jpegSize(bytes: Uint8Array): {
3
+ w: number;
4
+ h: number;
5
+ } | null;
6
+ /** The sampling factors the first start-of-frame segment states for each component, as hex pairs joined by commas ("22,11,11" is 4:2:0, the layout
7
+ * a browser's JPEG has; "11" is grayscale), or null when it states none. */
8
+ export declare function jpegSampling(bytes: Uint8Array): string | null;
@@ -0,0 +1,70 @@
1
+ /** The pixel size a JPEG states in its first start-of-frame segment, or null when it states none. */
2
+ export function jpegSize(bytes) {
3
+ if (bytes.length < 4 || bytes[0] !== 0xff || bytes[1] !== 0xd8)
4
+ return null;
5
+ let at = 2;
6
+ while (at + 4 <= bytes.length) {
7
+ if (bytes[at] !== 0xff) {
8
+ at += 1;
9
+ continue;
10
+ }
11
+ const marker = bytes[at + 1];
12
+ if (marker === 0xff) {
13
+ at += 1;
14
+ continue;
15
+ }
16
+ if (marker === 0xd8 || (marker >= 0xd0 && marker <= 0xd7) || marker === 0x01) {
17
+ at += 2;
18
+ continue;
19
+ }
20
+ if (marker === 0xd9 || marker === 0xda)
21
+ return null;
22
+ const length = (bytes[at + 2] << 8) | bytes[at + 3];
23
+ // C0..CF are start-of-frame markers, except C4 (tables), C8 (reserved) and CC (arithmetic conditioning).
24
+ if (marker >= 0xc0 && marker <= 0xcf && marker !== 0xc4 && marker !== 0xc8 && marker !== 0xcc) {
25
+ if (at + 9 > bytes.length)
26
+ return null;
27
+ const h = (bytes[at + 5] << 8) | bytes[at + 6];
28
+ const w = (bytes[at + 7] << 8) | bytes[at + 8];
29
+ return w > 0 && h > 0 ? { w, h } : null;
30
+ }
31
+ at += 2 + length;
32
+ }
33
+ return null;
34
+ }
35
+ /** The sampling factors the first start-of-frame segment states for each component, as hex pairs joined by commas ("22,11,11" is 4:2:0, the layout
36
+ * a browser's JPEG has; "11" is grayscale), or null when it states none. */
37
+ export function jpegSampling(bytes) {
38
+ if (bytes.length < 4 || bytes[0] !== 0xff || bytes[1] !== 0xd8)
39
+ return null;
40
+ let at = 2;
41
+ while (at + 4 <= bytes.length) {
42
+ if (bytes[at] !== 0xff) {
43
+ at += 1;
44
+ continue;
45
+ }
46
+ const marker = bytes[at + 1];
47
+ if (marker === 0xff) {
48
+ at += 1;
49
+ continue;
50
+ }
51
+ if (marker === 0xd8 || (marker >= 0xd0 && marker <= 0xd7) || marker === 0x01) {
52
+ at += 2;
53
+ continue;
54
+ }
55
+ if (marker === 0xd9 || marker === 0xda)
56
+ return null;
57
+ const length = (bytes[at + 2] << 8) | bytes[at + 3];
58
+ if (marker >= 0xc0 && marker <= 0xcf && marker !== 0xc4 && marker !== 0xc8 && marker !== 0xcc) {
59
+ const count = bytes[at + 9];
60
+ if (count === undefined || count < 1 || at + 10 + count * 3 > bytes.length)
61
+ return null;
62
+ const factors = [];
63
+ for (let i = 0; i < count; i += 1)
64
+ factors.push(bytes[at + 10 + i * 3 + 1].toString(16).padStart(2, "0"));
65
+ return factors.join(",");
66
+ }
67
+ at += 2 + length;
68
+ }
69
+ return null;
70
+ }
@@ -0,0 +1,107 @@
1
+ /** `frame.track`: written at export, one per session. The blobs are the WebM and its index, in that order. */
2
+ export type FrameTrackData = {
3
+ session: string;
4
+ fps: number;
5
+ from: number;
6
+ to: number;
7
+ };
8
+ export type FrameTrackLine = {
9
+ channel: "live";
10
+ source: "browser";
11
+ kind: "frame.track";
12
+ t: number;
13
+ conv: number;
14
+ data: FrameTrackData;
15
+ blobs: [string, string];
16
+ };
17
+ /** One stretch of the track: a frame shown, or the slate (`sha256` null) while the session had no frame to show. Times are the recording's. */
18
+ export type TrackSegment = {
19
+ start: number;
20
+ end: number;
21
+ sha256: string | null;
22
+ reason?: string;
23
+ };
24
+ export type TrackFrame = {
25
+ t: number;
26
+ ms: number;
27
+ sha256: string;
28
+ w: number;
29
+ h: number;
30
+ };
31
+ export type TrackGap = {
32
+ from: number;
33
+ to: number;
34
+ reason: string;
35
+ detail?: string;
36
+ };
37
+ export type TrackPlan = {
38
+ session: string;
39
+ conv: number;
40
+ fps: number;
41
+ from: number;
42
+ to: number;
43
+ width: number;
44
+ height: number;
45
+ segments: TrackSegment[];
46
+ frames: TrackFrame[];
47
+ gaps: TrackGap[];
48
+ };
49
+ /** The index blob: where each frame and each gap is, so a viewer maps a time in the recording (`t`) to a time in the video (`t - from`). */
50
+ export type TrackIndex = {
51
+ version: 1;
52
+ session: string;
53
+ fps: number;
54
+ from: number;
55
+ to: number;
56
+ width: number;
57
+ height: number;
58
+ codec: "vp9";
59
+ frames: TrackFrame[];
60
+ gaps: TrackGap[];
61
+ };
62
+ export declare class TrackExportError extends Error {
63
+ readonly code: "ffmpeg_missing" | "ffmpeg_failed" | "ffmpeg_timeout" | "aborted";
64
+ constructor(code: TrackExportError["code"], message: string);
65
+ }
66
+ /** The timeline of one session's lines. Each frame is shown until the next one; a gap puts the slate up (from the moment it was written, except
67
+ * `process_killed`, which starts at the last frame the earlier process saved: nothing is known after it) until the next frame or the end.
68
+ * Null when the session has no frame. `skip` names frames whose picture could not be read: they show the slate and are not listed. */
69
+ export declare function planTrack(lines: Iterable<unknown>, options?: {
70
+ fps?: number;
71
+ endAt?: number;
72
+ skip?: ReadonlySet<string>;
73
+ }): TrackPlan | null;
74
+ export type TrackOptions = {
75
+ /** The recording's lines; `frame` and `frame.gap` of every session are read, anything else is ignored. */
76
+ lines: Iterable<unknown>;
77
+ /** The bytes of a blob by its sha256 (the JPEG a `frame` line names); null or a throw: not there. */
78
+ readBlob: (sha256: string) => Uint8Array | null | undefined | Promise<Uint8Array | null | undefined>;
79
+ fps?: number;
80
+ /** When the recording ended, in its own ms: the last frame or slate is shown until then. */
81
+ endAt?: number;
82
+ /** The program to start; default `ffmpeg` on the PATH. */
83
+ ffmpeg?: string;
84
+ /** For each session's encoding; default 120 s. */
85
+ timeoutMs?: number;
86
+ signal?: AbortSignal;
87
+ };
88
+ export type Track = {
89
+ line: FrameTrackLine;
90
+ /** The video and the index, by sha256: the host keeps the first as `blobs/<sha256>.webm` and the second as `blobs/<sha256>.json`. */
91
+ blobs: Record<string, Uint8Array>;
92
+ files: {
93
+ webm: string;
94
+ index: string;
95
+ };
96
+ index: TrackIndex;
97
+ /** Frames the track could not use, shown as the slate: `missing` (no such blob), `hash_mismatch`, `not_a_jpeg`. */
98
+ skipped: Array<{
99
+ sha256: string;
100
+ reason: "missing" | "hash_mismatch" | "not_a_jpeg";
101
+ }>;
102
+ };
103
+ /** One WebM (VP9) and one index per session that has a frame. A session's pictures are read one at a time and written straight to a private temporary
104
+ * directory made for the session and removed after, so memory holds one picture at a time (not every picture of the recording), and ffmpeg reads
105
+ * only files there: it is started with a short environment, not given a URL, and nothing it writes reaches the recording but the video. `readBlob`
106
+ * may be asked for the same blob again by another session. */
107
+ export declare function exportFrameTracks(options: TrackOptions): Promise<Track[]>;