@parcha/pi-browser 0.1.0-beta.13 → 0.1.0-beta.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,259 @@
1
+ import { spawn } from "node:child_process";
2
+ import { createHash } from "node:crypto";
3
+ import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
4
+ import os from "node:os";
5
+ import path from "node:path";
6
+ import { jpegSampling, jpegSize } from "./jpeg.js";
7
+ export class TrackExportError extends Error {
8
+ code;
9
+ constructor(code, message) {
10
+ super(message);
11
+ this.name = "TrackExportError";
12
+ this.code = code;
13
+ }
14
+ }
15
+ const even = (n) => (n % 2 === 0 ? n : n + 1);
16
+ /** The timeline of one session's lines. Each frame is shown until the next one; a gap puts the slate up (from the moment it was written, except
17
+ * `process_killed`, which starts at the last frame the earlier process saved: nothing is known after it) until the next frame or the end.
18
+ * Null when the session has no frame. `skip` names frames whose picture could not be read: they show the slate and are not listed. */
19
+ export function planTrack(lines, options = {}) {
20
+ const fps = options.fps ?? 10;
21
+ const one = Math.max(1, Math.round(1000 / fps));
22
+ const rows = [...lines].map((row, order) => ({ row: row, order })).filter(({ row }) => (row.kind === "frame" || row.kind === "frame.gap") && typeof row.t === "number" && row.data && typeof row.data.session === "string");
23
+ const byTime = [...rows].sort((a, b) => a.row.t - b.row.t || a.order - b.order);
24
+ const first = byTime.find(({ row }) => row.kind === "frame");
25
+ if (!first)
26
+ return null;
27
+ const session = first.row.data.session;
28
+ const events = byTime.filter(({ row }) => row.data.session === session);
29
+ const from = first.row.t;
30
+ const segments = [];
31
+ const frames = [];
32
+ const gaps = [];
33
+ const state = { current: null, gap: null };
34
+ const close = (at) => {
35
+ const cur = state.current;
36
+ if (cur && at > cur.start)
37
+ segments.push({ start: cur.start, end: at, sha256: cur.sha256, ...(cur.reason ? { reason: cur.reason } : {}) });
38
+ state.current = null;
39
+ };
40
+ const closeGap = (at) => { if (state.gap)
41
+ gaps.push({ ...state.gap, to: at }); state.gap = null; };
42
+ let last = from;
43
+ for (const { row } of events) {
44
+ const t = row.t;
45
+ last = Math.max(last, t);
46
+ if (t < from)
47
+ continue; // a gap written before the first frame has no place in the track
48
+ if (row.kind === "frame") {
49
+ const sha = row.blobs?.[0] ?? "";
50
+ close(t);
51
+ closeGap(t);
52
+ if (options.skip?.has(sha)) {
53
+ state.current = { start: t, sha256: null, reason: "frame_unreadable" };
54
+ state.gap = { from: t, reason: "frame_unreadable" };
55
+ }
56
+ else {
57
+ state.current = { start: t, sha256: sha };
58
+ frames.push({ t, ms: t - from, sha256: sha, w: row.data.w, h: row.data.h });
59
+ }
60
+ continue;
61
+ }
62
+ const data = row.data;
63
+ const cur = state.current;
64
+ if (!cur)
65
+ continue;
66
+ const opened = (at) => { state.gap = { from: at, reason: data.reason, ...(data.detail ? { detail: data.detail } : {}) }; };
67
+ if (cur.sha256 === null) {
68
+ closeGap(t);
69
+ opened(t);
70
+ continue;
71
+ } // the slate is already up
72
+ const at = data.reason === "process_killed" ? Math.min(t, Math.max(cur.start, data.from)) : t;
73
+ close(at);
74
+ state.current = { start: at, sha256: null, reason: data.reason };
75
+ opened(at);
76
+ }
77
+ // The track ends where the recording does; only a track that would have no length at all is given one frame's time.
78
+ let to = Math.max(options.endAt ?? 0, last);
79
+ if (to - from < one)
80
+ to = from + one;
81
+ close(to);
82
+ closeGap(to);
83
+ const width = even(frames.reduce((max, f) => Math.max(max, f.w), 2));
84
+ const height = even(frames.reduce((max, f) => Math.max(max, f.h), 2));
85
+ return { session, conv: first.row.conv, fps, from, to, width, height, segments, frames, gaps };
86
+ }
87
+ const SLATE_COLOR = "0x1f2933";
88
+ const sha256 = (bytes) => createHash("sha256").update(bytes).digest("hex");
89
+ /** One WebM (VP9) and one index per session that has a frame. A session's pictures are read one at a time and written straight to a private temporary
90
+ * directory made for the session and removed after, so memory holds one picture at a time (not every picture of the recording), and ffmpeg reads
91
+ * only files there: it is started with a short environment, not given a URL, and nothing it writes reaches the recording but the video. `readBlob`
92
+ * may be asked for the same blob again by another session. */
93
+ export async function exportFrameTracks(options) {
94
+ const fps = options.fps ?? 10;
95
+ const lines = [...options.lines];
96
+ // The end of the recording is the last line of any kind, the same for every session.
97
+ const recordingEnd = lines.reduce((end, row) => (typeof row?.t === "number" && row.t > end ? row.t : end), 0);
98
+ const endAt = options.endAt ?? recordingEnd;
99
+ const sessions = new Map();
100
+ for (const row of lines) {
101
+ const session = row?.data?.session;
102
+ if ((row?.kind === "frame" || row?.kind === "frame.gap") && typeof session === "string")
103
+ (sessions.get(session) ?? sessions.set(session, []).get(session)).push(row);
104
+ }
105
+ const tracks = [];
106
+ for (const [, rows] of sessions) {
107
+ throwIfAborted(options.signal);
108
+ // One deadline for the session's whole export (every ffmpeg call of it): each call gets the time that is left.
109
+ const deadline = Date.now() + (options.timeoutMs ?? 120_000);
110
+ const dir = await mkdtemp(path.join(os.tmpdir(), "pib-track-"));
111
+ try {
112
+ const names = new Map();
113
+ const skip = new Set();
114
+ const skipped = [];
115
+ let n = 0;
116
+ for (const row of rows) {
117
+ if (row.kind !== "frame")
118
+ continue;
119
+ const hash = row.blobs?.[0];
120
+ if (!hash || names.has(hash) || skip.has(hash))
121
+ continue;
122
+ let bytes;
123
+ try {
124
+ bytes = await options.readBlob(hash);
125
+ }
126
+ catch {
127
+ bytes = null;
128
+ }
129
+ throwIfAborted(options.signal);
130
+ const reason = !bytes ? "missing" : sha256(bytes) !== hash ? "hash_mismatch" : jpegSize(bytes) === null ? "not_a_jpeg" : null;
131
+ if (reason || !bytes) {
132
+ skip.add(hash);
133
+ skipped.push({ sha256: hash, reason: reason ?? "missing" });
134
+ continue;
135
+ }
136
+ const file = path.join(dir, `f${n++}.jpg`);
137
+ await writeFile(file, bytes);
138
+ const size = jpegSize(bytes);
139
+ names.set(hash, { file, w: size.w, h: size.h, sampling: jpegSampling(bytes) });
140
+ }
141
+ let plan = null;
142
+ let webm = null;
143
+ for (let attempt = 0;; attempt += 1) {
144
+ plan = planTrack(rows, { fps, endAt, skip });
145
+ if (!plan)
146
+ break;
147
+ throwIfAborted(options.signal);
148
+ try {
149
+ webm = await encode(plan, names, dir, options, deadline);
150
+ break;
151
+ }
152
+ catch (error) {
153
+ // A picture whose header reads but whose data does not decode fails the whole encode: find it, show the slate for it, try once more.
154
+ if (attempt > 0 || !(error instanceof TrackExportError) || error.code !== "ffmpeg_failed")
155
+ throw error;
156
+ const undecodable = await findUndecodable(names, dir, options, deadline);
157
+ if (undecodable.length === 0)
158
+ throw error;
159
+ for (const hash of undecodable) {
160
+ names.delete(hash);
161
+ skip.add(hash);
162
+ skipped.push({ sha256: hash, reason: "not_a_jpeg" });
163
+ }
164
+ }
165
+ }
166
+ if (!plan || !webm)
167
+ continue;
168
+ const index = { version: 1, session: plan.session, fps: plan.fps, from: plan.from, to: plan.to, width: plan.width, height: plan.height, codec: "vp9", frames: plan.frames, gaps: plan.gaps };
169
+ const indexBytes = Buffer.from(`${JSON.stringify(index)}\n`, "utf8");
170
+ const files = { webm: sha256(webm), index: sha256(indexBytes) };
171
+ tracks.push({
172
+ line: { channel: "live", source: "browser", kind: "frame.track", t: plan.to, conv: plan.conv, data: { session: plan.session, fps: plan.fps, from: plan.from, to: plan.to }, blobs: [files.webm, files.index] },
173
+ blobs: { [files.webm]: webm, [files.index]: indexBytes },
174
+ files, index, skipped,
175
+ });
176
+ }
177
+ finally {
178
+ await rm(dir, { recursive: true, force: true });
179
+ }
180
+ }
181
+ return tracks.sort((a, b) => a.line.data.from - b.line.data.from);
182
+ }
183
+ const throwIfAborted = (signal) => { if (signal?.aborted)
184
+ throw new TrackExportError("aborted", "the export was aborted"); };
185
+ const quote = (file) => `'${file.replaceAll("'", "'\\''")}'`;
186
+ /** The layout a browser's JPEG has, and the one every picture is brought to: 4:2:0. */
187
+ const COMMON_SAMPLING = "22,11,11";
188
+ /** The pictures ffmpeg cannot decode, found one at a time (only after an encode has failed). */
189
+ async function findUndecodable(names, dir, options, deadline) {
190
+ const bad = [];
191
+ for (const [hash, picture] of names) {
192
+ throwIfAborted(options.signal);
193
+ try {
194
+ await run(options.ffmpeg ?? "ffmpeg", ["-hide_banner", "-loglevel", "error", "-nostdin", "-xerror", "-i", path.basename(picture.file), "-f", "null", "-"], dir, options, deadline);
195
+ }
196
+ catch (error) {
197
+ if (error instanceof TrackExportError && error.code === "ffmpeg_failed")
198
+ bad.push(hash);
199
+ else
200
+ throw error;
201
+ }
202
+ }
203
+ return bad;
204
+ }
205
+ async function encode(plan, names, dir, options, deadline) {
206
+ // The slate is made at the canvas's own size, so no frame is ever scaled: a smaller frame is centred on the canvas at the size it was recorded.
207
+ const slate = path.join(dir, "slate.jpg");
208
+ await run(options.ffmpeg ?? "ffmpeg", ["-hide_banner", "-loglevel", "error", "-nostdin", "-y", "-f", "lavfi", "-i", `color=c=${SLATE_COLOR}:s=${plan.width}x${plan.height}`, "-frames:v", "1", "slate.jpg"], dir, options, deadline);
209
+ // ffmpeg reconfigures its filter graph when a picture's size or pixel layout differs from the one before, and the picture it was holding is lost (the
210
+ // video then shows the next picture for its time). So every picture reaches the concat as the canvas's size and 4:2:0: one that is not is padded to
211
+ // the canvas, at the size it was recorded and centred, and converted, first.
212
+ const ready = new Map();
213
+ let fixed = 0;
214
+ for (const [hash, picture] of names) {
215
+ if (picture.w === plan.width && picture.h === plan.height && picture.sampling === COMMON_SAMPLING) {
216
+ ready.set(hash, picture.file);
217
+ continue;
218
+ }
219
+ throwIfAborted(options.signal);
220
+ const out = `n${fixed++}.jpg`;
221
+ await run(options.ffmpeg ?? "ffmpeg", ["-hide_banner", "-loglevel", "error", "-nostdin", "-y", "-i", path.basename(picture.file), "-vf", `pad=${plan.width}:${plan.height}:(ow-iw)/2:(oh-ih)/2:color=${SLATE_COLOR},format=yuvj420p`, "-frames:v", "1", "-q:v", "2", out], dir, options, deadline);
222
+ ready.set(hash, path.join(dir, out));
223
+ }
224
+ const list = [];
225
+ const file = (sha) => (sha === null ? slate : ready.get(sha));
226
+ for (const segment of plan.segments)
227
+ list.push(`file ${quote(file(segment.sha256))}`, `duration ${((segment.end - segment.start) / 1000).toFixed(3)}`);
228
+ list.push(`file ${quote(file(plan.segments.at(-1).sha256))}`); // the concat demuxer ignores the last duration unless its file is named again
229
+ await writeFile(path.join(dir, "list.txt"), `${list.join("\n")}\n`);
230
+ const filter = `fps=${plan.fps},format=yuv420p`;
231
+ const args = ["-hide_banner", "-loglevel", "error", "-nostdin", "-y", "-f", "concat", "-safe", "0", "-i", "list.txt", "-vf", filter, "-fps_mode", "cfr", "-c:v", "libvpx-vp9", "-b:v", "0", "-crf", "40", "-row-mt", "1", "-deadline", "good", "-cpu-used", "5", "-an", "out.webm"];
232
+ await run(options.ffmpeg ?? "ffmpeg", args, dir, options, deadline);
233
+ return new Uint8Array(await readFile(path.join(dir, "out.webm")));
234
+ }
235
+ function run(program, args, cwd, options, deadline) {
236
+ return new Promise((resolve, reject) => {
237
+ if (options.signal?.aborted)
238
+ return reject(new TrackExportError("aborted", "the export was aborted"));
239
+ const left = deadline - Date.now();
240
+ if (left <= 0)
241
+ return reject(new TrackExportError("ffmpeg_timeout", `the export did not finish within ${options.timeoutMs ?? 120_000} ms`));
242
+ // Only a short list of names reaches the program: not the host's keys.
243
+ const child = spawn(program, args, { cwd, env: { PATH: process.env.PATH ?? "/usr/bin:/bin", LANG: "C", LC_ALL: "C", HOME: cwd }, stdio: ["ignore", "ignore", "pipe"] });
244
+ let stderr = "";
245
+ let settled = false;
246
+ const finish = (error) => { if (settled)
247
+ return; settled = true; clearTimeout(timer); options.signal?.removeEventListener("abort", onAbort); if (error)
248
+ reject(error);
249
+ else
250
+ resolve(); };
251
+ const timer = setTimeout(() => { child.kill("SIGKILL"); finish(new TrackExportError("ffmpeg_timeout", `ffmpeg did not finish within ${options.timeoutMs ?? 120_000} ms of the session's export`)); }, left);
252
+ const onAbort = () => { child.kill("SIGKILL"); finish(new TrackExportError("aborted", "the export was aborted")); };
253
+ options.signal?.addEventListener("abort", onAbort, { once: true });
254
+ child.stderr.on("data", (chunk) => { if (stderr.length < 4_000)
255
+ stderr += chunk.toString("utf8"); });
256
+ child.once("error", (error) => finish(new TrackExportError("ffmpeg_missing", `could not start ffmpeg (${program}): ${error.code ?? error.message}`)));
257
+ child.once("close", (code) => finish(code === 0 ? undefined : new TrackExportError("ffmpeg_failed", `ffmpeg exited with ${code}: ${stderr.trim().slice(0, 1_000)}`)));
258
+ });
259
+ }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@parcha/pi-browser",
3
- "version": "0.1.0-beta.13",
3
+ "version": "0.1.0-beta.14",
4
4
  "description": "A durable browser for pi agents: session custody that survives crashes, effect and replay classes per tool, evidence with SHA-256, typed failures, and no credential in any model-visible string.",
5
5
  "keywords": [
6
6
  "pi-package",
@@ -84,9 +84,9 @@
84
84
  },
85
85
  "peerDependencies": {
86
86
  "@browserbasehq/sdk": ">=2.20.0 <3",
87
- "@earendil-works/chord": ">=1.1.0 <1.3.0",
87
+ "@earendil-works/chord": ">=1.1.0 <1.4.0",
88
88
  "@earendil-works/pi-coding-agent": "*",
89
- "@earendil-works/pi-durable": ">=1.1.0 <1.3.0"
89
+ "@earendil-works/pi-durable": ">=1.1.0 <1.4.0"
90
90
  },
91
91
  "peerDependenciesMeta": {
92
92
  "@browserbasehq/sdk": {