@parcha/pi-browser 0.1.0-beta.18 → 0.1.0-beta.20

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. package/CHANGELOG.md +5 -0
  2. package/README.md +177 -39
  3. package/dist/coding-agent-local.js +2 -1
  4. package/dist/coding-agent.d.ts +42 -2
  5. package/dist/coding-agent.js +159 -21
  6. package/dist/core/clip.d.ts +31 -0
  7. package/dist/core/clip.js +49 -0
  8. package/dist/core/contract.d.ts +4 -1
  9. package/dist/core/contract.js +40 -7
  10. package/dist/core/custody.d.ts +79 -5
  11. package/dist/core/custody.js +66 -5
  12. package/dist/core/decisions.d.ts +4 -0
  13. package/dist/core/dial.d.ts +7 -0
  14. package/dist/core/dial.js +41 -0
  15. package/dist/core/downloads.d.ts +1 -0
  16. package/dist/core/downloads.js +17 -5
  17. package/dist/core/driving-guard.d.ts +13 -0
  18. package/dist/core/driving-guard.js +61 -0
  19. package/dist/core/driving.d.ts +61 -0
  20. package/dist/core/driving.js +156 -0
  21. package/dist/core/effects.js +33 -13
  22. package/dist/core/evidence.d.ts +3 -1
  23. package/dist/core/failures.d.ts +5 -1
  24. package/dist/core/failures.js +20 -4
  25. package/dist/core/files.d.ts +8 -0
  26. package/dist/core/files.js +74 -0
  27. package/dist/core/find.d.ts +3 -1
  28. package/dist/core/find.js +2 -1
  29. package/dist/core/guard.d.ts +9 -1
  30. package/dist/core/guard.js +7 -2
  31. package/dist/core/host.d.ts +49 -1
  32. package/dist/core/images.d.ts +10 -2
  33. package/dist/core/images.js +4 -4
  34. package/dist/core/lease.d.ts +1 -1
  35. package/dist/core/lease.js +2 -1
  36. package/dist/core/session-control.d.ts +2 -0
  37. package/dist/core/session-control.js +5 -0
  38. package/dist/core/tools.d.ts +70 -4
  39. package/dist/core/tools.js +288 -45
  40. package/dist/core/unanswered.d.ts +3 -0
  41. package/dist/core/unanswered.js +17 -0
  42. package/dist/core/usage.d.ts +2 -0
  43. package/dist/core/usage.js +6 -0
  44. package/dist/core/web.d.ts +1 -1
  45. package/dist/core/web.js +9 -7
  46. package/dist/driver/extension-worker.d.ts +4 -2
  47. package/dist/driver/extension-worker.js +16 -10
  48. package/dist/driver/page-driver.d.ts +8 -0
  49. package/dist/driver/page-driver.js +128 -0
  50. package/dist/driver/stagehand.d.ts +3 -1
  51. package/dist/driver/stagehand.js +40 -66
  52. package/dist/durable.d.ts +16 -4
  53. package/dist/durable.js +415 -82
  54. package/dist/index.d.ts +2 -0
  55. package/dist/index.js +2 -0
  56. package/dist/live-view/index.d.ts +4 -0
  57. package/dist/live-view/index.js +4 -0
  58. package/dist/live-view/protocol.d.ts +152 -0
  59. package/dist/live-view/protocol.js +84 -0
  60. package/dist/live-view/provider.d.ts +6 -0
  61. package/dist/live-view/provider.js +23 -0
  62. package/dist/live-view/server.d.ts +50 -0
  63. package/dist/live-view/server.js +342 -0
  64. package/dist/live-view/stream.d.ts +7 -0
  65. package/dist/live-view/stream.js +388 -0
  66. package/dist/live-view/token.d.ts +6 -0
  67. package/dist/live-view/token.js +13 -0
  68. package/dist/live-view/viewer.d.ts +6 -0
  69. package/dist/live-view/viewer.js +350 -0
  70. package/dist/live-view/ws.d.ts +19 -0
  71. package/dist/live-view/ws.js +176 -0
  72. package/dist/providers/assigned.d.ts +18 -0
  73. package/dist/providers/assigned.js +76 -0
  74. package/dist/providers/cdp-call.js +12 -2
  75. package/dist/providers/cdp.js +37 -11
  76. package/dist/recorder/follow.d.ts +10 -0
  77. package/dist/recorder/follow.js +27 -2
  78. package/dist/recorder/frames.d.ts +7 -0
  79. package/dist/recorder/frames.js +73 -5
  80. package/dist/store.d.ts +22 -3
  81. package/dist/store.js +75 -4
  82. package/dist/testing/backend.d.ts +26 -0
  83. package/dist/testing/backend.js +21 -3
  84. package/dist/testing/driver.d.ts +18 -1
  85. package/dist/testing/driver.js +4 -0
  86. package/dist/web.js +2 -2
  87. package/package.json +10 -2
@@ -0,0 +1,342 @@
1
+ import { createServer } from "node:http";
2
+ import { drivingGates } from "../core/driving.js";
3
+ import { CLOSE_CODES, encodeFrame, LIMITS, parseClientMessage } from "./protocol.js";
4
+ import { openStream } from "./stream.js";
5
+ import { newToken, newViewerId, tokenMatches } from "./token.js";
6
+ import { viewerAssets } from "./viewer.js";
7
+ import { acceptWebSocket } from "./ws.js";
8
+ const ID = /^[A-Za-z0-9_-]{22}$/;
9
+ /** A viewer that cannot take frames as fast as they come drops them: past this many unsent bytes a frame is acknowledged unsent. */
10
+ const SLOW_VIEWER_BYTES = 512 * 1024;
11
+ /** Input messages are tiny: a whole message, fragments included, over this ends the connection (1009). */
12
+ const MAX_MESSAGE_BYTES = 64 * 1024;
13
+ /** A viewer whose messages the protocol does not know: one is dropped with a notice, and past this many within the window the connection is closed. */
14
+ /** While a person's command runs, their driving turn is renewed this often (the gate's window is 3 s by default). */
15
+ const KEEP_DRIVING_MS = 1_000;
16
+ /** Messages a viewer may have waiting behind the browser command that is running; the rest are dropped with `held busy`, and a viewer that keeps doing it is closed. */
17
+ const MAX_QUEUED_INPUT = 32;
18
+ const MAX_BAD_MESSAGES = 20;
19
+ const BAD_WINDOW_MS = 10_000;
20
+ export function createLiveView(options) {
21
+ const helloTimeoutMs = options.helloTimeoutMs ?? 5_000;
22
+ const maxPending = options.maxPending ?? 8;
23
+ const byTag = new Map();
24
+ const byId = new Map();
25
+ const mounted = options.baseUrl ? new URL(options.baseUrl) : null;
26
+ let origin = mounted?.origin ?? null;
27
+ const prefix = mounted ? mounted.pathname.replace(/\/+$/, "") : "";
28
+ let server = null;
29
+ let starting = null;
30
+ let closed = false;
31
+ /** The opens waiting for the server, by tag: a `release` or a `close` that comes meanwhile marks them cancelled, and each open takes its own mark off when it ends, so
32
+ * nothing is kept for a tag that has no open in flight. */
33
+ const opening = new Map();
34
+ const need = (value, what) => { if (value === undefined)
35
+ throw new Error(`the live view has no ${what}`); return value; };
36
+ const send = (conn, message) => { conn.send(JSON.stringify(message)); };
37
+ const refuse = (conn, reason) => { send(conn, { type: "refused", reason }); conn.close(CLOSE_CODES[reason], reason); };
38
+ /** The request's path, or null when its target cannot be read as one (`//[`): such a request is nobody's, so it is not this host's and nothing is thrown into the
39
+ * listener of the server that mounts it. */
40
+ const pathOf = (req) => { try {
41
+ return new URL(req.url ?? "/", "http://placeholder").pathname;
42
+ }
43
+ catch {
44
+ return null;
45
+ } };
46
+ /** The path after the mount, or null when the request is not under it ("/mount/liveness" is not under "/mount/live"). */
47
+ const relative = (pathname) => {
48
+ if (!pathname.startsWith(prefix))
49
+ return null;
50
+ const rest = pathname.slice(prefix.length);
51
+ return rest.startsWith("/") ? rest : null;
52
+ };
53
+ const wsOrigin = () => (origin ?? "").replace(/^http/, "ws");
54
+ const securityHeaders = () => ({
55
+ "Referrer-Policy": "no-referrer",
56
+ "Cache-Control": "no-store",
57
+ "X-Content-Type-Options": "nosniff",
58
+ "Content-Security-Policy": `default-src 'self'; script-src 'self'; style-src 'self'; img-src 'self' data: blob:; connect-src 'self' ${wsOrigin()}; base-uri 'none'; form-action 'none'; frame-ancestors ${options.hostOrigin ?? "'none'"}`,
59
+ });
60
+ const handle = (req, res) => {
61
+ const path = pathOf(req);
62
+ const rel = path === null ? null : relative(path);
63
+ if (rel === null)
64
+ return false;
65
+ const page = /^\/v\/([^/]+)$/.exec(rel);
66
+ const upgradeRoute = /^\/v\/([^/]+)\/ws$/.exec(rel);
67
+ if (!page && !upgradeRoute && rel !== "/viewer.js" && rel !== "/viewer.css")
68
+ return false;
69
+ const reply = (status, type, body) => { res.writeHead(status, { ...securityHeaders(), "Content-Type": type, "Content-Length": Buffer.byteLength(body) }); res.end(req.method === "HEAD" ? undefined : body); return true; };
70
+ if (req.method !== "GET" && req.method !== "HEAD")
71
+ return reply(405, "text/plain", "method not allowed");
72
+ if (upgradeRoute)
73
+ return reply(426, "text/plain", "upgrade required");
74
+ const viewer = options.viewer ?? viewerAssets;
75
+ if (rel === "/viewer.js")
76
+ return reply(200, "text/javascript; charset=utf-8", viewer.js);
77
+ if (rel === "/viewer.css")
78
+ return reply(200, "text/css; charset=utf-8", viewer.css);
79
+ const entry = ID.test(page[1]) ? byId.get(page[1]) : undefined;
80
+ if (!entry || entry.released)
81
+ return reply(404, "text/plain", "not found");
82
+ return reply(200, "text/html; charset=utf-8", viewer.html);
83
+ };
84
+ const upgrade = (req, socket, head) => {
85
+ const path = pathOf(req);
86
+ const rel = path === null ? null : relative(path);
87
+ const match = rel === null ? null : /^\/v\/([^/]+)\/ws$/.exec(rel);
88
+ if (!match)
89
+ return false;
90
+ const deny = (status) => { socket.end(`HTTP/1.1 ${status}\r\nConnection: close\r\nContent-Length: 0\r\n\r\n`); return true; };
91
+ // A browser always sends its page's origin on a WebSocket; another tab's script cannot forge it. Anything but the viewer's own is refused.
92
+ if (origin === null || req.headers.origin !== origin)
93
+ return deny("403 Forbidden");
94
+ const entry = ID.test(match[1]) ? byId.get(match[1]) : undefined;
95
+ if (!entry || entry.released)
96
+ return deny("404 Not Found");
97
+ if (entry.pending.size >= maxPending)
98
+ return deny("503 Service Unavailable");
99
+ const conn = acceptWebSocket(req, socket, head, { maxPayload: MAX_MESSAGE_BYTES });
100
+ if (!conn)
101
+ return true;
102
+ entry.pending.add(conn);
103
+ // The slot is freed when the connection is cut off, not when its socket finishes closing: a viewer arriving at that moment must not be counted out.
104
+ const timer = setTimeout(() => { if (entry.pending.delete(conn))
105
+ refuse(conn, "unauthorized"); }, helloTimeoutMs);
106
+ timer.unref();
107
+ conn.onClose = () => { clearTimeout(timer); entry.pending.delete(conn); if (entry.viewer?.conn === conn)
108
+ endViewer(entry); };
109
+ conn.onMessage = (data, binary) => {
110
+ if (entry.viewer?.conn === conn) {
111
+ onViewerMessage(entry, entry.viewer, data, binary);
112
+ return;
113
+ }
114
+ if (!entry.pending.has(conn))
115
+ return;
116
+ clearTimeout(timer);
117
+ onHello(entry, conn, data, binary);
118
+ };
119
+ return true;
120
+ };
121
+ /** The first message of a connection, and the only one that is read before the token is proven. */
122
+ const onHello = (entry, conn, data, binary) => {
123
+ entry.pending.delete(conn);
124
+ const message = binary || typeof data !== "string" ? null : parseClientMessage(data);
125
+ if (message?.type !== "hello" || !tokenMatches(entry.token, message.token) || entry.released)
126
+ return refuse(conn, "unauthorized");
127
+ // One authenticated viewer per session, told so, and the first is never replaced.
128
+ if (entry.viewer)
129
+ return refuse(conn, "viewer_in_use");
130
+ let viewer;
131
+ try {
132
+ const gate = (options.gates ?? drivingGates)(entry.tag);
133
+ const stream = (options.stream ?? openStream)(entry.cdpUrl, {
134
+ frame(jpeg, page, ack) {
135
+ if (conn.closed || conn.buffered > SLOW_VIEWER_BYTES) {
136
+ ack();
137
+ return;
138
+ }
139
+ if (!conn.send(encodeFrame(jpeg, page.w, page.h), ack))
140
+ ack();
141
+ },
142
+ page: (page) => send(conn, { type: "page", url: page.url, title: page.title }),
143
+ dialog: (dialog) => send(conn, dialog ? { type: "dialog", ...dialog } : { type: "dialog_closed" }),
144
+ closed: () => { if (entry.viewer?.conn === conn) {
145
+ send(conn, { type: "ended", reason: "browser_closed" });
146
+ conn.close(CLOSE_CODES.browser_closed, "browser_closed");
147
+ } },
148
+ });
149
+ viewer = { conn, stream, gate, unsubscribe: gate.onChange((drive) => send(conn, { type: "status", drive })), chain: Promise.resolve(), bad: [], queued: 0, tail: null };
150
+ }
151
+ catch {
152
+ conn.close(1011, "unavailable");
153
+ return;
154
+ }
155
+ entry.viewer = viewer;
156
+ // A person who sits still with the viewer open is still using the session: custody does not release it on its own while this is set.
157
+ viewer.gate.setViewing(true);
158
+ send(conn, { type: "ready", drive: viewer.gate.state() });
159
+ };
160
+ const onViewerMessage = (entry, viewer, data, binary) => {
161
+ const message = binary || typeof data !== "string" ? null : parseClientMessage(data);
162
+ // A message the protocol does not know, and one dropped because too many are waiting, are the same kind of excess: a viewer that keeps sending either is closed.
163
+ const excess = (reason) => {
164
+ const now = Date.now();
165
+ viewer.bad = [...viewer.bad.filter((at) => now - at < BAD_WINDOW_MS), now];
166
+ if (viewer.bad.length > MAX_BAD_MESSAGES) {
167
+ refuse(viewer.conn, "bad_message");
168
+ return;
169
+ }
170
+ send(viewer.conn, { type: "held", reason });
171
+ };
172
+ if (!message || message.type === "hello") {
173
+ excess("bad_message");
174
+ return;
175
+ }
176
+ // While a model call is in flight the input is dropped, not queued: a keystroke replayed after the model moved the page is worse than a lost one.
177
+ if (viewer.gate.input() === "agent_acting") {
178
+ send(viewer.conn, { type: "held", reason: "agent_acting" });
179
+ return;
180
+ }
181
+ // Mouse moves and scrolls arrive once per animation frame, so a slow browser command is always followed by a run of them. A run is combined into its last waiting
182
+ // message (the latest position; the scroll summed) instead of queued one by one: nothing grows, order against presses and keys is kept (only the LAST waiting
183
+ // message is ever merged into), and ordinary dragging is never counted as abuse.
184
+ const waiting = viewer.tail;
185
+ if (waiting && !waiting.started && message.type === "mouse" && message.action === "move" && waiting.message.type === "mouse" && waiting.message.action === "move") {
186
+ waiting.message = message;
187
+ return;
188
+ }
189
+ // Scrolls are summed only when they were made at the same point with the same modifiers: two scrolls over different controls (or one with Control held, a zoom) are two.
190
+ if (waiting && !waiting.started && message.type === "wheel" && waiting.message.type === "wheel" && waiting.message.x === message.x && waiting.message.y === message.y && (waiting.message.modifiers ?? 0) === (message.modifiers ?? 0)) {
191
+ const clamp = (n) => Math.max(-LIMITS.wheel, Math.min(LIMITS.wheel, n));
192
+ waiting.message = { ...message, deltaX: clamp(waiting.message.deltaX + message.deltaX), deltaY: clamp(waiting.message.deltaY + message.deltaY) };
193
+ return;
194
+ }
195
+ if (viewer.queued >= MAX_QUEUED_INPUT) {
196
+ // A move that does not fit is superseded by the next one a frame later: dropping it is not misbehaviour.
197
+ if (message.type === "mouse" && message.action === "move")
198
+ return;
199
+ excess("busy");
200
+ return;
201
+ }
202
+ const item = { message, started: false };
203
+ viewer.tail = item;
204
+ viewer.queued += 1;
205
+ viewer.chain = viewer.chain.then(async () => {
206
+ item.started = true;
207
+ if (viewer.tail === item)
208
+ viewer.tail = null;
209
+ const message = item.message;
210
+ if (entry.viewer !== viewer) {
211
+ viewer.queued -= 1;
212
+ return;
213
+ }
214
+ // Messages wait their turn behind a slow command, so the gate is asked again at delivery: the model may have taken the browser since the message arrived.
215
+ if (viewer.gate.input() === "agent_acting") {
216
+ send(viewer.conn, { type: "held", reason: "agent_acting" });
217
+ viewer.queued -= 1;
218
+ return;
219
+ }
220
+ // The person's turn must not lapse while their own command runs (a navigation can take seconds, the window is 3 s): it is renewed until the command returns.
221
+ const keep = setInterval(() => { viewer.gate.input(); }, KEEP_DRIVING_MS);
222
+ keep.unref();
223
+ try {
224
+ if (message.type === "nav")
225
+ await viewer.stream.nav(message);
226
+ else if (message.type === "dialog")
227
+ await viewer.stream.dialog(message);
228
+ else
229
+ await viewer.stream.input(message);
230
+ }
231
+ catch (error) {
232
+ send(viewer.conn, { type: "held", reason: error?.reason === "bad_message" ? "bad_message" : "no_page" });
233
+ }
234
+ finally {
235
+ clearInterval(keep);
236
+ viewer.queued -= 1;
237
+ }
238
+ });
239
+ };
240
+ const endViewer = (entry) => {
241
+ const viewer = entry.viewer;
242
+ if (!viewer)
243
+ return;
244
+ entry.viewer = null;
245
+ viewer.unsubscribe();
246
+ viewer.gate.setViewing(false);
247
+ try {
248
+ viewer.stream.close();
249
+ }
250
+ catch { /* the stream is gone either way */ }
251
+ };
252
+ const ensureServer = () => {
253
+ if (mounted || origin)
254
+ return Promise.resolve();
255
+ starting ??= new Promise((resolve, reject) => {
256
+ const s = createServer((req, res) => { if (!handle(req, res)) {
257
+ res.writeHead(404, { "Content-Length": 0 });
258
+ res.end();
259
+ } });
260
+ s.on("upgrade", (req, socket, head) => { if (!upgrade(req, socket, head))
261
+ socket.destroy(); });
262
+ s.once("error", reject);
263
+ // Loopback only, never 0.0.0.0: the host that wants more mounts the handler on its own server.
264
+ s.listen(0, "127.0.0.1", () => { s.unref(); server = s; origin = `http://127.0.0.1:${s.address().port}`; resolve(); });
265
+ });
266
+ return starting;
267
+ };
268
+ const release = async (tag) => {
269
+ for (const mark of opening.get(tag) ?? [])
270
+ mark.cancelled = true;
271
+ const entry = byTag.get(tag);
272
+ if (!entry)
273
+ return;
274
+ byTag.delete(tag);
275
+ byId.delete(entry.id);
276
+ entry.released = true;
277
+ for (const conn of [...entry.pending, ...(entry.viewer ? [entry.viewer.conn] : [])]) {
278
+ send(conn, { type: "ended", reason: "released" });
279
+ conn.close(CLOSE_CODES.released, "released");
280
+ }
281
+ entry.pending.clear();
282
+ const viewer = entry.viewer;
283
+ endViewer(entry);
284
+ // The gate is the session's, so it ends with it even when no viewer ever took it.
285
+ (options.gates ?? drivingGates)(tag).end();
286
+ void viewer;
287
+ };
288
+ return {
289
+ handle,
290
+ upgrade,
291
+ async open(tag, cdpUrl) {
292
+ if (closed)
293
+ throw new Error("the live view is closed");
294
+ const known = byTag.get(tag);
295
+ if (known?.view)
296
+ return known.view;
297
+ const mark = { cancelled: false };
298
+ const marks = opening.get(tag) ?? new Set();
299
+ marks.add(mark);
300
+ opening.set(tag, marks);
301
+ try {
302
+ await ensureServer();
303
+ // A close or a release that came while the server was starting is not undone by the open that was waiting for it.
304
+ if (closed || mark.cancelled)
305
+ throw new Error("the live view was closed, or the session released, while its viewer was being opened");
306
+ }
307
+ finally {
308
+ marks.delete(mark);
309
+ if (marks.size === 0 && opening.get(tag) === marks)
310
+ opening.delete(tag);
311
+ }
312
+ const raced = byTag.get(tag);
313
+ if (raced?.view)
314
+ return raced.view;
315
+ const entry = { tag, id: newViewerId(), token: newToken(), cdpUrl, view: null, released: false, viewer: null, pending: new Set() };
316
+ const url = `${origin}${prefix}/v/${entry.id}#t=${entry.token}`;
317
+ entry.view = { fullscreen: url, framed: url, pages: [] };
318
+ byTag.set(tag, entry);
319
+ byId.set(entry.id, entry);
320
+ return entry.view;
321
+ },
322
+ known: (tag) => { const entry = byTag.get(tag); return entry && !entry.released ? entry.view : null; },
323
+ driving: (tag) => { const entry = byTag.get(tag); return !!entry?.viewer && (options.gates ?? drivingGates)(tag).state() === "person"; },
324
+ tracked: () => ({ sessions: byTag.size, opening: [...opening.values()].reduce((n, marks) => n + marks.size, 0) }),
325
+ release,
326
+ async close() {
327
+ closed = true;
328
+ for (const marks of opening.values())
329
+ for (const mark of marks)
330
+ mark.cancelled = true;
331
+ // An open that is starting the server finishes (and is refused) before the server is looked for: nothing is left listening.
332
+ if (starting)
333
+ await starting.catch(() => undefined);
334
+ for (const tag of [...byTag.keys()])
335
+ await release(tag);
336
+ const s = server;
337
+ server = null;
338
+ if (s)
339
+ await new Promise((resolve) => { s.close(() => resolve()); s.closeAllConnections(); });
340
+ },
341
+ };
342
+ }
@@ -0,0 +1,7 @@
1
+ import type { StreamFactory } from "./protocol.js";
2
+ /** Why one input or nav message was not delivered: the server tells the viewer (`held`). */
3
+ export declare class StreamRefusal extends Error {
4
+ readonly reason: "no_page" | "bad_message";
5
+ constructor(reason: "no_page" | "bad_message");
6
+ }
7
+ export declare const openStream: StreamFactory;