@parcha/pi-browser 0.1.0-beta.30 → 0.1.0-beta.32

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/CHANGELOG.md +52 -0
  2. package/NOTICE +4 -4
  3. package/README.md +41 -12
  4. package/dist/coding-agent.d.ts +1 -0
  5. package/dist/coding-agent.js +5 -4
  6. package/dist/core/contract.d.ts +9 -0
  7. package/dist/core/contract.js +15 -3
  8. package/dist/core/custody.d.ts +7 -2
  9. package/dist/core/custody.js +7 -3
  10. package/dist/core/downloads.d.ts +7 -0
  11. package/dist/core/downloads.js +13 -1
  12. package/dist/core/driving-guard.d.ts +20 -0
  13. package/dist/core/driving-guard.js +175 -23
  14. package/dist/core/driving.d.ts +56 -2
  15. package/dist/core/driving.js +189 -10
  16. package/dist/core/effects.d.ts +80 -6
  17. package/dist/core/effects.js +374 -41
  18. package/dist/core/failures.d.ts +15 -1
  19. package/dist/core/failures.js +7 -0
  20. package/dist/core/host.d.ts +11 -2
  21. package/dist/core/tools.d.ts +14 -0
  22. package/dist/core/tools.js +42 -8
  23. package/dist/driver/page-driver.js +77 -4
  24. package/dist/driver/sealed.d.ts +4 -12
  25. package/dist/driver/sealed.js +23 -37
  26. package/dist/driver/stagehand.d.ts +10 -15
  27. package/dist/driver/stagehand.js +19 -17
  28. package/dist/durable/result.d.ts +1 -2
  29. package/dist/durable/result.js +2 -2
  30. package/dist/durable/store.d.ts +4 -1
  31. package/dist/durable/store.js +16 -11
  32. package/dist/durable.d.ts +17 -1
  33. package/dist/durable.js +41 -14
  34. package/dist/live-view/index.d.ts +2 -1
  35. package/dist/live-view/index.js +1 -0
  36. package/dist/live-view/protocol.d.ts +38 -6
  37. package/dist/live-view/protocol.js +2 -0
  38. package/dist/live-view/server.d.ts +24 -1
  39. package/dist/live-view/server.js +131 -11
  40. package/dist/live-view/stream.js +16 -3
  41. package/dist/live-view/viewer.js +106 -13
  42. package/dist/providers/cdp.d.ts +0 -1
  43. package/dist/providers/cdp.js +33 -11
  44. package/dist/providers/worker-extension.d.ts +6 -0
  45. package/dist/providers/worker-extension.js +17 -4
  46. package/package.json +10 -10
  47. package/dist/driver/extension-worker.d.ts +0 -9
  48. package/dist/driver/extension-worker.js +0 -185
@@ -1,5 +1,5 @@
1
1
  import { createServer } from "node:http";
2
- import { drivingGates } from "../core/driving.js";
2
+ import { drivingGates, epochOf, isHeld, isHolding } from "../core/driving.js";
3
3
  import { CLOSE_CODES, encodeFrame, LIMITS, parseClientMessage } from "./protocol.js";
4
4
  import { openStream } from "./stream.js";
5
5
  import { newToken, newViewerId, tokenMatches } from "./token.js";
@@ -18,6 +18,7 @@ const MAX_QUEUED_INPUT = 32;
18
18
  const MAX_BAD_MESSAGES = 20;
19
19
  const BAD_WINDOW_MS = 10_000;
20
20
  export function createLiveView(options) {
21
+ const takeOver = options.readOnly ? undefined : options.takeOver;
21
22
  const helloTimeoutMs = options.helloTimeoutMs ?? 5_000;
22
23
  const maxPending = options.maxPending ?? 8;
23
24
  const byTag = new Map();
@@ -138,6 +139,7 @@ export function createLiveView(options) {
138
139
  ack();
139
140
  },
140
141
  page: (page) => send(conn, { type: "page", url: page.url, title: page.title }),
142
+ noPage: () => send(conn, { type: "no_page" }),
141
143
  dialog: (dialog) => send(conn, dialog ? { type: "dialog", ...dialog } : { type: "dialog_closed" }),
142
144
  closed: () => { if (entry.viewer?.conn === conn) {
143
145
  send(conn, { type: "ended", reason: "browser_closed" });
@@ -151,9 +153,13 @@ export function createLiveView(options) {
151
153
  return;
152
154
  }
153
155
  entry.viewer = viewer;
154
- // A person who sits still with the viewer open is still using the session: custody does not release it on its own while this is set.
155
- viewer.gate.setViewing(true);
156
- send(conn, { type: "ready", drive: viewer.gate.state() });
156
+ // A person who sits still with the viewer open is still using the session: custody does not release it on its own while this is set. A read-only viewer
157
+ // only watches, so it keeps nothing open; a take-over viewer keeps it only by its hold, and its presence keeps the hold from its away window.
158
+ if (takeOver)
159
+ viewer.gate.setPresent?.(true);
160
+ else if (!options.readOnly)
161
+ viewer.gate.setViewing(true);
162
+ send(conn, { type: "ready", drive: viewer.gate.state(), ...(options.readOnly ? { readOnly: true } : takeOver ? { takeOver: true, held: isHolding(viewer.gate) } : {}) });
157
163
  };
158
164
  const onViewerMessage = (entry, viewer, data, binary) => {
159
165
  const message = binary || typeof data !== "string" ? null : parseClientMessage(data);
@@ -171,6 +177,24 @@ export function createLiveView(options) {
171
177
  excess("bad_message");
172
178
  return;
173
179
  }
180
+ // A read-only viewer's input never reaches the browser, whatever its page does: dropped here, and one that keeps sending it is closed as any excess is.
181
+ if (options.readOnly) {
182
+ excess("read_only");
183
+ return;
184
+ }
185
+ if (message.type === "take" || message.type === "give") {
186
+ if (takeOver)
187
+ hands(entry, viewer, message.type);
188
+ else
189
+ excess("bad_message");
190
+ return;
191
+ }
192
+ // A take-over viewer drives only while it holds the browser: anything else is dropped here, before the browser. A hover is not worth a notice.
193
+ if (takeOver && !isHeld(viewer.gate)) {
194
+ if (!(message.type === "mouse" && message.action === "move"))
195
+ send(viewer.conn, { type: "held", reason: "not_taken" });
196
+ return;
197
+ }
174
198
  // While a model call is in flight the input is dropped, not queued: a keystroke replayed after the model moved the page is worse than a lost one.
175
199
  if (viewer.gate.input() === "agent_acting") {
176
200
  send(viewer.conn, { type: "held", reason: "agent_acting" });
@@ -179,7 +203,9 @@ export function createLiveView(options) {
179
203
  // Mouse moves and scrolls arrive once per animation frame, so a slow browser command is always followed by a run of them. A run is combined into its last waiting
180
204
  // message (the latest position; the scroll summed) instead of queued one by one: nothing grows, order against presses and keys is kept (only the LAST waiting
181
205
  // message is ever merged into), and ordinary dragging is never counted as abuse.
182
- const waiting = viewer.tail;
206
+ // A message is merged only into one that arrived while the same hands held the browser.
207
+ const epoch = epochOf(viewer.gate);
208
+ const waiting = viewer.tail && viewer.tail.epoch === epoch ? viewer.tail : null;
183
209
  if (waiting && !waiting.started && message.type === "mouse" && message.action === "move" && waiting.message.type === "mouse" && waiting.message.action === "move") {
184
210
  waiting.message = message;
185
211
  return;
@@ -197,7 +223,7 @@ export function createLiveView(options) {
197
223
  excess("busy");
198
224
  return;
199
225
  }
200
- const item = { message, started: false };
226
+ const item = { message, started: false, epoch };
201
227
  viewer.tail = item;
202
228
  viewer.queued += 1;
203
229
  viewer.chain = viewer.chain.then(async () => {
@@ -209,18 +235,33 @@ export function createLiveView(options) {
209
235
  viewer.queued -= 1;
210
236
  return;
211
237
  }
238
+ // The hold may have ended while the message waited (handed back, idle, the agent's ask answered), or the browser changed
239
+ // hands and came back (a new hold, a page the message was not made for): it is not delivered.
240
+ if (takeOver && (!isHolding(viewer.gate) || epochOf(viewer.gate) !== item.epoch)) {
241
+ send(viewer.conn, { type: "held", reason: "not_taken" });
242
+ viewer.queued -= 1;
243
+ return;
244
+ }
212
245
  // Messages wait their turn behind a slow command, so the gate is asked again at delivery: the model may have taken the browser since the message arrived.
213
246
  if (viewer.gate.input() === "agent_acting") {
214
247
  send(viewer.conn, { type: "held", reason: "agent_acting" });
215
248
  viewer.queued -= 1;
216
249
  return;
217
250
  }
218
- // The person's turn must not lapse while their own command runs (a navigation can take seconds, the window is 3 s): it is renewed until the command returns.
219
- const keep = setInterval(() => { viewer.gate.input(); }, KEEP_DRIVING_MS);
251
+ // The gate knows the command runs: a hand-back meanwhile wakes the agent only once it returns (or is let go, `live` false from then on).
252
+ const command = viewer.gate.command?.();
253
+ const live = () => command?.live() ?? true;
254
+ // The person's turn must not lapse while their own command runs (a navigation can take seconds, the window is 3 s): it is renewed until the
255
+ // command returns, or until the gate lets it go after a hand-back (`live` false): past that a renewal would open a new turn for the person
256
+ // once the agent's call returns, refusing the agent's calls after they handed the browser back.
257
+ const keep = setInterval(() => { if (live())
258
+ viewer.gate.input();
259
+ else
260
+ clearInterval(keep); }, KEEP_DRIVING_MS);
220
261
  keep.unref();
221
262
  try {
222
263
  if (message.type === "nav")
223
- await viewer.stream.nav(message);
264
+ await viewer.stream.nav(message, live);
224
265
  else if (message.type === "dialog")
225
266
  await viewer.stream.dialog(message);
226
267
  else
@@ -231,17 +272,95 @@ export function createLiveView(options) {
231
272
  }
232
273
  finally {
233
274
  clearInterval(keep);
275
+ command?.done();
234
276
  viewer.queued -= 1;
235
277
  }
236
278
  });
237
279
  };
280
+ /** A take-over viewer's `take` or `give`. A take asks the host first (one at a time), and the hold then starts on the session's gate; however the
281
+ * hold ends, the host's `end` is called and the viewer of the moment is told. */
282
+ const hands = (entry, viewer, what) => {
283
+ const gate = viewer.gate;
284
+ if (what === "give") {
285
+ gate.give?.("given");
286
+ return;
287
+ }
288
+ if (isHolding(gate)) {
289
+ send(viewer.conn, { type: "hold", held: true });
290
+ return;
291
+ }
292
+ // Taken already, waiting for the agent's call to return.
293
+ if (isHeld(gate)) {
294
+ send(viewer.conn, { type: "held", reason: "taking" });
295
+ return;
296
+ }
297
+ // One take at a time is asked of the host: a second one meanwhile is told so.
298
+ if (entry.taking) {
299
+ send(viewer.conn, { type: "held", reason: "taking" });
300
+ return;
301
+ }
302
+ entry.taking = true;
303
+ void takeOver.take(entry.tag).then((answer) => {
304
+ entry.taking = false;
305
+ if ("refused" in answer) {
306
+ if (entry.viewer === viewer)
307
+ send(viewer.conn, { type: "take_refused", message: answer.refused });
308
+ return;
309
+ }
310
+ const ended = (why, message) => {
311
+ try {
312
+ answer.end(why);
313
+ }
314
+ catch { /* the host's own bookkeeping cannot keep the hold */ }
315
+ if (entry.viewer)
316
+ send(entry.viewer.conn, { type: "hold", held: false, reason: why, ...(message ? { message } : {}) });
317
+ };
318
+ // A session released, or a viewer gone, while the host answered: no hold starts, and the host lets go of what it took for one.
319
+ // The host, and the viewer of the moment, are told the hold is live: at once, or once the agent's call under way returns.
320
+ const started = () => {
321
+ try {
322
+ answer.started?.();
323
+ }
324
+ catch { /* the host's own bookkeeping cannot keep the hold */ }
325
+ if (entry.viewer)
326
+ send(entry.viewer.conn, { type: "hold", held: true });
327
+ };
328
+ // No hold starts (the session released, the viewer gone, or the browser held already when the host answered): the host
329
+ // lets go of what it took for one, and the viewer still there is told its take was refused.
330
+ const cancel = () => {
331
+ try {
332
+ (answer.cancel ?? (() => answer.end("released")))();
333
+ }
334
+ catch { /* as above */ }
335
+ };
336
+ if (entry.released || entry.viewer !== viewer)
337
+ return cancel();
338
+ const holder = (options.gates ?? drivingGates)(entry.tag);
339
+ // A registry from before holds has no take: it can never hand the browser over, which "held already" would not say.
340
+ if (typeof holder.take !== "function") {
341
+ cancel();
342
+ send(viewer.conn, { type: "take_refused", message: "This browser can't be handed over from the live view." });
343
+ }
344
+ else if (!holder.take(ended, started)) {
345
+ cancel();
346
+ send(viewer.conn, { type: "take_refused", message: "The browser could not be handed to you: it is held already. Try again in a moment." });
347
+ }
348
+ }, () => {
349
+ entry.taking = false;
350
+ if (entry.viewer === viewer)
351
+ send(viewer.conn, { type: "take_refused", message: "The browser could not be handed to you. Try again." });
352
+ });
353
+ };
238
354
  const endViewer = (entry) => {
239
355
  const viewer = entry.viewer;
240
356
  if (!viewer)
241
357
  return;
242
358
  entry.viewer = null;
243
359
  viewer.unsubscribe();
244
- viewer.gate.setViewing(false);
360
+ if (takeOver)
361
+ viewer.gate.setPresent?.(false);
362
+ else if (!options.readOnly)
363
+ viewer.gate.setViewing(false);
245
364
  try {
246
365
  viewer.stream.close();
247
366
  }
@@ -318,7 +437,8 @@ export function createLiveView(options) {
318
437
  return entry.view;
319
438
  },
320
439
  known: (tag) => { const entry = byTag.get(tag); return entry && !entry.released ? entry.view : null; },
321
- driving: (tag) => { const entry = byTag.get(tag); return !!entry?.viewer && (options.gates ?? drivingGates)(tag).state() === "person"; },
440
+ driving: (tag) => { const entry = byTag.get(tag); if (!entry)
441
+ return false; const gate = (options.gates ?? drivingGates)(tag); return isHeld(gate) || (!!entry.viewer && gate.state() === "person"); },
322
442
  tracked: () => ({ sessions: byTag.size, opening: [...opening.values()].reduce((n, marks) => n + marks.size, 0) }),
323
443
  release,
324
444
  async close() {
@@ -9,6 +9,8 @@ const STAND_IN_RETRY_MS = 1_000;
9
9
  const STAND_IN_TRIES = 6;
10
10
  /** Chrome sends a page's address when it moves but not its title when the document sets it, so the title is asked for at the end of a load and now and then. */
11
11
  const TITLE_POLL_MS = 1_500;
12
+ /** How long after the browser lists its pages the stream looks whether it follows one: none, and the viewer is told there is no page yet. */
13
+ const NO_PAGE_LOOK_MS = 250;
12
14
  /** A text message is one paste or one typed run: bound it so a viewer cannot make the browser take an unbounded insert. */
13
15
  const MAX_TEXT = 100_000;
14
16
  const BUTTON_BITS = { left: 1, right: 2, middle: 4 };
@@ -163,7 +165,15 @@ export const openStream = (cdpUrl, handlers) => {
163
165
  if (best)
164
166
  follow(best.id);
165
167
  };
166
- socket.onopen = () => { post("Target.setDiscoverTargets", { discover: true }); };
168
+ // A browser with no page once its pages are listed (a headless one that starts with none) has none to show yet. The list may come just before or just after
169
+ // the answer, so the stream looks a moment after the answer.
170
+ socket.onopen = () => {
171
+ void request("Target.setDiscoverTargets", { discover: true }).then(() => {
172
+ const look = setTimeout(() => { if (!current && !closing)
173
+ handlers.noPage?.(); }, NO_PAGE_LOOK_MS);
174
+ look.unref?.();
175
+ }, () => { });
176
+ };
167
177
  socket.onmessage = (event) => {
168
178
  let message;
169
179
  try {
@@ -256,6 +266,8 @@ export const openStream = (cdpUrl, handlers) => {
256
266
  handlers.dialog(null);
257
267
  }
258
268
  choose();
269
+ if (!current)
270
+ handlers.noPage?.();
259
271
  }
260
272
  }
261
273
  catch { /* a handler that throws must not end the browser's socket */ }
@@ -319,7 +331,7 @@ export const openStream = (cdpUrl, handlers) => {
319
331
  else
320
332
  return bad();
321
333
  },
322
- async nav(message) {
334
+ async nav(message, live = () => true) {
323
335
  const sessionId = page();
324
336
  if (message === null || typeof message !== "object" || message.type !== "nav")
325
337
  return bad();
@@ -330,7 +342,8 @@ export const openStream = (cdpUrl, handlers) => {
330
342
  if (message.action === "back") {
331
343
  const history = await request("Page.getNavigationHistory", {}, sessionId);
332
344
  const entry = history?.entries?.[Number(history?.currentIndex) - 1];
333
- if (!entry || !Number.isInteger(entry.id))
345
+ // A command let go meanwhile (handed back, and the agent's call went on past the bound) does not move the page under it.
346
+ if (!entry || !Number.isInteger(entry.id) || !live())
334
347
  return;
335
348
  await request("Page.navigateToHistoryEntry", { entryId: entry.id }, sessionId);
336
349
  return;
@@ -5,6 +5,9 @@
5
5
  // its first message; nothing else is sent before the server has answered. Frames arrive as binary [u16 w][u16 h][jpeg] and are drawn to the canvas; mouse, wheel, key and paste on the canvas go
6
6
  // back as protocol.ts messages in the page's own CSS pixels (the w and h of the latest frame), and the address bar and its back and reload buttons go back as `nav`. It writes no frame to
7
7
  // storage: a frame lives in the canvas and in the one decode in flight.
8
+ //
9
+ // A take-over viewer (`ready` with takeOver) shows Take over / Hand back in its own bar: it sends `take` and `give`, and nothing else until the host
10
+ // says it holds the browser (`hold`); the host drops anything else it sends outside a hold anyway (held not_taken).
8
11
  const html = `<!doctype html>
9
12
  <html lang="en">
10
13
  <head>
@@ -18,14 +21,17 @@ const html = `<!doctype html>
18
21
  <button id="back" type="button" title="Back" aria-label="Back">&#8592;</button>
19
22
  <button id="reload" type="button" title="Reload" aria-label="Reload">&#8635;</button>
20
23
  <form id="go" autocomplete="off"><input id="url" type="text" spellcheck="false" aria-label="Address" placeholder="https://"></form>
24
+ <button id="hands" type="button" hidden>Take over</button>
21
25
  <span id="drive" aria-live="polite"></span>
22
26
  </header>
27
+ <div id="watching" role="note" hidden><span id="watching-text">View only: you're watching the agent's browser; your clicks and keys are not sent.</span><span id="where"></span></div>
23
28
  <div id="notice" role="status" hidden></div>
24
29
  <div id="still" role="status" hidden>This page is in a background tab or has not drawn yet, so there is no picture until it is shown. Typing and clicking still work.</div>
25
30
  <div id="dialog" role="alertdialog" aria-modal="true" hidden>
26
31
  <div id="dialog-box">
27
32
  <p id="dialog-kind"></p>
28
33
  <p id="dialog-message"></p>
34
+ <p id="dialog-held" hidden>Take over to answer.</p>
29
35
  <input id="dialog-prompt" type="text" hidden aria-label="Answer">
30
36
  <div id="dialog-buttons"><button id="dialog-accept" type="button">OK</button><button id="dialog-dismiss" type="button">Cancel</button></div>
31
37
  </div>
@@ -46,21 +52,37 @@ body { display: flex; flex-direction: column; }
46
52
  #drive { min-width: 9em; text-align: right; color: #aaa; }
47
53
  #drive.agent { color: #f0b429; }
48
54
  #drive.person { color: #6cc070; }
55
+ .view-only #screen, .not-held #screen { cursor: not-allowed; }
56
+ #bar #hands { background: #1f4d2b; border-color: #2f6b3d; position: relative; z-index: 2; }
57
+ #bar #hands.holding { background: #5a3b00; border-color: #7a5410; }
58
+ #hands[hidden] { display: none; }
49
59
  #notice { padding: 6px 10px; background: #5a3b00; color: #ffe2a8; }
50
- #notice.fatal { background: #5a1a1a; color: #ffc4c4; }
51
- #stage { flex: 1; min-height: 0; display: flex; align-items: flex-start; justify-content: center; overflow: auto; }
52
- #screen { max-width: 100%; max-height: 100%; background: #000; outline: none; cursor: default; }
60
+ /* An end (the session released, the viewer refused or disconnected) reads at a glance, over a dimmed page. */
61
+ #notice.fatal { background: #5a1a1a; color: #ffc4c4; font-size: 18px; font-weight: 600; text-align: center; padding: 18px 16px; }
62
+ body.over #stage { opacity: 0.35; }
63
+ /* Watch only: nothing in the toolbar acts, so it is not drawn; the banner says what this view is. */
64
+ .view-only #bar { display: none; }
65
+ #watching { display: flex; flex-wrap: wrap; align-items: baseline; gap: 4px 16px; padding: 10px 14px; background: #17324d; color: #d6e9ff; font-size: 16px; font-weight: 600; border-bottom: 1px solid #24507a; }
66
+ #watching-text { flex: 0 1 auto; }
67
+ /* Where the agent is: text, never a field; a long address ends in an ellipsis and its whole is the tooltip. */
68
+ #where { flex: 1 1 14em; min-width: 0; text-align: right; font-weight: 400; font-size: 14px; color: #a9c7e6; white-space: nowrap; overflow: hidden; text-overflow: ellipsis; }
69
+ #watching[hidden] { display: none; }
70
+ /* The page fills the stage at its own shape: the script sizes the canvas to fit, so a point on it maps to the page exactly. */
71
+ #stage { flex: 1; min-height: 0; display: flex; align-items: center; justify-content: center; overflow: hidden; }
72
+ #screen { display: block; background: #000; outline: none; cursor: default; }
53
73
  #screen:focus, #stage:focus-within #screen { box-shadow: 0 0 0 2px #3b82f6; }
54
74
  #still { padding: 6px 12px; background: #3a3a1a; color: #f3e9a8; border-bottom: 1px solid #5a5a2a; }
55
75
  #sink { position: fixed; left: 0; bottom: 0; width: 1px; height: 1px; padding: 0; border: 0; opacity: 0; resize: none; overflow: hidden; pointer-events: none; font-size: 16px; }
56
- #dialog { position: fixed; inset: 0; background: rgba(0, 0, 0, 0.6); display: flex; align-items: center; justify-content: center; }
57
- #dialog[hidden], #notice[hidden], #dialog-prompt[hidden] { display: none; }
76
+ #dialog { position: fixed; inset: 0; z-index: 1; background: rgba(0, 0, 0, 0.6); display: flex; align-items: center; justify-content: center; }
77
+ #dialog[hidden], #notice[hidden], #dialog-prompt[hidden], #dialog-held[hidden] { display: none; }
78
+ #dialog-held { margin: 0 0 12px; color: #ffe2a8; }
58
79
  #dialog-box { background: #222; border: 1px solid #555; border-radius: 6px; padding: 16px; width: min(28em, 90vw); }
59
80
  #dialog-kind { margin: 0 0 6px; color: #aaa; text-transform: uppercase; font-size: 12px; }
60
81
  #dialog-message { margin: 0 0 12px; white-space: pre-wrap; overflow-wrap: anywhere; }
61
82
  #dialog-prompt { width: 100%; margin: 0 0 12px; background: #111; color: #eee; border: 1px solid #444; border-radius: 4px; padding: 5px 8px; font: inherit; }
62
83
  #dialog-buttons { display: flex; gap: 8px; justify-content: flex-end; }
63
84
  #dialog-buttons button { background: #333; color: #eee; border: 1px solid #444; border-radius: 4px; padding: 5px 14px; cursor: pointer; }
85
+ #dialog-buttons button:disabled, #dialog-prompt:disabled, #bar button:disabled, #url:disabled { opacity: 0.5; cursor: not-allowed; }
64
86
  `;
65
87
  const js = `"use strict";
66
88
  (function () {
@@ -75,6 +97,17 @@ const js = `"use strict";
75
97
  // The page is shown on a canvas, which cannot take text: the keyboard's focus sits on a hidden textarea beside it, so a virtual keyboard and an IME have something to type into.
76
98
  function focusScreen() { try { sink.focus({ preventScroll: true }); } catch (e) { canvas.focus(); } }
77
99
  var ctx = canvas.getContext("2d");
100
+ var stage = $("stage");
101
+ // The canvas drawn as large as the stage allows at the page's own shape. point() reads this box, so input stays exact.
102
+ function fit() {
103
+ if (!canvas.width || !canvas.height) return;
104
+ var box = stage.getBoundingClientRect();
105
+ var scale = Math.min(box.width / canvas.width, box.height / canvas.height);
106
+ if (!(scale > 0)) return;
107
+ canvas.style.width = Math.floor(canvas.width * scale) + "px";
108
+ canvas.style.height = Math.floor(canvas.height * scale) + "px";
109
+ }
110
+ if (typeof ResizeObserver === "function") new ResizeObserver(fit).observe(stage); else window.addEventListener("resize", fit);
78
111
  var urlBox = $("url");
79
112
  var notice = $("notice");
80
113
  var drive = $("drive");
@@ -85,7 +118,7 @@ const js = `"use strict";
85
118
  var shownUrl = null;
86
119
  // A page that gives no picture (a tab opened in the background, which Chrome does not paint) leaves the canvas as it was: after a while the viewer says so, and input still works.
87
120
  function expectPicture() { drewSince = false; stillEl.hidden = true; clearTimeout(stillTimer); stillTimer = setTimeout(function () { if (!drewSince && !fatal) stillEl.hidden = false; }, 2500); }
88
- function gotPicture() { drewSince = true; clearTimeout(stillTimer); stillEl.hidden = true; }
121
+ function gotPicture() { drewSince = true; clearTimeout(stillTimer); stillEl.hidden = true; if (noPage) { noPage = false; say(""); } }
89
122
  var fatal = false;
90
123
 
91
124
  function say(text, isFatal) {
@@ -93,10 +126,11 @@ const js = `"use strict";
93
126
  notice.hidden = false;
94
127
  notice.textContent = text;
95
128
  notice.className = isFatal ? "fatal" : "";
129
+ document.body.classList.toggle("over", !!isFatal);
96
130
  }
97
131
  function setDrive(state) {
98
132
  drive.className = state === "agent" || state === "person" ? state : "";
99
- drive.textContent = state === "agent" ? "the agent is acting" : state === "person" ? "you are driving" : "";
133
+ drive.textContent = readOnly ? "view only" : takeOver && holding ? "you have the browser" : takeOver && state === "person" ? "" : state === "agent" ? "the agent is acting" : state === "person" ? "you are driving" : "";
100
134
  }
101
135
 
102
136
  if (!token) { fatal = true; say("This link has no token. Open the live view from the link the host gave you.", true); return; }
@@ -111,7 +145,46 @@ const js = `"use strict";
111
145
 
112
146
  // Input before the host's ready is refused out loud and never queued: a keystroke replayed into a page the person has since lost sight of is worse than one that was not sent. A hover is not worth a notice.
113
147
  function stillConnecting() { if (ready || fatal || closed) return false; say(WAITING); return true; }
148
+ // A view-only viewer sends nothing but its hello: the server drops any input anyway (held read_only).
149
+ var readOnly = false;
150
+ var noPage = false;
151
+ var NO_PAGE = "No page yet: the browser has not opened a page. It shows here once it does.";
152
+ // The controls that act on the browser: the address bar and its buttons, and a page's dialog (its answer, OK or Cancel).
153
+ var CONTROLS = ["back", "reload", "url", "dialog-accept", "dialog-dismiss", "dialog-prompt"];
154
+ function viewOnly() {
155
+ document.body.classList.add("view-only");
156
+ $("watching").hidden = false;
157
+ // Nothing a watcher types or clicks is sent, so the no-picture notice promises no input.
158
+ stillEl.textContent = "This page is in a background tab or has not drawn yet, so there is no picture until it is shown.";
159
+ CONTROLS.forEach(function (id) { $(id).disabled = true; });
160
+ }
161
+ // A take-over viewer drives only while it holds the browser: Take over asks for it, Hand back gives it back.
162
+ var takeOver = false;
163
+ var holding = false;
164
+ // Whether the take of the moment went live: one that ends before it did (the agent's step ran on) never gave the person the browser.
165
+ var heldThisTake = false;
166
+ var handsEl = $("hands");
167
+ var NOT_TAKEN = "Take over to drive the browser: your input is not sent until you do.";
168
+ function showHands() {
169
+ handsEl.hidden = !takeOver;
170
+ handsEl.textContent = holding ? "Hand back" : "Take over";
171
+ handsEl.className = holding ? "holding" : "";
172
+ document.body.classList.toggle("not-held", takeOver && !holding);
173
+ // A view-only viewer's controls stay off (viewOnly); a take-over one's are on only while it holds the browser, a page's
174
+ // dialog's among them, so no button looks live that cannot act.
175
+ if (takeOver) CONTROLS.forEach(function (id) { $(id).disabled = !holding; });
176
+ // Take over sits above a page's dialog (the rest of the bar stays under it), so a dialog that came before the take
177
+ // never keeps the owner from taking over; it says how to answer it.
178
+ $("dialog-held").hidden = !(takeOver && !holding);
179
+ }
180
+ handsEl.addEventListener("click", function () {
181
+ if (!ready || !takeOver || ws.readyState !== 1) return;
182
+ ws.send(JSON.stringify({ type: holding ? "give" : "take" }));
183
+ if (!holding) { heldThisTake = false; say("Taking over the browser..."); }
184
+ });
114
185
  function send(message) {
186
+ if (readOnly) return;
187
+ if (takeOver && !holding) { if (ready && !(message.type === "mouse" && message.action === "move")) say(NOT_TAKEN); return; }
115
188
  if (!ready) { if (!(message.type === "mouse" && message.action === "move")) stillConnecting(); return; }
116
189
  if (ws.readyState === 1) ws.send(JSON.stringify(message));
117
190
  }
@@ -120,14 +193,18 @@ const js = `"use strict";
120
193
  // Frames: the latest only. A decode in flight and one waiting; a newer frame replaces the one waiting.
121
194
  var decoding = false;
122
195
  var waiting = null;
196
+ // Moved on by no_page: a decode begun before it is of the old page, and its picture is never drawn.
197
+ var generation = 0;
123
198
  function draw(buffer) {
124
199
  if (buffer.byteLength <= 4) { decoding = false; return; }
125
200
  var view = new DataView(buffer);
126
201
  var w = view.getUint16(0);
127
202
  var h = view.getUint16(2);
128
203
  var blob = new Blob([new Uint8Array(buffer, 4)], { type: "image/jpeg" });
204
+ var mine = generation;
129
205
  createImageBitmap(blob).then(function (bitmap) {
130
- if (canvas.width !== bitmap.width || canvas.height !== bitmap.height) { canvas.width = bitmap.width; canvas.height = bitmap.height; }
206
+ if (mine !== generation) { if (bitmap.close) bitmap.close(); return; }
207
+ if (canvas.width !== bitmap.width || canvas.height !== bitmap.height) { canvas.width = bitmap.width; canvas.height = bitmap.height; fit(); }
131
208
  ctx.drawImage(bitmap, 0, 0);
132
209
  gotPicture();
133
210
  if (bitmap.close) bitmap.close();
@@ -157,7 +234,10 @@ const js = `"use strict";
157
234
  $("dialog-accept").addEventListener("click", function () { answer(true); });
158
235
  $("dialog-dismiss").addEventListener("click", function () { answer(false); });
159
236
 
160
- var HELD = { agent_acting: "The agent is acting, so that input was held. Try again in a moment.", no_page: "There is no page to drive yet.", bad_message: "That input was not understood." };
237
+ var HELD = { agent_acting: "The agent is acting, so that input was held. Try again in a moment.", no_page: "There is no page to drive yet.", bad_message: "That input was not understood.", read_only: "This view is view-only: your input is not sent.", not_taken: NOT_TAKEN, taking: "Taking over the browser: wait a moment." };
238
+ var HOLD_ENDED = { given: "You handed the browser back to the agent.", answered: "The agent's request was answered, so the browser went back to the agent.", idle: "The browser went back to the agent: no input came from you for a while.", away: "The browser went back to the agent while this view was closed.", released: "The browser's session ended.", cap: "The browser went back to the agent: the run's spend cap has no room for more of your time." };
239
+ // Why a take ended before it went live: the person never had the browser, so it never "went back".
240
+ var NOT_STARTED = { idle: "the agent's step was still running when your time ran out", away: "this view was closed before the agent's step finished", released: "the browser's session ended", cap: "the run's spend cap has no room for more of your time", answered: "the agent's request was answered first", given: "you handed it back before it started" };
161
241
  var REFUSED = { unauthorized: "This link's token was not accepted.", viewer_in_use: "Another viewer already has this session.", released: "This session has been released.", bad_message: "The host did not understand this viewer." };
162
242
  ws.onmessage = function (event) {
163
243
  if (typeof event.data !== "string") {
@@ -168,16 +248,29 @@ const js = `"use strict";
168
248
  }
169
249
  var m;
170
250
  try { m = JSON.parse(event.data); } catch (e) { return; }
171
- if (m.type === "ready") { ready = true; setDrive(m.drive); say(""); focusScreen(); expectPicture(); }
172
- else if (m.type === "status") { setDrive(m.drive); if (m.drive !== "agent") say(""); else say("The agent is acting. Your input is held until it is done."); }
173
- else if (m.type === "page") { document.title = m.title ? m.title + " - Live view" : "Live view"; if (document.activeElement !== urlBox) urlBox.value = m.url; if (m.url !== shownUrl) { shownUrl = m.url; expectPicture(); } }
251
+ if (m.type === "ready") { ready = true; readOnly = m.readOnly === true; takeOver = !readOnly && m.takeOver === true; holding = takeOver && m.held === true; heldThisTake = holding; if (readOnly) viewOnly(); showHands(); setDrive(m.drive); say(""); focusScreen(); expectPicture(); }
252
+ else if (m.type === "hold") {
253
+ var started = heldThisTake;
254
+ holding = m.held === true;
255
+ if (holding) heldThisTake = true; else heldThisTake = false;
256
+ showHands(); setDrive(holding ? "person" : "idle");
257
+ var extra = typeof m.message === "string" && m.message ? " " + m.message : "";
258
+ say(holding ? "" : started ? (HOLD_ENDED[m.reason] || "The browser went back to the agent.") + extra : "Your take-over did not start: " + (NOT_STARTED[m.reason] || "the browser was not handed to you") + "." + extra);
259
+ if (holding) focusScreen();
260
+ }
261
+ else if (m.type === "take_refused") say(typeof m.message === "string" && m.message ? m.message : "The browser could not be handed to you.");
262
+ // The old page's picture goes with it, and so does the frame waiting; a decode in flight finishes without drawing.
263
+ else if (m.type === "no_page") { noPage = true; generation += 1; waiting = null; ctx.clearRect(0, 0, canvas.width, canvas.height); stillEl.hidden = true; clearTimeout(stillTimer); say(NO_PAGE); }
264
+ // "No page yet" stays until a page or a frame comes: who drives meanwhile changes nothing on a browser with no page.
265
+ else if (m.type === "status") { setDrive(m.drive); if (noPage || takeOver || readOnly) return; if (m.drive !== "agent") say(""); else say("The agent is acting. Your input is held until it is done."); }
266
+ else if (m.type === "page") { if (noPage) { noPage = false; say(""); } document.title = m.title ? m.title + " - Live view" : "Live view"; if (document.activeElement !== urlBox) urlBox.value = m.url; $("where").textContent = m.url; $("where").title = m.url; if (m.url !== shownUrl) { shownUrl = m.url; expectPicture(); } }
174
267
  else if (m.type === "dialog") showDialog(m);
175
268
  else if (m.type === "dialog_closed") hideDialog();
176
269
  else if (m.type === "held") say(HELD[m.reason] || "That input was held.");
177
270
  else if (m.type === "refused") { fatal = true; ready = false; stillEl.hidden = true; say(REFUSED[m.reason] || "The host refused this viewer.", true); }
178
271
  else if (m.type === "ended") { fatal = true; ready = false; say("This session has ended: the browser was released.", true); }
179
272
  };
180
- ws.onclose = function () { closed = true; ready = false; setDrive(""); if (!fatal) say("Disconnected. Open the live view link again.", true); };
273
+ ws.onclose = function () { closed = true; ready = false; holding = false; handsEl.disabled = true; setDrive(""); if (!fatal) say("Disconnected. Open the live view link again.", true); };
181
274
 
182
275
  // Input: the page's CSS pixels, from the canvas's own box and the latest frame's width and height.
183
276
  var lastPoint = null;
@@ -4,7 +4,6 @@ export type CdpOptions =
4
4
  /** An existing browser: `http://host:port` or a `ws://` debugger URL. Release leaves it running. */
5
5
  {
6
6
  endpoint: string;
7
- extensionId?: string;
8
7
  /** The host asserts this endpoint is the run's OWN browser (the launcher that owns its process and its stop started it for this run alone), not one other runs or people share. Only then is a session of it offered to a session vault. Never set it for a shared browser. */
9
8
  ownBrowser?: boolean;
10
9
  }
@@ -5,12 +5,13 @@ import os from "node:os";
5
5
  import { pipeline } from "node:stream/promises";
6
6
  import path from "node:path";
7
7
  import { BrowserFailureError, PROFILE_NAME, refusedError } from "../core/custody.js";
8
+ import { DownloadNotSettled } from "../core/downloads.js";
8
9
  import { browserFailure } from "../core/failures.js";
9
10
  import { cdpCall } from "./cdp-call.js";
10
11
  import { chromeTail, openChromeStderr, watchChromeStderr } from "./chrome-stderr.js";
11
12
  import { claimWriter, freeWriter, noteWriter } from "./profiles.js";
12
13
  import { chromeArgs } from "./chrome-args.js";
13
- import { COPY, copyExtension, copyId, isLoaded, loadConfined } from "./worker-extension.js";
14
+ import { COPY, copyExtension, copyId, isLoaded, loadConfined, soleStagehand } from "./worker-extension.js";
14
15
  const CAPS = { timeoutModel: "wall-clock", maxLifetimeS: 86_400, survivesDisconnect: true, releaseIsAsync: false, profiles: false, extension: "loaded-locally", settings: { proxies: false, verified: false, geolocation: false, region: false, context: false } };
15
16
  const downloadsDir = (profileRoot, tag) => path.join(profileDir(profileRoot, tag) + ".downloads");
16
17
  const READY_MS = 20_000;
@@ -314,7 +315,7 @@ export function cdpProvider(options, internals = {}) {
314
315
  /** The private TMPDIR of each lease this provider started, by tag: the profile's marker is the only record a restarted process has, and a profile
315
316
  * that something else removed first takes it along. */
316
317
  const privateTmps = new Map();
317
- /** The confined extension each lease has been given, by tag, loaded on its first attach: its id is handed to Stagehand as the attach target's extension. */
318
+ /** The confined extension each lease has been given, by tag, loaded on its first attach: the only Stagehand extension its browser keeps. */
318
319
  const extensionIds = new Map();
319
320
  /** A lease whose private TMPDIR is gone (or never made) keeps neither its map entry nor its record. */
320
321
  const forgetTmp = async (root, tag) => {
@@ -498,7 +499,7 @@ export function cdpProvider(options, internals = {}) {
498
499
  },
499
500
  async attach(ref) {
500
501
  if ("endpoint" in options)
501
- return { sdkCdpUrl: await debuggerUrl(options.endpoint), ...(options.extensionId ? { extensionId: options.extensionId } : {}) };
502
+ return { sdkCdpUrl: await debuggerUrl(options.endpoint) };
502
503
  const local = parse(ref.id);
503
504
  if (!local)
504
505
  throw new Error("not a cdp lease");
@@ -510,8 +511,12 @@ export function cdpProvider(options, internals = {}) {
510
511
  throw new Error(`the browser on port ${local.port} is not the browser this lease's profile records`);
511
512
  // A Chrome this host did not start outlived the host that did: its stderr file is bounded here too.
512
513
  watchChromeStderr(privateTmps.get(local.tag) ?? await recordedTmp(options.chrome.profileRoot, local.tag) ?? await ownedTmp(profileDir(options.chrome.profileRoot, local.tag), local.tag), local.pid);
514
+ // Stagehand attaches to the one extension carrying its name: on every attach, any other (one loaded since the last) goes.
513
515
  const extensionId = await confinedExtension(local.tag, url);
514
- return { sdkCdpUrl: url, ...(extensionId ? { extensionId } : {}) };
516
+ if (extensionId)
517
+ await soleStagehand(url, extensionId);
518
+ // The extension lives in this browser and dials it on loopback, never through whatever the host puts in front of the driver.
519
+ return { sdkCdpUrl: url, extensionCdpUrl: url };
515
520
  },
516
521
  async release(ref) {
517
522
  if (ref.id.startsWith("endpoint:")) {
@@ -590,7 +595,10 @@ export function cdpProvider(options, internals = {}) {
590
595
  },
591
596
  };
592
597
  }
593
- /** Finished files only: Chrome writes `.crdownload` while it works. */
598
+ /** Chrome writes a download to a `.crdownload` file and renames it to its name once done; at the end (sandboxed, under load)
599
+ * it first creates that name empty for a moment, the bytes still in the `.crdownload`. Both are listed as `downloading`
600
+ * and never read: a `.crdownload` (under its name without the extension), and an empty file while any `.crdownload` is
601
+ * there. A read checks after it ends that the file it opened is still the file at that name, whole. */
594
602
  function localDownloads(profileRoot) {
595
603
  const inside = (ref, name) => { if (name !== path.basename(name) || !name)
596
604
  throw new Error("not a download name"); return path.join(downloadsDir(profileRoot, ref.tag), name); };
@@ -598,26 +606,40 @@ function localDownloads(profileRoot) {
598
606
  async list(ref) {
599
607
  const dir = downloadsDir(profileRoot, ref.tag);
600
608
  const rows = [];
601
- for (const name of await readdir(dir).catch(() => [])) {
602
- if (name.endsWith(".crdownload") || name.startsWith("."))
609
+ const names = (await readdir(dir).catch(() => [])).filter((name) => !name.startsWith("."));
610
+ const writing = names.some((name) => name.endsWith(".crdownload"));
611
+ const finals = new Set(names.filter((name) => !name.endsWith(".crdownload")));
612
+ for (const name of names) {
613
+ const partial = name.endsWith(".crdownload");
614
+ const shown = partial ? name.slice(0, -".crdownload".length) : name;
615
+ // A `.crdownload` named for a final name that is there too is that same download, told once.
616
+ if (partial && finals.has(shown))
603
617
  continue;
604
618
  const info = await stat(path.join(dir, name)).catch(() => null);
605
- if (info?.isFile())
606
- rows.push({ name, sizeBytes: info.size, modifiedAt: info.mtime.toISOString() });
619
+ if (!info?.isFile())
620
+ continue;
621
+ const row = { name: shown, sizeBytes: info.size, modifiedAt: info.mtime.toISOString() };
622
+ rows.push(partial || (writing && info.size === 0) ? { ...row, downloading: true } : row);
607
623
  }
608
624
  return rows.sort((a, b) => a.name.localeCompare(b.name));
609
625
  },
610
626
  async read(ref, name, maxBytes) {
611
- const file = await open(inside(ref, name), "r");
627
+ const where = inside(ref, name);
628
+ const file = await open(where, "r");
629
+ const opened = await file.stat().catch(async (error) => { await file.close().catch(() => undefined); throw error; });
612
630
  return (async function* () {
613
631
  try {
614
632
  let total = 0;
615
- for await (const chunk of file.createReadStream()) {
633
+ for await (const chunk of file.createReadStream({ autoClose: false })) {
616
634
  total += chunk.byteLength;
617
635
  if (total > maxBytes)
618
636
  throw new Error(`larger than the ${maxBytes}-byte limit`);
619
637
  yield chunk;
620
638
  }
639
+ // The whole file it opened, still at its name: Chrome renames a finished download over the name it created empty.
640
+ const now = await stat(where).catch(() => null);
641
+ if (total !== opened.size || !now || now.dev !== opened.dev || now.ino !== opened.ino || now.size !== opened.size || now.mtimeMs !== opened.mtimeMs)
642
+ throw new DownloadNotSettled(name);
621
643
  }
622
644
  finally {
623
645
  await file.close().catch(() => undefined);