@melaya/runner 1.0.117 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,1021 @@
1
+ // packages/runner/src/browserBridge.ts
2
+ //
3
+ // Melaya Browser, Phase 1 (plan Sections 3, 4, 7, 8, 9, 10): the governed
4
+ // browser control bridge. Generalizes the localhost-HTTP-shell +
5
+ // per-session-bearer-token + lazy-Playwright pattern proven by
6
+ // lumaBrowserBridge.ts, and deliberately does NOT inherit its two known
7
+ // bugs: screenshots are RECOMPRESSED (JPEG quality ladder + CSS-pixel
8
+ // scale), never base64-sliced (luma :330), and teardown closes ONLY
9
+ // owned browsers, never an externally attached user browser (luma :756).
10
+ //
11
+ // Connection modes:
12
+ // LAUNCH - dedicated-profile browser (chrome | edge | brave |
13
+ // chromium, caller-selected) via launchPersistentContext on
14
+ // a Melaya-owned user-data-dir. NEVER the real default
15
+ // profile (Chrome 136+ ignores the debug port there).
16
+ // ATTACH - chromium.connectOverCDP(ws) to a Melaya-launched
17
+ // controllable browser. Ownership tracked as "attached";
18
+ // teardown only severs the CDP connection.
19
+ //
20
+ // HTTP contract (shared verbatim with the Python tool agent):
21
+ // POST /browser/current_target -> {ok, target:{ref, origin, title?}}
22
+ // POST /browser/get_screen_tree -> {ok, tree, generation, url, title, nodes}
23
+ // POST /browser/screenshot -> {ok, image_b64, media_type}
24
+ // POST /browser/act -> {ok, result, tree, generation} |
25
+ // {ok:false, error:{code, message}}
26
+ // POST /browser/run -> {ok, result, console[], tree}
27
+ //
28
+ // Every act is ACT-AND-OBSERVE: it returns a fresh snapshot so the model
29
+ // never acts twice on a stale world.
30
+ import { createServer } from "node:http";
31
+ import { request as httpRequest } from "node:http";
32
+ import { request as httpsRequest } from "node:https";
33
+ import { randomBytes, timingSafeEqual } from "node:crypto";
34
+ import { mkdirSync } from "node:fs";
35
+ import { join } from "node:path";
36
+ import { tmpdir } from "node:os";
37
+ import { buildOriginPolicy, evaluateUrlResolved, checkEffect, enforceOnContext, enforceOnPage, disableWebRtcViaCdp, } from "./browserAuthz.js";
38
+ import { SessionManager, SessionError, spaceUserDataDir, } from "./sessionManager.js";
39
+ import { ensureEngine } from "./browserProvisioner.js";
40
+ import { runSandboxedScript } from "./codeWorker.js";
41
+ // ---------------------------------------------------------------------
42
+ // Limits and constants
43
+ // ---------------------------------------------------------------------
44
+ const MAX_BODY_BYTES = 512 * 1024; // /browser/run scripts included
45
+ const NODE_CAP = 240; // snapshot node cap (Section 8)
46
+ const SCREENSHOT_MAX_B64 = 1_400_000; // ~1 MB binary; ladder recompresses
47
+ const ACT_SETTLE_MS = 500;
48
+ const NAV_TIMEOUT_MS = 25_000;
49
+ // Roles that read as actionable in the AX tree.
50
+ const CLICK_ROLES = new Set([
51
+ "button", "link", "menuitem", "menuitemcheckbox", "menuitemradio", "tab",
52
+ "checkbox", "radio", "switch", "option", "treeitem", "listbox", "combobox",
53
+ "slider", "spinbutton", "disclosuretriangle", "togglebutton",
54
+ ]);
55
+ const EDIT_ROLES = new Set(["textbox", "searchbox", "combobox", "spinbutton", "textfield", "textarea"]);
56
+ // Non-actionable roles still worth showing when named (context anchors).
57
+ const CONTEXT_ROLES = new Set([
58
+ "heading", "img", "image", "alert", "alertdialog", "dialog", "status",
59
+ "tabpanel", "cell", "columnheader", "rowheader", "statictext", "text",
60
+ "listitem", "article", "banner", "navigation", "main", "form", "search",
61
+ ]);
62
+ /** Runner-side minimum effect per act kind. The tool layer may DECLARE a
63
+ * higher typed effect (message/publish/purchase/...) which is then
64
+ * checked against the grant; the runner cannot infer those from a raw
65
+ * click, so kind minima keep the second gate sound without blocking
66
+ * read-ish interactions (full classification is the server's first
67
+ * gate, plan Section 6). */
68
+ const KIND_MIN_EFFECT = {
69
+ navigate: "navigate", back: "navigate", forward: "navigate",
70
+ click: "read", dblclick: "read", hover: "read", scroll: "read",
71
+ input_text: "read", press_key: "read", select_option: "read",
72
+ wait: "read",
73
+ };
74
+ // ---------------------------------------------------------------------
75
+ // Bridge
76
+ // ---------------------------------------------------------------------
77
+ export async function startBrowserBridge(opts) {
78
+ const log = opts.log;
79
+ const byToken = new Map();
80
+ const byRunId = new Map();
81
+ const sessions = new SessionManager({ log: (m) => opts.verbose && log(m) });
82
+ // Per-lease transient state the sessionManager types stay clean of.
83
+ const frameMaps = new WeakMap();
84
+ const cdpByPage = new WeakMap();
85
+ let playwrightMod = null;
86
+ async function pw() {
87
+ if (!playwrightMod)
88
+ playwrightMod = await import("playwright");
89
+ return playwrightMod;
90
+ }
91
+ // -- Session establishment (lazy, once per run) -----------------------
92
+ async function ensureSession(reg) {
93
+ const existing = sessions.peekSession(reg.spec.runId);
94
+ if (existing && existing.state !== "closed" && existing.state !== "crashed")
95
+ return existing;
96
+ if (reg.sessionInit)
97
+ return reg.sessionInit;
98
+ reg.sessionInit = (async () => {
99
+ const playwright = await pw();
100
+ const space = reg.spec.space ?? { kind: "ephemeral" };
101
+ const hooks = {
102
+ onViolation: (v) => {
103
+ reg.violations.push({ url: v.url.slice(0, 300), code: v.code, surface: v.surface, at: Date.now() });
104
+ if (reg.violations.length > 200)
105
+ reg.violations.shift();
106
+ log(`[authz] DENY ${v.code} (${v.surface}) ${v.url.slice(0, 120)}`);
107
+ },
108
+ isCancelled: () => reg.cancelled,
109
+ };
110
+ if (reg.spec.mode === "attach") {
111
+ const ws = String(reg.spec.cdpWsEndpoint || "");
112
+ if (!/^wss?:\/\/(127\.0\.0\.1|localhost|\[::1\])[:/]/.test(ws)) {
113
+ throw new BridgeError("attach_endpoint_invalid", "attach requires a loopback CDP websocket endpoint");
114
+ }
115
+ const rec = sessions.createSession({
116
+ runId: reg.spec.runId, ownership: "attached",
117
+ engine: reg.spec.engine || "chromium", space,
118
+ });
119
+ try {
120
+ const browser = await playwright.chromium.connectOverCDP(ws, { timeout: 15_000 });
121
+ const context = browser.contexts()[0];
122
+ if (!context)
123
+ throw new BridgeError("attach_no_context", "attached browser exposes no context");
124
+ sessions.attachHandles(rec, { browser, context });
125
+ const page = context.pages()[0] ?? (await context.newPage());
126
+ const lease = sessions.leaseTarget(rec, reg.spec.grant.target.ref, page);
127
+ // Attach mode: enforce on the LEASED page (and its popups)
128
+ // only — we do not take over routing for the user's whole
129
+ // externally owned context (plan Section 7 ownership rule;
130
+ // context-wide routing is applied on owned contexts below).
131
+ await enforceOnPage(page, reg.policy, hooks);
132
+ void lease;
133
+ return rec;
134
+ }
135
+ catch (e) {
136
+ rec.state = "crashed";
137
+ await sessions.teardownRun(reg.spec.runId, "attach_failed");
138
+ throw e instanceof BridgeError ? e : new BridgeError("attach_failed", String(e?.message || e));
139
+ }
140
+ }
141
+ // LAUNCH mode: dedicated profile, never the browser's default dir.
142
+ const engineId = (reg.spec.engine || "chrome");
143
+ const engine = await ensureEngine(engineId, { log });
144
+ const { dir: userDataDir, ephemeral } = spaceUserDataDir(space, reg.spec.runId);
145
+ const rec = sessions.createSession({
146
+ runId: reg.spec.runId, ownership: "owned", engine: engineId, space,
147
+ });
148
+ try {
149
+ const context = await playwright.chromium.launchPersistentContext(userDataDir, {
150
+ executablePath: engine.executablePath,
151
+ headless: reg.spec.headless === true,
152
+ viewport: { width: 1280, height: 800 },
153
+ acceptDownloads: false, // downloads default-denied (Section 10)
154
+ args: [
155
+ "--no-first-run",
156
+ "--no-default-browser-check",
157
+ "--disable-background-networking",
158
+ "--disable-sync",
159
+ ],
160
+ });
161
+ sessions.attachHandles(rec, {
162
+ context,
163
+ ephemeralUserDataDir: ephemeral ? userDataDir : null,
164
+ });
165
+ await enforceOnContext(context, reg.policy, hooks);
166
+ const page = context.pages()[0] ?? (await context.newPage());
167
+ sessions.leaseTarget(rec, reg.spec.grant.target.ref, page);
168
+ log(`browser session up: run=${reg.spec.runId.slice(0, 10)} engine=${engineId} owned profile=${space.kind}`);
169
+ return rec;
170
+ }
171
+ catch (e) {
172
+ rec.state = "crashed";
173
+ await sessions.teardownRun(reg.spec.runId, "launch_failed");
174
+ throw e instanceof BridgeError ? e : new BridgeError("launch_failed", String(e?.message || e));
175
+ }
176
+ })();
177
+ reg.sessionInit.catch(() => { reg.sessionInit = null; });
178
+ return reg.sessionInit;
179
+ }
180
+ async function getLease(reg) {
181
+ if (reg.cancelled)
182
+ throw new BridgeError("run_cancelled", "run was cancelled/torn down");
183
+ const rec = await ensureSession(reg);
184
+ sessions.touch(rec);
185
+ const lease = sessions.getLease(rec, reg.spec.grant.target.ref);
186
+ return { rec, lease };
187
+ }
188
+ async function getCdp(rec, page) {
189
+ const cached = cdpByPage.get(page);
190
+ if (cached)
191
+ return cached;
192
+ const context = rec.context;
193
+ const cdp = await context.newCDPSession(page);
194
+ cdpByPage.set(page, cdp);
195
+ page.on("close", () => cdpByPage.delete(page));
196
+ // Belt-and-suspenders: also disable WebRTC at the CDP level (the init
197
+ // script neutralises it in JS; this kills it at the browser level too).
198
+ void disableWebRtcViaCdp(cdp);
199
+ return cdp;
200
+ }
201
+ async function captureSnapshot(rec, lease, scope) {
202
+ const page = lease.page;
203
+ const generation = sessions.beginSnapshot(lease);
204
+ const frames = new Map();
205
+ frames.set("main", page.mainFrame());
206
+ frameMaps.set(lease, frames);
207
+ const viewport = page.viewportSize() ?? { width: 1280, height: 800 };
208
+ const collected = [];
209
+ // Main target: covers the top document + all SAME-PROCESS iframes
210
+ // (the flattened DOMSnapshot + AX tree include them) + shadow DOM
211
+ // (the AX tree pierces shadow roots natively).
212
+ try {
213
+ const cdp = await getCdp(rec, page);
214
+ await collectFromTarget(cdp, "main", { x: 0, y: 0 }, viewport, collected);
215
+ }
216
+ catch (e) {
217
+ if (opts.verbose)
218
+ log(`snapshot main-target failed: ${e?.message || e}`);
219
+ }
220
+ // OOPIF descent: frames living in other renderer processes have
221
+ // their own CDP targets; newCDPSession(frame) succeeds only for
222
+ // those (throws for same-process frames, which are already covered).
223
+ let frameIdx = 0;
224
+ for (const frame of page.frames()) {
225
+ if (frame === page.mainFrame())
226
+ continue;
227
+ frameIdx += 1;
228
+ let fcdp = null;
229
+ try {
230
+ fcdp = await rec.context.newCDPSession(frame);
231
+ }
232
+ catch {
233
+ continue; // same-process frame -> already in the main capture
234
+ }
235
+ try {
236
+ const el = await frame.frameElement().catch(() => null);
237
+ const box = el ? await el.boundingBox().catch(() => null) : null;
238
+ if (el)
239
+ await el.dispose().catch(() => { });
240
+ if (!box)
241
+ continue;
242
+ const key = `f${frameIdx}`;
243
+ frames.set(key, frame);
244
+ await collectFromTarget(fcdp, key, { x: box.x, y: box.y }, viewport, collected);
245
+ }
246
+ catch { /* skip this OOPIF, keep the rest */ }
247
+ finally {
248
+ await fcdp?.detach().catch(() => { });
249
+ }
250
+ }
251
+ // Scope filter (optional substring over role/name), rank, cap.
252
+ const scopeLc = (scope || "").trim().toLowerCase();
253
+ let candidates = scopeLc
254
+ ? collected.filter((n) => n.role.toLowerCase().includes(scopeLc) || n.name.toLowerCase().includes(scopeLc))
255
+ : collected;
256
+ const total = candidates.length;
257
+ candidates = candidates
258
+ .sort((a, b) => (Number(!(a.click || a.edit)) - Number(!(b.click || b.edit))) || a.order - b.order)
259
+ .slice(0, NODE_CAP)
260
+ .sort((a, b) => a.order - b.order);
261
+ // Bind refs and render one line per node.
262
+ const lines = [];
263
+ let refNo = 0;
264
+ for (const n of candidates) {
265
+ refNo += 1;
266
+ const ref = `@e${refNo}`;
267
+ sessions.bindRef(lease, {
268
+ ref,
269
+ targetRef: lease.ref,
270
+ frameKey: n.frameKey,
271
+ backendNodeId: n.backendNodeId,
272
+ snapshotGeneration: generation,
273
+ documentGeneration: lease.documentGeneration,
274
+ role: n.role,
275
+ name: n.name,
276
+ });
277
+ const flags = `${n.click ? " click" : ""}${n.edit ? " edit" : ""}${n.covered ? " COVERED" : ""}`;
278
+ const name = n.name ? ` "${n.name.slice(0, 80).replace(/"/g, "'")}"` : ' ""';
279
+ const aria = n.aria ? ` ~${n.aria.slice(0, 40)}` : "";
280
+ const domId = n.domId ? ` #${n.domId.slice(0, 40)}` : "";
281
+ const href = n.href ? ` @${n.href.slice(0, 100)}` : "";
282
+ lines.push(`${ref} <${n.role}>${flags}${name}${aria}${domId}${href} <${Math.round(n.l)},${Math.round(n.t)},${Math.round(n.r)},${Math.round(n.b)}>`);
283
+ }
284
+ const url = page.url();
285
+ const title = await page.title().catch(() => "");
286
+ const header = `url=${url} title=${title.slice(0, 120)} gen=${generation} nodes=${lines.length}/${total}`;
287
+ let tree = `${header}\n${lines.join("\n")}`;
288
+ if (lines.length <= 2) {
289
+ tree += `\n[vision] The page exposes almost no accessible nodes (canvas/video/empty page). Call POST /browser/screenshot and act with coordinates: {"kind":"click","x":0.5,"y":0.4,"unit":"fraction"} (fractions of the viewport) or CSS pixel x/y.`;
290
+ }
291
+ return { tree, generation, url, title, nodes: lines.length };
292
+ }
293
+ async function collectFromTarget(cdp, frameKey, frameOffset, viewport, out) {
294
+ const STYLE_FILTER = ["display", "visibility", "opacity"];
295
+ const [snap, ax] = await Promise.all([
296
+ cdp.send("DOMSnapshot.captureSnapshot", {
297
+ computedStyles: STYLE_FILTER,
298
+ includePaintOrder: true,
299
+ }),
300
+ (async () => {
301
+ await cdp.send("Accessibility.enable").catch(() => { });
302
+ return cdp.send("Accessibility.getFullAXTree", {});
303
+ })(),
304
+ ]);
305
+ const strings = snap.strings;
306
+ const docs = snap.documents;
307
+ // Per-document viewport offset: root = frameOffset - scroll; child
308
+ // (same-process iframe) = parentOffset + iframe bounds - child scroll.
309
+ const docOffset = docs.map(() => null);
310
+ if (docs.length > 0) {
311
+ docOffset[0] = {
312
+ x: frameOffset.x - (docs[0].scrollOffsetX ?? 0),
313
+ y: frameOffset.y - (docs[0].scrollOffsetY ?? 0),
314
+ };
315
+ }
316
+ const layoutByDoc = docs.map((d) => {
317
+ const m = new Map();
318
+ for (let i = 0; i < d.layout.nodeIndex.length; i++) {
319
+ const styleIdx = d.layout.styles[i] || [];
320
+ const sv = (k) => {
321
+ const si = styleIdx[k];
322
+ return si !== undefined && si >= 0 ? String(strings[si] ?? "") : "";
323
+ };
324
+ const display = sv(0);
325
+ const visibility = sv(1);
326
+ const opacity = sv(2);
327
+ m.set(d.layout.nodeIndex[i], {
328
+ bounds: d.layout.bounds[i],
329
+ hidden: display === "none" || visibility === "hidden" || visibility === "collapse",
330
+ opacity0: opacity !== "" && Number(opacity) === 0,
331
+ paintOrder: d.layout.paintOrders?.[i] ?? 0,
332
+ nodeIndex: d.layout.nodeIndex[i],
333
+ });
334
+ }
335
+ return m;
336
+ });
337
+ // Resolve child-document offsets via contentDocumentIndex links.
338
+ for (let di = 0; di < docs.length; di++) {
339
+ const cdi = docs[di].nodes.contentDocumentIndex;
340
+ if (!cdi)
341
+ continue;
342
+ for (let k = 0; k < cdi.index.length; k++) {
343
+ const iframeNodeIdx = cdi.index[k];
344
+ const childDoc = cdi.value[k];
345
+ if (childDoc === di || childDoc >= docs.length)
346
+ continue;
347
+ const parentOff = docOffset[di];
348
+ const hit = layoutByDoc[di].get(iframeNodeIdx);
349
+ if (!parentOff || !hit)
350
+ continue;
351
+ docOffset[childDoc] = {
352
+ x: parentOff.x + hit.bounds[0] - (docs[childDoc].scrollOffsetX ?? 0),
353
+ y: parentOff.y + hit.bounds[1] - (docs[childDoc].scrollOffsetY ?? 0),
354
+ };
355
+ }
356
+ }
357
+ const domByBackendId = new Map();
358
+ const clickableByDoc = docs.map((d) => new Set(d.nodes.isClickable?.index ?? []));
359
+ for (let di = 0; di < docs.length; di++) {
360
+ const ids = docs[di].nodes.backendNodeId ?? [];
361
+ for (let ni = 0; ni < ids.length; ni++) {
362
+ domByBackendId.set(ids[ni], { doc: di, layout: layoutByDoc[di].get(ni) ?? null, nodeIdx: ni });
363
+ }
364
+ }
365
+ const attr = (di, ni, wanted) => {
366
+ const attrs = docs[di].nodes.attributes?.[ni] ?? [];
367
+ for (let i = 0; i + 1 < attrs.length; i += 2) {
368
+ if (strings[attrs[i]] === wanted)
369
+ return String(strings[attrs[i + 1]] ?? "");
370
+ }
371
+ return "";
372
+ };
373
+ const isAncestor = (di, maybeAncestor, node) => {
374
+ const parents = docs[di].nodes.parentIndex ?? [];
375
+ let cur = node;
376
+ for (let hops = 0; hops < 500 && cur !== undefined && cur >= 0; hops++) {
377
+ if (cur === maybeAncestor)
378
+ return true;
379
+ cur = parents[cur];
380
+ }
381
+ return false;
382
+ };
383
+ // AX walk (DFS from roots) in document order.
384
+ const axById = new Map();
385
+ for (const n of ax.nodes)
386
+ axById.set(n.nodeId, n);
387
+ const hasParent = new Set();
388
+ for (const n of ax.nodes)
389
+ for (const c of n.childIds ?? [])
390
+ hasParent.add(c);
391
+ const roots = ax.nodes.filter((n) => !hasParent.has(n.nodeId) && !n.parentId);
392
+ let order = out.length * 1000;
393
+ const seenBackend = new Set();
394
+ const visit = (n, depth) => {
395
+ if (!n || depth > 80)
396
+ return;
397
+ order += 1;
398
+ if (!n.ignored) {
399
+ const role = String(n.role?.value ?? "").toLowerCase();
400
+ const name = String(n.name?.value ?? "").trim();
401
+ const backendId = n.backendDOMNodeId;
402
+ if (backendId && !seenBackend.has(backendId)) {
403
+ const dom = domByBackendId.get(backendId);
404
+ const domClickable = dom ? clickableByDoc[dom.doc].has(dom.nodeIdx) : false;
405
+ const click = CLICK_ROLES.has(role) || domClickable;
406
+ const edit = EDIT_ROLES.has(role);
407
+ const interesting = click || edit || (name.length > 0 && (CONTEXT_ROLES.has(role) || role === "iframe"));
408
+ if (interesting && dom?.layout && !dom.layout.hidden && !dom.layout.opacity0) {
409
+ const off = docOffset[dom.doc];
410
+ if (off) {
411
+ const [bx, by, bw, bh] = dom.layout.bounds;
412
+ const l = bx + off.x, t = by + off.y, r = l + bw, b = t + bh;
413
+ const inViewport = bw > 0 && bh > 0 && r > 0 && b > 0 && l < viewport.width && t < viewport.height;
414
+ if (inViewport) {
415
+ seenBackend.add(backendId);
416
+ // COVERED heuristic: some non-ancestor, non-descendant
417
+ // layout node with a HIGHER paint order fully contains
418
+ // the center point. Conservative: any doubt -> not set.
419
+ let covered = false;
420
+ try {
421
+ const cx = bx + bw / 2, cy = by + bh / 2;
422
+ for (const hit of layoutByDoc[dom.doc].values()) {
423
+ if (hit.nodeIndex === dom.nodeIdx || hit.hidden || hit.opacity0)
424
+ continue;
425
+ if (hit.paintOrder <= (dom.layout.paintOrder))
426
+ continue;
427
+ const [hx, hy, hw, hh] = hit.bounds;
428
+ if (cx >= hx && cx <= hx + hw && cy >= hy && cy <= hy + hh) {
429
+ if (!isAncestor(dom.doc, hit.nodeIndex, dom.nodeIdx) && !isAncestor(dom.doc, dom.nodeIdx, hit.nodeIndex)) {
430
+ covered = true;
431
+ break;
432
+ }
433
+ }
434
+ }
435
+ }
436
+ catch { /* unknown -> leave uncovered (conservative) */ }
437
+ out.push({
438
+ frameKey,
439
+ backendNodeId: backendId,
440
+ role: role || "node",
441
+ name,
442
+ aria: String(n.description?.value ?? "").trim(),
443
+ domId: attr(dom.doc, dom.nodeIdx, "id"),
444
+ href: role === "link" ? attr(dom.doc, dom.nodeIdx, "href") : "",
445
+ click, edit, covered,
446
+ l, t, r, b,
447
+ order,
448
+ });
449
+ }
450
+ }
451
+ }
452
+ }
453
+ }
454
+ for (const cid of n.childIds ?? [])
455
+ visit(axById.get(cid), depth + 1);
456
+ };
457
+ for (const root of roots)
458
+ visit(root, 0);
459
+ }
460
+ // -- Screenshot (recompress, NEVER slice base64) -----------------------
461
+ async function takeScreenshot(lease) {
462
+ const page = lease.page;
463
+ // Quality ladder: recompress smaller instead of truncating. scale:
464
+ // "css" downscales HiDPI captures to CSS-pixel resolution.
465
+ for (const quality of [60, 40, 25, 12]) {
466
+ const buf = await page.screenshot({ type: "jpeg", quality, fullPage: false, scale: "css", timeout: 15_000 });
467
+ const b64 = buf.toString("base64");
468
+ if (b64.length <= SCREENSHOT_MAX_B64 || quality === 12) {
469
+ return { image_b64: b64, media_type: "image/jpeg" };
470
+ }
471
+ }
472
+ throw new BridgeError("screenshot_failed", "unreachable");
473
+ }
474
+ async function resolveActionPoint(reg, rec, lease, args) {
475
+ const page = lease.page;
476
+ const viewport = page.viewportSize() ?? { width: 1280, height: 800 };
477
+ if (args.ref) {
478
+ const binding = sessions.resolveRef(lease, args.ref);
479
+ const frames = frameMaps.get(lease);
480
+ const frame = frames?.get(binding.frameKey);
481
+ // Element center via CDP box model on the owning target, plus the
482
+ // frame's main-viewport offset for OOPIFs.
483
+ let localCdp;
484
+ let offset = { x: 0, y: 0 };
485
+ let detachAfter = false;
486
+ if (binding.frameKey === "main" || !frame) {
487
+ localCdp = await getCdp(rec, page);
488
+ }
489
+ else {
490
+ localCdp = await rec.context.newCDPSession(frame);
491
+ detachAfter = true;
492
+ const el = await frame.frameElement().catch(() => null);
493
+ const box = el ? await el.boundingBox().catch(() => null) : null;
494
+ if (el)
495
+ await el.dispose().catch(() => { });
496
+ if (box)
497
+ offset = { x: box.x, y: box.y };
498
+ }
499
+ try {
500
+ await localCdp.send("DOM.scrollIntoViewIfNeeded", { backendNodeId: binding.backendNodeId }).catch(() => { });
501
+ const bm = await localCdp.send("DOM.getBoxModel", { backendNodeId: binding.backendNodeId });
502
+ const q = bm.model.content; // 4 corner points, viewport coords of the owning target
503
+ const cx = (q[0] + q[2] + q[4] + q[6]) / 4 + offset.x;
504
+ const cy = (q[1] + q[3] + q[5] + q[7]) / 4 + offset.y;
505
+ return { x: cx, y: cy };
506
+ }
507
+ finally {
508
+ if (detachAfter)
509
+ await localCdp.detach().catch(() => { });
510
+ }
511
+ }
512
+ if (typeof args.x === "number" && typeof args.y === "number") {
513
+ const fraction = args.unit === "fraction" || (args.x >= 0 && args.x <= 1 && args.y >= 0 && args.y <= 1);
514
+ return fraction
515
+ ? { x: args.x * viewport.width, y: args.y * viewport.height }
516
+ : { x: args.x, y: args.y };
517
+ }
518
+ throw new BridgeError("act_args_invalid", `kind '${args.kind}' needs a ref or x/y coordinates`);
519
+ }
520
+ async function performAct(reg, args) {
521
+ const kind = String(args.kind || "");
522
+ if (!(kind in KIND_MIN_EFFECT)) {
523
+ throw new BridgeError("act_kind_unknown", `unknown act kind '${kind}'`);
524
+ }
525
+ // Grant second gate: declared effect (if the tool layer classified
526
+ // one) AND the kind's minimum effect must both clear the grant.
527
+ const effects = new Set([KIND_MIN_EFFECT[kind]]);
528
+ if (args.effect)
529
+ effects.add(String(args.effect));
530
+ for (const eff of effects) {
531
+ const d = checkEffect(eff, reg.spec.grant);
532
+ if (!d.allowed)
533
+ throw new BridgeError(d.code, d.message);
534
+ }
535
+ const { rec, lease } = await getLease(reg);
536
+ return sessions.runOnTarget(lease, async () => {
537
+ if (reg.cancelled)
538
+ throw new BridgeError("run_cancelled", "run was cancelled/torn down");
539
+ const page = lease.page;
540
+ switch (kind) {
541
+ case "navigate": {
542
+ const url = String(args.url || "");
543
+ const d = await evaluateUrlResolved(url, reg.policy);
544
+ if (!d.allowed)
545
+ throw new BridgeError(d.code, d.message);
546
+ await page.goto(url, { waitUntil: "domcontentloaded", timeout: NAV_TIMEOUT_MS });
547
+ return { navigated: page.url() };
548
+ }
549
+ case "back": {
550
+ await page.goBack({ waitUntil: "domcontentloaded", timeout: NAV_TIMEOUT_MS }).catch(() => { });
551
+ return { url: page.url() };
552
+ }
553
+ case "forward": {
554
+ await page.goForward({ waitUntil: "domcontentloaded", timeout: NAV_TIMEOUT_MS }).catch(() => { });
555
+ return { url: page.url() };
556
+ }
557
+ case "click":
558
+ case "dblclick":
559
+ case "hover": {
560
+ const pt = await resolveActionPoint(reg, rec, lease, args);
561
+ if (kind === "hover")
562
+ await page.mouse.move(pt.x, pt.y);
563
+ else
564
+ await page.mouse.click(pt.x, pt.y, { clickCount: kind === "dblclick" ? 2 : 1 });
565
+ return { at: { x: Math.round(pt.x), y: Math.round(pt.y) } };
566
+ }
567
+ case "input_text": {
568
+ const text = String(args.text ?? args.value ?? "");
569
+ if (text.length > 20_000)
570
+ throw new BridgeError("act_args_invalid", "text exceeds 20000 chars");
571
+ const pt = await resolveActionPoint(reg, rec, lease, args);
572
+ await page.mouse.click(pt.x, pt.y);
573
+ // Clear-then-type: select-all + type preserves site key handlers.
574
+ await page.keyboard.press(process.platform === "darwin" ? "Meta+A" : "Control+A").catch(() => { });
575
+ await page.keyboard.type(text, { delay: 8 });
576
+ return { typed: text.length };
577
+ }
578
+ case "press_key": {
579
+ const key = String(args.key || "");
580
+ if (!/^[A-Za-z0-9+]{1,32}$/.test(key))
581
+ throw new BridgeError("act_args_invalid", `invalid key '${key}'`);
582
+ await page.keyboard.press(key);
583
+ return { pressed: key };
584
+ }
585
+ case "scroll": {
586
+ if (args.ref) {
587
+ const binding = sessions.resolveRef(lease, args.ref);
588
+ const cdp = await getCdp(rec, page);
589
+ await cdp.send("DOM.scrollIntoViewIfNeeded", { backendNodeId: binding.backendNodeId }).catch(() => { });
590
+ return { scrolled: "into_view" };
591
+ }
592
+ const dy = Math.max(-4000, Math.min(4000, Number(args.dy ?? 600)));
593
+ await page.mouse.wheel(0, dy);
594
+ return { scrolled: dy };
595
+ }
596
+ case "select_option": {
597
+ if (!args.ref)
598
+ throw new BridgeError("act_args_invalid", "select_option requires a ref");
599
+ const binding = sessions.resolveRef(lease, args.ref);
600
+ const value = String(args.value ?? "");
601
+ const cdp = await getCdp(rec, page);
602
+ const resolved = await cdp.send("DOM.resolveNode", { backendNodeId: binding.backendNodeId });
603
+ await cdp.send("Runtime.callFunctionOn", {
604
+ objectId: resolved.object.objectId,
605
+ functionDeclaration: `function(v){ this.value = v; this.dispatchEvent(new Event('input',{bubbles:true})); this.dispatchEvent(new Event('change',{bubbles:true})); }`,
606
+ arguments: [{ value }],
607
+ });
608
+ return { selected: value };
609
+ }
610
+ case "wait": {
611
+ const ms = Math.min(Math.max(0, Number(args.ms ?? 500)), 15_000);
612
+ await page.waitForTimeout(ms);
613
+ return { waited: ms };
614
+ }
615
+ default:
616
+ throw new BridgeError("act_kind_unknown", `unhandled kind '${kind}'`);
617
+ }
618
+ });
619
+ }
620
+ async function actAndObserve(reg, args) {
621
+ const result = await performAct(reg, args);
622
+ const { rec, lease } = await getLease(reg);
623
+ await lease.page.waitForLoadState("domcontentloaded", { timeout: 5_000 }).catch(() => { });
624
+ await lease.page.waitForTimeout(ACT_SETTLE_MS);
625
+ const snap = await captureSnapshot(rec, lease);
626
+ return { ok: true, result, tree: snap.tree, generation: snap.generation };
627
+ }
628
+ // -- Governed facade for the code worker --------------------------------
629
+ function buildFacade(reg) {
630
+ return async (op, args) => {
631
+ if (reg.cancelled)
632
+ throw new BridgeError("run_cancelled", "run was cancelled/torn down");
633
+ switch (op) {
634
+ case "current_target": {
635
+ const { lease } = await getLease(reg);
636
+ return currentTarget(reg, lease);
637
+ }
638
+ case "get_screen_tree": {
639
+ const { rec, lease } = await getLease(reg);
640
+ const snap = await captureSnapshot(rec, lease, typeof args["scope"] === "string" ? args["scope"] : undefined);
641
+ return snap;
642
+ }
643
+ case "screenshot": {
644
+ const { lease } = await getLease(reg);
645
+ return takeScreenshot(lease);
646
+ }
647
+ case "act":
648
+ return actAndObserve(reg, args);
649
+ case "wait": {
650
+ const { lease } = await getLease(reg);
651
+ const ms = Math.min(Math.max(0, Number(args["ms"] ?? 500)), 10_000);
652
+ await lease.page.waitForTimeout(ms);
653
+ return { waited: ms };
654
+ }
655
+ default:
656
+ throw new BridgeError("facade_op_unknown", `op '${op}' is not part of the governed facade`);
657
+ }
658
+ };
659
+ }
660
+ function currentTarget(reg, lease) {
661
+ let origin = "";
662
+ try {
663
+ origin = new URL(lease.page.url()).origin;
664
+ }
665
+ catch {
666
+ origin = "";
667
+ }
668
+ return {
669
+ ok: true,
670
+ target: {
671
+ ref: lease.ref,
672
+ origin,
673
+ // Title is optional metadata; kept local-only (never persisted
674
+ // by the runner) per plan Section 10 target-metadata rule.
675
+ title: undefined,
676
+ },
677
+ };
678
+ }
679
+ // sessionId -> WatchLease
680
+ const watchLeases = new Map();
681
+ function postFrame(lease, jpegB64) {
682
+ // Never send if the lease has been deactivated concurrently.
683
+ if (!lease.active)
684
+ return;
685
+ const body = JSON.stringify({ sessionId: lease.sessionId, runId: lease.runId, jpegB64 });
686
+ const url = lease.framePostUrl;
687
+ try {
688
+ const u = new URL(url);
689
+ const isHttps = u.protocol === "https:";
690
+ const req = (isHttps ? httpsRequest : httpRequest)({
691
+ hostname: u.hostname,
692
+ port: u.port || (isHttps ? 443 : 80),
693
+ path: u.pathname + u.search,
694
+ method: "POST",
695
+ headers: {
696
+ "Content-Type": "application/json",
697
+ "Content-Length": Buffer.byteLength(body),
698
+ "Authorization": lease.runAuthHeader,
699
+ },
700
+ }, (res) => { res.resume(); /* drain */ });
701
+ req.on("error", () => { });
702
+ req.end(body);
703
+ }
704
+ catch {
705
+ // URL parse or socket error — silently skip this frame.
706
+ }
707
+ }
708
+ async function captureAndSchedule(lease) {
709
+ if (!lease.active)
710
+ return;
711
+ const reg = byRunId.get(lease.runId);
712
+ if (!reg || reg.cancelled) {
713
+ lease.active = false;
714
+ return;
715
+ }
716
+ try {
717
+ const rec = sessions.peekSession(lease.runId);
718
+ if (!rec || rec.state === "crashed" || rec.state === "closed") {
719
+ lease.active = false;
720
+ return;
721
+ }
722
+ const target = sessions.getLease(rec, reg.spec.grant.target.ref);
723
+ const { image_b64: jpegB64 } = await takeScreenshot(target);
724
+ const changed = jpegB64 !== lease.lastJpegB64;
725
+ if (changed) {
726
+ lease.lastJpegB64 = jpegB64;
727
+ postFrame(lease, jpegB64);
728
+ }
729
+ // Adaptive FPS: 250 ms (4 fps) when changed, 500 ms (2 fps) when
730
+ // the page is static. The cap is 4 fps; do not go lower than 2 fps
731
+ // so the viewer sees a heartbeat even on a static page.
732
+ const nextMs = changed ? 250 : 500;
733
+ if (lease.active) {
734
+ lease.timer = setTimeout(() => { void captureAndSchedule(lease); }, nextMs);
735
+ }
736
+ }
737
+ catch {
738
+ // Screenshot failed (page crashed, navigating, etc.) — retry later.
739
+ if (lease.active) {
740
+ lease.timer = setTimeout(() => { void captureAndSchedule(lease); }, 1000);
741
+ }
742
+ }
743
+ }
744
+ function stopWatchLease(sessionId) {
745
+ const lease = watchLeases.get(sessionId);
746
+ if (!lease)
747
+ return;
748
+ lease.active = false;
749
+ if (lease.timer) {
750
+ clearTimeout(lease.timer);
751
+ lease.timer = null;
752
+ }
753
+ watchLeases.delete(sessionId);
754
+ }
755
+ function setWatchLease(sessionId, active, framePostUrl, runAuthHeader) {
756
+ if (!active) {
757
+ stopWatchLease(sessionId);
758
+ return;
759
+ }
760
+ // Find which runId corresponds to this sessionId (session ids map 1:1
761
+ // to runIds in Phase 1 — one session per run).
762
+ let matchedRunId = "";
763
+ for (const [runId, reg] of byRunId) {
764
+ const rec = sessions.peekSession(runId);
765
+ if (rec && rec.id === sessionId) {
766
+ matchedRunId = runId;
767
+ break;
768
+ }
769
+ // Also accept runId directly as sessionId (used when the server
770
+ // addresses the watch by runId rather than the session UUID).
771
+ if (runId === sessionId) {
772
+ matchedRunId = runId;
773
+ break;
774
+ }
775
+ }
776
+ if (!matchedRunId) {
777
+ log(`[watch] browser:watch active=true for unknown sessionId ${sessionId.slice(0, 16)} — ignored`);
778
+ return;
779
+ }
780
+ // Stop any existing lease for this session before starting a new one.
781
+ stopWatchLease(sessionId);
782
+ const lease = {
783
+ runId: matchedRunId,
784
+ sessionId,
785
+ framePostUrl,
786
+ runAuthHeader,
787
+ active: true,
788
+ timer: null,
789
+ lastJpegB64: "",
790
+ };
791
+ watchLeases.set(sessionId, lease);
792
+ log(`[watch] frame producer started for session ${sessionId.slice(0, 16)} run=${matchedRunId.slice(0, 10)}`);
793
+ void captureAndSchedule(lease);
794
+ }
795
+ // -- HTTP shell ---------------------------------------------------------
796
+ const server = createServer(async (req, res) => {
797
+ res.setHeader("X-Frame-Options", "DENY");
798
+ res.setHeader("Cache-Control", "no-store");
799
+ res.setHeader("Content-Type", "application/json");
800
+ const respond = (status, body) => {
801
+ res.statusCode = status;
802
+ res.end(JSON.stringify(body));
803
+ };
804
+ const fail = (status, code, message) => respond(status, { ok: false, error: { code, message } });
805
+ if (req.method !== "POST")
806
+ return fail(405, "method_not_allowed", "POST only");
807
+ // Per-run bearer token, constant-time compare.
808
+ const auth = String(req.headers["authorization"] || "");
809
+ const reg = findRegistrationByAuth(auth);
810
+ if (!reg)
811
+ return fail(401, "unauthorized", "missing or unknown bearer token");
812
+ if (reg.cancelled)
813
+ return fail(410, "run_cancelled", "run was torn down");
814
+ // Grants are short-lived; the SESSION may outlive grant exp (leases
815
+ // have their own TTLs) but ops after exp + tolerance are refused so
816
+ // a leaked token cannot outlive its authorization.
817
+ if (Date.now() / 1000 > reg.spec.grant.exp + 30) {
818
+ return fail(403, "grant_expired", "the browser grant for this run has expired");
819
+ }
820
+ const route = req.url || "";
821
+ let payload;
822
+ try {
823
+ payload = await readJsonBody(req);
824
+ }
825
+ catch (e) {
826
+ const code = e?.code || "invalid_json";
827
+ return fail(code === "payload_too_large" ? 413 : 400, code, String(e?.message || e));
828
+ }
829
+ try {
830
+ switch (route) {
831
+ case "/browser/current_target": {
832
+ const { lease } = await getLease(reg);
833
+ return respond(200, currentTarget(reg, lease));
834
+ }
835
+ case "/browser/get_screen_tree": {
836
+ const { rec, lease } = await getLease(reg);
837
+ const snap = await captureSnapshot(rec, lease, typeof payload["scope"] === "string" ? String(payload["scope"]) : undefined);
838
+ return respond(200, { ok: true, ...snap });
839
+ }
840
+ case "/browser/screenshot": {
841
+ const { lease } = await getLease(reg);
842
+ const shot = await takeScreenshot(lease);
843
+ return respond(200, { ok: true, ...shot });
844
+ }
845
+ case "/browser/act": {
846
+ const out = await actAndObserve(reg, payload);
847
+ return respond(200, out);
848
+ }
849
+ case "/browser/run": {
850
+ if (!reg.spec.codeMode) {
851
+ return fail(403, "code_mode_disabled", "code mode is not enabled for this run (Phase 2 capability)");
852
+ }
853
+ const script = String(payload["script"] || "");
854
+ if (!script.trim())
855
+ return fail(400, "script_missing", "body.script is required");
856
+ if (script.length > 64 * 1024)
857
+ return fail(413, "script_too_large", "script exceeds 64KB");
858
+ const scratch = join(tmpdir(), `melaya-codeworker-${reg.spec.runId}`);
859
+ mkdirSync(scratch, { recursive: true });
860
+ const result = await runSandboxedScript({
861
+ script,
862
+ scratchDir: scratch,
863
+ facade: buildFacade(reg),
864
+ onTrace: (t) => {
865
+ reg.traces.push(t);
866
+ if (reg.traces.length > 500)
867
+ reg.traces.shift();
868
+ if (opts.verbose)
869
+ log(`[code-worker trace] #${t.seq} ${t.op} ${t.ok ? "ok" : `FAIL ${t.error}`} (${t.ms}ms)`);
870
+ },
871
+ log: (m) => opts.verbose && log(m),
872
+ });
873
+ // Fresh observation after the script, per the run contract.
874
+ let tree = "";
875
+ try {
876
+ const { rec, lease } = await getLease(reg);
877
+ tree = (await captureSnapshot(rec, lease)).tree;
878
+ }
879
+ catch { /* session may have crashed mid-script */ }
880
+ return respond(200, {
881
+ ok: result.ok,
882
+ result: result.result ?? null,
883
+ console: result.console,
884
+ tree,
885
+ traces: result.traces,
886
+ sandbox: result.sandbox,
887
+ ...(result.error ? { error: result.error } : {}),
888
+ });
889
+ }
890
+ default:
891
+ return fail(404, "not_found", `unknown route ${route}`);
892
+ }
893
+ }
894
+ catch (e) {
895
+ if (e instanceof SessionError)
896
+ return fail(409, e.code, e.message);
897
+ if (e instanceof BridgeError)
898
+ return fail(422, e.code, e.message);
899
+ const msg = String(e?.message || e);
900
+ log(`bridge internal error on ${route}: ${msg}`);
901
+ return fail(500, "bridge_error", msg.slice(0, 500));
902
+ }
903
+ });
904
+ function findRegistrationByAuth(authHeader) {
905
+ if (!authHeader.startsWith("Bearer "))
906
+ return null;
907
+ const presented = Buffer.from(authHeader.slice(7));
908
+ for (const [token, reg] of byToken) {
909
+ const expected = Buffer.from(token);
910
+ if (presented.length === expected.length && timingSafeEqual(presented, expected))
911
+ return reg;
912
+ }
913
+ return null;
914
+ }
915
+ async function readJsonBody(req) {
916
+ const chunks = [];
917
+ let total = 0;
918
+ for await (const chunk of req) {
919
+ total += chunk.length;
920
+ if (total > MAX_BODY_BYTES)
921
+ throw new BridgeError("payload_too_large", `body exceeds ${MAX_BODY_BYTES} bytes`);
922
+ chunks.push(chunk);
923
+ }
924
+ if (total === 0)
925
+ return {};
926
+ let parsed;
927
+ try {
928
+ parsed = JSON.parse(Buffer.concat(chunks).toString("utf-8"));
929
+ }
930
+ catch {
931
+ throw new BridgeError("invalid_json", "body is not valid JSON");
932
+ }
933
+ if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
934
+ throw new BridgeError("invalid_json", "body must be a JSON object");
935
+ }
936
+ return parsed;
937
+ }
938
+ await new Promise((resolve, reject) => {
939
+ server.once("error", reject);
940
+ server.listen(0, "127.0.0.1", () => resolve());
941
+ });
942
+ const addr = server.address();
943
+ if (!addr || typeof addr === "string") {
944
+ server.close();
945
+ throw new BridgeError("bridge_listen_failed", "could not bind the browser bridge to 127.0.0.1");
946
+ }
947
+ const port = addr.port;
948
+ log(`browser bridge listening on http://127.0.0.1:${port}`);
949
+ const registerRun = (spec) => {
950
+ if (byRunId.has(spec.runId)) {
951
+ throw new BridgeError("duplicate_run", `run ${spec.runId} is already registered on the bridge`);
952
+ }
953
+ const token = randomBytes(24).toString("hex");
954
+ const reg = {
955
+ spec,
956
+ token,
957
+ policy: buildOriginPolicy(spec.grant.originScopes),
958
+ cancelled: false,
959
+ violations: [],
960
+ sessionInit: null,
961
+ traces: [],
962
+ };
963
+ byToken.set(token, reg);
964
+ byRunId.set(spec.runId, reg);
965
+ log(`browser run registered: ${spec.runId.slice(0, 10)} mode=${spec.mode} engine=${spec.engine || "chrome"} scopes=${spec.grant.originScopes.length} ceiling=${spec.grant.effectCeiling}`);
966
+ return { token };
967
+ };
968
+ const teardownRun = async (runId, reason) => {
969
+ const reg = byRunId.get(runId);
970
+ if (!reg)
971
+ return;
972
+ reg.cancelled = true; // cancels in-flight ops at their gates
973
+ byRunId.delete(runId);
974
+ byToken.delete(reg.token);
975
+ // Stop any active watch-lease frame producer for this run so the
976
+ // periodic screenshot timer does not fire on a dead session.
977
+ for (const [sid, lease] of watchLeases) {
978
+ if (lease.runId === runId)
979
+ stopWatchLease(sid);
980
+ }
981
+ await sessions.teardownRun(runId, reason);
982
+ // The grant object becomes unreachable here (revocation-by-forget:
983
+ // the jti was already burned at verification, the claims held only
984
+ // in this registration are dropped, and the bearer token dies).
985
+ log(`browser run torn down: ${runId.slice(0, 10)} reason=${reason}`);
986
+ };
987
+ const teardownAll = async (reason) => {
988
+ const ids = [...byRunId.keys()];
989
+ await Promise.all(ids.map((id) => teardownRun(id, reason)));
990
+ };
991
+ return {
992
+ url: `http://127.0.0.1:${port}`,
993
+ port,
994
+ registerRun,
995
+ teardownRun,
996
+ teardownAll,
997
+ hasRun(runId) {
998
+ return byRunId.has(runId);
999
+ },
1000
+ setWatchLease,
1001
+ async shutdown() {
1002
+ // Stop all watch-lease producers before closing sessions.
1003
+ for (const sid of [...watchLeases.keys()])
1004
+ stopWatchLease(sid);
1005
+ await teardownAll("bridge_shutdown");
1006
+ sessions.dispose();
1007
+ await new Promise((r) => server.close(() => r()));
1008
+ },
1009
+ };
1010
+ }
1011
+ // ---------------------------------------------------------------------
1012
+ // Errors
1013
+ // ---------------------------------------------------------------------
1014
+ export class BridgeError extends Error {
1015
+ code;
1016
+ constructor(code, message) {
1017
+ super(message);
1018
+ this.name = "BridgeError";
1019
+ this.code = code;
1020
+ }
1021
+ }