@tomflow/proflow-execution-browser-extension 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. package/README.md +7 -0
  2. package/conformance.json +1 -0
  3. package/deployment/browser-extension.json +6 -0
  4. package/dist/deployment/adapter.d.ts +61 -0
  5. package/dist/deployment/adapter.js +47 -0
  6. package/dist/deployment/descriptor.d.ts +103 -0
  7. package/dist/deployment/descriptor.js +109 -0
  8. package/dist/extension/background.d.ts +1 -0
  9. package/dist/extension/background.js +752 -0
  10. package/dist/extension/content.d.ts +1 -0
  11. package/dist/extension/content.js +90 -0
  12. package/dist/extension/options.d.ts +1 -0
  13. package/dist/extension/options.js +68 -0
  14. package/dist/extension/side-panel.d.ts +1 -0
  15. package/dist/extension/side-panel.js +262 -0
  16. package/dist/src/bridge.d.ts +26 -0
  17. package/dist/src/bridge.js +288 -0
  18. package/dist/src/collaboration-carrier.d.ts +65 -0
  19. package/dist/src/collaboration-carrier.js +138 -0
  20. package/dist/src/index.d.ts +137 -0
  21. package/dist/src/index.js +779 -0
  22. package/dist/src/runtime-composition.d.ts +97 -0
  23. package/dist/src/runtime-composition.js +124 -0
  24. package/dist/src/system-observer.d.ts +86 -0
  25. package/dist/src/system-observer.js +252 -0
  26. package/dist/src/task-observer.d.ts +118 -0
  27. package/dist/src/task-observer.js +105 -0
  28. package/dist/src/vision.d.ts +73 -0
  29. package/dist/src/vision.js +82 -0
  30. package/extension/background.ts +997 -0
  31. package/extension/content.ts +138 -0
  32. package/extension/options.html +54 -0
  33. package/extension/options.ts +98 -0
  34. package/extension/side-panel.html +77 -0
  35. package/extension/side-panel.ts +349 -0
  36. package/manifest.json +20 -0
  37. package/package.json +58 -0
  38. package/proflow.module.json +127 -0
  39. package/self-install.mjs +27 -0
@@ -0,0 +1,779 @@
1
+ import { randomUUID } from "node:crypto";
2
+ import { browserCapabilityIds, executeCapabilityRequestSchema, } from "@tomflow/proflow-execution-contracts";
3
+ import { deferVisionObservation, isVisionObservationVerified, parseCapturedScreenshot, } from "./vision.js";
4
+ export { BrowserRealityBridgeError, createBrowserRealityBridgeServer, } from "./bridge.js";
5
+ export { createCollaborationCarrierApplication, } from "./collaboration-carrier.js";
6
+ export { createSystemObserver, } from "./system-observer.js";
7
+ export { createTaskObserver, } from "./task-observer.js";
8
+ export { deferVisionObservation, isVisionObservationVerified, parseCapturedScreenshot, VISION_OBSERVATION_MIN_CONFIDENCE, visionMimeTypes, visionRecommendedNext, } from "./vision.js";
9
+ export class ExecutionBrowserError extends Error {
10
+ code;
11
+ retryable = false;
12
+ constructor(code, message) {
13
+ super(message);
14
+ this.name = "ExecutionBrowserError";
15
+ this.code = code;
16
+ }
17
+ }
18
+ const browserCapabilities = new Set(browserCapabilityIds);
19
+ const workerWakeTriggerTypes = new Set([
20
+ "NODE_READY",
21
+ "REOPEN",
22
+ "EXECUTION_RESULT_READY",
23
+ "PEER_REPLY_READY",
24
+ "RECOVERY_RESUME",
25
+ ]);
26
+ function parseCarrierIdentity(raw) {
27
+ let url;
28
+ try {
29
+ url = new URL(raw);
30
+ }
31
+ catch {
32
+ throw new ExecutionBrowserError("PRECONDITION_FAILED", "CARRIER_URL_INVALID");
33
+ }
34
+ if (url.protocol !== "https:" || url.hostname !== "chatgpt.com")
35
+ throw new ExecutionBrowserError("PRECONDITION_FAILED", "CARRIER_URL_INVALID");
36
+ const segments = url.pathname.split("/").filter(Boolean);
37
+ if (segments[0] !== "g" || !segments[1]?.startsWith("g-"))
38
+ throw new ExecutionBrowserError("PRECONDITION_FAILED", "ROLE_URL_INVALID");
39
+ if (segments.length === 2)
40
+ return { roleRef: segments[1], workerRef: null };
41
+ if (segments[2] !== "c" ||
42
+ !segments[3] ||
43
+ segments[3].length > 512 ||
44
+ !/^[A-Za-z0-9_-]+$/.test(segments[3]) ||
45
+ segments.length !== 4)
46
+ throw new ExecutionBrowserError("PRECONDITION_FAILED", "WORKER_URL_INVALID");
47
+ return { roleRef: segments[1], workerRef: segments[3] };
48
+ }
49
+ function browserEvidence(idFactory, observation, verified) {
50
+ return {
51
+ kind: "browser",
52
+ evidenceRef: `evidence:${idFactory()}`,
53
+ targetRef: `tab:${observation.tabId}`,
54
+ observationRef: `observation:${idFactory()}`,
55
+ verified,
56
+ };
57
+ }
58
+ export function createExecutionBrowserExtension(options) {
59
+ const idFactory = options.idFactory ?? randomUUID;
60
+ const now = options.now ?? (() => new Date());
61
+ const extensionInstanceId = `extension:${idFactory()}`;
62
+ const sessions = new Map();
63
+ const lanes = new Map();
64
+ let recoveryCompleted = false;
65
+ let writeTail = Promise.resolve();
66
+ const serializeWrite = async (operation) => {
67
+ let release;
68
+ const previous = writeTail;
69
+ writeTail = new Promise((resolve) => {
70
+ release = resolve;
71
+ });
72
+ await previous;
73
+ try {
74
+ return await operation();
75
+ }
76
+ finally {
77
+ release();
78
+ }
79
+ };
80
+ const registerContentSession = (observation) => {
81
+ const identity = parseCarrierIdentity(observation.url);
82
+ sessions.set(observation.tabId, structuredClone(observation));
83
+ if (identity.workerRef) {
84
+ lanes.set(`${identity.roleRef}:${identity.workerRef}`, {
85
+ roleRef: identity.roleRef,
86
+ workerRef: identity.workerRef,
87
+ tabId: observation.tabId,
88
+ pageState: observation.pageState,
89
+ activityKind: observation.activityKind,
90
+ currentExecutionRef: null,
91
+ continuationRef: null,
92
+ lastProgressAt: observation.observedAt,
93
+ });
94
+ }
95
+ };
96
+ const matchingTab = async (roleRef, workerRef) => {
97
+ for (const tab of await options.browser.listTabs()) {
98
+ try {
99
+ const identity = parseCarrierIdentity(tab.url);
100
+ if (identity.roleRef === roleRef && identity.workerRef === workerRef) {
101
+ registerContentSession(tab);
102
+ return tab;
103
+ }
104
+ }
105
+ catch {
106
+ /* an unrelated tab is not a carrier identity */
107
+ }
108
+ }
109
+ return null;
110
+ };
111
+ const ensureRestored = async (taskId, roleRef, workerRef, expectedConversationLocator) => {
112
+ const bound = await options.task.getWorkerBinding(taskId, roleRef);
113
+ if (!bound || bound.workerRef !== workerRef)
114
+ throw new ExecutionBrowserError("PRECONDITION_FAILED", "WORKER_BINDING_MISMATCH");
115
+ if (!bound.conversationLocator)
116
+ throw new ExecutionBrowserError("PRECONDITION_FAILED", "CONVERSATION_LOCATOR_REQUIRED");
117
+ if (expectedConversationLocator !== undefined &&
118
+ expectedConversationLocator !== bound.conversationLocator)
119
+ throw new ExecutionBrowserError("PRECONDITION_FAILED", "CONVERSATION_LOCATOR_MISMATCH");
120
+ const existing = await matchingTab(roleRef, workerRef);
121
+ if (existing?.url === bound.conversationLocator)
122
+ return existing;
123
+ const opened = await options.browser.open(bound.conversationLocator);
124
+ const observed = await options.browser.observe(opened.tabId);
125
+ const identity = parseCarrierIdentity(observed.url);
126
+ if (identity.roleRef !== roleRef || identity.workerRef !== workerRef)
127
+ throw new ExecutionBrowserError("PRECONDITION_FAILED", "RESTORE_IDENTITY_MISMATCH");
128
+ if (observed.url !== bound.conversationLocator)
129
+ throw new ExecutionBrowserError("PRECONDITION_FAILED", "RESTORE_LOCATOR_MISMATCH");
130
+ registerContentSession(observed);
131
+ return observed;
132
+ };
133
+ const browserPrecondition = (request) => {
134
+ const precondition = {
135
+ kind: "browser",
136
+ capability: request.capability,
137
+ ...(request.taskId ? { taskId: request.taskId } : {}),
138
+ ...(request.roleRef ? { roleRef: request.roleRef } : {}),
139
+ ...(request.workerRef ? { workerRef: request.workerRef } : {}),
140
+ };
141
+ if (request.capability === "worker.create")
142
+ return {
143
+ ...precondition,
144
+ roleRef: request.input.roleRef,
145
+ roleUrl: request.input.roleUrl,
146
+ fingerprint: request.input.bootstrapFingerprint,
147
+ };
148
+ if (request.capability === "worker.restore")
149
+ return {
150
+ ...precondition,
151
+ roleRef: request.input.roleRef,
152
+ workerRef: request.input.workerRef,
153
+ conversationUrl: request.input.conversationUrl,
154
+ };
155
+ if (request.capability === "worker.wake")
156
+ return {
157
+ ...precondition,
158
+ roleRef: request.input.roleRef,
159
+ workerRef: request.input.workerRef,
160
+ fingerprint: request.input.fingerprint,
161
+ };
162
+ if (request.capability === "collaboration.deliver")
163
+ return {
164
+ ...precondition,
165
+ roleRef: request.input.roleRef,
166
+ workerRef: request.input.workerRef,
167
+ fingerprint: request.input.contentFingerprint,
168
+ messageRef: request.input.messageRef,
169
+ };
170
+ if ("targetRef" in request.input)
171
+ precondition.targetRef = request.input.targetRef;
172
+ if (request.capability === "browser.submit")
173
+ precondition.fingerprint = request.input.fingerprint;
174
+ if (request.capability === "browser.navigate")
175
+ precondition.expectedUrl = request.input.url;
176
+ return precondition;
177
+ };
178
+ const assertNotAborted = (invocation) => {
179
+ if (invocation.signal?.aborted)
180
+ throw new ExecutionBrowserError("CANCELLED", "EXECUTION_ABORTED_BEFORE_BROWSER_EFFECT");
181
+ };
182
+ const effectStarted = async (invocation) => {
183
+ assertNotAborted(invocation);
184
+ const precondition = browserPrecondition(invocation.request);
185
+ if (!invocation.onEffectStarted)
186
+ throw new ExecutionBrowserError("PRECONDITION_FAILED", "DURABLE_EFFECT_BOUNDARY_REQUIRED");
187
+ await invocation.onEffectStarted(precondition);
188
+ assertNotAborted(invocation);
189
+ return precondition;
190
+ };
191
+ const visionObservation = async (shot, observationContext) => {
192
+ let image;
193
+ try {
194
+ image = parseCapturedScreenshot(shot);
195
+ }
196
+ catch (error) {
197
+ return deferVisionObservation("VISION_IMAGE_INVALID", error instanceof Error ? error.message : "screenshot image is invalid");
198
+ }
199
+ if (!options.vision)
200
+ return deferVisionObservation("VISION_PORT_UNAVAILABLE", "no Browser Vision port is injected");
201
+ try {
202
+ return await options.vision.inspect({ image, observationContext });
203
+ }
204
+ catch (error) {
205
+ return deferVisionObservation("VISION_INFERENCE_FAILED", error instanceof Error ? error.message : "vision inference failed");
206
+ }
207
+ };
208
+ const result = (capabilityResult, observation, effectApplied, precondition) => ({
209
+ result: capabilityResult,
210
+ evidence: [browserEvidence(idFactory, observation, true)],
211
+ artifacts: [],
212
+ precondition: precondition ?? {
213
+ kind: "browser",
214
+ capability: capabilityResult.capability,
215
+ },
216
+ effectApplied,
217
+ successful: true,
218
+ });
219
+ const execute = async (raw) => {
220
+ assertNotAborted(raw);
221
+ const request = executeCapabilityRequestSchema.parse(raw.request);
222
+ if (!browserCapabilities.has(request.capability))
223
+ throw new ExecutionBrowserError("EXECUTOR_UNAVAILABLE", "BROWSER_CAPABILITY_REQUIRED");
224
+ if (!request.taskId &&
225
+ [
226
+ "worker.create",
227
+ "worker.restore",
228
+ "worker.wake",
229
+ "collaboration.deliver",
230
+ ].includes(request.capability))
231
+ throw new ExecutionBrowserError("PRECONDITION_FAILED", "TASK_ID_REQUIRED");
232
+ const taskId = request.taskId ?? "";
233
+ if (request.capability === "worker.create")
234
+ return serializeWrite(async () => {
235
+ if (await options.task.getWorkerBinding(taskId, request.input.roleRef))
236
+ throw new ExecutionBrowserError("PRECONDITION_FAILED", "WORKER_ALREADY_BOUND");
237
+ const roleIdentity = parseCarrierIdentity(request.input.roleUrl);
238
+ if (roleIdentity.roleRef !== request.input.roleRef ||
239
+ roleIdentity.workerRef)
240
+ throw new ExecutionBrowserError("PRECONDITION_FAILED", "ROLE_URL_MISMATCH");
241
+ const precondition = await effectStarted(raw);
242
+ const opened = await options.browser.open(request.input.roleUrl);
243
+ await options.browser.submit(opened.tabId, `WORKER_BIND ${request.input.bootstrapFingerprint}`, request.input.bootstrapFingerprint);
244
+ const observed = await options.browser.observe(opened.tabId);
245
+ const identity = parseCarrierIdentity(observed.url);
246
+ if (identity.roleRef !== request.input.roleRef || !identity.workerRef)
247
+ throw new ExecutionBrowserError("UNKNOWN_SIDE_EFFECT", "CREATE_REALITY_UNCONFIRMED");
248
+ await options.task.bindWorker({
249
+ taskId,
250
+ roleRef: identity.roleRef,
251
+ workerRef: identity.workerRef,
252
+ conversationLocator: observed.url,
253
+ });
254
+ registerContentSession(observed);
255
+ return result({
256
+ capability: "worker.create",
257
+ data: {
258
+ roleRef: identity.roleRef,
259
+ workerRef: identity.workerRef,
260
+ conversationUrl: observed.url,
261
+ verified: true,
262
+ },
263
+ }, observed, true, precondition);
264
+ });
265
+ if (request.capability === "worker.restore") {
266
+ const observed = await ensureRestored(taskId, request.input.roleRef, request.input.workerRef, request.input.conversationUrl);
267
+ return result({
268
+ capability: "worker.restore",
269
+ data: {
270
+ roleRef: request.input.roleRef,
271
+ workerRef: request.input.workerRef,
272
+ restored: true,
273
+ },
274
+ }, observed, false);
275
+ }
276
+ if (request.capability === "worker.wake")
277
+ return serializeWrite(async () => {
278
+ if (!workerWakeTriggerTypes.has(request.input.trigger))
279
+ throw new ExecutionBrowserError("PRECONDITION_FAILED", "WAKE_TRIGGER_TYPE_INVALID");
280
+ const observed = await ensureRestored(taskId, request.input.roleRef, request.input.workerRef);
281
+ if (observed.pageState === "BUSY" || observed.pageState === "BLOCKED")
282
+ throw new ExecutionBrowserError("PRECONDITION_FAILED", "PAGE_NOT_WRITABLE");
283
+ const precondition = await effectStarted(raw);
284
+ const trigger = JSON.stringify({
285
+ protocol: "proflow.agent.browser-trigger.v1",
286
+ triggerRef: request.input.fingerprint,
287
+ triggerType: request.input.trigger,
288
+ taskId,
289
+ nodeId: request.input.nodeId,
290
+ runNo: request.input.runNo,
291
+ roleRef: request.input.roleRef,
292
+ workerRef: request.input.workerRef,
293
+ occurredAt: now().toISOString(),
294
+ fingerprint: request.input.fingerprint,
295
+ payload: { trigger: request.input.trigger },
296
+ });
297
+ const after = await options.browser.submit(observed.tabId, trigger, request.input.fingerprint);
298
+ if (!(await options.browser.hasMessage(after.tabId, request.input.fingerprint)))
299
+ throw new ExecutionBrowserError("UNKNOWN_SIDE_EFFECT", "WAKE_REALITY_UNCONFIRMED");
300
+ registerContentSession(after);
301
+ return result({
302
+ capability: "worker.wake",
303
+ data: {
304
+ roleRef: request.input.roleRef,
305
+ workerRef: request.input.workerRef,
306
+ triggerFingerprint: request.input.fingerprint,
307
+ delivered: true,
308
+ },
309
+ }, after, true, precondition);
310
+ });
311
+ if (request.capability === "collaboration.deliver")
312
+ return serializeWrite(async () => {
313
+ const message = await options.agent.getPendingMessage(request.input.messageRef);
314
+ if (message.messageId !== request.input.messageRef ||
315
+ message.taskId !== taskId ||
316
+ message.targetRoleRef !== request.input.roleRef ||
317
+ message.targetWorkerRef !== request.input.workerRef ||
318
+ message.status !== "PENDING")
319
+ throw new ExecutionBrowserError("PRECONDITION_FAILED", "DELIVERY_OWNER_FACT_MISMATCH");
320
+ const observed = await ensureRestored(taskId, request.input.roleRef, request.input.workerRef);
321
+ const precondition = await effectStarted(raw);
322
+ const trigger = JSON.stringify({
323
+ protocol: "proflow.agent.browser-trigger.v1",
324
+ triggerRef: message.messageId,
325
+ triggerType: message.kind === "REPLY" ? "PEER_REPLY_READY" : "PEER_MESSAGE",
326
+ taskId,
327
+ roleRef: message.targetRoleRef,
328
+ workerRef: message.targetWorkerRef,
329
+ occurredAt: now().toISOString(),
330
+ fingerprint: request.input.contentFingerprint,
331
+ payload: { collaboration: message },
332
+ });
333
+ const after = await options.browser.submit(observed.tabId, trigger, request.input.contentFingerprint);
334
+ if (!(await options.browser.hasMessage(after.tabId, request.input.contentFingerprint)))
335
+ throw new ExecutionBrowserError("UNKNOWN_SIDE_EFFECT", "DELIVERY_REALITY_UNCONFIRMED");
336
+ const evidence = browserEvidence(idFactory, after, true);
337
+ return {
338
+ ...result({
339
+ capability: "collaboration.deliver",
340
+ data: {
341
+ messageRef: request.input.messageRef,
342
+ delivered: true,
343
+ evidenceRef: evidence.evidenceRef,
344
+ },
345
+ }, after, true, precondition),
346
+ evidence: [evidence],
347
+ };
348
+ });
349
+ const target = "targetRef" in request.input
350
+ ? request.input.targetRef
351
+ : request.workerRef;
352
+ if (!target)
353
+ throw new ExecutionBrowserError("PRECONDITION_FAILED", "BROWSER_TARGET_REQUIRED");
354
+ const numericTab = Number(target.replace(/^tab:/, ""));
355
+ const observed = Number.isInteger(numericTab)
356
+ ? await options.browser.observe(numericTab)
357
+ : request.roleRef && request.workerRef
358
+ ? await ensureRestored(taskId, request.roleRef, request.workerRef)
359
+ : null;
360
+ if (!observed)
361
+ throw new ExecutionBrowserError("PRECONDITION_FAILED", "BROWSER_TARGET_NOT_FOUND");
362
+ if (request.capability === "browser.observe") {
363
+ const needsVisionFallback = observed.pageState === "UNKNOWN" ||
364
+ observed.activityKind === "RECOVERING";
365
+ if (!needsVisionFallback)
366
+ return result({
367
+ capability: "browser.observe",
368
+ data: {
369
+ targetRef: target,
370
+ verified: true,
371
+ observationRef: `observation:${idFactory()}`,
372
+ },
373
+ }, observed, false);
374
+ // Deterministic DOM/runtime observation is always primary. Only an
375
+ // explicitly ambiguous/recovery state may escalate to screenshot Vision.
376
+ // The typed Vision result remains bounded diagnostic evidence and never
377
+ // mutates Task/Execution/Approval authority.
378
+ const shot = await options.browser.screenshot(observed.tabId);
379
+ const vision = await visionObservation(shot, {
380
+ targetRef: target,
381
+ pageState: observed.pageState,
382
+ activityKind: observed.activityKind,
383
+ observedAt: observed.observedAt,
384
+ });
385
+ const visionVerified = isVisionObservationVerified(vision);
386
+ const observationRef = vision.status === "OBSERVED" ? vision.observationRef : shot.evidenceRef;
387
+ return {
388
+ ...result({
389
+ capability: "browser.observe",
390
+ data: {
391
+ targetRef: target,
392
+ verified: visionVerified,
393
+ observationRef,
394
+ visionFallback: "REAL_EXTERNAL_PENDING",
395
+ },
396
+ }, observed, false),
397
+ evidence: [
398
+ {
399
+ kind: "browser",
400
+ evidenceRef: shot.evidenceRef,
401
+ targetRef: target,
402
+ observationRef: shot.evidenceRef,
403
+ verified: visionVerified,
404
+ },
405
+ ],
406
+ artifacts: [
407
+ {
408
+ ref: shot.evidenceRef,
409
+ path: "",
410
+ bytes: shot.sizeBytes,
411
+ stream: "report",
412
+ kind: "output",
413
+ hash: shot.hash,
414
+ mime: shot.mimeType,
415
+ metadata: {
416
+ source: "browser.observe.vision-fallback",
417
+ trigger: {
418
+ pageState: observed.pageState,
419
+ activityKind: observed.activityKind,
420
+ },
421
+ vision,
422
+ },
423
+ },
424
+ ],
425
+ };
426
+ }
427
+ if (request.capability === "browser.screenshot") {
428
+ const shot = await options.browser.screenshot(observed.tabId);
429
+ // The screenshot is the ambiguity/recovery fallback: hand the real
430
+ // captured image bytes to the injected Vision port and retain only the
431
+ // bounded typed observation. `visionFallback` stays REAL_EXTERNAL_PENDING
432
+ // as the honest marker that the physical-phone Vision E2E is not wired;
433
+ // the code wiring (capture → typed Vision port → typed observation) is
434
+ // complete and recorded in the artifact metadata below.
435
+ const vision = await visionObservation(shot, {
436
+ targetRef: target,
437
+ pageState: observed.pageState,
438
+ activityKind: observed.activityKind,
439
+ observedAt: observed.observedAt,
440
+ });
441
+ return {
442
+ ...result({
443
+ capability: "browser.screenshot",
444
+ data: {
445
+ targetRef: target,
446
+ verified: true,
447
+ observationRef: shot.evidenceRef,
448
+ mimeType: shot.mimeType,
449
+ sizeBytes: shot.sizeBytes,
450
+ hash: shot.hash,
451
+ visionFallback: "REAL_EXTERNAL_PENDING",
452
+ },
453
+ }, observed, false),
454
+ evidence: [
455
+ {
456
+ kind: "browser",
457
+ evidenceRef: shot.evidenceRef,
458
+ targetRef: target,
459
+ observationRef: shot.evidenceRef,
460
+ verified: true,
461
+ },
462
+ ],
463
+ artifacts: [
464
+ {
465
+ ref: shot.evidenceRef,
466
+ path: "",
467
+ bytes: shot.sizeBytes,
468
+ stream: "report",
469
+ kind: "output",
470
+ hash: shot.hash,
471
+ mime: shot.mimeType,
472
+ metadata: {
473
+ source: "browser.screenshot",
474
+ vision,
475
+ },
476
+ },
477
+ ],
478
+ };
479
+ }
480
+ if (request.capability === "browser.verify") {
481
+ const verified = await options.browser.hasMessage(observed.tabId, request.input.expectedFingerprint);
482
+ return result({
483
+ capability: "browser.verify",
484
+ data: {
485
+ targetRef: target,
486
+ verified,
487
+ observationRef: `observation:${idFactory()}`,
488
+ },
489
+ }, observed, false);
490
+ }
491
+ if (!options.browser.perform)
492
+ throw new ExecutionBrowserError("EXECUTOR_UNAVAILABLE", "BROWSER_PRIMITIVE_UNAVAILABLE");
493
+ return serializeWrite(async () => {
494
+ const precondition = await effectStarted(raw);
495
+ const after = await options.browser.perform?.(request, observed.tabId);
496
+ if (!after)
497
+ throw new ExecutionBrowserError("UNKNOWN_SIDE_EFFECT", "BROWSER_RESULT_MISSING");
498
+ return result({
499
+ capability: request.capability,
500
+ data: {
501
+ targetRef: target,
502
+ verified: true,
503
+ observationRef: `observation:${idFactory()}`,
504
+ },
505
+ }, after, true, precondition);
506
+ });
507
+ };
508
+ const reconcile = async (requestRaw, preconditionRaw) => {
509
+ const request = executeCapabilityRequestSchema.parse(requestRaw);
510
+ if (preconditionRaw.kind !== "browser" ||
511
+ preconditionRaw.capability !== request.capability)
512
+ return { state: "UNKNOWN", evidence: [] };
513
+ const precondition = preconditionRaw;
514
+ const observeTarget = async () => {
515
+ if (precondition.roleRef && precondition.workerRef)
516
+ return matchingTab(precondition.roleRef, precondition.workerRef);
517
+ if (precondition.targetRef) {
518
+ const numericTab = Number(precondition.targetRef.replace(/^tab:/, ""));
519
+ if (Number.isInteger(numericTab)) {
520
+ try {
521
+ return await options.browser.observe(numericTab);
522
+ }
523
+ catch {
524
+ return null;
525
+ }
526
+ }
527
+ }
528
+ return null;
529
+ };
530
+ if (request.capability === "worker.create") {
531
+ const roleRef = precondition.roleRef;
532
+ const taskId = precondition.taskId;
533
+ if (!roleRef || !taskId || !precondition.fingerprint)
534
+ return { state: "UNKNOWN", evidence: [] };
535
+ const boundWorker = await options.task.getWorkerBinding(taskId, roleRef);
536
+ if (boundWorker) {
537
+ const observed = await matchingTab(roleRef, boundWorker.workerRef);
538
+ if (!observed)
539
+ return { state: "UNKNOWN", evidence: [] };
540
+ if (!(await options.browser.hasMessage(observed.tabId, precondition.fingerprint)))
541
+ return { state: "UNKNOWN", evidence: [] };
542
+ const evidence = browserEvidence(idFactory, observed, true);
543
+ return {
544
+ state: "APPLIED",
545
+ evidence: [evidence],
546
+ result: {
547
+ capability: "worker.create",
548
+ data: {
549
+ roleRef,
550
+ workerRef: boundWorker.workerRef,
551
+ conversationUrl: observed.url,
552
+ verified: true,
553
+ },
554
+ },
555
+ };
556
+ }
557
+ const candidates = [];
558
+ for (const tab of await options.browser.listTabs()) {
559
+ try {
560
+ const identity = parseCarrierIdentity(tab.url);
561
+ if (identity.roleRef === roleRef &&
562
+ identity.workerRef !== null &&
563
+ (await options.browser.hasMessage(tab.tabId, precondition.fingerprint)))
564
+ candidates.push(tab);
565
+ }
566
+ catch {
567
+ // Ignore unrelated/non-carrier tabs.
568
+ }
569
+ }
570
+ if (candidates.length !== 1)
571
+ return { state: "UNKNOWN", evidence: [] };
572
+ const observed = candidates[0];
573
+ if (!observed)
574
+ return { state: "UNKNOWN", evidence: [] };
575
+ const identity = parseCarrierIdentity(observed.url);
576
+ if (!identity.workerRef)
577
+ return { state: "UNKNOWN", evidence: [] };
578
+ await options.task.bindWorker({
579
+ taskId,
580
+ roleRef,
581
+ workerRef: identity.workerRef,
582
+ conversationLocator: observed.url,
583
+ });
584
+ const evidence = browserEvidence(idFactory, observed, true);
585
+ return {
586
+ state: "APPLIED",
587
+ evidence: [evidence],
588
+ result: {
589
+ capability: "worker.create",
590
+ data: {
591
+ roleRef,
592
+ workerRef: identity.workerRef,
593
+ conversationUrl: observed.url,
594
+ verified: true,
595
+ },
596
+ },
597
+ };
598
+ }
599
+ if (request.capability === "worker.wake") {
600
+ if (!precondition.roleRef ||
601
+ !precondition.workerRef ||
602
+ !precondition.fingerprint)
603
+ return { state: "UNKNOWN", evidence: [] };
604
+ const observed = await matchingTab(precondition.roleRef, precondition.workerRef);
605
+ if (!observed)
606
+ return { state: "UNKNOWN", evidence: [] };
607
+ const delivered = await options.browser.hasMessage(observed.tabId, precondition.fingerprint);
608
+ const evidence = browserEvidence(idFactory, observed, delivered);
609
+ if (!delivered)
610
+ return { state: "NOT_APPLIED", evidence: [evidence] };
611
+ return {
612
+ state: "APPLIED",
613
+ evidence: [evidence],
614
+ result: {
615
+ capability: "worker.wake",
616
+ data: {
617
+ roleRef: precondition.roleRef,
618
+ workerRef: precondition.workerRef,
619
+ triggerFingerprint: precondition.fingerprint,
620
+ delivered: true,
621
+ },
622
+ },
623
+ };
624
+ }
625
+ if (request.capability === "collaboration.deliver") {
626
+ if (!precondition.roleRef ||
627
+ !precondition.workerRef ||
628
+ !precondition.fingerprint ||
629
+ !precondition.messageRef)
630
+ return { state: "UNKNOWN", evidence: [] };
631
+ const observed = await matchingTab(precondition.roleRef, precondition.workerRef);
632
+ if (!observed)
633
+ return { state: "UNKNOWN", evidence: [] };
634
+ const delivered = await options.browser.hasMessage(observed.tabId, precondition.fingerprint);
635
+ const evidence = browserEvidence(idFactory, observed, delivered);
636
+ if (!delivered)
637
+ return { state: "NOT_APPLIED", evidence: [evidence] };
638
+ return {
639
+ state: "APPLIED",
640
+ evidence: [evidence],
641
+ result: {
642
+ capability: "collaboration.deliver",
643
+ data: {
644
+ messageRef: precondition.messageRef,
645
+ delivered: true,
646
+ evidenceRef: evidence.evidenceRef,
647
+ },
648
+ },
649
+ };
650
+ }
651
+ if (request.capability === "browser.submit" && precondition.fingerprint) {
652
+ const observed = await observeTarget();
653
+ if (!observed)
654
+ return { state: "UNKNOWN", evidence: [] };
655
+ const delivered = await options.browser.hasMessage(observed.tabId, precondition.fingerprint);
656
+ const evidence = browserEvidence(idFactory, observed, delivered);
657
+ if (!delivered)
658
+ return { state: "NOT_APPLIED", evidence: [evidence] };
659
+ return {
660
+ state: "APPLIED",
661
+ evidence: [evidence],
662
+ result: {
663
+ capability: "browser.submit",
664
+ data: {
665
+ targetRef: precondition.targetRef ?? `tab:${observed.tabId}`,
666
+ verified: true,
667
+ observationRef: evidence.observationRef,
668
+ },
669
+ },
670
+ };
671
+ }
672
+ if (request.capability === "browser.navigate" && precondition.expectedUrl) {
673
+ const observed = await observeTarget();
674
+ if (!observed || observed.url !== precondition.expectedUrl)
675
+ return { state: "UNKNOWN", evidence: [] };
676
+ const evidence = browserEvidence(idFactory, observed, true);
677
+ return {
678
+ state: "APPLIED",
679
+ evidence: [evidence],
680
+ result: {
681
+ capability: "browser.navigate",
682
+ data: {
683
+ targetRef: precondition.targetRef ?? `tab:${observed.tabId}`,
684
+ verified: true,
685
+ observationRef: evidence.observationRef,
686
+ },
687
+ },
688
+ };
689
+ }
690
+ // click/input/upload and other writes lack a durable postcondition that can
691
+ // prove the exact effect after restart. Never infer APPLIED from tab presence.
692
+ return { state: "UNKNOWN", evidence: [] };
693
+ };
694
+ return Object.freeze({
695
+ extensionInstanceId,
696
+ parseCarrierIdentity,
697
+ registerContentSession,
698
+ isContentSessionCurrent(tabId, contentInstanceId) {
699
+ return sessions.get(tabId)?.contentInstanceId === contentInstanceId;
700
+ },
701
+ classifyProgress(input) {
702
+ if (input.legitimateWait)
703
+ return "EXPECTED_WAIT";
704
+ if (input.pageState === "IDLE" && input.nodeInProgress)
705
+ return "PROGRESS_GAP";
706
+ if (input.pageState === "BUSY" &&
707
+ input.millisecondsWithoutProgress > 60_000)
708
+ return "RUNTIME_STALL";
709
+ return "NORMAL";
710
+ },
711
+ async inspectScreenshot(tabId, observationContext) {
712
+ const shot = await options.browser.screenshot(tabId);
713
+ return visionObservation(shot, observationContext);
714
+ },
715
+ async handlePermissionFallback(tabId, continuationRef) {
716
+ const observed = await options.browser.observe(tabId);
717
+ const shot = await options.browser.screenshot(tabId);
718
+ const identity = parseCarrierIdentity(observed.url);
719
+ if (identity.workerRef) {
720
+ const lane = lanes.get(`${identity.roleRef}:${identity.workerRef}`);
721
+ if (lane)
722
+ lanes.set(`${identity.roleRef}:${identity.workerRef}`, {
723
+ ...lane,
724
+ pageState: "BLOCKED",
725
+ activityKind: "WAITING_HUMAN",
726
+ continuationRef,
727
+ });
728
+ }
729
+ return {
730
+ status: "WAITING_HUMAN",
731
+ continuationRef,
732
+ evidenceRef: shot.evidenceRef,
733
+ };
734
+ },
735
+ getSidePanelSnapshot() {
736
+ const snapshot = {
737
+ extensionInstanceId,
738
+ observedAt: now().toISOString(),
739
+ sessions: [...sessions.values()].map((item) => ({
740
+ tabId: item.tabId,
741
+ windowId: item.windowId,
742
+ url: item.url,
743
+ contentInstanceId: item.contentInstanceId,
744
+ pageState: item.pageState,
745
+ activityKind: item.activityKind,
746
+ })),
747
+ lanes: [...lanes.values()].map((item) => ({ ...item })),
748
+ };
749
+ Object.freeze(snapshot.sessions);
750
+ Object.freeze(snapshot.lanes);
751
+ return Object.freeze(snapshot);
752
+ },
753
+ execute,
754
+ observePrecondition: async (request) => browserPrecondition(request),
755
+ reconcile,
756
+ async readArtifact() {
757
+ throw new ExecutionBrowserError("EXECUTOR_UNAVAILABLE", "ARTIFACT_OWNED_BY_EXECUTION_RUNTIME");
758
+ },
759
+ async recoveryScan(unfinished) {
760
+ if (recoveryCompleted)
761
+ return { status: "ALREADY_COMPLETED", reconciled: [] };
762
+ recoveryCompleted = true;
763
+ for (const tab of await options.browser.listTabs()) {
764
+ try {
765
+ registerContentSession(tab);
766
+ }
767
+ catch {
768
+ /* unrelated tab */
769
+ }
770
+ }
771
+ const reconciled = [];
772
+ for (const item of unfinished)
773
+ reconciled.push(item.effectStarted
774
+ ? await reconcile(item.request, browserPrecondition(item.request))
775
+ : { state: "NOT_APPLIED", evidence: [] });
776
+ return { status: "COMPLETED", reconciled };
777
+ },
778
+ });
779
+ }