humanish 0.97.0 → 0.98.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (144) hide show
  1. package/AGENTS.md +86 -79
  2. package/CONTRIBUTING.md +7 -2
  3. package/README.md +7 -1
  4. package/dist/actor-contract.d.ts +35 -1
  5. package/dist/actor-contract.js +38 -0
  6. package/dist/actor-contract.js.map +1 -1
  7. package/dist/adapter-extension.js +1 -0
  8. package/dist/adapter-extension.js.map +1 -1
  9. package/dist/browser-control-client.d.ts +14 -0
  10. package/dist/browser-control-client.js +134 -0
  11. package/dist/browser-control-client.js.map +1 -0
  12. package/dist/browser-control-dispatcher.d.ts +14 -0
  13. package/dist/browser-control-dispatcher.js +109 -0
  14. package/dist/browser-control-dispatcher.js.map +1 -0
  15. package/dist/browser-control-protocol.d.ts +371 -0
  16. package/dist/browser-control-protocol.js +155 -0
  17. package/dist/browser-control-protocol.js.map +1 -0
  18. package/dist/browser-control-transport.d.ts +24 -0
  19. package/dist/browser-control-transport.js +156 -0
  20. package/dist/browser-control-transport.js.map +1 -0
  21. package/dist/comms-lease-store.d.ts +1 -0
  22. package/dist/comms-lease-store.js +9 -3
  23. package/dist/comms-lease-store.js.map +1 -1
  24. package/dist/computer-use-actor.d.ts +2 -2
  25. package/dist/computer-use-actor.js +6 -1
  26. package/dist/computer-use-actor.js.map +1 -1
  27. package/dist/computer-use.d.ts +23 -1
  28. package/dist/computer-use.js +253 -70
  29. package/dist/computer-use.js.map +1 -1
  30. package/dist/cua-actor-lab.d.ts +6 -1
  31. package/dist/cua-actor-lab.js +55 -15
  32. package/dist/cua-actor-lab.js.map +1 -1
  33. package/dist/cua-executor-error.d.ts +31 -0
  34. package/dist/cua-executor-error.js +48 -0
  35. package/dist/cua-executor-error.js.map +1 -0
  36. package/dist/cua-provider-error.d.ts +12 -0
  37. package/dist/cua-provider-error.js +28 -0
  38. package/dist/cua-provider-error.js.map +1 -0
  39. package/dist/desktop-session.js +3 -1
  40. package/dist/desktop-session.js.map +1 -1
  41. package/dist/doctor-lab.d.ts +2 -0
  42. package/dist/doctor-lab.js +25 -7
  43. package/dist/doctor-lab.js.map +1 -1
  44. package/dist/e2b-desktop-executor.d.ts +1 -24
  45. package/dist/e2b-desktop-executor.js +2 -127
  46. package/dist/e2b-desktop-executor.js.map +1 -1
  47. package/dist/e2b-terminal-lab.js +1 -0
  48. package/dist/e2b-terminal-lab.js.map +1 -1
  49. package/dist/frame-signature.d.ts +24 -0
  50. package/dist/frame-signature.js +128 -0
  51. package/dist/frame-signature.js.map +1 -0
  52. package/dist/guest-bootstrap.d.ts +43 -0
  53. package/dist/guest-bootstrap.js +240 -0
  54. package/dist/guest-bootstrap.js.map +1 -0
  55. package/dist/guest-browser-tools.d.ts +8 -0
  56. package/dist/guest-browser-tools.js +66 -0
  57. package/dist/guest-browser-tools.js.map +1 -0
  58. package/dist/guest-chromium-text.d.ts +27 -0
  59. package/dist/guest-chromium-text.js +281 -0
  60. package/dist/guest-chromium-text.js.map +1 -0
  61. package/dist/guest-desktop-executor.d.ts +22 -0
  62. package/dist/guest-desktop-executor.js +177 -0
  63. package/dist/guest-desktop-executor.js.map +1 -0
  64. package/dist/guest-desktop-native.d.ts +14 -0
  65. package/dist/guest-desktop-native.js +131 -0
  66. package/dist/guest-desktop-native.js.map +1 -0
  67. package/dist/guest-runtime-desktop.d.ts +35 -0
  68. package/dist/guest-runtime-desktop.js +231 -0
  69. package/dist/guest-runtime-desktop.js.map +1 -0
  70. package/dist/guest-runtime-main.d.ts +1 -0
  71. package/dist/guest-runtime-main.js +31 -0
  72. package/dist/guest-runtime-main.js.map +1 -0
  73. package/dist/guest-runtime-revision.d.ts +1 -0
  74. package/dist/guest-runtime-revision.js +3 -0
  75. package/dist/guest-runtime-revision.js.map +1 -0
  76. package/dist/guest-runtime.d.ts +25 -0
  77. package/dist/guest-runtime.js +96 -0
  78. package/dist/guest-runtime.js.map +1 -0
  79. package/dist/lab-config.js +10 -3
  80. package/dist/lab-config.js.map +1 -1
  81. package/dist/lab-engine.js +6 -0
  82. package/dist/lab-engine.js.map +1 -1
  83. package/dist/lab-summary.d.ts +2 -0
  84. package/dist/lab-summary.js +5 -0
  85. package/dist/lab-summary.js.map +1 -1
  86. package/dist/local-agent-cli.js +1 -1
  87. package/dist/local-agent-cli.js.map +1 -1
  88. package/dist/local-firecracker-desktop.d.ts +13 -0
  89. package/dist/local-firecracker-desktop.js +150 -0
  90. package/dist/local-firecracker-desktop.js.map +1 -0
  91. package/dist/local-firecracker-study.d.ts +9 -0
  92. package/dist/local-firecracker-study.js +93 -0
  93. package/dist/local-firecracker-study.js.map +1 -0
  94. package/dist/local-runtime-config.d.ts +6 -0
  95. package/dist/local-runtime-config.js +56 -0
  96. package/dist/local-runtime-config.js.map +1 -0
  97. package/dist/local-runtime-release.d.ts +3 -0
  98. package/dist/local-runtime-release.js +8 -0
  99. package/dist/local-runtime-release.js.map +1 -0
  100. package/dist/local-runtime.d.ts +25 -0
  101. package/dist/local-runtime.js +113 -0
  102. package/dist/local-runtime.js.map +1 -0
  103. package/dist/observer-app.html +4 -4
  104. package/dist/pricing.d.ts +22 -1
  105. package/dist/pricing.js +22 -0
  106. package/dist/pricing.js.map +1 -1
  107. package/dist/program.js +26 -0
  108. package/dist/program.js.map +1 -1
  109. package/dist/restricted-codex-participant-policy.d.ts +39 -0
  110. package/dist/restricted-codex-participant-policy.js +69 -0
  111. package/dist/restricted-codex-participant-policy.js.map +1 -0
  112. package/dist/restricted-codex-participant-run.d.ts +20 -0
  113. package/dist/restricted-codex-participant-run.js +78 -0
  114. package/dist/restricted-codex-participant-run.js.map +1 -0
  115. package/dist/restricted-codex-participant.d.ts +14 -0
  116. package/dist/restricted-codex-participant.js +178 -0
  117. package/dist/restricted-codex-participant.js.map +1 -0
  118. package/dist/restricted-codex-session.d.ts +2 -2
  119. package/dist/restricted-codex-session.js +11 -17
  120. package/dist/restricted-codex-session.js.map +1 -1
  121. package/dist/run-detail.js +4 -2
  122. package/dist/run-detail.js.map +1 -1
  123. package/dist/run.d.ts +12 -5
  124. package/dist/run.js +17 -1
  125. package/dist/run.js.map +1 -1
  126. package/dist/shared-world-lab.js +2 -2
  127. package/dist/shared-world-lab.js.map +1 -1
  128. package/dist/study-analysis-job.d.ts +2 -2
  129. package/dist/study-analysis-validation.d.ts +14 -14
  130. package/dist/study-costs.js +6 -0
  131. package/dist/study-costs.js.map +1 -1
  132. package/dist/tui-app.js +37 -37
  133. package/docs/architecture/browser-control.md +117 -0
  134. package/docs/architecture/guest-desktop.md +87 -0
  135. package/docs/architecture/local-browser-runtime.md +100 -0
  136. package/docs/architecture/restricted-codex-analysis.md +4 -2
  137. package/docs/architecture/runtime-broker-core.md +30 -0
  138. package/docs/contracts/schemas.md +1 -1
  139. package/docs/goals/current.md +24 -8
  140. package/docs/product/open-source-install-experience.md +7 -0
  141. package/docs/ramp/README.md +24 -13
  142. package/docs/release/0.98.0-local-browser-studies.md +31 -0
  143. package/package.json +4 -2
  144. package/skills/humanish/SKILL.md +23 -3
@@ -1,9 +1,12 @@
1
1
  import { ACTOR_TRACE_SCHEMA } from "./actor-contract.js";
2
+ import { CuaProviderError, isCuaProviderError } from "./cua-provider-error.js";
2
3
  import { classifyCuaAction, summarizeAffordanceUse } from "./affordance.js";
3
4
  import { commandFailureInfo, isCommandExitError } from "./command-failure.js";
4
5
  import { evaluateStopWhen } from "./stop-conditions.js";
5
6
  import { TaskTracker } from "./tasks.js";
6
7
  import { isCuaAdmissionLimitError } from "./cua-admission-limit.js";
8
+ import { CuaExecutorError, isCuaExecutorError } from "./cua-executor-error.js";
9
+ export const CUA_PROVIDER_CLEANUP_GRACE_MS = 5000;
7
10
  /** Runtime validation is also required for third-party provider ports. */
8
11
  export function validClosingReport(value) {
9
12
  if (value === null || typeof value !== "object" || Array.isArray(value))
@@ -329,6 +332,11 @@ function raceSettle(promise, remainingMs, signal) {
329
332
  * before its ref is recorded, so the trace is public-safe by construction.
330
333
  */
331
334
  export async function runComputerUseLoop(options) {
335
+ if (options.provider.executionProfile?.billing === "account-unknown" &&
336
+ (options.maxUsd !== undefined || options.overRunBudget !== undefined || options.estimateTurnCostUsd !== undefined ||
337
+ options.provider.modelSettings?.maxOutputTokens !== undefined)) {
338
+ throw new CuaProviderError("request_rejected", { dispatched: false, usageComplete: false, cleanup: "confirmed" });
339
+ }
332
340
  const { instructions, provider, executor, persona, redaction, timeoutMs, now, signal, idleSteps = DEFAULT_IDLE_STEPS, noProgressSteps = DEFAULT_NO_PROGRESS_STEPS, acknowledgeSafetyChecks = () => null, redactScreenshots = false, scrubText = (text) => text, writeScreenshot = async (name) => `screenshots/${name}`, stopWhen, dwell, sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)), tasks, maxUsd, overRunBudget, estimateTurnCostUsd, requireReportedUsageForSpendCap = false, onObservedUrl, onMessage, onScreenshot, onTrace } = options;
333
341
  const noProgressRecoverySteps = Math.min(Math.max(1, noProgressSteps - 1), 3);
334
342
  const idleRecoverySteps = Math.min(Math.max(1, idleSteps - 1), 3);
@@ -341,6 +349,14 @@ export async function runComputerUseLoop(options) {
341
349
  const turnTimeoutMs = options.turnTimeoutMs ?? DEFAULT_TURN_TIMEOUT_MS;
342
350
  const observationTimeoutMs = options.observationTimeoutMs ?? DEFAULT_OBSERVATION_TIMEOUT_MS;
343
351
  const items = [];
352
+ const providerRequests = [];
353
+ let providerCleanupUnconfirmed = false;
354
+ const liveMetadata = () => ({
355
+ ...(provider.executionProfile === undefined ? {} : { executionProfile: provider.executionProfile }),
356
+ ...(provider.requestPolicy === "fail_closed" ? { providerRequests: providerRequests.map(row => ({ ...row })),
357
+ historyTurnsOmitted: provider.historyTurnsOmitted ?? 0 } : {})
358
+ });
359
+ const flushTrace = () => onTrace?.(items.slice(), runningUsage(), liveMetadata());
344
360
  // The ONE recording choke point (#441): every trace item is stamped `at` from the
345
361
  // loop's injected clock as it is recorded, so timed playback reads recorded facts
346
362
  // (deterministic in tests via the injected `now`).
@@ -368,6 +384,7 @@ export async function runComputerUseLoop(options) {
368
384
  // zero material actions is still an honest failure (timed_out). Kept beside counts.materialActions
369
385
  // so the evidence self-describes the distinction.
370
386
  let materialActions = 0;
387
+ let interruptedActionOutcome = false;
371
388
  let seq = 0;
372
389
  let usageInput = 0;
373
390
  let usageCachedInput = 0;
@@ -382,7 +399,16 @@ export async function runComputerUseLoop(options) {
382
399
  // was always a number (0 included), and arithmetic on a suddenly-undefined field yields NaN —
383
400
  // which comparison operators swallow silently (red-team finding: a stale study-budget guard
384
401
  // would run uncapped without a sound).
385
- const runningUsage = () => ({
402
+ const accountUsage = () => ({
403
+ ...(usageTurns.some(t => t.input !== undefined) ? { input: usageInput } : {}),
404
+ ...(usageTurns.some(t => t.output !== undefined) ? { output: usageOutput } : {}),
405
+ ...(usageTurns.some(t => t.cachedInput !== undefined) ? { cachedInput: usageCachedInput } : {}),
406
+ ...(usageTurns.some(t => t.cacheWriteInput !== undefined) ? { cacheWriteInput: usageCacheWriteInput } : {}),
407
+ ...(usageTurns.length > 0 ? { turns: usageTurns.map(t => ({ ...t })) } : {}),
408
+ ...(providerRequests.length > 0 && providerRequests.every(r => r.usageComplete && completeTurnUsage(r.usage))
409
+ ? { total: usageInput + usageOutput } : {})
410
+ });
411
+ const runningUsage = () => provider.executionProfile?.billing === "account-unknown" ? accountUsage() : ({
386
412
  input: usageInput,
387
413
  output: usageOutput,
388
414
  cachedInput: usageCachedInput,
@@ -405,7 +431,7 @@ export async function runComputerUseLoop(options) {
405
431
  counts[key] = (counts[key] ?? 0) + 1;
406
432
  };
407
433
  const recordUsage = (turn, interaction = true) => {
408
- if (interaction && !completeTurnUsage(turn.usage))
434
+ if (interaction && (!completeTurnUsage(turn.usage) || turn.providerRequest?.usageComplete === false))
409
435
  incompleteInteractionUsage = true;
410
436
  const raw = turn.usage;
411
437
  if (raw === undefined)
@@ -425,6 +451,85 @@ export async function runComputerUseLoop(options) {
425
451
  usageOutput += usage.output ?? 0;
426
452
  usageTurns.push(usage);
427
453
  };
454
+ /** Marked providers own one attempt through settlement; an outer race never retries it. */
455
+ const markedRequest = async (request, kind, capMs) => {
456
+ const controller = new AbortController();
457
+ const onAbort = () => controller.abort();
458
+ if (signal?.aborted)
459
+ controller.abort();
460
+ else
461
+ signal?.addEventListener("abort", onAbort, { once: true });
462
+ let settlement;
463
+ let failure;
464
+ let failed = false;
465
+ let accepted;
466
+ const pending = Promise.resolve().then(() => {
467
+ if (controller.signal.aborted)
468
+ throw new CuaProviderError("cancelled", { dispatched: false, usageComplete: false, cleanup: "confirmed" });
469
+ return kind === "debrief" ? provider.debrief(request, controller.signal) : provider.nextTurn(request, controller.signal);
470
+ }).then(turn => { settlement = { turn }; return turn; }, (error) => { settlement = { error }; throw error; });
471
+ void pending.catch(() => undefined);
472
+ try {
473
+ accepted = await raceBounded(`participant ${kind}`, pending, remaining(), capMs, signal);
474
+ const r = accepted.providerRequest;
475
+ if (!r || r.dispatched !== true || typeof r.usageComplete !== "boolean" || r.cleanup !== "confirmed") {
476
+ throw new CuaProviderError("invalid_response", { dispatched: "unknown", usageComplete: false, cleanup: "unconfirmed" }, accepted.usage);
477
+ }
478
+ }
479
+ catch (error) {
480
+ failed = true;
481
+ failure = error;
482
+ }
483
+ finally {
484
+ controller.abort();
485
+ signal?.removeEventListener("abort", onAbort);
486
+ if (settlement === undefined) {
487
+ let timer;
488
+ try {
489
+ await Promise.race([pending.catch(() => undefined), new Promise(resolve => {
490
+ timer = setTimeout(resolve, CUA_PROVIDER_CLEANUP_GRACE_MS);
491
+ })]);
492
+ }
493
+ finally {
494
+ clearTimeout(timer);
495
+ }
496
+ }
497
+ }
498
+ const final = settlement;
499
+ const typed = isCuaProviderError(failure) ? failure : final && "error" in final && isCuaProviderError(final.error) ? final.error : undefined;
500
+ const turn = final && "turn" in final ? final.turn : undefined;
501
+ const raw = typed?.receipt ?? turn?.providerRequest;
502
+ const receipt = raw && (typeof raw.dispatched === "boolean" || raw.dispatched === "unknown") &&
503
+ typeof raw.usageComplete === "boolean" && ["confirmed", "unconfirmed"].includes(raw.cleanup) ? { dispatched: raw.dispatched, usageComplete: raw.usageComplete, cleanup: raw.cleanup }
504
+ : { dispatched: "unknown", usageComplete: false, cleanup: "unconfirmed" };
505
+ const rawUsage = typed?.usage ?? turn?.usage;
506
+ const usage = rawUsage === undefined ? undefined : Object.fromEntries(["input", "output", "cachedInput", "cacheWriteInput", "total"].flatMap(key => {
507
+ const value = rawUsage[key];
508
+ return validTokenCount(value) ? [[key, value]] : [];
509
+ }));
510
+ // restricted-codex-session sets dispatched only after initialize/config/account/
511
+ // thread/MCP admission, immediately before turn/start; it is not a success claim.
512
+ providerRequests.push({ ordinal: providerRequests.length + 1, kind, ...receipt,
513
+ profileVerified: provider.executionProfile !== undefined && receipt.dispatched === true,
514
+ ...(typed ? { errorCode: typed.code } : {}), ...(usage === undefined ? {} : { usage: { ...usage } }) });
515
+ if (receipt.cleanup !== "confirmed") {
516
+ providerCleanupUnconfirmed = true;
517
+ record({ id: nextId("notice"), kind: "notice", lifecycle: "completed", status: "error",
518
+ title: "participant request cleanup unconfirmed", text: "The request did not confirm cleanup within the settlement boundary. No further participant request or action is admitted." });
519
+ }
520
+ if (kind === "interaction" && receipt.dispatched !== false && (!receipt.usageComplete || !completeTurnUsage(usage)))
521
+ unreportedInteractionUsage = true;
522
+ if (failed) {
523
+ if (usage)
524
+ recordUsage({ actions: [], pendingSafetyChecks: [], done: false, usage, providerRequest: receipt }, kind === "interaction");
525
+ if (failure instanceof CuaAbortError || failure instanceof CuaDeadlineError)
526
+ throw failure;
527
+ if (failure instanceof CuaStallError)
528
+ throw new CuaProviderError("timeout", receipt, usage);
529
+ throw typed ?? new CuaProviderError("process_failed", receipt, usage);
530
+ }
531
+ return accepted;
532
+ };
428
533
  // Whether any observation this run surfaced structured appState (a non-vision/state executor).
429
534
  // RUNTIME-ONLY signal: used solely to self-describe in redaction.notes that app state drove
430
535
  // progress detection and was NOT written to the trace — the appState itself never persists.
@@ -517,6 +622,7 @@ export async function runComputerUseLoop(options) {
517
622
  // Loop-local state. Declared here (before the try) so the initial observe + the per-turn
518
623
  // frame guard can fail closed cleanly while these still scope across the loop.
519
624
  let previousResponseId;
625
+ let previousExecution;
520
626
  let closingObservation;
521
627
  let closingTrigger;
522
628
  let debrief;
@@ -535,8 +641,8 @@ export async function runComputerUseLoop(options) {
535
641
  let pendingAcks;
536
642
  try {
537
643
  currentPhase = "observing initial UI state";
538
- // A stalled observe is retried once (#480); a second stall is the substrate telling us it is
539
- // gone, and that ends the lane with its own name rather than an unexplained deadline.
644
+ // Legacy executors retry a stalled observe once (#480). A transport that cannot safely
645
+ // replay pending requests opts out, so even a shorter outer bound stops without retrying.
540
646
  const observeBounded = async (label) => {
541
647
  try {
542
648
  return await raceBounded(`observe (${label})`, executor.observe(), remaining(), observationTimeoutMs, signal);
@@ -544,6 +650,11 @@ export async function runComputerUseLoop(options) {
544
650
  catch (error) {
545
651
  if (!(error instanceof CuaStallError))
546
652
  throw error;
653
+ if (executor.stallRecovery === "fail_closed") {
654
+ // The outer bound says nothing about whether the still-pending request completed.
655
+ // The owning session will close it; this loop must not send a replacement request.
656
+ throw new CuaExecutorError("deadline_exceeded", "outcome_uncertain");
657
+ }
547
658
  record({
548
659
  id: nextId("notice"),
549
660
  kind: "notice",
@@ -598,7 +709,7 @@ export async function runComputerUseLoop(options) {
598
709
  if (frameObservation.screenshot !== undefined)
599
710
  onScreenshot?.(frameObservation.screenshot);
600
711
  await maybeRecordScreenshot(frameObservation, `dwell-${frames.toString().padStart(2, "0")}`);
601
- onTrace?.(items.slice(), runningUsage());
712
+ flushTrace();
602
713
  observeTasks(frameObservation, turnNumber);
603
714
  }
604
715
  dwellHeldMs = now() - dwellStartedAtMs;
@@ -628,7 +739,7 @@ export async function runComputerUseLoop(options) {
628
739
  if (frameGuardTripped(observation))
629
740
  throw new CuaFrameGuardStop();
630
741
  await maybeRecordScreenshot(observation, "turn-00-start");
631
- onTrace?.(items.slice(), runningUsage());
742
+ flushTrace();
632
743
  observeTasks(observation, 0);
633
744
  const initialDwell = await dwellIfDue(observation, 0);
634
745
  if (initialDwell !== undefined) {
@@ -680,7 +791,7 @@ export async function runComputerUseLoop(options) {
680
791
  break;
681
792
  }
682
793
  const turnNumber = (counts.turns ?? 0) + 1;
683
- const request = { instructions, observation };
794
+ const request = { instructions, observation, ...(provider.requestPolicy !== "fail_closed" || previousExecution === undefined ? {} : { previousExecution }) };
684
795
  if (previousResponseId !== undefined)
685
796
  request.previousResponseId = previousResponseId;
686
797
  if (contextHint !== undefined)
@@ -696,68 +807,73 @@ export async function runComputerUseLoop(options) {
696
807
  let turn;
697
808
  // A timeout race alone does not cancel its losing provider promise. Strict accounting
698
809
  // owns this signal so a delayed transport failure cannot retry after the loop has ended.
699
- const requestController = requiresUsage ? new AbortController() : undefined;
700
- const onRequestAbort = () => requestController?.abort();
701
- if (requestController) {
702
- if (signal?.aborted)
703
- requestController.abort();
704
- else
705
- signal?.addEventListener("abort", onRequestAbort, { once: true });
810
+ if (provider.requestPolicy === "fail_closed") {
811
+ turn = await markedRequest(request, "interaction", turnTimeoutMs);
706
812
  }
707
- const requestSignal = requestController?.signal ?? signal ?? neverAbort;
708
- try {
709
- turn = await raceBounded(`provider turn ${turnNumber}`, provider.nextTurn(request, requestSignal), remaining(), turnTimeoutMs, signal);
710
- }
711
- catch (error) {
712
- if (isCuaAdmissionLimitError(error)) {
713
- stopForAdmissionLimit();
714
- break;
715
- }
716
- // A thrown request may have been billed without returning usage. Admission refusal is
717
- // the explicit no-dispatch exception above; strict capped routes cannot safely retry.
718
- if (requiresUsage) {
719
- stopForUnreportedUsage();
720
- break;
813
+ else {
814
+ const requestController = requiresUsage ? new AbortController() : undefined;
815
+ const onRequestAbort = () => requestController?.abort();
816
+ if (requestController) {
817
+ if (signal?.aborted)
818
+ requestController.abort();
819
+ else
820
+ signal?.addEventListener("abort", onRequestAbort, { once: true });
721
821
  }
722
- if (!(error instanceof CuaStallError))
723
- throw error;
724
- unreportedInteractionUsage = true;
725
- record({
726
- id: nextId("notice"),
727
- kind: "notice",
728
- lifecycle: "completed",
729
- status: "warn",
730
- title: "provider turn stalled; retrying once",
731
- text: `${error.what} produced nothing within ${error.afterMs}ms; sending the same observation again`
732
- });
822
+ const requestSignal = requestController?.signal ?? signal ?? neverAbort;
733
823
  try {
734
- turn = await raceBounded(`provider turn ${turnNumber} (retry)`, provider.nextTurn(request, signal ?? neverAbort), remaining(), turnTimeoutMs, signal);
824
+ turn = await raceBounded(`provider turn ${turnNumber}`, provider.nextTurn(request, requestSignal), remaining(), turnTimeoutMs, signal);
735
825
  }
736
- catch (retryError) {
737
- if (isCuaAdmissionLimitError(retryError)) {
826
+ catch (error) {
827
+ if (isCuaAdmissionLimitError(error)) {
738
828
  stopForAdmissionLimit();
739
829
  break;
740
830
  }
741
- if (!(retryError instanceof CuaStallError))
742
- throw retryError;
831
+ // A thrown request may have been billed without returning usage. Admission refusal is
832
+ // the explicit no-dispatch exception above; strict capped routes cannot safely retry.
833
+ if (requiresUsage) {
834
+ stopForUnreportedUsage();
835
+ break;
836
+ }
837
+ if (!(error instanceof CuaStallError))
838
+ throw error;
743
839
  unreportedInteractionUsage = true;
744
- completionReason = "harness_error";
745
- reason = `provider turn ${turnNumber} stalled twice (${retryError.afterMs}ms each); the model produced no turn and the lane was ended rather than left to run out its budget`;
746
840
  record({
747
841
  id: nextId("notice"),
748
842
  kind: "notice",
749
843
  lifecycle: "completed",
750
- status: "error",
751
- title: "provider turn stalled twice",
752
- text: reason
844
+ status: "warn",
845
+ title: "provider turn stalled; retrying once",
846
+ text: `${error.what} produced nothing within ${error.afterMs}ms; sending the same observation again`
753
847
  });
754
- break;
848
+ try {
849
+ turn = await raceBounded(`provider turn ${turnNumber} (retry)`, provider.nextTurn(request, signal ?? neverAbort), remaining(), turnTimeoutMs, signal);
850
+ }
851
+ catch (retryError) {
852
+ if (isCuaAdmissionLimitError(retryError)) {
853
+ stopForAdmissionLimit();
854
+ break;
855
+ }
856
+ if (!(retryError instanceof CuaStallError))
857
+ throw retryError;
858
+ unreportedInteractionUsage = true;
859
+ completionReason = "harness_error";
860
+ reason = `provider turn ${turnNumber} stalled twice (${retryError.afterMs}ms each); the model produced no turn and the lane was ended rather than left to run out its budget`;
861
+ record({
862
+ id: nextId("notice"),
863
+ kind: "notice",
864
+ lifecycle: "completed",
865
+ status: "error",
866
+ title: "provider turn stalled twice",
867
+ text: reason
868
+ });
869
+ break;
870
+ }
871
+ }
872
+ finally {
873
+ if (requestController)
874
+ signal?.removeEventListener("abort", onRequestAbort);
875
+ requestController?.abort();
755
876
  }
756
- }
757
- finally {
758
- if (requestController)
759
- signal?.removeEventListener("abort", onRequestAbort);
760
- requestController?.abort();
761
877
  }
762
878
  bump("turns");
763
879
  previousResponseId = turn.responseId ?? previousResponseId;
@@ -906,21 +1022,27 @@ export async function runComputerUseLoop(options) {
906
1022
  // FINAL state — and a task completed by that state read as incomplete. The first live
907
1023
  // study caught it: both participants reached the dashboard, said so, and the funnel
908
1024
  // reported 0/2. One guarded closing observation feeds the tracker; a failed observe
909
- // changes nothing (the funnel stays honest about what it saw), and no screenshot or
910
- // stop evaluation rides it — the session is already over.
1025
+ // ordinarily changes nothing (the funnel stays honest about what it saw), and no
1026
+ // screenshot or stop evaluation rides it — the session is already over. An explicit
1027
+ // executor failure still fails the harness instead of disappearing behind completion.
911
1028
  if (taskTracker !== undefined) {
912
1029
  try {
1030
+ currentPhase = "observing closing task state";
913
1031
  const closing = await observeBounded("closing");
914
1032
  observeTasks(closing, turnNumber);
915
1033
  }
916
- catch {
917
- // Best-effort by design.
1034
+ catch (error) {
1035
+ // Ordinary closing observations remain best-effort. A declared executor failure
1036
+ // must not disappear behind a participant's reported success.
1037
+ if (isCuaExecutorError(error))
1038
+ throw error;
918
1039
  }
919
1040
  }
920
1041
  break;
921
1042
  }
922
1043
  const idleThisTurn = isIdleTurn(turn.actions);
923
- for (const action of turn.actions) {
1044
+ previousExecution = { actions: [] };
1045
+ for (const [actionIndex, action] of turn.actions.entries()) {
924
1046
  if (signal?.aborted)
925
1047
  throw new CuaAbortError();
926
1048
  const actionTitle = describeCuaAction(action);
@@ -959,6 +1081,22 @@ export async function runComputerUseLoop(options) {
959
1081
  else
960
1082
  await raceBounded(`idle action ${actionTitle}`, pending, remaining(), idleBound, signal);
961
1083
  }
1084
+ catch (error) {
1085
+ if (error instanceof CuaStallError && executor.stallRecovery === "fail_closed") {
1086
+ throw new CuaExecutorError("deadline_exceeded", "outcome_uncertain");
1087
+ }
1088
+ if (error instanceof CuaDeadlineError || error instanceof CuaAbortError) {
1089
+ // The loop's deadline/abort may win before the executor can report whether its
1090
+ // write reached the desktop. Cancellation alone does not establish rollback.
1091
+ interruptedActionOutcome = true;
1092
+ record({
1093
+ id: nextId("notice"), kind: "notice", lifecycle: "completed", status: "warn",
1094
+ title: "action outcome uncertain",
1095
+ text: redactNarration(`action: ${actionTitle}; disposition: outcome_uncertain; the loop stopped waiting before execution was acknowledged; the action was not retried`)
1096
+ });
1097
+ }
1098
+ throw error;
1099
+ }
962
1100
  finally {
963
1101
  signal?.removeEventListener("abort", onAbort);
964
1102
  // A deadline also closes async executor preparation, so a late pointer read
@@ -966,6 +1104,7 @@ export async function runComputerUseLoop(options) {
966
1104
  actionController.abort();
967
1105
  }
968
1106
  };
1107
+ let actionSkipped = false;
969
1108
  try {
970
1109
  if (isIdleAction(action)) {
971
1110
  // Observation actions only look (#480). A `wait` that hangs inside the SDK has, by
@@ -977,6 +1116,7 @@ export async function runComputerUseLoop(options) {
977
1116
  catch (error) {
978
1117
  if (!(error instanceof CuaStallError))
979
1118
  throw error;
1119
+ actionSkipped = true;
980
1120
  record({
981
1121
  id: nextId("notice"),
982
1122
  kind: "notice",
@@ -998,8 +1138,17 @@ export async function runComputerUseLoop(options) {
998
1138
  // (e.g. a Ctrl+Minus keypress exiting 2), so one flaky desktop command
999
1139
  // must not end the whole run. Everything else — a raceSettle deadline
1000
1140
  // (CuaDeadlineError) or abort (CuaAbortError), a sandbox-gone failure,
1001
- // any non-CommandExitError — is RE-THROWN so the existing fatal handling
1002
- // (actor_error / timed_out / harness_error) stays byte-identical.
1141
+ // any non-CommandExitError — is rethrown. A typed executor declaration must also
1142
+ // bypass command recovery even if an adapter has changed its ordinary Error metadata.
1143
+ if (isCuaExecutorError(error)) {
1144
+ previousExecution.actions.push({ index: actionIndex, status: error.disposition });
1145
+ if (error.disposition === "not_dispatched" && !isIdleAction(action)) {
1146
+ materialActions -= 1;
1147
+ counts.materialActions = materialActions;
1148
+ lastMaterialActionTitle = priorMaterialActionTitle;
1149
+ }
1150
+ throw error;
1151
+ }
1003
1152
  if (!isCommandExitError(error))
1004
1153
  throw error;
1005
1154
  if (!isIdleAction(action)) {
@@ -1017,6 +1166,7 @@ export async function runComputerUseLoop(options) {
1017
1166
  // counts.materialActions is 0 (honest-evidence invariant).
1018
1167
  lastMaterialActionTitle = priorMaterialActionTitle;
1019
1168
  }
1169
+ previousExecution.actions.push({ index: actionIndex, status: "skipped" });
1020
1170
  const { exitCode, stderrTail } = commandFailureInfo(error);
1021
1171
  record({
1022
1172
  id: nextId("notice"),
@@ -1036,6 +1186,7 @@ export async function runComputerUseLoop(options) {
1036
1186
  });
1037
1187
  continue;
1038
1188
  }
1189
+ previousExecution.actions.push({ index: actionIndex, status: actionSkipped ? "skipped" : "completed" });
1039
1190
  record({
1040
1191
  id: nextId("ui_action"),
1041
1192
  kind: "ui_action",
@@ -1062,7 +1213,7 @@ export async function runComputerUseLoop(options) {
1062
1213
  if (frameGuardTripped(observation))
1063
1214
  break;
1064
1215
  await maybeRecordScreenshot(observation, `turn-${turnNumber.toString().padStart(2, "0")}`);
1065
- onTrace?.(items.slice(), runningUsage());
1216
+ flushTrace();
1066
1217
  observeTasks(observation, turnNumber);
1067
1218
  const dwellOutcome = await dwellIfDue(observation, turnNumber);
1068
1219
  if (dwellOutcome !== undefined) {
@@ -1189,7 +1340,9 @@ export async function runComputerUseLoop(options) {
1189
1340
  stopCause = "time_limit";
1190
1341
  if (materialActions > 0) {
1191
1342
  completionReason = "budget_reached";
1192
- reason = `reached the ${timeoutMs}ms time budget after productive activity (${materialActions} material action(s), ${counts.turns} turn(s))`;
1343
+ reason = interruptedActionOutcome
1344
+ ? `reached the ${timeoutMs}ms time budget with ${materialActions} material action attempt(s), ${counts.turns} turn(s); the latest action outcome is uncertain`
1345
+ : `reached the ${timeoutMs}ms time budget after productive activity (${materialActions} material action(s), ${counts.turns} turn(s))`;
1193
1346
  }
1194
1347
  else {
1195
1348
  completionReason = "timed_out";
@@ -1201,6 +1354,27 @@ export async function runComputerUseLoop(options) {
1201
1354
  reason = "run aborted by the harness";
1202
1355
  stopCause = "harness_aborted";
1203
1356
  }
1357
+ else if (isCuaProviderError(error)) {
1358
+ completionReason = "harness_error";
1359
+ reason = `participant provider error: ${error.code}; cleanup: ${error.receipt.cleanup}`;
1360
+ record({ id: nextId("notice"), kind: "notice", lifecycle: "completed", status: "error",
1361
+ title: "participant provider error", text: reason });
1362
+ }
1363
+ else if (isCuaExecutorError(error)) {
1364
+ completionReason = "harness_error";
1365
+ reason = `desktop executor error: ${error.code}; disposition: ${error.disposition}`;
1366
+ record({
1367
+ id: nextId("notice"), kind: "notice", lifecycle: "completed", status: "error",
1368
+ title: "desktop executor error",
1369
+ text: [
1370
+ `phase: ${redactNarration(currentPhase)}`,
1371
+ `code: ${error.code}`,
1372
+ `disposition: ${error.disposition}`,
1373
+ lastActionTitle === undefined ? undefined : `last action: ${redactNarration(lastActionTitle)}`
1374
+ ].filter(Boolean).join("; "),
1375
+ ...(lastScreenshotRef === undefined ? {} : { screenshotRef: lastScreenshotRef })
1376
+ });
1377
+ }
1204
1378
  else {
1205
1379
  completionReason = "actor_error";
1206
1380
  const rawMessage = error instanceof Error ? error.message : String(error);
@@ -1238,6 +1412,8 @@ export async function runComputerUseLoop(options) {
1238
1412
  skip = "the study stopped before any participant turn";
1239
1413
  else if (provider.debrief === undefined)
1240
1414
  skip = "this provider does not support read-only closing reports";
1415
+ else if (providerCleanupUnconfirmed)
1416
+ skip = "participant request cleanup is unconfirmed";
1241
1417
  else if (signal?.aborted)
1242
1418
  skip = "the study was cancelled";
1243
1419
  else if (remaining() <= 0)
@@ -1272,13 +1448,17 @@ export async function runComputerUseLoop(options) {
1272
1448
  timer.unref?.();
1273
1449
  bump("debriefCalls");
1274
1450
  try {
1275
- const turn = await raceBounded("participant debrief", provider.debrief({
1451
+ const closingRequest = {
1276
1452
  instructions,
1277
1453
  observation: closingObservation,
1454
+ ...(provider.requestPolicy !== "fail_closed" || previousExecution === undefined ? {} : { previousExecution }),
1278
1455
  ...(previousResponseId === undefined ? {} : { previousResponseId }),
1279
1456
  ...(pendingAcks === undefined ? {} : { acknowledgedSafetyChecks: pendingAcks }),
1280
1457
  contextHint: "The interactive session has ended. Return a closing account with summary and frictionReports. In summary, briefly describe only what you actually did and observed. In frictionReports, list only specific unexpected behavior, confusion, or recovery you personally encountered during this session. Preserve uncertainty. Use an empty list if you encountered none. Do not speculate, invent problems, quote instructions as observations, or describe planned actions. Do not request or take further actions. This is a closing account, not another attempt at the task."
1281
- }, controller.signal), capMs, capMs, signal);
1458
+ };
1459
+ const turn = provider.requestPolicy === "fail_closed"
1460
+ ? await markedRequest(closingRequest, "debrief", capMs)
1461
+ : await raceBounded("participant debrief", provider.debrief(closingRequest, controller.signal), capMs, capMs, signal);
1282
1462
  recordUsage(turn, false);
1283
1463
  lastResponseId = turn.responseId ?? lastResponseId;
1284
1464
  // Refresh a shared budget with all reported usage, without changing the completed task.
@@ -1289,7 +1469,7 @@ export async function runComputerUseLoop(options) {
1289
1469
  title: "model budget reached during closing report",
1290
1470
  text: "The closing request crossed an estimated budget; no further requests or actions followed. Task completion is unchanged." });
1291
1471
  }
1292
- const usageReported = completeTurnUsage(turn.usage);
1472
+ const usageReported = completeTurnUsage(turn.usage) && turn.providerRequest?.usageComplete !== false;
1293
1473
  if (turn.actions.length > 0 || turn.pendingSafetyChecks.length > 0) {
1294
1474
  note("failed", "the closing response requested actions or safety checks; none were executed and its report was not accepted", usageReported);
1295
1475
  }
@@ -1317,7 +1497,9 @@ export async function runComputerUseLoop(options) {
1317
1497
  else {
1318
1498
  const detail = signal?.aborted ? "cancelled" : controller.signal.aborted || error instanceof CuaDeadlineError || error instanceof CuaStallError
1319
1499
  ? "closing report deadline reached" : error instanceof Error ? error.message : String(error);
1320
- note("failed", `${detail}; closing request usage is unreported`, false);
1500
+ const closingReceipt = provider.requestPolicy === "fail_closed" ? providerRequests.at(-1) : undefined;
1501
+ const usageReported = closingReceipt?.kind === "debrief" && closingReceipt.usageComplete && completeTurnUsage(closingReceipt.usage);
1502
+ note("failed", `${detail}; closing request usage is ${usageReported ? "reported" : "unreported"}`, usageReported);
1321
1503
  }
1322
1504
  }
1323
1505
  finally {
@@ -1327,7 +1509,7 @@ export async function runComputerUseLoop(options) {
1327
1509
  }
1328
1510
  }
1329
1511
  }
1330
- onTrace?.(items.slice(), runningUsage());
1512
+ flushTrace();
1331
1513
  }
1332
1514
  const completedAtMs = now();
1333
1515
  const status = statusForCompletionReason(completionReason);
@@ -1352,6 +1534,7 @@ export async function runComputerUseLoop(options) {
1352
1534
  const trace = {
1353
1535
  schema: ACTOR_TRACE_SCHEMA,
1354
1536
  provider: provider.id,
1537
+ ...liveMetadata(),
1355
1538
  ...(provider.version === undefined ? {} : { providerVersion: provider.version }),
1356
1539
  protocol: "cua-loop",
1357
1540
  lane: "computer-use",
@@ -1388,7 +1571,7 @@ export async function runComputerUseLoop(options) {
1388
1571
  ...(taskTracker === undefined ? {} : { taskFunnel: taskTracker.funnel() }),
1389
1572
  ...(sawUsage
1390
1573
  ? {
1391
- tokenUsage: {
1574
+ tokenUsage: provider.executionProfile?.billing === "account-unknown" ? accountUsage() : {
1392
1575
  input: usageInput,
1393
1576
  output: usageOutput,
1394
1577
  // Recorded only when the provider actually reported it, so a reader can tell "no cache