humanish 0.97.0 → 0.98.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +86 -79
- package/CONTRIBUTING.md +7 -2
- package/README.md +7 -1
- package/dist/actor-contract.d.ts +35 -1
- package/dist/actor-contract.js +38 -0
- package/dist/actor-contract.js.map +1 -1
- package/dist/adapter-extension.js +1 -0
- package/dist/adapter-extension.js.map +1 -1
- package/dist/browser-control-client.d.ts +14 -0
- package/dist/browser-control-client.js +134 -0
- package/dist/browser-control-client.js.map +1 -0
- package/dist/browser-control-dispatcher.d.ts +14 -0
- package/dist/browser-control-dispatcher.js +109 -0
- package/dist/browser-control-dispatcher.js.map +1 -0
- package/dist/browser-control-protocol.d.ts +371 -0
- package/dist/browser-control-protocol.js +155 -0
- package/dist/browser-control-protocol.js.map +1 -0
- package/dist/browser-control-transport.d.ts +24 -0
- package/dist/browser-control-transport.js +156 -0
- package/dist/browser-control-transport.js.map +1 -0
- package/dist/comms-lease-store.d.ts +1 -0
- package/dist/comms-lease-store.js +9 -3
- package/dist/comms-lease-store.js.map +1 -1
- package/dist/computer-use-actor.d.ts +2 -2
- package/dist/computer-use-actor.js +6 -1
- package/dist/computer-use-actor.js.map +1 -1
- package/dist/computer-use.d.ts +23 -1
- package/dist/computer-use.js +253 -70
- package/dist/computer-use.js.map +1 -1
- package/dist/cua-actor-lab.d.ts +6 -1
- package/dist/cua-actor-lab.js +55 -15
- package/dist/cua-actor-lab.js.map +1 -1
- package/dist/cua-executor-error.d.ts +31 -0
- package/dist/cua-executor-error.js +48 -0
- package/dist/cua-executor-error.js.map +1 -0
- package/dist/cua-provider-error.d.ts +12 -0
- package/dist/cua-provider-error.js +28 -0
- package/dist/cua-provider-error.js.map +1 -0
- package/dist/desktop-session.js +3 -1
- package/dist/desktop-session.js.map +1 -1
- package/dist/doctor-lab.d.ts +2 -0
- package/dist/doctor-lab.js +25 -7
- package/dist/doctor-lab.js.map +1 -1
- package/dist/e2b-desktop-executor.d.ts +1 -24
- package/dist/e2b-desktop-executor.js +2 -127
- package/dist/e2b-desktop-executor.js.map +1 -1
- package/dist/e2b-terminal-lab.js +1 -0
- package/dist/e2b-terminal-lab.js.map +1 -1
- package/dist/frame-signature.d.ts +24 -0
- package/dist/frame-signature.js +128 -0
- package/dist/frame-signature.js.map +1 -0
- package/dist/guest-bootstrap.d.ts +43 -0
- package/dist/guest-bootstrap.js +240 -0
- package/dist/guest-bootstrap.js.map +1 -0
- package/dist/guest-browser-tools.d.ts +8 -0
- package/dist/guest-browser-tools.js +66 -0
- package/dist/guest-browser-tools.js.map +1 -0
- package/dist/guest-chromium-text.d.ts +27 -0
- package/dist/guest-chromium-text.js +281 -0
- package/dist/guest-chromium-text.js.map +1 -0
- package/dist/guest-desktop-executor.d.ts +22 -0
- package/dist/guest-desktop-executor.js +177 -0
- package/dist/guest-desktop-executor.js.map +1 -0
- package/dist/guest-desktop-native.d.ts +14 -0
- package/dist/guest-desktop-native.js +131 -0
- package/dist/guest-desktop-native.js.map +1 -0
- package/dist/guest-runtime-desktop.d.ts +35 -0
- package/dist/guest-runtime-desktop.js +231 -0
- package/dist/guest-runtime-desktop.js.map +1 -0
- package/dist/guest-runtime-main.d.ts +1 -0
- package/dist/guest-runtime-main.js +31 -0
- package/dist/guest-runtime-main.js.map +1 -0
- package/dist/guest-runtime-revision.d.ts +1 -0
- package/dist/guest-runtime-revision.js +3 -0
- package/dist/guest-runtime-revision.js.map +1 -0
- package/dist/guest-runtime.d.ts +25 -0
- package/dist/guest-runtime.js +96 -0
- package/dist/guest-runtime.js.map +1 -0
- package/dist/lab-config.js +10 -3
- package/dist/lab-config.js.map +1 -1
- package/dist/lab-engine.js +6 -0
- package/dist/lab-engine.js.map +1 -1
- package/dist/lab-summary.d.ts +2 -0
- package/dist/lab-summary.js +5 -0
- package/dist/lab-summary.js.map +1 -1
- package/dist/local-agent-cli.js +1 -1
- package/dist/local-agent-cli.js.map +1 -1
- package/dist/local-firecracker-desktop.d.ts +13 -0
- package/dist/local-firecracker-desktop.js +150 -0
- package/dist/local-firecracker-desktop.js.map +1 -0
- package/dist/local-firecracker-study.d.ts +9 -0
- package/dist/local-firecracker-study.js +93 -0
- package/dist/local-firecracker-study.js.map +1 -0
- package/dist/local-runtime-config.d.ts +6 -0
- package/dist/local-runtime-config.js +56 -0
- package/dist/local-runtime-config.js.map +1 -0
- package/dist/local-runtime-release.d.ts +3 -0
- package/dist/local-runtime-release.js +8 -0
- package/dist/local-runtime-release.js.map +1 -0
- package/dist/local-runtime.d.ts +25 -0
- package/dist/local-runtime.js +113 -0
- package/dist/local-runtime.js.map +1 -0
- package/dist/observer-app.html +4 -4
- package/dist/pricing.d.ts +22 -1
- package/dist/pricing.js +22 -0
- package/dist/pricing.js.map +1 -1
- package/dist/program.js +26 -0
- package/dist/program.js.map +1 -1
- package/dist/restricted-codex-participant-policy.d.ts +39 -0
- package/dist/restricted-codex-participant-policy.js +69 -0
- package/dist/restricted-codex-participant-policy.js.map +1 -0
- package/dist/restricted-codex-participant-run.d.ts +20 -0
- package/dist/restricted-codex-participant-run.js +78 -0
- package/dist/restricted-codex-participant-run.js.map +1 -0
- package/dist/restricted-codex-participant.d.ts +14 -0
- package/dist/restricted-codex-participant.js +178 -0
- package/dist/restricted-codex-participant.js.map +1 -0
- package/dist/restricted-codex-session.d.ts +2 -2
- package/dist/restricted-codex-session.js +11 -17
- package/dist/restricted-codex-session.js.map +1 -1
- package/dist/run-detail.js +4 -2
- package/dist/run-detail.js.map +1 -1
- package/dist/run.d.ts +12 -5
- package/dist/run.js +17 -1
- package/dist/run.js.map +1 -1
- package/dist/shared-world-lab.js +2 -2
- package/dist/shared-world-lab.js.map +1 -1
- package/dist/study-analysis-job.d.ts +2 -2
- package/dist/study-analysis-validation.d.ts +14 -14
- package/dist/study-costs.js +6 -0
- package/dist/study-costs.js.map +1 -1
- package/dist/tui-app.js +37 -37
- package/docs/architecture/browser-control.md +117 -0
- package/docs/architecture/guest-desktop.md +87 -0
- package/docs/architecture/local-browser-runtime.md +100 -0
- package/docs/architecture/restricted-codex-analysis.md +4 -2
- package/docs/architecture/runtime-broker-core.md +30 -0
- package/docs/contracts/schemas.md +1 -1
- package/docs/goals/current.md +24 -8
- package/docs/product/open-source-install-experience.md +7 -0
- package/docs/ramp/README.md +24 -13
- package/docs/release/0.98.0-local-browser-studies.md +31 -0
- package/package.json +4 -2
- package/skills/humanish/SKILL.md +23 -3
package/dist/computer-use.js
CHANGED
|
@@ -1,9 +1,12 @@
|
|
|
1
1
|
import { ACTOR_TRACE_SCHEMA } from "./actor-contract.js";
|
|
2
|
+
import { CuaProviderError, isCuaProviderError } from "./cua-provider-error.js";
|
|
2
3
|
import { classifyCuaAction, summarizeAffordanceUse } from "./affordance.js";
|
|
3
4
|
import { commandFailureInfo, isCommandExitError } from "./command-failure.js";
|
|
4
5
|
import { evaluateStopWhen } from "./stop-conditions.js";
|
|
5
6
|
import { TaskTracker } from "./tasks.js";
|
|
6
7
|
import { isCuaAdmissionLimitError } from "./cua-admission-limit.js";
|
|
8
|
+
import { CuaExecutorError, isCuaExecutorError } from "./cua-executor-error.js";
|
|
9
|
+
export const CUA_PROVIDER_CLEANUP_GRACE_MS = 5000;
|
|
7
10
|
/** Runtime validation is also required for third-party provider ports. */
|
|
8
11
|
export function validClosingReport(value) {
|
|
9
12
|
if (value === null || typeof value !== "object" || Array.isArray(value))
|
|
@@ -329,6 +332,11 @@ function raceSettle(promise, remainingMs, signal) {
|
|
|
329
332
|
* before its ref is recorded, so the trace is public-safe by construction.
|
|
330
333
|
*/
|
|
331
334
|
export async function runComputerUseLoop(options) {
|
|
335
|
+
if (options.provider.executionProfile?.billing === "account-unknown" &&
|
|
336
|
+
(options.maxUsd !== undefined || options.overRunBudget !== undefined || options.estimateTurnCostUsd !== undefined ||
|
|
337
|
+
options.provider.modelSettings?.maxOutputTokens !== undefined)) {
|
|
338
|
+
throw new CuaProviderError("request_rejected", { dispatched: false, usageComplete: false, cleanup: "confirmed" });
|
|
339
|
+
}
|
|
332
340
|
const { instructions, provider, executor, persona, redaction, timeoutMs, now, signal, idleSteps = DEFAULT_IDLE_STEPS, noProgressSteps = DEFAULT_NO_PROGRESS_STEPS, acknowledgeSafetyChecks = () => null, redactScreenshots = false, scrubText = (text) => text, writeScreenshot = async (name) => `screenshots/${name}`, stopWhen, dwell, sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)), tasks, maxUsd, overRunBudget, estimateTurnCostUsd, requireReportedUsageForSpendCap = false, onObservedUrl, onMessage, onScreenshot, onTrace } = options;
|
|
333
341
|
const noProgressRecoverySteps = Math.min(Math.max(1, noProgressSteps - 1), 3);
|
|
334
342
|
const idleRecoverySteps = Math.min(Math.max(1, idleSteps - 1), 3);
|
|
@@ -341,6 +349,14 @@ export async function runComputerUseLoop(options) {
|
|
|
341
349
|
const turnTimeoutMs = options.turnTimeoutMs ?? DEFAULT_TURN_TIMEOUT_MS;
|
|
342
350
|
const observationTimeoutMs = options.observationTimeoutMs ?? DEFAULT_OBSERVATION_TIMEOUT_MS;
|
|
343
351
|
const items = [];
|
|
352
|
+
const providerRequests = [];
|
|
353
|
+
let providerCleanupUnconfirmed = false;
|
|
354
|
+
const liveMetadata = () => ({
|
|
355
|
+
...(provider.executionProfile === undefined ? {} : { executionProfile: provider.executionProfile }),
|
|
356
|
+
...(provider.requestPolicy === "fail_closed" ? { providerRequests: providerRequests.map(row => ({ ...row })),
|
|
357
|
+
historyTurnsOmitted: provider.historyTurnsOmitted ?? 0 } : {})
|
|
358
|
+
});
|
|
359
|
+
const flushTrace = () => onTrace?.(items.slice(), runningUsage(), liveMetadata());
|
|
344
360
|
// The ONE recording choke point (#441): every trace item is stamped `at` from the
|
|
345
361
|
// loop's injected clock as it is recorded, so timed playback reads recorded facts
|
|
346
362
|
// (deterministic in tests via the injected `now`).
|
|
@@ -368,6 +384,7 @@ export async function runComputerUseLoop(options) {
|
|
|
368
384
|
// zero material actions is still an honest failure (timed_out). Kept beside counts.materialActions
|
|
369
385
|
// so the evidence self-describes the distinction.
|
|
370
386
|
let materialActions = 0;
|
|
387
|
+
let interruptedActionOutcome = false;
|
|
371
388
|
let seq = 0;
|
|
372
389
|
let usageInput = 0;
|
|
373
390
|
let usageCachedInput = 0;
|
|
@@ -382,7 +399,16 @@ export async function runComputerUseLoop(options) {
|
|
|
382
399
|
// was always a number (0 included), and arithmetic on a suddenly-undefined field yields NaN —
|
|
383
400
|
// which comparison operators swallow silently (red-team finding: a stale study-budget guard
|
|
384
401
|
// would run uncapped without a sound).
|
|
385
|
-
const
|
|
402
|
+
const accountUsage = () => ({
|
|
403
|
+
...(usageTurns.some(t => t.input !== undefined) ? { input: usageInput } : {}),
|
|
404
|
+
...(usageTurns.some(t => t.output !== undefined) ? { output: usageOutput } : {}),
|
|
405
|
+
...(usageTurns.some(t => t.cachedInput !== undefined) ? { cachedInput: usageCachedInput } : {}),
|
|
406
|
+
...(usageTurns.some(t => t.cacheWriteInput !== undefined) ? { cacheWriteInput: usageCacheWriteInput } : {}),
|
|
407
|
+
...(usageTurns.length > 0 ? { turns: usageTurns.map(t => ({ ...t })) } : {}),
|
|
408
|
+
...(providerRequests.length > 0 && providerRequests.every(r => r.usageComplete && completeTurnUsage(r.usage))
|
|
409
|
+
? { total: usageInput + usageOutput } : {})
|
|
410
|
+
});
|
|
411
|
+
const runningUsage = () => provider.executionProfile?.billing === "account-unknown" ? accountUsage() : ({
|
|
386
412
|
input: usageInput,
|
|
387
413
|
output: usageOutput,
|
|
388
414
|
cachedInput: usageCachedInput,
|
|
@@ -405,7 +431,7 @@ export async function runComputerUseLoop(options) {
|
|
|
405
431
|
counts[key] = (counts[key] ?? 0) + 1;
|
|
406
432
|
};
|
|
407
433
|
const recordUsage = (turn, interaction = true) => {
|
|
408
|
-
if (interaction && !completeTurnUsage(turn.usage))
|
|
434
|
+
if (interaction && (!completeTurnUsage(turn.usage) || turn.providerRequest?.usageComplete === false))
|
|
409
435
|
incompleteInteractionUsage = true;
|
|
410
436
|
const raw = turn.usage;
|
|
411
437
|
if (raw === undefined)
|
|
@@ -425,6 +451,85 @@ export async function runComputerUseLoop(options) {
|
|
|
425
451
|
usageOutput += usage.output ?? 0;
|
|
426
452
|
usageTurns.push(usage);
|
|
427
453
|
};
|
|
454
|
+
/** Marked providers own one attempt through settlement; an outer race never retries it. */
|
|
455
|
+
const markedRequest = async (request, kind, capMs) => {
|
|
456
|
+
const controller = new AbortController();
|
|
457
|
+
const onAbort = () => controller.abort();
|
|
458
|
+
if (signal?.aborted)
|
|
459
|
+
controller.abort();
|
|
460
|
+
else
|
|
461
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
462
|
+
let settlement;
|
|
463
|
+
let failure;
|
|
464
|
+
let failed = false;
|
|
465
|
+
let accepted;
|
|
466
|
+
const pending = Promise.resolve().then(() => {
|
|
467
|
+
if (controller.signal.aborted)
|
|
468
|
+
throw new CuaProviderError("cancelled", { dispatched: false, usageComplete: false, cleanup: "confirmed" });
|
|
469
|
+
return kind === "debrief" ? provider.debrief(request, controller.signal) : provider.nextTurn(request, controller.signal);
|
|
470
|
+
}).then(turn => { settlement = { turn }; return turn; }, (error) => { settlement = { error }; throw error; });
|
|
471
|
+
void pending.catch(() => undefined);
|
|
472
|
+
try {
|
|
473
|
+
accepted = await raceBounded(`participant ${kind}`, pending, remaining(), capMs, signal);
|
|
474
|
+
const r = accepted.providerRequest;
|
|
475
|
+
if (!r || r.dispatched !== true || typeof r.usageComplete !== "boolean" || r.cleanup !== "confirmed") {
|
|
476
|
+
throw new CuaProviderError("invalid_response", { dispatched: "unknown", usageComplete: false, cleanup: "unconfirmed" }, accepted.usage);
|
|
477
|
+
}
|
|
478
|
+
}
|
|
479
|
+
catch (error) {
|
|
480
|
+
failed = true;
|
|
481
|
+
failure = error;
|
|
482
|
+
}
|
|
483
|
+
finally {
|
|
484
|
+
controller.abort();
|
|
485
|
+
signal?.removeEventListener("abort", onAbort);
|
|
486
|
+
if (settlement === undefined) {
|
|
487
|
+
let timer;
|
|
488
|
+
try {
|
|
489
|
+
await Promise.race([pending.catch(() => undefined), new Promise(resolve => {
|
|
490
|
+
timer = setTimeout(resolve, CUA_PROVIDER_CLEANUP_GRACE_MS);
|
|
491
|
+
})]);
|
|
492
|
+
}
|
|
493
|
+
finally {
|
|
494
|
+
clearTimeout(timer);
|
|
495
|
+
}
|
|
496
|
+
}
|
|
497
|
+
}
|
|
498
|
+
const final = settlement;
|
|
499
|
+
const typed = isCuaProviderError(failure) ? failure : final && "error" in final && isCuaProviderError(final.error) ? final.error : undefined;
|
|
500
|
+
const turn = final && "turn" in final ? final.turn : undefined;
|
|
501
|
+
const raw = typed?.receipt ?? turn?.providerRequest;
|
|
502
|
+
const receipt = raw && (typeof raw.dispatched === "boolean" || raw.dispatched === "unknown") &&
|
|
503
|
+
typeof raw.usageComplete === "boolean" && ["confirmed", "unconfirmed"].includes(raw.cleanup) ? { dispatched: raw.dispatched, usageComplete: raw.usageComplete, cleanup: raw.cleanup }
|
|
504
|
+
: { dispatched: "unknown", usageComplete: false, cleanup: "unconfirmed" };
|
|
505
|
+
const rawUsage = typed?.usage ?? turn?.usage;
|
|
506
|
+
const usage = rawUsage === undefined ? undefined : Object.fromEntries(["input", "output", "cachedInput", "cacheWriteInput", "total"].flatMap(key => {
|
|
507
|
+
const value = rawUsage[key];
|
|
508
|
+
return validTokenCount(value) ? [[key, value]] : [];
|
|
509
|
+
}));
|
|
510
|
+
// restricted-codex-session sets dispatched only after initialize/config/account/
|
|
511
|
+
// thread/MCP admission, immediately before turn/start; it is not a success claim.
|
|
512
|
+
providerRequests.push({ ordinal: providerRequests.length + 1, kind, ...receipt,
|
|
513
|
+
profileVerified: provider.executionProfile !== undefined && receipt.dispatched === true,
|
|
514
|
+
...(typed ? { errorCode: typed.code } : {}), ...(usage === undefined ? {} : { usage: { ...usage } }) });
|
|
515
|
+
if (receipt.cleanup !== "confirmed") {
|
|
516
|
+
providerCleanupUnconfirmed = true;
|
|
517
|
+
record({ id: nextId("notice"), kind: "notice", lifecycle: "completed", status: "error",
|
|
518
|
+
title: "participant request cleanup unconfirmed", text: "The request did not confirm cleanup within the settlement boundary. No further participant request or action is admitted." });
|
|
519
|
+
}
|
|
520
|
+
if (kind === "interaction" && receipt.dispatched !== false && (!receipt.usageComplete || !completeTurnUsage(usage)))
|
|
521
|
+
unreportedInteractionUsage = true;
|
|
522
|
+
if (failed) {
|
|
523
|
+
if (usage)
|
|
524
|
+
recordUsage({ actions: [], pendingSafetyChecks: [], done: false, usage, providerRequest: receipt }, kind === "interaction");
|
|
525
|
+
if (failure instanceof CuaAbortError || failure instanceof CuaDeadlineError)
|
|
526
|
+
throw failure;
|
|
527
|
+
if (failure instanceof CuaStallError)
|
|
528
|
+
throw new CuaProviderError("timeout", receipt, usage);
|
|
529
|
+
throw typed ?? new CuaProviderError("process_failed", receipt, usage);
|
|
530
|
+
}
|
|
531
|
+
return accepted;
|
|
532
|
+
};
|
|
428
533
|
// Whether any observation this run surfaced structured appState (a non-vision/state executor).
|
|
429
534
|
// RUNTIME-ONLY signal: used solely to self-describe in redaction.notes that app state drove
|
|
430
535
|
// progress detection and was NOT written to the trace — the appState itself never persists.
|
|
@@ -517,6 +622,7 @@ export async function runComputerUseLoop(options) {
|
|
|
517
622
|
// Loop-local state. Declared here (before the try) so the initial observe + the per-turn
|
|
518
623
|
// frame guard can fail closed cleanly while these still scope across the loop.
|
|
519
624
|
let previousResponseId;
|
|
625
|
+
let previousExecution;
|
|
520
626
|
let closingObservation;
|
|
521
627
|
let closingTrigger;
|
|
522
628
|
let debrief;
|
|
@@ -535,8 +641,8 @@ export async function runComputerUseLoop(options) {
|
|
|
535
641
|
let pendingAcks;
|
|
536
642
|
try {
|
|
537
643
|
currentPhase = "observing initial UI state";
|
|
538
|
-
//
|
|
539
|
-
//
|
|
644
|
+
// Legacy executors retry a stalled observe once (#480). A transport that cannot safely
|
|
645
|
+
// replay pending requests opts out, so even a shorter outer bound stops without retrying.
|
|
540
646
|
const observeBounded = async (label) => {
|
|
541
647
|
try {
|
|
542
648
|
return await raceBounded(`observe (${label})`, executor.observe(), remaining(), observationTimeoutMs, signal);
|
|
@@ -544,6 +650,11 @@ export async function runComputerUseLoop(options) {
|
|
|
544
650
|
catch (error) {
|
|
545
651
|
if (!(error instanceof CuaStallError))
|
|
546
652
|
throw error;
|
|
653
|
+
if (executor.stallRecovery === "fail_closed") {
|
|
654
|
+
// The outer bound says nothing about whether the still-pending request completed.
|
|
655
|
+
// The owning session will close it; this loop must not send a replacement request.
|
|
656
|
+
throw new CuaExecutorError("deadline_exceeded", "outcome_uncertain");
|
|
657
|
+
}
|
|
547
658
|
record({
|
|
548
659
|
id: nextId("notice"),
|
|
549
660
|
kind: "notice",
|
|
@@ -598,7 +709,7 @@ export async function runComputerUseLoop(options) {
|
|
|
598
709
|
if (frameObservation.screenshot !== undefined)
|
|
599
710
|
onScreenshot?.(frameObservation.screenshot);
|
|
600
711
|
await maybeRecordScreenshot(frameObservation, `dwell-${frames.toString().padStart(2, "0")}`);
|
|
601
|
-
|
|
712
|
+
flushTrace();
|
|
602
713
|
observeTasks(frameObservation, turnNumber);
|
|
603
714
|
}
|
|
604
715
|
dwellHeldMs = now() - dwellStartedAtMs;
|
|
@@ -628,7 +739,7 @@ export async function runComputerUseLoop(options) {
|
|
|
628
739
|
if (frameGuardTripped(observation))
|
|
629
740
|
throw new CuaFrameGuardStop();
|
|
630
741
|
await maybeRecordScreenshot(observation, "turn-00-start");
|
|
631
|
-
|
|
742
|
+
flushTrace();
|
|
632
743
|
observeTasks(observation, 0);
|
|
633
744
|
const initialDwell = await dwellIfDue(observation, 0);
|
|
634
745
|
if (initialDwell !== undefined) {
|
|
@@ -680,7 +791,7 @@ export async function runComputerUseLoop(options) {
|
|
|
680
791
|
break;
|
|
681
792
|
}
|
|
682
793
|
const turnNumber = (counts.turns ?? 0) + 1;
|
|
683
|
-
const request = { instructions, observation };
|
|
794
|
+
const request = { instructions, observation, ...(provider.requestPolicy !== "fail_closed" || previousExecution === undefined ? {} : { previousExecution }) };
|
|
684
795
|
if (previousResponseId !== undefined)
|
|
685
796
|
request.previousResponseId = previousResponseId;
|
|
686
797
|
if (contextHint !== undefined)
|
|
@@ -696,68 +807,73 @@ export async function runComputerUseLoop(options) {
|
|
|
696
807
|
let turn;
|
|
697
808
|
// A timeout race alone does not cancel its losing provider promise. Strict accounting
|
|
698
809
|
// owns this signal so a delayed transport failure cannot retry after the loop has ended.
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
if (requestController) {
|
|
702
|
-
if (signal?.aborted)
|
|
703
|
-
requestController.abort();
|
|
704
|
-
else
|
|
705
|
-
signal?.addEventListener("abort", onRequestAbort, { once: true });
|
|
810
|
+
if (provider.requestPolicy === "fail_closed") {
|
|
811
|
+
turn = await markedRequest(request, "interaction", turnTimeoutMs);
|
|
706
812
|
}
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
}
|
|
716
|
-
// A thrown request may have been billed without returning usage. Admission refusal is
|
|
717
|
-
// the explicit no-dispatch exception above; strict capped routes cannot safely retry.
|
|
718
|
-
if (requiresUsage) {
|
|
719
|
-
stopForUnreportedUsage();
|
|
720
|
-
break;
|
|
813
|
+
else {
|
|
814
|
+
const requestController = requiresUsage ? new AbortController() : undefined;
|
|
815
|
+
const onRequestAbort = () => requestController?.abort();
|
|
816
|
+
if (requestController) {
|
|
817
|
+
if (signal?.aborted)
|
|
818
|
+
requestController.abort();
|
|
819
|
+
else
|
|
820
|
+
signal?.addEventListener("abort", onRequestAbort, { once: true });
|
|
721
821
|
}
|
|
722
|
-
|
|
723
|
-
throw error;
|
|
724
|
-
unreportedInteractionUsage = true;
|
|
725
|
-
record({
|
|
726
|
-
id: nextId("notice"),
|
|
727
|
-
kind: "notice",
|
|
728
|
-
lifecycle: "completed",
|
|
729
|
-
status: "warn",
|
|
730
|
-
title: "provider turn stalled; retrying once",
|
|
731
|
-
text: `${error.what} produced nothing within ${error.afterMs}ms; sending the same observation again`
|
|
732
|
-
});
|
|
822
|
+
const requestSignal = requestController?.signal ?? signal ?? neverAbort;
|
|
733
823
|
try {
|
|
734
|
-
turn = await raceBounded(`provider turn ${turnNumber}
|
|
824
|
+
turn = await raceBounded(`provider turn ${turnNumber}`, provider.nextTurn(request, requestSignal), remaining(), turnTimeoutMs, signal);
|
|
735
825
|
}
|
|
736
|
-
catch (
|
|
737
|
-
if (isCuaAdmissionLimitError(
|
|
826
|
+
catch (error) {
|
|
827
|
+
if (isCuaAdmissionLimitError(error)) {
|
|
738
828
|
stopForAdmissionLimit();
|
|
739
829
|
break;
|
|
740
830
|
}
|
|
741
|
-
|
|
742
|
-
|
|
831
|
+
// A thrown request may have been billed without returning usage. Admission refusal is
|
|
832
|
+
// the explicit no-dispatch exception above; strict capped routes cannot safely retry.
|
|
833
|
+
if (requiresUsage) {
|
|
834
|
+
stopForUnreportedUsage();
|
|
835
|
+
break;
|
|
836
|
+
}
|
|
837
|
+
if (!(error instanceof CuaStallError))
|
|
838
|
+
throw error;
|
|
743
839
|
unreportedInteractionUsage = true;
|
|
744
|
-
completionReason = "harness_error";
|
|
745
|
-
reason = `provider turn ${turnNumber} stalled twice (${retryError.afterMs}ms each); the model produced no turn and the lane was ended rather than left to run out its budget`;
|
|
746
840
|
record({
|
|
747
841
|
id: nextId("notice"),
|
|
748
842
|
kind: "notice",
|
|
749
843
|
lifecycle: "completed",
|
|
750
|
-
status: "
|
|
751
|
-
title: "provider turn stalled
|
|
752
|
-
text:
|
|
844
|
+
status: "warn",
|
|
845
|
+
title: "provider turn stalled; retrying once",
|
|
846
|
+
text: `${error.what} produced nothing within ${error.afterMs}ms; sending the same observation again`
|
|
753
847
|
});
|
|
754
|
-
|
|
848
|
+
try {
|
|
849
|
+
turn = await raceBounded(`provider turn ${turnNumber} (retry)`, provider.nextTurn(request, signal ?? neverAbort), remaining(), turnTimeoutMs, signal);
|
|
850
|
+
}
|
|
851
|
+
catch (retryError) {
|
|
852
|
+
if (isCuaAdmissionLimitError(retryError)) {
|
|
853
|
+
stopForAdmissionLimit();
|
|
854
|
+
break;
|
|
855
|
+
}
|
|
856
|
+
if (!(retryError instanceof CuaStallError))
|
|
857
|
+
throw retryError;
|
|
858
|
+
unreportedInteractionUsage = true;
|
|
859
|
+
completionReason = "harness_error";
|
|
860
|
+
reason = `provider turn ${turnNumber} stalled twice (${retryError.afterMs}ms each); the model produced no turn and the lane was ended rather than left to run out its budget`;
|
|
861
|
+
record({
|
|
862
|
+
id: nextId("notice"),
|
|
863
|
+
kind: "notice",
|
|
864
|
+
lifecycle: "completed",
|
|
865
|
+
status: "error",
|
|
866
|
+
title: "provider turn stalled twice",
|
|
867
|
+
text: reason
|
|
868
|
+
});
|
|
869
|
+
break;
|
|
870
|
+
}
|
|
871
|
+
}
|
|
872
|
+
finally {
|
|
873
|
+
if (requestController)
|
|
874
|
+
signal?.removeEventListener("abort", onRequestAbort);
|
|
875
|
+
requestController?.abort();
|
|
755
876
|
}
|
|
756
|
-
}
|
|
757
|
-
finally {
|
|
758
|
-
if (requestController)
|
|
759
|
-
signal?.removeEventListener("abort", onRequestAbort);
|
|
760
|
-
requestController?.abort();
|
|
761
877
|
}
|
|
762
878
|
bump("turns");
|
|
763
879
|
previousResponseId = turn.responseId ?? previousResponseId;
|
|
@@ -906,21 +1022,27 @@ export async function runComputerUseLoop(options) {
|
|
|
906
1022
|
// FINAL state — and a task completed by that state read as incomplete. The first live
|
|
907
1023
|
// study caught it: both participants reached the dashboard, said so, and the funnel
|
|
908
1024
|
// reported 0/2. One guarded closing observation feeds the tracker; a failed observe
|
|
909
|
-
// changes nothing (the funnel stays honest about what it saw), and no
|
|
910
|
-
// stop evaluation rides it — the session is already over.
|
|
1025
|
+
// ordinarily changes nothing (the funnel stays honest about what it saw), and no
|
|
1026
|
+
// screenshot or stop evaluation rides it — the session is already over. An explicit
|
|
1027
|
+
// executor failure still fails the harness instead of disappearing behind completion.
|
|
911
1028
|
if (taskTracker !== undefined) {
|
|
912
1029
|
try {
|
|
1030
|
+
currentPhase = "observing closing task state";
|
|
913
1031
|
const closing = await observeBounded("closing");
|
|
914
1032
|
observeTasks(closing, turnNumber);
|
|
915
1033
|
}
|
|
916
|
-
catch {
|
|
917
|
-
//
|
|
1034
|
+
catch (error) {
|
|
1035
|
+
// Ordinary closing observations remain best-effort. A declared executor failure
|
|
1036
|
+
// must not disappear behind a participant's reported success.
|
|
1037
|
+
if (isCuaExecutorError(error))
|
|
1038
|
+
throw error;
|
|
918
1039
|
}
|
|
919
1040
|
}
|
|
920
1041
|
break;
|
|
921
1042
|
}
|
|
922
1043
|
const idleThisTurn = isIdleTurn(turn.actions);
|
|
923
|
-
|
|
1044
|
+
previousExecution = { actions: [] };
|
|
1045
|
+
for (const [actionIndex, action] of turn.actions.entries()) {
|
|
924
1046
|
if (signal?.aborted)
|
|
925
1047
|
throw new CuaAbortError();
|
|
926
1048
|
const actionTitle = describeCuaAction(action);
|
|
@@ -959,6 +1081,22 @@ export async function runComputerUseLoop(options) {
|
|
|
959
1081
|
else
|
|
960
1082
|
await raceBounded(`idle action ${actionTitle}`, pending, remaining(), idleBound, signal);
|
|
961
1083
|
}
|
|
1084
|
+
catch (error) {
|
|
1085
|
+
if (error instanceof CuaStallError && executor.stallRecovery === "fail_closed") {
|
|
1086
|
+
throw new CuaExecutorError("deadline_exceeded", "outcome_uncertain");
|
|
1087
|
+
}
|
|
1088
|
+
if (error instanceof CuaDeadlineError || error instanceof CuaAbortError) {
|
|
1089
|
+
// The loop's deadline/abort may win before the executor can report whether its
|
|
1090
|
+
// write reached the desktop. Cancellation alone does not establish rollback.
|
|
1091
|
+
interruptedActionOutcome = true;
|
|
1092
|
+
record({
|
|
1093
|
+
id: nextId("notice"), kind: "notice", lifecycle: "completed", status: "warn",
|
|
1094
|
+
title: "action outcome uncertain",
|
|
1095
|
+
text: redactNarration(`action: ${actionTitle}; disposition: outcome_uncertain; the loop stopped waiting before execution was acknowledged; the action was not retried`)
|
|
1096
|
+
});
|
|
1097
|
+
}
|
|
1098
|
+
throw error;
|
|
1099
|
+
}
|
|
962
1100
|
finally {
|
|
963
1101
|
signal?.removeEventListener("abort", onAbort);
|
|
964
1102
|
// A deadline also closes async executor preparation, so a late pointer read
|
|
@@ -966,6 +1104,7 @@ export async function runComputerUseLoop(options) {
|
|
|
966
1104
|
actionController.abort();
|
|
967
1105
|
}
|
|
968
1106
|
};
|
|
1107
|
+
let actionSkipped = false;
|
|
969
1108
|
try {
|
|
970
1109
|
if (isIdleAction(action)) {
|
|
971
1110
|
// Observation actions only look (#480). A `wait` that hangs inside the SDK has, by
|
|
@@ -977,6 +1116,7 @@ export async function runComputerUseLoop(options) {
|
|
|
977
1116
|
catch (error) {
|
|
978
1117
|
if (!(error instanceof CuaStallError))
|
|
979
1118
|
throw error;
|
|
1119
|
+
actionSkipped = true;
|
|
980
1120
|
record({
|
|
981
1121
|
id: nextId("notice"),
|
|
982
1122
|
kind: "notice",
|
|
@@ -998,8 +1138,17 @@ export async function runComputerUseLoop(options) {
|
|
|
998
1138
|
// (e.g. a Ctrl+Minus keypress exiting 2), so one flaky desktop command
|
|
999
1139
|
// must not end the whole run. Everything else — a raceSettle deadline
|
|
1000
1140
|
// (CuaDeadlineError) or abort (CuaAbortError), a sandbox-gone failure,
|
|
1001
|
-
// any non-CommandExitError — is
|
|
1002
|
-
//
|
|
1141
|
+
// any non-CommandExitError — is rethrown. A typed executor declaration must also
|
|
1142
|
+
// bypass command recovery even if an adapter has changed its ordinary Error metadata.
|
|
1143
|
+
if (isCuaExecutorError(error)) {
|
|
1144
|
+
previousExecution.actions.push({ index: actionIndex, status: error.disposition });
|
|
1145
|
+
if (error.disposition === "not_dispatched" && !isIdleAction(action)) {
|
|
1146
|
+
materialActions -= 1;
|
|
1147
|
+
counts.materialActions = materialActions;
|
|
1148
|
+
lastMaterialActionTitle = priorMaterialActionTitle;
|
|
1149
|
+
}
|
|
1150
|
+
throw error;
|
|
1151
|
+
}
|
|
1003
1152
|
if (!isCommandExitError(error))
|
|
1004
1153
|
throw error;
|
|
1005
1154
|
if (!isIdleAction(action)) {
|
|
@@ -1017,6 +1166,7 @@ export async function runComputerUseLoop(options) {
|
|
|
1017
1166
|
// counts.materialActions is 0 (honest-evidence invariant).
|
|
1018
1167
|
lastMaterialActionTitle = priorMaterialActionTitle;
|
|
1019
1168
|
}
|
|
1169
|
+
previousExecution.actions.push({ index: actionIndex, status: "skipped" });
|
|
1020
1170
|
const { exitCode, stderrTail } = commandFailureInfo(error);
|
|
1021
1171
|
record({
|
|
1022
1172
|
id: nextId("notice"),
|
|
@@ -1036,6 +1186,7 @@ export async function runComputerUseLoop(options) {
|
|
|
1036
1186
|
});
|
|
1037
1187
|
continue;
|
|
1038
1188
|
}
|
|
1189
|
+
previousExecution.actions.push({ index: actionIndex, status: actionSkipped ? "skipped" : "completed" });
|
|
1039
1190
|
record({
|
|
1040
1191
|
id: nextId("ui_action"),
|
|
1041
1192
|
kind: "ui_action",
|
|
@@ -1062,7 +1213,7 @@ export async function runComputerUseLoop(options) {
|
|
|
1062
1213
|
if (frameGuardTripped(observation))
|
|
1063
1214
|
break;
|
|
1064
1215
|
await maybeRecordScreenshot(observation, `turn-${turnNumber.toString().padStart(2, "0")}`);
|
|
1065
|
-
|
|
1216
|
+
flushTrace();
|
|
1066
1217
|
observeTasks(observation, turnNumber);
|
|
1067
1218
|
const dwellOutcome = await dwellIfDue(observation, turnNumber);
|
|
1068
1219
|
if (dwellOutcome !== undefined) {
|
|
@@ -1189,7 +1340,9 @@ export async function runComputerUseLoop(options) {
|
|
|
1189
1340
|
stopCause = "time_limit";
|
|
1190
1341
|
if (materialActions > 0) {
|
|
1191
1342
|
completionReason = "budget_reached";
|
|
1192
|
-
reason =
|
|
1343
|
+
reason = interruptedActionOutcome
|
|
1344
|
+
? `reached the ${timeoutMs}ms time budget with ${materialActions} material action attempt(s), ${counts.turns} turn(s); the latest action outcome is uncertain`
|
|
1345
|
+
: `reached the ${timeoutMs}ms time budget after productive activity (${materialActions} material action(s), ${counts.turns} turn(s))`;
|
|
1193
1346
|
}
|
|
1194
1347
|
else {
|
|
1195
1348
|
completionReason = "timed_out";
|
|
@@ -1201,6 +1354,27 @@ export async function runComputerUseLoop(options) {
|
|
|
1201
1354
|
reason = "run aborted by the harness";
|
|
1202
1355
|
stopCause = "harness_aborted";
|
|
1203
1356
|
}
|
|
1357
|
+
else if (isCuaProviderError(error)) {
|
|
1358
|
+
completionReason = "harness_error";
|
|
1359
|
+
reason = `participant provider error: ${error.code}; cleanup: ${error.receipt.cleanup}`;
|
|
1360
|
+
record({ id: nextId("notice"), kind: "notice", lifecycle: "completed", status: "error",
|
|
1361
|
+
title: "participant provider error", text: reason });
|
|
1362
|
+
}
|
|
1363
|
+
else if (isCuaExecutorError(error)) {
|
|
1364
|
+
completionReason = "harness_error";
|
|
1365
|
+
reason = `desktop executor error: ${error.code}; disposition: ${error.disposition}`;
|
|
1366
|
+
record({
|
|
1367
|
+
id: nextId("notice"), kind: "notice", lifecycle: "completed", status: "error",
|
|
1368
|
+
title: "desktop executor error",
|
|
1369
|
+
text: [
|
|
1370
|
+
`phase: ${redactNarration(currentPhase)}`,
|
|
1371
|
+
`code: ${error.code}`,
|
|
1372
|
+
`disposition: ${error.disposition}`,
|
|
1373
|
+
lastActionTitle === undefined ? undefined : `last action: ${redactNarration(lastActionTitle)}`
|
|
1374
|
+
].filter(Boolean).join("; "),
|
|
1375
|
+
...(lastScreenshotRef === undefined ? {} : { screenshotRef: lastScreenshotRef })
|
|
1376
|
+
});
|
|
1377
|
+
}
|
|
1204
1378
|
else {
|
|
1205
1379
|
completionReason = "actor_error";
|
|
1206
1380
|
const rawMessage = error instanceof Error ? error.message : String(error);
|
|
@@ -1238,6 +1412,8 @@ export async function runComputerUseLoop(options) {
|
|
|
1238
1412
|
skip = "the study stopped before any participant turn";
|
|
1239
1413
|
else if (provider.debrief === undefined)
|
|
1240
1414
|
skip = "this provider does not support read-only closing reports";
|
|
1415
|
+
else if (providerCleanupUnconfirmed)
|
|
1416
|
+
skip = "participant request cleanup is unconfirmed";
|
|
1241
1417
|
else if (signal?.aborted)
|
|
1242
1418
|
skip = "the study was cancelled";
|
|
1243
1419
|
else if (remaining() <= 0)
|
|
@@ -1272,13 +1448,17 @@ export async function runComputerUseLoop(options) {
|
|
|
1272
1448
|
timer.unref?.();
|
|
1273
1449
|
bump("debriefCalls");
|
|
1274
1450
|
try {
|
|
1275
|
-
const
|
|
1451
|
+
const closingRequest = {
|
|
1276
1452
|
instructions,
|
|
1277
1453
|
observation: closingObservation,
|
|
1454
|
+
...(provider.requestPolicy !== "fail_closed" || previousExecution === undefined ? {} : { previousExecution }),
|
|
1278
1455
|
...(previousResponseId === undefined ? {} : { previousResponseId }),
|
|
1279
1456
|
...(pendingAcks === undefined ? {} : { acknowledgedSafetyChecks: pendingAcks }),
|
|
1280
1457
|
contextHint: "The interactive session has ended. Return a closing account with summary and frictionReports. In summary, briefly describe only what you actually did and observed. In frictionReports, list only specific unexpected behavior, confusion, or recovery you personally encountered during this session. Preserve uncertainty. Use an empty list if you encountered none. Do not speculate, invent problems, quote instructions as observations, or describe planned actions. Do not request or take further actions. This is a closing account, not another attempt at the task."
|
|
1281
|
-
}
|
|
1458
|
+
};
|
|
1459
|
+
const turn = provider.requestPolicy === "fail_closed"
|
|
1460
|
+
? await markedRequest(closingRequest, "debrief", capMs)
|
|
1461
|
+
: await raceBounded("participant debrief", provider.debrief(closingRequest, controller.signal), capMs, capMs, signal);
|
|
1282
1462
|
recordUsage(turn, false);
|
|
1283
1463
|
lastResponseId = turn.responseId ?? lastResponseId;
|
|
1284
1464
|
// Refresh a shared budget with all reported usage, without changing the completed task.
|
|
@@ -1289,7 +1469,7 @@ export async function runComputerUseLoop(options) {
|
|
|
1289
1469
|
title: "model budget reached during closing report",
|
|
1290
1470
|
text: "The closing request crossed an estimated budget; no further requests or actions followed. Task completion is unchanged." });
|
|
1291
1471
|
}
|
|
1292
|
-
const usageReported = completeTurnUsage(turn.usage);
|
|
1472
|
+
const usageReported = completeTurnUsage(turn.usage) && turn.providerRequest?.usageComplete !== false;
|
|
1293
1473
|
if (turn.actions.length > 0 || turn.pendingSafetyChecks.length > 0) {
|
|
1294
1474
|
note("failed", "the closing response requested actions or safety checks; none were executed and its report was not accepted", usageReported);
|
|
1295
1475
|
}
|
|
@@ -1317,7 +1497,9 @@ export async function runComputerUseLoop(options) {
|
|
|
1317
1497
|
else {
|
|
1318
1498
|
const detail = signal?.aborted ? "cancelled" : controller.signal.aborted || error instanceof CuaDeadlineError || error instanceof CuaStallError
|
|
1319
1499
|
? "closing report deadline reached" : error instanceof Error ? error.message : String(error);
|
|
1320
|
-
|
|
1500
|
+
const closingReceipt = provider.requestPolicy === "fail_closed" ? providerRequests.at(-1) : undefined;
|
|
1501
|
+
const usageReported = closingReceipt?.kind === "debrief" && closingReceipt.usageComplete && completeTurnUsage(closingReceipt.usage);
|
|
1502
|
+
note("failed", `${detail}; closing request usage is ${usageReported ? "reported" : "unreported"}`, usageReported);
|
|
1321
1503
|
}
|
|
1322
1504
|
}
|
|
1323
1505
|
finally {
|
|
@@ -1327,7 +1509,7 @@ export async function runComputerUseLoop(options) {
|
|
|
1327
1509
|
}
|
|
1328
1510
|
}
|
|
1329
1511
|
}
|
|
1330
|
-
|
|
1512
|
+
flushTrace();
|
|
1331
1513
|
}
|
|
1332
1514
|
const completedAtMs = now();
|
|
1333
1515
|
const status = statusForCompletionReason(completionReason);
|
|
@@ -1352,6 +1534,7 @@ export async function runComputerUseLoop(options) {
|
|
|
1352
1534
|
const trace = {
|
|
1353
1535
|
schema: ACTOR_TRACE_SCHEMA,
|
|
1354
1536
|
provider: provider.id,
|
|
1537
|
+
...liveMetadata(),
|
|
1355
1538
|
...(provider.version === undefined ? {} : { providerVersion: provider.version }),
|
|
1356
1539
|
protocol: "cua-loop",
|
|
1357
1540
|
lane: "computer-use",
|
|
@@ -1388,7 +1571,7 @@ export async function runComputerUseLoop(options) {
|
|
|
1388
1571
|
...(taskTracker === undefined ? {} : { taskFunnel: taskTracker.funnel() }),
|
|
1389
1572
|
...(sawUsage
|
|
1390
1573
|
? {
|
|
1391
|
-
tokenUsage: {
|
|
1574
|
+
tokenUsage: provider.executionProfile?.billing === "account-unknown" ? accountUsage() : {
|
|
1392
1575
|
input: usageInput,
|
|
1393
1576
|
output: usageOutput,
|
|
1394
1577
|
// Recorded only when the provider actually reported it, so a reader can tell "no cache
|