humanish 0.120.1 → 0.121.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/dist/cli/commands/study-format.js +25 -4
- package/dist/routes/computer-use/bundle.d.ts +3 -0
- package/dist/routes/computer-use/bundle.js +2 -0
- package/dist/routes/computer-use/fanout-bundle.js +20 -2
- package/dist/routes/computer-use/live-phase.d.ts +1 -0
- package/dist/routes/computer-use/participant-deps.d.ts +1 -1
- package/dist/routes/computer-use/participant-deps.js +3 -1
- package/dist/routes/computer-use/participant-execution.d.ts +8 -7
- package/dist/routes/computer-use/participant-execution.js +93 -71
- package/dist/routes/computer-use/result.js +56 -7
- package/dist/routes/computer-use/setup.d.ts +1 -1
- package/dist/routes/computer-use/single-bundle.d.ts +3 -0
- package/dist/routes/computer-use/single-bundle.js +3 -2
- package/dist/routes/computer-use/types.d.ts +16 -2
- package/dist/run/host-suspension.d.ts +56 -0
- package/dist/run/host-suspension.js +106 -0
- package/dist/run/outcomes.js +3 -1
- package/dist/run/run.d.ts +15 -2
- package/dist/run/run.js +15 -8
- package/dist/run/status.d.ts +8 -2
- package/dist/run/status.js +11 -8
- package/dist/study/study-deps.d.ts +6 -0
- package/dist/tui-app.js +35 -35
- package/docs/contracts/run-bundle.md +19 -0
- package/docs/contracts/schemas.md +2 -1
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -8,6 +8,21 @@ The Unreleased section holds the full notes for the next version until it is tag
|
|
|
8
8
|
|
|
9
9
|
## Unreleased
|
|
10
10
|
|
|
11
|
+
## 0.121.0: One failed desktop fails alone, host sleep named as the cause (2026-10-10)
|
|
12
|
+
|
|
13
|
+
humanish 0.121.0 keeps one participant's failed desktop from costing a whole hosted study, and
|
|
14
|
+
names a host that slept mid-run as the likely cause of the failures after it. A computer-use study
|
|
15
|
+
on an `app-url` subject no longer holds its participants behind the first one's desktop, and a
|
|
16
|
+
participant that fails before its session no longer stops the participants queued behind it; a
|
|
17
|
+
study whose subject is built in each sandbox still lets one participant set it up first, and the
|
|
18
|
+
next one takes over when the first never got a desktop. When the machine running a study sleeps,
|
|
19
|
+
the run records the suspension as a `host.suspended` event, and the run outcome, `humanish run`,
|
|
20
|
+
the review and the `--json` result say the failures of participants running through it are
|
|
21
|
+
likely its effect, while each participant's record keeps its own error. The review's participant
|
|
22
|
+
line lists interruption causes in a fixed order.
|
|
23
|
+
|
|
24
|
+
[Release notes](https://github.com/danielgwilson/humanish/releases/tag/v0.121.0)
|
|
25
|
+
|
|
11
26
|
## 0.120.1: Analysis requests under Bun run past 5 minutes (2026-10-09)
|
|
12
27
|
|
|
13
28
|
humanish 0.120.1 lets an analysis request from a Bun process run longer than 5 minutes. Under Bun,
|
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
import { participantCaption } from "../../run/participant-caption.js";
|
|
2
2
|
import { formatCuaDiagnostics, formatCuaStopCause } from "../../routes/computer-use/diagnostics.js";
|
|
3
3
|
import { subjectName } from "../../run/subject-name.js";
|
|
4
|
+
import { cli } from "../invocation.js";
|
|
4
5
|
/**
|
|
5
6
|
* A run's first lines: the command that ran, whether it was a dry run, how it ended, and its route.
|
|
6
7
|
* `runOk` is the run's own ok, which automaticAnalysisEnvelope keeps when an analysis that did not
|
|
7
|
-
* complete turns `ok` false. That run finished, and the line says so.
|
|
8
|
+
* complete turns `ok` false. That run finished, and the line says so. A host suspension follows,
|
|
9
|
+
* since it may explain every failure listed after it.
|
|
8
10
|
*/
|
|
9
11
|
function runHeader(result, route) {
|
|
10
12
|
const kind = result.dryRun === true ? "dry run" : result.dryRun === false ? "live run" : "run";
|
|
@@ -13,7 +15,24 @@ function runHeader(result, route) {
|
|
|
13
15
|
: result.runOk === true
|
|
14
16
|
? "finished; the analysis did not complete"
|
|
15
17
|
: "failed";
|
|
16
|
-
return [
|
|
18
|
+
return [
|
|
19
|
+
`humanish run ${result.studyId}: ${kind} ${ending}`,
|
|
20
|
+
`route: ${route}`,
|
|
21
|
+
...(result.hostSuspension === undefined ? [] : [result.hostSuspension.summary]),
|
|
22
|
+
];
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* What to run after a host suspension failed participants: the rerun of those that did not pass,
|
|
26
|
+
* or the study again for a run with one participant.
|
|
27
|
+
*/
|
|
28
|
+
function suspensionNextStep(result, participants) {
|
|
29
|
+
if ((result.hostSuspension?.participantIds.length ?? 0) === 0)
|
|
30
|
+
return [];
|
|
31
|
+
return participants > 1
|
|
32
|
+
? [
|
|
33
|
+
`Rerun the participants that did not pass: ${cli(`run ${result.studyId} --rerun-failed-from ${result.runId}`)}`,
|
|
34
|
+
]
|
|
35
|
+
: [`Run it again: ${cli(`run ${result.studyId}`)}`];
|
|
17
36
|
}
|
|
18
37
|
/**
|
|
19
38
|
* A run's human output: its lines on stdout and, when it failed with an error, that error on
|
|
@@ -167,8 +186,10 @@ export function formatScriptedStudyHuman(result, subject) {
|
|
|
167
186
|
}
|
|
168
187
|
/** `subject` is the study's declared subject, which names it when the result has no URL. */
|
|
169
188
|
export function formatCuaStudyHuman(result, subject) {
|
|
189
|
+
const participants = result.lanes ?? [];
|
|
170
190
|
return withError(result.error, [
|
|
171
191
|
...runHeader(result, "computer-use"),
|
|
192
|
+
...suspensionNextStep(result, participants.length),
|
|
172
193
|
`run: ${result.runId}`,
|
|
173
194
|
`actor: ${result.actor}`,
|
|
174
195
|
`subject: ${subjectLine(result.appUrl, subject)}`,
|
|
@@ -181,8 +202,8 @@ export function formatCuaStudyHuman(result, subject) {
|
|
|
181
202
|
? [`rerun: ${result.rerun.selectedLaneIds.join(", ")} from ${result.rerun.sourceRunId}`]
|
|
182
203
|
: []),
|
|
183
204
|
...(result.diagnostics ? [`diagnostic: ${formatCuaDiagnostics(result.diagnostics)}`] : []),
|
|
184
|
-
...participantLines(
|
|
185
|
-
...(result.session &&
|
|
205
|
+
...participantLines(participants),
|
|
206
|
+
...(result.session && participants.length <= 1
|
|
186
207
|
? [
|
|
187
208
|
`session: ${participantStatus(result.session.status)} (${result.session.completionReason})${result.session.stopCause ? ` · ${formatCuaStopCause(result.session.stopCause)}` : ""} · ${result.session.reason}`,
|
|
188
209
|
`screenshots: ${result.session.screenshots}`,
|
|
@@ -2,6 +2,7 @@ import type { CuaActorDescriptor } from "../../actors/registry.js";
|
|
|
2
2
|
import type { ComputerUsePlan } from "../../study/plan-types.js";
|
|
3
3
|
import type { BundleRun, RunBundle, RunRerunLineage, RunSubjectProvenance } from "../../run/bundle.js";
|
|
4
4
|
import { type Judgment } from "../../run/judge.js";
|
|
5
|
+
import type { HostSuspensionReading } from "../../run/host-suspension.js";
|
|
5
6
|
import type { CuaParticipantPlan, DesktopParticipantRun, ParticipantRunOutcome } from "./types.js";
|
|
6
7
|
/** What every bundle of one run shares, in progress or final. */
|
|
7
8
|
export interface CuaRunBundleBase {
|
|
@@ -35,6 +36,8 @@ interface CuaRunBundleState {
|
|
|
35
36
|
subjects: RunSubjectProvenance[];
|
|
36
37
|
aggregateSubject: RunSubjectProvenance;
|
|
37
38
|
failFastReason?: string;
|
|
39
|
+
/** The final bundle's reading of the run's host suspensions. */
|
|
40
|
+
hostSuspension?: HostSuspensionReading;
|
|
38
41
|
inProgress?: true;
|
|
39
42
|
}
|
|
40
43
|
/**
|
|
@@ -61,6 +61,7 @@ export function buildCuaRunBundle(base, state) {
|
|
|
61
61
|
participantPlan: base.participantPlan,
|
|
62
62
|
...(base.rerun === undefined ? {} : { rerun: base.rerun }),
|
|
63
63
|
...(state.failFastReason === undefined ? {} : { failFastReason: state.failFastReason }),
|
|
64
|
+
...(state.hostSuspension === undefined ? {} : { hostSuspension: state.hostSuspension }),
|
|
64
65
|
...(base.publicRepo === undefined ? {} : { publicRepo: base.publicRepo }),
|
|
65
66
|
...inProgress,
|
|
66
67
|
});
|
|
@@ -117,6 +118,7 @@ function singleParticipantArgs(base, state) {
|
|
|
117
118
|
}),
|
|
118
119
|
source: base.source,
|
|
119
120
|
...(state.inProgress === undefined ? {} : { inProgress: state.inProgress }),
|
|
121
|
+
...(state.hostSuspension === undefined ? {} : { hostSuspension: state.hostSuspension }),
|
|
120
122
|
subject: state.aggregateSubject,
|
|
121
123
|
...(desktopTemplate === undefined ? {} : { desktopTemplate }),
|
|
122
124
|
...(outcome?.desktopBrowser === undefined ? {} : { desktopBrowser: outcome.desktopBrowser }),
|
|
@@ -49,7 +49,25 @@ function fanoutSummary(args, facts) {
|
|
|
49
49
|
return `${rerun}Dry run: ${people} would each use their own copy of ${subjectName(publicSafeAppUrlLabel(args.appUrl), args.aggregateSubject)}. No desktops were launched and $0 was spent.`;
|
|
50
50
|
}
|
|
51
51
|
const tasks = facts.tasks === undefined ? "" : ` Tasks: ${facts.tasks}.`;
|
|
52
|
-
|
|
52
|
+
// A host suspension comes first: it may explain everything after it.
|
|
53
|
+
const host = args.hostSuspension === undefined ? "" : `${args.hostSuspension.summary} `;
|
|
54
|
+
return `${host}${rerun}${people} took part, each in their own copy of the app: ${facts.outcomes}${gapsListClause(count - facts.passed, count)}.${tasks}`;
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* The review's gaps: one line for the participants a host suspension likely failed, then a line
|
|
58
|
+
* for each other participant that did not pass.
|
|
59
|
+
*/
|
|
60
|
+
function fanoutGaps(args, judgment) {
|
|
61
|
+
const suspended = new Set(args.hostSuspension?.participantIds ?? []);
|
|
62
|
+
const others = judgment.participants.flatMap((participant, index) => participant.gapLine === undefined || suspended.has(args.specs[index].planned.id)
|
|
63
|
+
? []
|
|
64
|
+
: [participant.gapLine]);
|
|
65
|
+
if (suspended.size === 0)
|
|
66
|
+
return others;
|
|
67
|
+
return [
|
|
68
|
+
`${[...suspended].join(", ")}: failed after the host was suspended, likely its effect. Each participant's record keeps its own error.`,
|
|
69
|
+
...others,
|
|
70
|
+
];
|
|
53
71
|
}
|
|
54
72
|
function fanoutReview(args, streams, judgment) {
|
|
55
73
|
const { outcomes } = args;
|
|
@@ -91,7 +109,7 @@ function fanoutReview(args, streams, judgment) {
|
|
|
91
109
|
? ["Live fan-out session is still running."]
|
|
92
110
|
: args.dryRun
|
|
93
111
|
? ["Live fan-out session not yet run (dry run only)."]
|
|
94
|
-
:
|
|
112
|
+
: fanoutGaps(args, judgment),
|
|
95
113
|
}, streams);
|
|
96
114
|
return review;
|
|
97
115
|
}
|
|
@@ -38,6 +38,7 @@ export declare function runStudyParticipants(setup: CuaRunSetup, participants: C
|
|
|
38
38
|
rerun?: import("../../run/bundle.js").RunRerunLineage;
|
|
39
39
|
observer?: import("../../index.js").ObserverResult;
|
|
40
40
|
diagnostics?: import("./diagnostics.js").CuaDiagnostics;
|
|
41
|
+
hostSuspension?: import("../../run/host-suspension.js").HostSuspensionReading;
|
|
41
42
|
warnings: string[];
|
|
42
43
|
error?: {
|
|
43
44
|
code: import("./types.js").CuaActorStudyErrorCode;
|
|
@@ -3,7 +3,7 @@ import type { StudyDeps } from "../../study/study-deps.js";
|
|
|
3
3
|
import type { LiveTraceFlush } from "./live-flush.js";
|
|
4
4
|
import type { CuaParticipantDeps } from "./types.js";
|
|
5
5
|
/** The deps a desktop route builds once per run. Each participant adds the app URL it opens. */
|
|
6
|
-
export type DesktopParticipantDeps = Omit<CuaParticipantDeps, "
|
|
6
|
+
export type DesktopParticipantDeps = Omit<CuaParticipantDeps, "signalReady" | "appUrl" | "onObservedUrl">;
|
|
7
7
|
/** What every participant of a desktop run reads. Each route supplies all of it. */
|
|
8
8
|
interface DesktopRunFacts {
|
|
9
9
|
plan: Pick<ComputerUsePlan | SharedWorldPlan, "studyId" | "residual" | "caps" | "dryRun">;
|
|
@@ -37,7 +37,9 @@ export function desktopParticipantDeps(run, overrides) {
|
|
|
37
37
|
redactScreenshots: run.redactScreenshots,
|
|
38
38
|
scrubKnownValues: run.scrubKnownValues,
|
|
39
39
|
runSession: run.runSession,
|
|
40
|
-
|
|
40
|
+
// The run's own clock unless a test sets another, so a participant's times and the run's host
|
|
41
|
+
// suspensions read one clock.
|
|
42
|
+
now: seams.now ?? seams.hostClock?.now ?? Date.now,
|
|
41
43
|
onStream: run.onStream,
|
|
42
44
|
reportSubjectPhase: run.reportSubjectPhase,
|
|
43
45
|
};
|
|
@@ -12,17 +12,18 @@ export declare function makeParticipantWriteScreenshot(artifactRoot: PreparedOut
|
|
|
12
12
|
export declare function runCuaParticipant(spec: DesktopParticipantRun, deps: CuaParticipantDeps): Promise<ParticipantRunOutcome>;
|
|
13
13
|
/**
|
|
14
14
|
* Run N>1 E2B participants on the study's schedule (runOnSchedule: each at its start offset, at
|
|
15
|
-
* most `concurrency` at once), behind
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
* by id; nothing here
|
|
15
|
+
* most `concurrency` at once), behind the pipeline gate when the subject is set up in each sandbox,
|
|
16
|
+
* and session fail-fast on harness errors only (queued participants become `blocked` with a pinned
|
|
17
|
+
* reason + a fail-fast event; mission verdicts never trip it). A participant that stops before its
|
|
18
|
+
* session has failed on its own and trips no fail-fast. A closed gate or a fail-fast ends every
|
|
19
|
+
* wait for a later start at once. Each participant tears down its own sandbox by id; nothing here
|
|
20
|
+
* ever enumerates.
|
|
20
21
|
*
|
|
21
22
|
* Exported for the total-runner tests: the injectable runner lets a test make one participant
|
|
22
23
|
* throw (the exact class the guard exists for) without a live sandbox. Production always uses
|
|
23
24
|
* the default.
|
|
24
25
|
*/
|
|
25
|
-
export declare function runCuaParticipants(runs: DesktopParticipantRun[], deps: Omit<CuaParticipantDeps, "
|
|
26
|
+
export declare function runCuaParticipants(runs: DesktopParticipantRun[], deps: Omit<CuaParticipantDeps, "signalReady">, concurrency: number, runParticipant?: typeof runCuaParticipant): Promise<{
|
|
26
27
|
outcomes: ParticipantRunOutcome[];
|
|
27
28
|
failFastReason?: string;
|
|
28
29
|
}>;
|
|
@@ -32,7 +33,7 @@ export declare function toParticipantResult(spec: DesktopParticipantRun, outcome
|
|
|
32
33
|
* Run every participant of a live run. The in-process route drives its single participant in this
|
|
33
34
|
* process; one hosted participant runs alone; a fan-out runs at the plan's concurrency and may stop early.
|
|
34
35
|
*/
|
|
35
|
-
export declare function runAllCuaParticipants(runs: readonly DesktopParticipantRun[], deps: Omit<CuaParticipantDeps, "
|
|
36
|
+
export declare function runAllCuaParticipants(runs: readonly DesktopParticipantRun[], deps: Omit<CuaParticipantDeps, "signalReady">, participantPlan: CuaParticipantPlan, inProcess: boolean): Promise<{
|
|
36
37
|
outcomes: ParticipantRunOutcome[];
|
|
37
38
|
failFastReason: string | undefined;
|
|
38
39
|
}>;
|
|
@@ -56,7 +56,6 @@ export async function runCuaParticipant(spec, deps) {
|
|
|
56
56
|
// model turn only to stop at its first one, so it is skipped before its desktop exists.
|
|
57
57
|
const crossed = deps.runBudget?.crossed();
|
|
58
58
|
if (crossed !== undefined) {
|
|
59
|
-
deps.signalProvisioned?.(false);
|
|
60
59
|
return skippedOutcome(spec, `skipped: study budget reached before this participant started: ${crossed}; no desktop was created`);
|
|
61
60
|
}
|
|
62
61
|
let model = {};
|
|
@@ -67,23 +66,13 @@ export async function runCuaParticipant(spec, deps) {
|
|
|
67
66
|
let sessionError;
|
|
68
67
|
let providerCleanupError;
|
|
69
68
|
let providerPolicyError;
|
|
70
|
-
let provisioned = false;
|
|
71
|
-
let signaled = false;
|
|
72
|
-
const signal = (ok) => {
|
|
73
|
-
if (!signaled && deps.signalProvisioned) {
|
|
74
|
-
signaled = true;
|
|
75
|
-
deps.signalProvisioned(ok);
|
|
76
|
-
}
|
|
77
|
-
};
|
|
78
69
|
const desktop = deps.createDesktop?.(spec, warnings, deps.artifactRoot) ??
|
|
79
70
|
createE2BParticipantDesktop(spec, deps, warnings);
|
|
80
71
|
try {
|
|
81
72
|
await desktop.prepare();
|
|
82
73
|
const ready = await desktop.openSession();
|
|
83
74
|
model = await startParticipantModel(spec, deps, ready.executor);
|
|
84
|
-
|
|
85
|
-
provisioned = true;
|
|
86
|
-
signal(true);
|
|
75
|
+
deps.signalReady?.();
|
|
87
76
|
session = await deps.runSession(participantSessionOptions(spec, deps, ready, model.provider, writeScreenshot));
|
|
88
77
|
}
|
|
89
78
|
catch (error) {
|
|
@@ -102,18 +91,12 @@ export async function runCuaParticipant(spec, deps) {
|
|
|
102
91
|
: closed.refusal.startsWith("HUMANISH_CLAUDE_PARTICIPANT_")
|
|
103
92
|
? `Claude Code's stream showed a call the participant may not make after its last turn (${closed.refusal}); the participant was stopped.`
|
|
104
93
|
: `Codex output after the participant's last request could not be checked against the item policy (${closed.refusal}).`));
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
failed: sessionError !== undefined ||
|
|
112
|
-
providerCleanupError !== undefined ||
|
|
113
|
-
providerPolicyError !== undefined ||
|
|
114
|
-
session === undefined,
|
|
115
|
-
});
|
|
116
|
-
}
|
|
94
|
+
await desktop.finalize({
|
|
95
|
+
failed: sessionError !== undefined ||
|
|
96
|
+
providerCleanupError !== undefined ||
|
|
97
|
+
providerPolicyError !== undefined ||
|
|
98
|
+
session === undefined,
|
|
99
|
+
});
|
|
117
100
|
}
|
|
118
101
|
const { released, desktopFailure, ...desktopEvidence } = desktop.snapshot();
|
|
119
102
|
// A desktop that died under the participant explains the failure better than the transport
|
|
@@ -147,15 +130,73 @@ export async function runCuaParticipant(spec, deps) {
|
|
|
147
130
|
selfReportedBlocker,
|
|
148
131
|
reportedFriction,
|
|
149
132
|
harnessError,
|
|
133
|
+
endedAt: deps.now(),
|
|
134
|
+
};
|
|
135
|
+
}
|
|
136
|
+
/**
|
|
137
|
+
* Holds a fan-out's participants back until one of them is ready, when each sets up the subject in
|
|
138
|
+
* its own sandbox the same way (a clone or local tree built and served, a desktop CLI installed),
|
|
139
|
+
* so a subject that fails to set up fails once. The first participant to enter holds the gate and
|
|
140
|
+
* the others wait. Any participant that becomes ready opens it. A holder that never got a desktop
|
|
141
|
+
* learned nothing about the subject, so the next waiting participant holds it instead. A holder
|
|
142
|
+
* whose desktop came up and then failed before its session closes it, and `onClose` runs.
|
|
143
|
+
*/
|
|
144
|
+
function pipelineGate(open, onClose) {
|
|
145
|
+
let state = open ? "open" : "vacant";
|
|
146
|
+
let holder;
|
|
147
|
+
const waiting = [];
|
|
148
|
+
const settle = (turn) => {
|
|
149
|
+
for (const participant of waiting.splice(0))
|
|
150
|
+
participant.turn(turn);
|
|
151
|
+
};
|
|
152
|
+
return {
|
|
153
|
+
/** The participant that holds the gate, or held it when it closed. */
|
|
154
|
+
holder: () => holder,
|
|
155
|
+
enter(id) {
|
|
156
|
+
if (state === "open")
|
|
157
|
+
return Promise.resolve("start");
|
|
158
|
+
if (state === "closed")
|
|
159
|
+
return Promise.resolve("closed");
|
|
160
|
+
if (state === "vacant") {
|
|
161
|
+
state = "held";
|
|
162
|
+
holder = id;
|
|
163
|
+
return Promise.resolve("hold");
|
|
164
|
+
}
|
|
165
|
+
return new Promise((turn) => waiting.push({ id, turn }));
|
|
166
|
+
},
|
|
167
|
+
ready() {
|
|
168
|
+
if (state !== "held")
|
|
169
|
+
return;
|
|
170
|
+
state = "open";
|
|
171
|
+
settle("start");
|
|
172
|
+
},
|
|
173
|
+
failed(id, { handOver }) {
|
|
174
|
+
if (state !== "held" || holder !== id)
|
|
175
|
+
return;
|
|
176
|
+
if (handOver) {
|
|
177
|
+
const next = waiting.shift();
|
|
178
|
+
if (next === undefined)
|
|
179
|
+
state = "vacant";
|
|
180
|
+
else {
|
|
181
|
+
holder = next.id;
|
|
182
|
+
next.turn("hold");
|
|
183
|
+
}
|
|
184
|
+
return;
|
|
185
|
+
}
|
|
186
|
+
state = "closed";
|
|
187
|
+
settle("closed");
|
|
188
|
+
onClose();
|
|
189
|
+
},
|
|
150
190
|
};
|
|
151
191
|
}
|
|
152
192
|
/**
|
|
153
193
|
* Run N>1 E2B participants on the study's schedule (runOnSchedule: each at its start offset, at
|
|
154
|
-
* most `concurrency` at once), behind
|
|
155
|
-
*
|
|
156
|
-
*
|
|
157
|
-
*
|
|
158
|
-
* by id; nothing here
|
|
194
|
+
* most `concurrency` at once), behind the pipeline gate when the subject is set up in each sandbox,
|
|
195
|
+
* and session fail-fast on harness errors only (queued participants become `blocked` with a pinned
|
|
196
|
+
* reason + a fail-fast event; mission verdicts never trip it). A participant that stops before its
|
|
197
|
+
* session has failed on its own and trips no fail-fast. A closed gate or a fail-fast ends every
|
|
198
|
+
* wait for a later start at once. Each participant tears down its own sandbox by id; nothing here
|
|
199
|
+
* ever enumerates.
|
|
159
200
|
*
|
|
160
201
|
* Exported for the total-runner tests: the injectable runner lets a test make one participant
|
|
161
202
|
* throw (the exact class the guard exists for) without a live sandbox. Production always uses
|
|
@@ -164,40 +205,26 @@ export async function runCuaParticipant(spec, deps) {
|
|
|
164
205
|
export async function runCuaParticipants(runs, deps, concurrency, runParticipant = runCuaParticipant) {
|
|
165
206
|
const failFast = { tripped: false, reason: "" };
|
|
166
207
|
const stopping = new AbortController();
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
let rejectGate;
|
|
170
|
-
const gate = new Promise((resolve, reject) => {
|
|
171
|
-
resolveGate = resolve;
|
|
172
|
-
rejectGate = () => {
|
|
173
|
-
reject(new Error("gate"));
|
|
174
|
-
stopping.abort();
|
|
175
|
-
};
|
|
176
|
-
});
|
|
177
|
-
// The gate is rejected on the first participant's provisioning failure; swallow the unhandled
|
|
178
|
-
// rejection if no later participant ever awaits it (concurrency could let the first finish
|
|
179
|
-
// alone).
|
|
180
|
-
gate.catch(() => undefined);
|
|
208
|
+
// An app-url subject is only opened, so one participant's start says nothing about another's.
|
|
209
|
+
const gate = pipelineGate(deps.subject.kind === "app-url", () => stopping.abort());
|
|
181
210
|
const outcomes = await runOnSchedule(runs, {
|
|
182
211
|
startAfterMs: (spec) => spec.planned.startAfterMs,
|
|
183
212
|
slots: concurrency,
|
|
184
213
|
now: deps.now,
|
|
185
214
|
signal: stopping.signal,
|
|
186
|
-
}, async (spec, _index, {
|
|
187
|
-
if (order === 0)
|
|
188
|
-
gateHolder = spec.planned.id;
|
|
189
|
-
if (order > 0) {
|
|
190
|
-
try {
|
|
191
|
-
await gate;
|
|
192
|
-
}
|
|
193
|
-
catch {
|
|
194
|
-
return skippedOutcome(spec, `skipped: participant ${gateHolder} failed to provision its world (pipeline gate)`, { scheduledAt });
|
|
195
|
-
}
|
|
196
|
-
}
|
|
215
|
+
}, async (spec, _index, { scheduledAt }) => {
|
|
197
216
|
if (failFast.tripped) {
|
|
198
217
|
return skippedOutcome(spec, `skipped: ${failFast.reason}`, { scheduledAt });
|
|
199
218
|
}
|
|
219
|
+
// A participant trips fail-fast only after the gate opened, or as it closes the gate, so no
|
|
220
|
+
// participant is still waiting at the gate when fail-fast skips the rest.
|
|
221
|
+
const id = spec.planned.id;
|
|
222
|
+
if ((await gate.enter(id)) === "closed") {
|
|
223
|
+
return skippedOutcome(spec, `skipped: participant ${gate.holder()} failed to provision its world (pipeline gate)`, { scheduledAt });
|
|
224
|
+
}
|
|
200
225
|
const arrival = { scheduledAt, startedAt: deps.now() };
|
|
226
|
+
let ready = false;
|
|
227
|
+
let threw = false;
|
|
201
228
|
// The participant runner is total: every exit path returns a recorded outcome. Without this
|
|
202
229
|
// guard, one participant's late throw (e.g. its trace write hitting ENOSPC after its own sandbox was
|
|
203
230
|
// already torn down) rejected the whole map while sibling workers kept launching sandboxes
|
|
@@ -206,25 +233,14 @@ export async function runCuaParticipants(runs, deps, concurrency, runParticipant
|
|
|
206
233
|
try {
|
|
207
234
|
outcome = await runParticipant(spec, {
|
|
208
235
|
...deps,
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
resolveGate?.();
|
|
214
|
-
}
|
|
215
|
-
else {
|
|
216
|
-
rejectGate?.();
|
|
217
|
-
}
|
|
218
|
-
},
|
|
219
|
-
}
|
|
220
|
-
: {}),
|
|
236
|
+
signalReady: () => {
|
|
237
|
+
ready = true;
|
|
238
|
+
gate.ready();
|
|
239
|
+
},
|
|
221
240
|
});
|
|
222
241
|
}
|
|
223
242
|
catch (error) {
|
|
224
|
-
|
|
225
|
-
// blocked rather than leaving them awaiting a gate that will never settle.
|
|
226
|
-
if (order === 0)
|
|
227
|
-
rejectGate?.();
|
|
243
|
+
threw = true;
|
|
228
244
|
const detail = redactText(toErrorMessage(error));
|
|
229
245
|
outcome = {
|
|
230
246
|
spec,
|
|
@@ -239,9 +255,15 @@ export async function runCuaParticipants(runs, deps, concurrency, runParticipant
|
|
|
239
255
|
reportedFriction: false,
|
|
240
256
|
harnessError: true,
|
|
241
257
|
sessionError: `participant runner threw outside the session guard: ${detail}`,
|
|
258
|
+
endedAt: deps.now(),
|
|
242
259
|
};
|
|
243
260
|
}
|
|
244
|
-
|
|
261
|
+
// A participant that stopped before its session failed on its own and trips no fail-fast.
|
|
262
|
+
// Without a desktop it says nothing about the subject, so the next participant starts first.
|
|
263
|
+
// A runner that threw is a harness defect: it closes the gate and trips fail-fast.
|
|
264
|
+
if (!ready)
|
|
265
|
+
gate.failed(id, { handOver: !threw && outcome.sandboxId === undefined });
|
|
266
|
+
if (outcome.harnessError && (ready || threw) && !failFast.tripped) {
|
|
245
267
|
failFast.tripped = true;
|
|
246
268
|
failFast.reason = `a prior participant (${outcome.spec.planned.id}) ended in a harness error (fail-fast)`;
|
|
247
269
|
stopping.abort();
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { adapterScoreFailureMessage, applyBrowserScorer } from "../../study/adapter-extension.js";
|
|
2
2
|
import { redactText } from "../../evidence/redaction.js";
|
|
3
|
-
import { foldScorerFailures, judgeExecution, judgeParticipantRecords, OUTCOME_POLICIES, resultOk, } from "../../run/judge.js";
|
|
3
|
+
import { foldScorerFailures, judgeExecution, judgeParticipantRecords, OUTCOME_POLICIES, participantHarnessFailed, resultOk, } from "../../run/judge.js";
|
|
4
|
+
import { readHostSuspensions, } from "../../run/host-suspension.js";
|
|
4
5
|
import { validatePreparedRunArtifactPaths } from "../../run/paths.js";
|
|
5
6
|
import { participantExecutionFailures, participantOutcomeOk, participantFactsOf, } from "./participant-facts.js";
|
|
6
7
|
import { participantCapWarning } from "./participant-model.js";
|
|
@@ -53,13 +54,39 @@ function buildParticipantSummary(outcomes, participantCount, participantPlan, dr
|
|
|
53
54
|
waves: participantPlan.waves,
|
|
54
55
|
};
|
|
55
56
|
}
|
|
57
|
+
/**
|
|
58
|
+
* Each participant as the host-suspension reading reads it: a harness error, its session's time
|
|
59
|
+
* limit or a skip, when it started, and when its runner returned, which a setup failure with no
|
|
60
|
+
* session also records.
|
|
61
|
+
*/
|
|
62
|
+
function suspendedParticipants(outcomes) {
|
|
63
|
+
return outcomes.map((outcome) => {
|
|
64
|
+
const facts = participantFactsOf(outcome);
|
|
65
|
+
const trace = outcome.session?.trace;
|
|
66
|
+
const failure = facts.skipped
|
|
67
|
+
? "skipped"
|
|
68
|
+
: participantHarnessFailed(facts)
|
|
69
|
+
? "harness"
|
|
70
|
+
: trace?.stopCause === "time_limit"
|
|
71
|
+
? "time-limit"
|
|
72
|
+
: undefined;
|
|
73
|
+
const startedAtMs = outcome.arrival?.startedAt ?? (trace === undefined ? undefined : Date.parse(trace.startedAt));
|
|
74
|
+
const endedAtMs = outcome.endedAt ?? (trace === undefined ? undefined : Date.parse(trace.completedAt));
|
|
75
|
+
return {
|
|
76
|
+
id: outcome.spec.planned.id,
|
|
77
|
+
...(failure === undefined ? {} : { failure }),
|
|
78
|
+
...(startedAtMs === undefined ? {} : { startedAtMs }),
|
|
79
|
+
...(endedAtMs === undefined ? {} : { endedAtMs }),
|
|
80
|
+
};
|
|
81
|
+
});
|
|
82
|
+
}
|
|
56
83
|
/**
|
|
57
84
|
* The computer-use study result for a finished run. The run passes only when the Observer rendered,
|
|
58
85
|
* every participant passed (dry-run participants pass as contracts), and no adapter score or declared scorer
|
|
59
86
|
* verdict failed; otherwise the error names the first reason.
|
|
60
87
|
*/
|
|
61
88
|
function cuaStudyResult(args) {
|
|
62
|
-
const { studyId, cwd, runId, appUrl, dryRun, participantRuns, outcomes, subjects, aggregateSubject, participantPlan, rerunLineage, bundle, observer, receivingWarnings, aggregateWarnings, adapterWarnings, } = args;
|
|
89
|
+
const { studyId, cwd, runId, appUrl, dryRun, participantRuns, outcomes, subjects, aggregateSubject, participantPlan, rerunLineage, bundle, observer, hostSuspension, receivingWarnings, aggregateWarnings, adapterWarnings, } = args;
|
|
63
90
|
const participantCount = participantRuns.length;
|
|
64
91
|
// Participant-level pass: dry-run participants are contract-ok; live ones need a passed,
|
|
65
92
|
// engaged session.
|
|
@@ -77,6 +104,10 @@ function cuaStudyResult(args) {
|
|
|
77
104
|
const participantResults = participantRuns.map((spec, index) => toParticipantResult(spec, outcomes?.[index], subjects[index], dryRun));
|
|
78
105
|
const summary = buildParticipantSummary(outcomes, participantCount, participantPlan, dryRun);
|
|
79
106
|
const firstOutcome = outcomes?.[0];
|
|
107
|
+
// A failure the host suspension likely caused names the suspension first.
|
|
108
|
+
const suspended = hostSuspension !== undefined && hostSuspension.participantIds.length > 0
|
|
109
|
+
? hostSuspension.summary
|
|
110
|
+
: undefined;
|
|
80
111
|
const errorResult = (() => {
|
|
81
112
|
if (ok)
|
|
82
113
|
return undefined;
|
|
@@ -87,24 +118,31 @@ function cuaStudyResult(args) {
|
|
|
87
118
|
const outcome = firstOutcome;
|
|
88
119
|
return {
|
|
89
120
|
code: outcome?.failureCode ?? "HUMANISH_COMPUTER_USE_FAILED",
|
|
90
|
-
message: outcome?.sessionError ??
|
|
121
|
+
message: `${suspended === undefined ? "" : `${suspended} `}${outcome?.sessionError ??
|
|
91
122
|
outcome?.providerCleanupError ??
|
|
92
123
|
outcome?.providerPolicyError ??
|
|
93
124
|
(outcome?.noEngagement || outcome?.selfReportedBlocker || observer.ok
|
|
94
125
|
? (participantResults[0]?.error?.message ??
|
|
95
126
|
judgeParticipantRecords([participantFactsOf(outcome)]).participants[0]
|
|
96
127
|
.notPassedMessage)
|
|
97
|
-
: (observer.error?.message ?? "Observer failed for the computer-use run."))
|
|
128
|
+
: (observer.error?.message ?? "Observer failed for the computer-use run."))}`,
|
|
98
129
|
};
|
|
99
130
|
}
|
|
100
131
|
const failing = (outcomes ?? []).find((outcome) => !participantOk(outcome));
|
|
132
|
+
const firstFailure = suspended !== undefined
|
|
133
|
+
? `; ${suspended.replace(/\.$/, "")}`
|
|
134
|
+
: failing?.sessionError !== undefined
|
|
135
|
+
? `; first failure: ${failing.sessionError}`
|
|
136
|
+
: failing === undefined && args.outcome.execution.failures[0] !== undefined
|
|
137
|
+
? `; ${args.outcome.execution.failures[0].message}`
|
|
138
|
+
: "";
|
|
101
139
|
// A participant whose desktop failed for a known cause gives the run that cause's code.
|
|
102
140
|
const codedFailure = (outcomes ?? []).find((outcome) => outcome.failureCode !== undefined);
|
|
103
141
|
const code = codedFailure?.failureCode ?? "HUMANISH_COMPUTER_USE_FAILED";
|
|
104
142
|
return {
|
|
105
143
|
code,
|
|
106
144
|
message: observer.ok
|
|
107
|
-
? `Fan-out run failed: ${summary.passed}/${plural(participantCount, "participant")} passed (${summary.skipped} skipped, ${plural(summary.harnessErrors, "harness error")}, ${summary.hollow} without engagement)${
|
|
145
|
+
? `Fan-out run failed: ${summary.passed}/${plural(participantCount, "participant")} passed (${summary.skipped} skipped, ${plural(summary.harnessErrors, "harness error")}, ${summary.hollow} without engagement)${firstFailure}.`
|
|
108
146
|
: (observer.error?.message ?? "Observer failed for the computer-use fan-out run."),
|
|
109
147
|
};
|
|
110
148
|
})();
|
|
@@ -149,6 +187,7 @@ function cuaStudyResult(args) {
|
|
|
149
187
|
laneSummary: summary,
|
|
150
188
|
...(rerunLineage === undefined ? {} : { rerun: rerunLineage }),
|
|
151
189
|
observer,
|
|
190
|
+
...(hostSuspension === undefined ? {} : { hostSuspension }),
|
|
152
191
|
warnings,
|
|
153
192
|
...(errorResult === undefined ? {} : { error: errorResult }),
|
|
154
193
|
};
|
|
@@ -175,6 +214,8 @@ export async function finishCuaRun(setup, finish, ran) {
|
|
|
175
214
|
];
|
|
176
215
|
// One judgment for the whole run: the bundle's verdict and the result's ok both read it.
|
|
177
216
|
const judgment = judgeComputerUseRun(bundleBase, { dryRun, outcomes });
|
|
217
|
+
// One reading of the host suspensions: the review, the outcome and the result all name it.
|
|
218
|
+
const hostSuspension = readHostSuspensions(run.hostSuspensions(), suspendedParticipants(outcomes ?? []), Date.parse(run.createdAt));
|
|
178
219
|
const bundle = buildCuaRunBundle(bundleBase, {
|
|
179
220
|
judgment,
|
|
180
221
|
dryRun,
|
|
@@ -182,6 +223,7 @@ export async function finishCuaRun(setup, finish, ran) {
|
|
|
182
223
|
subjects,
|
|
183
224
|
aggregateSubject,
|
|
184
225
|
...(failFastReason === undefined ? {} : { failFastReason }),
|
|
226
|
+
...(hostSuspension === undefined ? {} : { hostSuspension }),
|
|
185
227
|
});
|
|
186
228
|
const adapterWarnings = [];
|
|
187
229
|
const scorerResult = await applyBrowserScorer({
|
|
@@ -206,8 +248,14 @@ export async function finishCuaRun(setup, finish, ran) {
|
|
|
206
248
|
if (receiving)
|
|
207
249
|
bundle.commsReceiving = receiving.snapshot();
|
|
208
250
|
const policy = OUTCOME_POLICIES["computer-use"];
|
|
209
|
-
// FinishedRun.renderObserver adds an Observer that did not render.
|
|
210
|
-
|
|
251
|
+
// FinishedRun.renderObserver adds an Observer that did not render. A suspension that likely
|
|
252
|
+
// caused participant failures is the run's first failure, ahead of each one it explains.
|
|
253
|
+
const execution = judgeExecution([
|
|
254
|
+
...(hostSuspension !== undefined && hostSuspension.participantIds.length > 0
|
|
255
|
+
? [{ kind: "run", message: hostSuspension.summary }]
|
|
256
|
+
: []),
|
|
257
|
+
...participantExecutionFailures(outcomes, runId),
|
|
258
|
+
], policy);
|
|
211
259
|
const finished = await run.finish(bundle, {
|
|
212
260
|
ok: resultOk({ judgment, execution, scorerFailures: scorerResult.failures, policy }),
|
|
213
261
|
execution,
|
|
@@ -236,6 +284,7 @@ export async function finishCuaRun(setup, finish, ran) {
|
|
|
236
284
|
bundle,
|
|
237
285
|
outcome: finished.outcome,
|
|
238
286
|
observer,
|
|
287
|
+
hostSuspension,
|
|
239
288
|
receivingWarnings,
|
|
240
289
|
aggregateWarnings,
|
|
241
290
|
adapterWarnings,
|
|
@@ -38,7 +38,7 @@ export interface CuaRunSetup {
|
|
|
38
38
|
/** What only runStudyParticipants reads. */
|
|
39
39
|
export interface CuaParticipantsSetup {
|
|
40
40
|
env: Record<string, string | undefined>;
|
|
41
|
-
deps: Omit<CuaParticipantDeps, "
|
|
41
|
+
deps: Omit<CuaParticipantDeps, "signalReady">;
|
|
42
42
|
/** Filled by runStudyParticipants on a live run; deps.onTrace reads it. */
|
|
43
43
|
liveTrace: {
|
|
44
44
|
flush?: LiveTraceFlush["flush"];
|
|
@@ -7,6 +7,7 @@ import { type BundleRun, type RunBundle, type RunFeedbackCandidate, type RunProv
|
|
|
7
7
|
import { type RunDesktopGeometry, type RunStream } from "../../run/streams.js";
|
|
8
8
|
import { type Verdict } from "../../run/judge.js";
|
|
9
9
|
import { type DesktopUsage } from "../../run/cost-summary.js";
|
|
10
|
+
import type { HostSuspensionReading } from "../../run/host-suspension.js";
|
|
10
11
|
/**
|
|
11
12
|
* The bundle of a single-participant run: one participant and no rerun. buildCuaRunBundle
|
|
12
13
|
* (bundle.ts) maps the run's base and state into these arguments.
|
|
@@ -92,4 +93,6 @@ export declare function buildSingleParticipantBundle(args: {
|
|
|
92
93
|
* desktopDurationMs). Absent when no sandbox ran (in-process/dry-run) → no desktop cost line. */
|
|
93
94
|
desktopMinutes?: number;
|
|
94
95
|
desktopUsage?: DesktopUsage;
|
|
96
|
+
/** The run's host suspensions and the failure they likely caused. */
|
|
97
|
+
hostSuspension?: HostSuspensionReading;
|
|
95
98
|
}): RunBundle;
|
|
@@ -332,10 +332,11 @@ function singleReview(args, view, stream) {
|
|
|
332
332
|
}
|
|
333
333
|
: {}),
|
|
334
334
|
...(singleRunTasks === undefined ? {} : { tasks: singleRunTasks }),
|
|
335
|
-
|
|
335
|
+
// A host suspension comes first: it may explain everything after it.
|
|
336
|
+
summary: `${args.hostSuspension === undefined ? "" : `${args.hostSuspension.summary} `}${singleSummary(args, view, {
|
|
336
337
|
...(singleRunTasks === undefined ? {} : { tasks: singleRunTasks }),
|
|
337
338
|
...(credibilityNote === undefined ? {} : { credibilityNote }),
|
|
338
|
-
})
|
|
339
|
+
})}`,
|
|
339
340
|
gaps: args.session || args.sessionError !== undefined
|
|
340
341
|
? []
|
|
341
342
|
: args.inProgress === true
|