humanish 0.120.1 → 0.121.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -8,6 +8,21 @@ The Unreleased section holds the full notes for the next version until it is tag
8
8
 
9
9
  ## Unreleased
10
10
 
11
+ ## 0.121.0: One failed desktop fails alone, host sleep named as the cause (2026-10-10)
12
+
13
+ humanish 0.121.0 keeps one participant's failed desktop from costing a whole hosted study, and
14
+ names a host that slept mid-run as the likely cause of the failures after it. A computer-use study
15
+ on an `app-url` subject no longer holds its participants behind the first one's desktop, and a
16
+ participant that fails before its session no longer stops the participants queued behind it; a
17
+ study whose subject is built in each sandbox still lets one participant set it up first, and the
18
+ next one takes over when the first never got a desktop. When the machine running a study sleeps,
19
+ the run records the suspension as a `host.suspended` event, and the run outcome, `humanish run`,
20
+ the review and the `--json` result say the failures of participants running through it are
21
+ likely its effect, while each participant's record keeps its own error. The review's participant
22
+ line lists interruption causes in a fixed order.
23
+
24
+ [Release notes](https://github.com/danielgwilson/humanish/releases/tag/v0.121.0)
25
+
11
26
  ## 0.120.1: Analysis requests under Bun run past 5 minutes (2026-10-09)
12
27
 
13
28
  humanish 0.120.1 lets an analysis request from a Bun process run longer than 5 minutes. Under Bun,
@@ -1,10 +1,12 @@
1
1
  import { participantCaption } from "../../run/participant-caption.js";
2
2
  import { formatCuaDiagnostics, formatCuaStopCause } from "../../routes/computer-use/diagnostics.js";
3
3
  import { subjectName } from "../../run/subject-name.js";
4
+ import { cli } from "../invocation.js";
4
5
  /**
5
6
  * A run's first lines: the command that ran, whether it was a dry run, how it ended, and its route.
6
7
  * `runOk` is the run's own ok, which automaticAnalysisEnvelope keeps when an analysis that did not
7
- * complete turns `ok` false. That run finished, and the line says so.
8
+ * complete turns `ok` false. That run finished, and the line says so. A host suspension follows,
9
+ * since it may explain every failure listed after it.
8
10
  */
9
11
  function runHeader(result, route) {
10
12
  const kind = result.dryRun === true ? "dry run" : result.dryRun === false ? "live run" : "run";
@@ -13,7 +15,24 @@ function runHeader(result, route) {
13
15
  : result.runOk === true
14
16
  ? "finished; the analysis did not complete"
15
17
  : "failed";
16
- return [`humanish run ${result.studyId}: ${kind} ${ending}`, `route: ${route}`];
18
+ return [
19
+ `humanish run ${result.studyId}: ${kind} ${ending}`,
20
+ `route: ${route}`,
21
+ ...(result.hostSuspension === undefined ? [] : [result.hostSuspension.summary]),
22
+ ];
23
+ }
24
+ /**
25
+ * What to run after a host suspension failed participants: the rerun of those that did not pass,
26
+ * or the study again for a run with one participant.
27
+ */
28
+ function suspensionNextStep(result, participants) {
29
+ if ((result.hostSuspension?.participantIds.length ?? 0) === 0)
30
+ return [];
31
+ return participants > 1
32
+ ? [
33
+ `Rerun the participants that did not pass: ${cli(`run ${result.studyId} --rerun-failed-from ${result.runId}`)}`,
34
+ ]
35
+ : [`Run it again: ${cli(`run ${result.studyId}`)}`];
17
36
  }
18
37
  /**
19
38
  * A run's human output: its lines on stdout and, when it failed with an error, that error on
@@ -167,8 +186,10 @@ export function formatScriptedStudyHuman(result, subject) {
167
186
  }
168
187
  /** `subject` is the study's declared subject, which names it when the result has no URL. */
169
188
  export function formatCuaStudyHuman(result, subject) {
189
+ const participants = result.lanes ?? [];
170
190
  return withError(result.error, [
171
191
  ...runHeader(result, "computer-use"),
192
+ ...suspensionNextStep(result, participants.length),
172
193
  `run: ${result.runId}`,
173
194
  `actor: ${result.actor}`,
174
195
  `subject: ${subjectLine(result.appUrl, subject)}`,
@@ -181,8 +202,8 @@ export function formatCuaStudyHuman(result, subject) {
181
202
  ? [`rerun: ${result.rerun.selectedLaneIds.join(", ")} from ${result.rerun.sourceRunId}`]
182
203
  : []),
183
204
  ...(result.diagnostics ? [`diagnostic: ${formatCuaDiagnostics(result.diagnostics)}`] : []),
184
- ...participantLines(result.lanes ?? []),
185
- ...(result.session && (result.lanes?.length ?? 0) <= 1
205
+ ...participantLines(participants),
206
+ ...(result.session && participants.length <= 1
186
207
  ? [
187
208
  `session: ${participantStatus(result.session.status)} (${result.session.completionReason})${result.session.stopCause ? ` · ${formatCuaStopCause(result.session.stopCause)}` : ""} · ${result.session.reason}`,
188
209
  `screenshots: ${result.session.screenshots}`,
@@ -2,6 +2,7 @@ import type { CuaActorDescriptor } from "../../actors/registry.js";
2
2
  import type { ComputerUsePlan } from "../../study/plan-types.js";
3
3
  import type { BundleRun, RunBundle, RunRerunLineage, RunSubjectProvenance } from "../../run/bundle.js";
4
4
  import { type Judgment } from "../../run/judge.js";
5
+ import type { HostSuspensionReading } from "../../run/host-suspension.js";
5
6
  import type { CuaParticipantPlan, DesktopParticipantRun, ParticipantRunOutcome } from "./types.js";
6
7
  /** What every bundle of one run shares, in progress or final. */
7
8
  export interface CuaRunBundleBase {
@@ -35,6 +36,8 @@ interface CuaRunBundleState {
35
36
  subjects: RunSubjectProvenance[];
36
37
  aggregateSubject: RunSubjectProvenance;
37
38
  failFastReason?: string;
39
+ /** The final bundle's reading of the run's host suspensions. */
40
+ hostSuspension?: HostSuspensionReading;
38
41
  inProgress?: true;
39
42
  }
40
43
  /**
@@ -61,6 +61,7 @@ export function buildCuaRunBundle(base, state) {
61
61
  participantPlan: base.participantPlan,
62
62
  ...(base.rerun === undefined ? {} : { rerun: base.rerun }),
63
63
  ...(state.failFastReason === undefined ? {} : { failFastReason: state.failFastReason }),
64
+ ...(state.hostSuspension === undefined ? {} : { hostSuspension: state.hostSuspension }),
64
65
  ...(base.publicRepo === undefined ? {} : { publicRepo: base.publicRepo }),
65
66
  ...inProgress,
66
67
  });
@@ -117,6 +118,7 @@ function singleParticipantArgs(base, state) {
117
118
  }),
118
119
  source: base.source,
119
120
  ...(state.inProgress === undefined ? {} : { inProgress: state.inProgress }),
121
+ ...(state.hostSuspension === undefined ? {} : { hostSuspension: state.hostSuspension }),
120
122
  subject: state.aggregateSubject,
121
123
  ...(desktopTemplate === undefined ? {} : { desktopTemplate }),
122
124
  ...(outcome?.desktopBrowser === undefined ? {} : { desktopBrowser: outcome.desktopBrowser }),
@@ -49,7 +49,25 @@ function fanoutSummary(args, facts) {
49
49
  return `${rerun}Dry run: ${people} would each use their own copy of ${subjectName(publicSafeAppUrlLabel(args.appUrl), args.aggregateSubject)}. No desktops were launched and $0 was spent.`;
50
50
  }
51
51
  const tasks = facts.tasks === undefined ? "" : ` Tasks: ${facts.tasks}.`;
52
- return `${rerun}${people} took part, each in their own copy of the app: ${facts.outcomes}${gapsListClause(count - facts.passed, count)}.${tasks}`;
52
+ // A host suspension comes first: it may explain everything after it.
53
+ const host = args.hostSuspension === undefined ? "" : `${args.hostSuspension.summary} `;
54
+ return `${host}${rerun}${people} took part, each in their own copy of the app: ${facts.outcomes}${gapsListClause(count - facts.passed, count)}.${tasks}`;
55
+ }
56
+ /**
57
+ * The review's gaps: one line for the participants a host suspension likely failed, then a line
58
+ * for each other participant that did not pass.
59
+ */
60
+ function fanoutGaps(args, judgment) {
61
+ const suspended = new Set(args.hostSuspension?.participantIds ?? []);
62
+ const others = judgment.participants.flatMap((participant, index) => participant.gapLine === undefined || suspended.has(args.specs[index].planned.id)
63
+ ? []
64
+ : [participant.gapLine]);
65
+ if (suspended.size === 0)
66
+ return others;
67
+ return [
68
+ `${[...suspended].join(", ")}: failed after the host was suspended, likely its effect. Each participant's record keeps its own error.`,
69
+ ...others,
70
+ ];
53
71
  }
54
72
  function fanoutReview(args, streams, judgment) {
55
73
  const { outcomes } = args;
@@ -91,7 +109,7 @@ function fanoutReview(args, streams, judgment) {
91
109
  ? ["Live fan-out session is still running."]
92
110
  : args.dryRun
93
111
  ? ["Live fan-out session not yet run (dry run only)."]
94
- : judgment.participants.flatMap((participant) => participant.gapLine === undefined ? [] : [participant.gapLine]),
112
+ : fanoutGaps(args, judgment),
95
113
  }, streams);
96
114
  return review;
97
115
  }
@@ -38,6 +38,7 @@ export declare function runStudyParticipants(setup: CuaRunSetup, participants: C
38
38
  rerun?: import("../../run/bundle.js").RunRerunLineage;
39
39
  observer?: import("../../index.js").ObserverResult;
40
40
  diagnostics?: import("./diagnostics.js").CuaDiagnostics;
41
+ hostSuspension?: import("../../run/host-suspension.js").HostSuspensionReading;
41
42
  warnings: string[];
42
43
  error?: {
43
44
  code: import("./types.js").CuaActorStudyErrorCode;
@@ -3,7 +3,7 @@ import type { StudyDeps } from "../../study/study-deps.js";
3
3
  import type { LiveTraceFlush } from "./live-flush.js";
4
4
  import type { CuaParticipantDeps } from "./types.js";
5
5
  /** The deps a desktop route builds once per run. Each participant adds the app URL it opens. */
6
- export type DesktopParticipantDeps = Omit<CuaParticipantDeps, "signalProvisioned" | "appUrl" | "onObservedUrl">;
6
+ export type DesktopParticipantDeps = Omit<CuaParticipantDeps, "signalReady" | "appUrl" | "onObservedUrl">;
7
7
  /** What every participant of a desktop run reads. Each route supplies all of it. */
8
8
  interface DesktopRunFacts {
9
9
  plan: Pick<ComputerUsePlan | SharedWorldPlan, "studyId" | "residual" | "caps" | "dryRun">;
@@ -37,7 +37,9 @@ export function desktopParticipantDeps(run, overrides) {
37
37
  redactScreenshots: run.redactScreenshots,
38
38
  scrubKnownValues: run.scrubKnownValues,
39
39
  runSession: run.runSession,
40
- now: seams.now ?? Date.now,
40
+ // The run's own clock unless a test sets another, so a participant's times and the run's host
41
+ // suspensions read one clock.
42
+ now: seams.now ?? seams.hostClock?.now ?? Date.now,
41
43
  onStream: run.onStream,
42
44
  reportSubjectPhase: run.reportSubjectPhase,
43
45
  };
@@ -12,17 +12,18 @@ export declare function makeParticipantWriteScreenshot(artifactRoot: PreparedOut
12
12
  export declare function runCuaParticipant(spec: DesktopParticipantRun, deps: CuaParticipantDeps): Promise<ParticipantRunOutcome>;
13
13
  /**
14
14
  * Run N>1 E2B participants on the study's schedule (runOnSchedule: each at its start offset, at
15
- * most `concurrency` at once), behind a pipeline gate (the first to start provisions before the
16
- * rest start) and session fail-fast on harness errors only (queued participants become `blocked`
17
- * with a pinned reason + a fail-fast event; mission verdicts never trip it). A gate failure or a
18
- * fail-fast ends every wait for a later start at once. Each participant tears down its own sandbox
19
- * by id; nothing here ever enumerates.
15
+ * most `concurrency` at once), behind the pipeline gate when the subject is set up in each sandbox,
16
+ * and session fail-fast on harness errors only (queued participants become `blocked` with a pinned
17
+ * reason + a fail-fast event; mission verdicts never trip it). A participant that stops before its
18
+ * session has failed on its own and trips no fail-fast. A closed gate or a fail-fast ends every
19
+ * wait for a later start at once. Each participant tears down its own sandbox by id; nothing here
20
+ * ever enumerates.
20
21
  *
21
22
  * Exported for the total-runner tests: the injectable runner lets a test make one participant
22
23
  * throw (the exact class the guard exists for) without a live sandbox. Production always uses
23
24
  * the default.
24
25
  */
25
- export declare function runCuaParticipants(runs: DesktopParticipantRun[], deps: Omit<CuaParticipantDeps, "signalProvisioned">, concurrency: number, runParticipant?: typeof runCuaParticipant): Promise<{
26
+ export declare function runCuaParticipants(runs: DesktopParticipantRun[], deps: Omit<CuaParticipantDeps, "signalReady">, concurrency: number, runParticipant?: typeof runCuaParticipant): Promise<{
26
27
  outcomes: ParticipantRunOutcome[];
27
28
  failFastReason?: string;
28
29
  }>;
@@ -32,7 +33,7 @@ export declare function toParticipantResult(spec: DesktopParticipantRun, outcome
32
33
  * Run every participant of a live run. The in-process route drives its single participant in this
33
34
  * process; one hosted participant runs alone; a fan-out runs at the plan's concurrency and may stop early.
34
35
  */
35
- export declare function runAllCuaParticipants(runs: readonly DesktopParticipantRun[], deps: Omit<CuaParticipantDeps, "signalProvisioned">, participantPlan: CuaParticipantPlan, inProcess: boolean): Promise<{
36
+ export declare function runAllCuaParticipants(runs: readonly DesktopParticipantRun[], deps: Omit<CuaParticipantDeps, "signalReady">, participantPlan: CuaParticipantPlan, inProcess: boolean): Promise<{
36
37
  outcomes: ParticipantRunOutcome[];
37
38
  failFastReason: string | undefined;
38
39
  }>;
@@ -56,7 +56,6 @@ export async function runCuaParticipant(spec, deps) {
56
56
  // model turn only to stop at its first one, so it is skipped before its desktop exists.
57
57
  const crossed = deps.runBudget?.crossed();
58
58
  if (crossed !== undefined) {
59
- deps.signalProvisioned?.(false);
60
59
  return skippedOutcome(spec, `skipped: study budget reached before this participant started: ${crossed}; no desktop was created`);
61
60
  }
62
61
  let model = {};
@@ -67,23 +66,13 @@ export async function runCuaParticipant(spec, deps) {
67
66
  let sessionError;
68
67
  let providerCleanupError;
69
68
  let providerPolicyError;
70
- let provisioned = false;
71
- let signaled = false;
72
- const signal = (ok) => {
73
- if (!signaled && deps.signalProvisioned) {
74
- signaled = true;
75
- deps.signalProvisioned(ok);
76
- }
77
- };
78
69
  const desktop = deps.createDesktop?.(spec, warnings, deps.artifactRoot) ??
79
70
  createE2BParticipantDesktop(spec, deps, warnings);
80
71
  try {
81
72
  await desktop.prepare();
82
73
  const ready = await desktop.openSession();
83
74
  model = await startParticipantModel(spec, deps, ready.executor);
84
- // World is ready: release the pipeline gate so the remaining participants may start.
85
- provisioned = true;
86
- signal(true);
75
+ deps.signalReady?.();
87
76
  session = await deps.runSession(participantSessionOptions(spec, deps, ready, model.provider, writeScreenshot));
88
77
  }
89
78
  catch (error) {
@@ -102,18 +91,12 @@ export async function runCuaParticipant(spec, deps) {
102
91
  : closed.refusal.startsWith("HUMANISH_CLAUDE_PARTICIPANT_")
103
92
  ? `Claude Code's stream showed a call the participant may not make after its last turn (${closed.refusal}); the participant was stopped.`
104
93
  : `Codex output after the participant's last request could not be checked against the item policy (${closed.refusal}).`));
105
- try {
106
- if (!provisioned)
107
- signal(false);
108
- }
109
- finally {
110
- await desktop.finalize({
111
- failed: sessionError !== undefined ||
112
- providerCleanupError !== undefined ||
113
- providerPolicyError !== undefined ||
114
- session === undefined,
115
- });
116
- }
94
+ await desktop.finalize({
95
+ failed: sessionError !== undefined ||
96
+ providerCleanupError !== undefined ||
97
+ providerPolicyError !== undefined ||
98
+ session === undefined,
99
+ });
117
100
  }
118
101
  const { released, desktopFailure, ...desktopEvidence } = desktop.snapshot();
119
102
  // A desktop that died under the participant explains the failure better than the transport
@@ -147,15 +130,73 @@ export async function runCuaParticipant(spec, deps) {
147
130
  selfReportedBlocker,
148
131
  reportedFriction,
149
132
  harnessError,
133
+ endedAt: deps.now(),
134
+ };
135
+ }
136
+ /**
137
+ * Holds a fan-out's participants back until one of them is ready, when each sets up the subject in
138
+ * its own sandbox the same way (a clone or local tree built and served, a desktop CLI installed),
139
+ * so a subject that fails to set up fails once. The first participant to enter holds the gate and
140
+ * the others wait. Any participant that becomes ready opens it. A holder that never got a desktop
141
+ * learned nothing about the subject, so the next waiting participant holds it instead. A holder
142
+ * whose desktop came up and then failed before its session closes it, and `onClose` runs.
143
+ */
144
+ function pipelineGate(open, onClose) {
145
+ let state = open ? "open" : "vacant";
146
+ let holder;
147
+ const waiting = [];
148
+ const settle = (turn) => {
149
+ for (const participant of waiting.splice(0))
150
+ participant.turn(turn);
151
+ };
152
+ return {
153
+ /** The participant that holds the gate, or held it when it closed. */
154
+ holder: () => holder,
155
+ enter(id) {
156
+ if (state === "open")
157
+ return Promise.resolve("start");
158
+ if (state === "closed")
159
+ return Promise.resolve("closed");
160
+ if (state === "vacant") {
161
+ state = "held";
162
+ holder = id;
163
+ return Promise.resolve("hold");
164
+ }
165
+ return new Promise((turn) => waiting.push({ id, turn }));
166
+ },
167
+ ready() {
168
+ if (state !== "held")
169
+ return;
170
+ state = "open";
171
+ settle("start");
172
+ },
173
+ failed(id, { handOver }) {
174
+ if (state !== "held" || holder !== id)
175
+ return;
176
+ if (handOver) {
177
+ const next = waiting.shift();
178
+ if (next === undefined)
179
+ state = "vacant";
180
+ else {
181
+ holder = next.id;
182
+ next.turn("hold");
183
+ }
184
+ return;
185
+ }
186
+ state = "closed";
187
+ settle("closed");
188
+ onClose();
189
+ },
150
190
  };
151
191
  }
152
192
  /**
153
193
  * Run N>1 E2B participants on the study's schedule (runOnSchedule: each at its start offset, at
154
- * most `concurrency` at once), behind a pipeline gate (the first to start provisions before the
155
- * rest start) and session fail-fast on harness errors only (queued participants become `blocked`
156
- * with a pinned reason + a fail-fast event; mission verdicts never trip it). A gate failure or a
157
- * fail-fast ends every wait for a later start at once. Each participant tears down its own sandbox
158
- * by id; nothing here ever enumerates.
194
+ * most `concurrency` at once), behind the pipeline gate when the subject is set up in each sandbox,
195
+ * and session fail-fast on harness errors only (queued participants become `blocked` with a pinned
196
+ * reason + a fail-fast event; mission verdicts never trip it). A participant that stops before its
197
+ * session has failed on its own and trips no fail-fast. A closed gate or a fail-fast ends every
198
+ * wait for a later start at once. Each participant tears down its own sandbox by id; nothing here
199
+ * ever enumerates.
159
200
  *
160
201
  * Exported for the total-runner tests: the injectable runner lets a test make one participant
161
202
  * throw (the exact class the guard exists for) without a live sandbox. Production always uses
@@ -164,40 +205,26 @@ export async function runCuaParticipant(spec, deps) {
164
205
  export async function runCuaParticipants(runs, deps, concurrency, runParticipant = runCuaParticipant) {
165
206
  const failFast = { tripped: false, reason: "" };
166
207
  const stopping = new AbortController();
167
- let gateHolder = runs[0]?.planned.id ?? "lane-01";
168
- let resolveGate;
169
- let rejectGate;
170
- const gate = new Promise((resolve, reject) => {
171
- resolveGate = resolve;
172
- rejectGate = () => {
173
- reject(new Error("gate"));
174
- stopping.abort();
175
- };
176
- });
177
- // The gate is rejected on the first participant's provisioning failure; swallow the unhandled
178
- // rejection if no later participant ever awaits it (concurrency could let the first finish
179
- // alone).
180
- gate.catch(() => undefined);
208
+ // An app-url subject is only opened, so one participant's start says nothing about another's.
209
+ const gate = pipelineGate(deps.subject.kind === "app-url", () => stopping.abort());
181
210
  const outcomes = await runOnSchedule(runs, {
182
211
  startAfterMs: (spec) => spec.planned.startAfterMs,
183
212
  slots: concurrency,
184
213
  now: deps.now,
185
214
  signal: stopping.signal,
186
- }, async (spec, _index, { order, scheduledAt }) => {
187
- if (order === 0)
188
- gateHolder = spec.planned.id;
189
- if (order > 0) {
190
- try {
191
- await gate;
192
- }
193
- catch {
194
- return skippedOutcome(spec, `skipped: participant ${gateHolder} failed to provision its world (pipeline gate)`, { scheduledAt });
195
- }
196
- }
215
+ }, async (spec, _index, { scheduledAt }) => {
197
216
  if (failFast.tripped) {
198
217
  return skippedOutcome(spec, `skipped: ${failFast.reason}`, { scheduledAt });
199
218
  }
219
+ // A participant trips fail-fast only after the gate opened, or as it closes the gate, so no
220
+ // participant is still waiting at the gate when fail-fast skips the rest.
221
+ const id = spec.planned.id;
222
+ if ((await gate.enter(id)) === "closed") {
223
+ return skippedOutcome(spec, `skipped: participant ${gate.holder()} failed to provision its world (pipeline gate)`, { scheduledAt });
224
+ }
200
225
  const arrival = { scheduledAt, startedAt: deps.now() };
226
+ let ready = false;
227
+ let threw = false;
201
228
  // The participant runner is total: every exit path returns a recorded outcome. Without this
202
229
  // guard, one participant's late throw (e.g. its trace write hitting ENOSPC after its own sandbox was
203
230
  // already torn down) rejected the whole map while sibling workers kept launching sandboxes
@@ -206,25 +233,14 @@ export async function runCuaParticipants(runs, deps, concurrency, runParticipant
206
233
  try {
207
234
  outcome = await runParticipant(spec, {
208
235
  ...deps,
209
- ...(order === 0
210
- ? {
211
- signalProvisioned: (ok) => {
212
- if (ok) {
213
- resolveGate?.();
214
- }
215
- else {
216
- rejectGate?.();
217
- }
218
- },
219
- }
220
- : {}),
236
+ signalReady: () => {
237
+ ready = true;
238
+ gate.ready();
239
+ },
221
240
  });
222
241
  }
223
242
  catch (error) {
224
- // The first participant may have thrown before signaling the provisioning gate; release the followers as
225
- // blocked rather than leaving them awaiting a gate that will never settle.
226
- if (order === 0)
227
- rejectGate?.();
243
+ threw = true;
228
244
  const detail = redactText(toErrorMessage(error));
229
245
  outcome = {
230
246
  spec,
@@ -239,9 +255,15 @@ export async function runCuaParticipants(runs, deps, concurrency, runParticipant
239
255
  reportedFriction: false,
240
256
  harnessError: true,
241
257
  sessionError: `participant runner threw outside the session guard: ${detail}`,
258
+ endedAt: deps.now(),
242
259
  };
243
260
  }
244
- if (outcome.harnessError && !failFast.tripped) {
261
+ // A participant that stopped before its session failed on its own and trips no fail-fast.
262
+ // Without a desktop it says nothing about the subject, so the next participant starts first.
263
+ // A runner that threw is a harness defect: it closes the gate and trips fail-fast.
264
+ if (!ready)
265
+ gate.failed(id, { handOver: !threw && outcome.sandboxId === undefined });
266
+ if (outcome.harnessError && (ready || threw) && !failFast.tripped) {
245
267
  failFast.tripped = true;
246
268
  failFast.reason = `a prior participant (${outcome.spec.planned.id}) ended in a harness error (fail-fast)`;
247
269
  stopping.abort();
@@ -1,6 +1,7 @@
1
1
  import { adapterScoreFailureMessage, applyBrowserScorer } from "../../study/adapter-extension.js";
2
2
  import { redactText } from "../../evidence/redaction.js";
3
- import { foldScorerFailures, judgeExecution, judgeParticipantRecords, OUTCOME_POLICIES, resultOk, } from "../../run/judge.js";
3
+ import { foldScorerFailures, judgeExecution, judgeParticipantRecords, OUTCOME_POLICIES, participantHarnessFailed, resultOk, } from "../../run/judge.js";
4
+ import { readHostSuspensions, } from "../../run/host-suspension.js";
4
5
  import { validatePreparedRunArtifactPaths } from "../../run/paths.js";
5
6
  import { participantExecutionFailures, participantOutcomeOk, participantFactsOf, } from "./participant-facts.js";
6
7
  import { participantCapWarning } from "./participant-model.js";
@@ -53,13 +54,39 @@ function buildParticipantSummary(outcomes, participantCount, participantPlan, dr
53
54
  waves: participantPlan.waves,
54
55
  };
55
56
  }
57
+ /**
58
+ * Each participant as the host-suspension reading reads it: a harness error, its session's time
59
+ * limit or a skip, when it started, and when its runner returned, which a setup failure with no
60
+ * session also records.
61
+ */
62
+ function suspendedParticipants(outcomes) {
63
+ return outcomes.map((outcome) => {
64
+ const facts = participantFactsOf(outcome);
65
+ const trace = outcome.session?.trace;
66
+ const failure = facts.skipped
67
+ ? "skipped"
68
+ : participantHarnessFailed(facts)
69
+ ? "harness"
70
+ : trace?.stopCause === "time_limit"
71
+ ? "time-limit"
72
+ : undefined;
73
+ const startedAtMs = outcome.arrival?.startedAt ?? (trace === undefined ? undefined : Date.parse(trace.startedAt));
74
+ const endedAtMs = outcome.endedAt ?? (trace === undefined ? undefined : Date.parse(trace.completedAt));
75
+ return {
76
+ id: outcome.spec.planned.id,
77
+ ...(failure === undefined ? {} : { failure }),
78
+ ...(startedAtMs === undefined ? {} : { startedAtMs }),
79
+ ...(endedAtMs === undefined ? {} : { endedAtMs }),
80
+ };
81
+ });
82
+ }
56
83
  /**
57
84
  * The computer-use study result for a finished run. The run passes only when the Observer rendered,
58
85
  * every participant passed (dry-run participants pass as contracts), and no adapter score or declared scorer
59
86
  * verdict failed; otherwise the error names the first reason.
60
87
  */
61
88
  function cuaStudyResult(args) {
62
- const { studyId, cwd, runId, appUrl, dryRun, participantRuns, outcomes, subjects, aggregateSubject, participantPlan, rerunLineage, bundle, observer, receivingWarnings, aggregateWarnings, adapterWarnings, } = args;
89
+ const { studyId, cwd, runId, appUrl, dryRun, participantRuns, outcomes, subjects, aggregateSubject, participantPlan, rerunLineage, bundle, observer, hostSuspension, receivingWarnings, aggregateWarnings, adapterWarnings, } = args;
63
90
  const participantCount = participantRuns.length;
64
91
  // Participant-level pass: dry-run participants are contract-ok; live ones need a passed,
65
92
  // engaged session.
@@ -77,6 +104,10 @@ function cuaStudyResult(args) {
77
104
  const participantResults = participantRuns.map((spec, index) => toParticipantResult(spec, outcomes?.[index], subjects[index], dryRun));
78
105
  const summary = buildParticipantSummary(outcomes, participantCount, participantPlan, dryRun);
79
106
  const firstOutcome = outcomes?.[0];
107
+ // A failure the host suspension likely caused names the suspension first.
108
+ const suspended = hostSuspension !== undefined && hostSuspension.participantIds.length > 0
109
+ ? hostSuspension.summary
110
+ : undefined;
80
111
  const errorResult = (() => {
81
112
  if (ok)
82
113
  return undefined;
@@ -87,24 +118,31 @@ function cuaStudyResult(args) {
87
118
  const outcome = firstOutcome;
88
119
  return {
89
120
  code: outcome?.failureCode ?? "HUMANISH_COMPUTER_USE_FAILED",
90
- message: outcome?.sessionError ??
121
+ message: `${suspended === undefined ? "" : `${suspended} `}${outcome?.sessionError ??
91
122
  outcome?.providerCleanupError ??
92
123
  outcome?.providerPolicyError ??
93
124
  (outcome?.noEngagement || outcome?.selfReportedBlocker || observer.ok
94
125
  ? (participantResults[0]?.error?.message ??
95
126
  judgeParticipantRecords([participantFactsOf(outcome)]).participants[0]
96
127
  .notPassedMessage)
97
- : (observer.error?.message ?? "Observer failed for the computer-use run.")),
128
+ : (observer.error?.message ?? "Observer failed for the computer-use run."))}`,
98
129
  };
99
130
  }
100
131
  const failing = (outcomes ?? []).find((outcome) => !participantOk(outcome));
132
+ const firstFailure = suspended !== undefined
133
+ ? `; ${suspended.replace(/\.$/, "")}`
134
+ : failing?.sessionError !== undefined
135
+ ? `; first failure: ${failing.sessionError}`
136
+ : failing === undefined && args.outcome.execution.failures[0] !== undefined
137
+ ? `; ${args.outcome.execution.failures[0].message}`
138
+ : "";
101
139
  // A participant whose desktop failed for a known cause gives the run that cause's code.
102
140
  const codedFailure = (outcomes ?? []).find((outcome) => outcome.failureCode !== undefined);
103
141
  const code = codedFailure?.failureCode ?? "HUMANISH_COMPUTER_USE_FAILED";
104
142
  return {
105
143
  code,
106
144
  message: observer.ok
107
- ? `Fan-out run failed: ${summary.passed}/${plural(participantCount, "participant")} passed (${summary.skipped} skipped, ${plural(summary.harnessErrors, "harness error")}, ${summary.hollow} without engagement)${failing?.sessionError !== undefined ? `; first failure: ${failing.sessionError}` : failing === undefined && args.outcome.execution.failures[0] !== undefined ? `; ${args.outcome.execution.failures[0].message}` : ""}.`
145
+ ? `Fan-out run failed: ${summary.passed}/${plural(participantCount, "participant")} passed (${summary.skipped} skipped, ${plural(summary.harnessErrors, "harness error")}, ${summary.hollow} without engagement)${firstFailure}.`
108
146
  : (observer.error?.message ?? "Observer failed for the computer-use fan-out run."),
109
147
  };
110
148
  })();
@@ -149,6 +187,7 @@ function cuaStudyResult(args) {
149
187
  laneSummary: summary,
150
188
  ...(rerunLineage === undefined ? {} : { rerun: rerunLineage }),
151
189
  observer,
190
+ ...(hostSuspension === undefined ? {} : { hostSuspension }),
152
191
  warnings,
153
192
  ...(errorResult === undefined ? {} : { error: errorResult }),
154
193
  };
@@ -175,6 +214,8 @@ export async function finishCuaRun(setup, finish, ran) {
175
214
  ];
176
215
  // One judgment for the whole run: the bundle's verdict and the result's ok both read it.
177
216
  const judgment = judgeComputerUseRun(bundleBase, { dryRun, outcomes });
217
+ // One reading of the host suspensions: the review, the outcome and the result all name it.
218
+ const hostSuspension = readHostSuspensions(run.hostSuspensions(), suspendedParticipants(outcomes ?? []), Date.parse(run.createdAt));
178
219
  const bundle = buildCuaRunBundle(bundleBase, {
179
220
  judgment,
180
221
  dryRun,
@@ -182,6 +223,7 @@ export async function finishCuaRun(setup, finish, ran) {
182
223
  subjects,
183
224
  aggregateSubject,
184
225
  ...(failFastReason === undefined ? {} : { failFastReason }),
226
+ ...(hostSuspension === undefined ? {} : { hostSuspension }),
185
227
  });
186
228
  const adapterWarnings = [];
187
229
  const scorerResult = await applyBrowserScorer({
@@ -206,8 +248,14 @@ export async function finishCuaRun(setup, finish, ran) {
206
248
  if (receiving)
207
249
  bundle.commsReceiving = receiving.snapshot();
208
250
  const policy = OUTCOME_POLICIES["computer-use"];
209
- // FinishedRun.renderObserver adds an Observer that did not render.
210
- const execution = judgeExecution(participantExecutionFailures(outcomes, runId), policy);
251
+ // FinishedRun.renderObserver adds an Observer that did not render. A suspension that likely
252
+ // caused participant failures is the run's first failure, ahead of each one it explains.
253
+ const execution = judgeExecution([
254
+ ...(hostSuspension !== undefined && hostSuspension.participantIds.length > 0
255
+ ? [{ kind: "run", message: hostSuspension.summary }]
256
+ : []),
257
+ ...participantExecutionFailures(outcomes, runId),
258
+ ], policy);
211
259
  const finished = await run.finish(bundle, {
212
260
  ok: resultOk({ judgment, execution, scorerFailures: scorerResult.failures, policy }),
213
261
  execution,
@@ -236,6 +284,7 @@ export async function finishCuaRun(setup, finish, ran) {
236
284
  bundle,
237
285
  outcome: finished.outcome,
238
286
  observer,
287
+ hostSuspension,
239
288
  receivingWarnings,
240
289
  aggregateWarnings,
241
290
  adapterWarnings,
@@ -38,7 +38,7 @@ export interface CuaRunSetup {
38
38
  /** What only runStudyParticipants reads. */
39
39
  export interface CuaParticipantsSetup {
40
40
  env: Record<string, string | undefined>;
41
- deps: Omit<CuaParticipantDeps, "signalProvisioned">;
41
+ deps: Omit<CuaParticipantDeps, "signalReady">;
42
42
  /** Filled by runStudyParticipants on a live run; deps.onTrace reads it. */
43
43
  liveTrace: {
44
44
  flush?: LiveTraceFlush["flush"];
@@ -7,6 +7,7 @@ import { type BundleRun, type RunBundle, type RunFeedbackCandidate, type RunProv
7
7
  import { type RunDesktopGeometry, type RunStream } from "../../run/streams.js";
8
8
  import { type Verdict } from "../../run/judge.js";
9
9
  import { type DesktopUsage } from "../../run/cost-summary.js";
10
+ import type { HostSuspensionReading } from "../../run/host-suspension.js";
10
11
  /**
11
12
  * The bundle of a single-participant run: one participant and no rerun. buildCuaRunBundle
12
13
  * (bundle.ts) maps the run's base and state into these arguments.
@@ -92,4 +93,6 @@ export declare function buildSingleParticipantBundle(args: {
92
93
  * desktopDurationMs). Absent when no sandbox ran (in-process/dry-run) → no desktop cost line. */
93
94
  desktopMinutes?: number;
94
95
  desktopUsage?: DesktopUsage;
96
+ /** The run's host suspensions and the failure they likely caused. */
97
+ hostSuspension?: HostSuspensionReading;
95
98
  }): RunBundle;
@@ -332,10 +332,11 @@ function singleReview(args, view, stream) {
332
332
  }
333
333
  : {}),
334
334
  ...(singleRunTasks === undefined ? {} : { tasks: singleRunTasks }),
335
- summary: singleSummary(args, view, {
335
+ // A host suspension comes first: it may explain everything after it.
336
+ summary: `${args.hostSuspension === undefined ? "" : `${args.hostSuspension.summary} `}${singleSummary(args, view, {
336
337
  ...(singleRunTasks === undefined ? {} : { tasks: singleRunTasks }),
337
338
  ...(credibilityNote === undefined ? {} : { credibilityNote }),
338
- }),
339
+ })}`,
339
340
  gaps: args.session || args.sessionError !== undefined
340
341
  ? []
341
342
  : args.inProgress === true