humanish 0.120.0 → 0.121.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +27 -0
- package/dist/analysis/provider.d.ts +6 -1
- package/dist/analysis/provider.js +4 -0
- package/dist/cli/commands/study-format.js +25 -4
- package/dist/routes/computer-use/bundle.d.ts +3 -0
- package/dist/routes/computer-use/bundle.js +2 -0
- package/dist/routes/computer-use/fanout-bundle.js +20 -2
- package/dist/routes/computer-use/live-phase.d.ts +1 -0
- package/dist/routes/computer-use/participant-deps.d.ts +1 -1
- package/dist/routes/computer-use/participant-deps.js +3 -1
- package/dist/routes/computer-use/participant-execution.d.ts +8 -7
- package/dist/routes/computer-use/participant-execution.js +93 -71
- package/dist/routes/computer-use/result.js +56 -7
- package/dist/routes/computer-use/setup.d.ts +1 -1
- package/dist/routes/computer-use/single-bundle.d.ts +3 -0
- package/dist/routes/computer-use/single-bundle.js +3 -2
- package/dist/routes/computer-use/types.d.ts +16 -2
- package/dist/run/host-suspension.d.ts +56 -0
- package/dist/run/host-suspension.js +106 -0
- package/dist/run/outcomes.js +3 -1
- package/dist/run/run.d.ts +15 -2
- package/dist/run/run.js +15 -8
- package/dist/run/status.d.ts +8 -2
- package/dist/run/status.js +11 -8
- package/dist/study/study-deps.d.ts +6 -0
- package/dist/tui-app.js +35 -35
- package/docs/contracts/run-bundle.md +19 -0
- package/docs/contracts/schemas.md +2 -1
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -8,6 +8,33 @@ The Unreleased section holds the full notes for the next version until it is tag
|
|
|
8
8
|
|
|
9
9
|
## Unreleased
|
|
10
10
|
|
|
11
|
+
## 0.121.0: One failed desktop fails alone, host sleep named as the cause (2026-10-10)
|
|
12
|
+
|
|
13
|
+
humanish 0.121.0 keeps one participant's failed desktop from costing a whole hosted study, and
|
|
14
|
+
names a host that slept mid-run as the likely cause of the failures after it. A computer-use study
|
|
15
|
+
on an `app-url` subject no longer holds its participants behind the first one's desktop, and a
|
|
16
|
+
participant that fails before its session no longer stops the participants queued behind it; a
|
|
17
|
+
study whose subject is built in each sandbox still lets one participant set it up first, and the
|
|
18
|
+
next one takes over when the first never got a desktop. When the machine running a study sleeps,
|
|
19
|
+
the run records the suspension as a `host.suspended` event, and the run outcome, `humanish run`,
|
|
20
|
+
the review and the `--json` result say the failures of participants running through it are
|
|
21
|
+
likely its effect, while each participant's record keeps its own error. The review's participant
|
|
22
|
+
line lists interruption causes in a fixed order.
|
|
23
|
+
|
|
24
|
+
[Release notes](https://github.com/danielgwilson/humanish/releases/tag/v0.121.0)
|
|
25
|
+
|
|
26
|
+
## 0.120.1: Analysis requests under Bun run past 5 minutes (2026-10-09)
|
|
27
|
+
|
|
28
|
+
humanish 0.120.1 lets an analysis request from a Bun process run longer than 5 minutes. Under Bun,
|
|
29
|
+
`undici` is Bun's built-in module, whose `fetch` ignores the analysis provider's dispatcher and
|
|
30
|
+
ended each request after 300 seconds, so an analysis whose model took longer, as a 34-participant
|
|
31
|
+
run's cohort requests did at 325 to 405 seconds, failed with `analysis_provider_network_error`.
|
|
32
|
+
The provider now passes `timeout: false`, which Bun reads and Node's undici ignores, and the
|
|
33
|
+
request's own `timeoutMs` ends it under either runtime. A `TimeoutError` from `fetch` is recorded
|
|
34
|
+
as `provider_client_timeout`.
|
|
35
|
+
|
|
36
|
+
[Release notes](https://github.com/danielgwilson/humanish/releases/tag/v0.120.1)
|
|
37
|
+
|
|
11
38
|
## 0.120.0: Analysis completes under Bun, runs past 16 participants print a bounded list, Observer counts participants by status (2026-10-09)
|
|
12
39
|
|
|
13
40
|
humanish 0.120.0 completes an analysis run from a Bun process, prints a run of more than 16
|
|
@@ -28,7 +28,7 @@ export interface AnalysisProviderResult {
|
|
|
28
28
|
dispatched: boolean;
|
|
29
29
|
/** Omitted preserves complete API-response accounting; account snapshots must state this. */
|
|
30
30
|
usageComplete?: boolean;
|
|
31
|
-
errorCode: "invalid_request" | "provider_http_error" | "provider_network_error" | "invalid_response" | "provider_headers_timeout" | "provider_body_timeout" | "provider_connect_timeout" | "provider_connection_reset" | "provider_dns_error" | "codex_busy" | "codex_unavailable" | "codex_unsupported_version" | "codex_incompatible_release" | "codex_unsupported_platform" | "codex_login_required" | "codex_unsupported_auth" | "codex_unsafe_configuration" | "codex_model_unavailable" | "codex_protocol_error" | "codex_tool_call" | "codex_process_failed" | "codex_cleanup_failed" | "response_too_large" | "output_incomplete" | "refusal" | "cancelled" | "timeout" | null;
|
|
31
|
+
errorCode: "invalid_request" | "provider_http_error" | "provider_network_error" | "invalid_response" | "provider_headers_timeout" | "provider_client_timeout" | "provider_body_timeout" | "provider_connect_timeout" | "provider_connection_reset" | "provider_dns_error" | "codex_busy" | "codex_unavailable" | "codex_unsupported_version" | "codex_incompatible_release" | "codex_unsupported_platform" | "codex_login_required" | "codex_unsupported_auth" | "codex_unsafe_configuration" | "codex_model_unavailable" | "codex_protocol_error" | "codex_tool_call" | "codex_process_failed" | "codex_cleanup_failed" | "response_too_large" | "output_incomplete" | "refusal" | "cancelled" | "timeout" | null;
|
|
32
32
|
httpStatus?: number;
|
|
33
33
|
/** Codex only: notification methods humanish does not know that carried no item, by count. */
|
|
34
34
|
unknownNotifications?: Readonly<Record<string, number>>;
|
|
@@ -52,6 +52,11 @@ export type AnalysisFetch = (url: string, init: {
|
|
|
52
52
|
headers: Record<string, string>;
|
|
53
53
|
body: string;
|
|
54
54
|
dispatcher: Dispatcher;
|
|
55
|
+
/**
|
|
56
|
+
* Bun's fetch ignores `dispatcher` and ends a request after 300 s unless this is false;
|
|
57
|
+
* Node's undici fetch ignores the field. The caller's timeoutMs governs either way.
|
|
58
|
+
*/
|
|
59
|
+
timeout: false;
|
|
55
60
|
}) => Promise<{
|
|
56
61
|
ok: boolean;
|
|
57
62
|
status: number;
|
|
@@ -34,6 +34,9 @@ function networkErrorCode(error) {
|
|
|
34
34
|
const value = record(current);
|
|
35
35
|
if (typeof value.code === "string" && Object.hasOwn(codes, value.code))
|
|
36
36
|
return codes[value.code];
|
|
37
|
+
// Bun's fetch ends a request with a TimeoutError of its own, with no code.
|
|
38
|
+
if (value.name === "TimeoutError")
|
|
39
|
+
return "provider_client_timeout";
|
|
37
40
|
current = value.cause;
|
|
38
41
|
}
|
|
39
42
|
return "provider_network_error";
|
|
@@ -177,6 +180,7 @@ export function createAnalysisProvider(options) {
|
|
|
177
180
|
headers: { Authorization: `Bearer ${options.apiKey}`, "Content-Type": "application/json" },
|
|
178
181
|
body,
|
|
179
182
|
dispatcher,
|
|
183
|
+
timeout: false,
|
|
180
184
|
});
|
|
181
185
|
if (!response.ok) {
|
|
182
186
|
// Never read provider error prose: it may echo evidence or credentials.
|
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
import { participantCaption } from "../../run/participant-caption.js";
|
|
2
2
|
import { formatCuaDiagnostics, formatCuaStopCause } from "../../routes/computer-use/diagnostics.js";
|
|
3
3
|
import { subjectName } from "../../run/subject-name.js";
|
|
4
|
+
import { cli } from "../invocation.js";
|
|
4
5
|
/**
|
|
5
6
|
* A run's first lines: the command that ran, whether it was a dry run, how it ended, and its route.
|
|
6
7
|
* `runOk` is the run's own ok, which automaticAnalysisEnvelope keeps when an analysis that did not
|
|
7
|
-
* complete turns `ok` false. That run finished, and the line says so.
|
|
8
|
+
* complete turns `ok` false. That run finished, and the line says so. A host suspension follows,
|
|
9
|
+
* since it may explain every failure listed after it.
|
|
8
10
|
*/
|
|
9
11
|
function runHeader(result, route) {
|
|
10
12
|
const kind = result.dryRun === true ? "dry run" : result.dryRun === false ? "live run" : "run";
|
|
@@ -13,7 +15,24 @@ function runHeader(result, route) {
|
|
|
13
15
|
: result.runOk === true
|
|
14
16
|
? "finished; the analysis did not complete"
|
|
15
17
|
: "failed";
|
|
16
|
-
return [
|
|
18
|
+
return [
|
|
19
|
+
`humanish run ${result.studyId}: ${kind} ${ending}`,
|
|
20
|
+
`route: ${route}`,
|
|
21
|
+
...(result.hostSuspension === undefined ? [] : [result.hostSuspension.summary]),
|
|
22
|
+
];
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* What to run after a host suspension failed participants: the rerun of those that did not pass,
|
|
26
|
+
* or the study again for a run with one participant.
|
|
27
|
+
*/
|
|
28
|
+
function suspensionNextStep(result, participants) {
|
|
29
|
+
if ((result.hostSuspension?.participantIds.length ?? 0) === 0)
|
|
30
|
+
return [];
|
|
31
|
+
return participants > 1
|
|
32
|
+
? [
|
|
33
|
+
`Rerun the participants that did not pass: ${cli(`run ${result.studyId} --rerun-failed-from ${result.runId}`)}`,
|
|
34
|
+
]
|
|
35
|
+
: [`Run it again: ${cli(`run ${result.studyId}`)}`];
|
|
17
36
|
}
|
|
18
37
|
/**
|
|
19
38
|
* A run's human output: its lines on stdout and, when it failed with an error, that error on
|
|
@@ -167,8 +186,10 @@ export function formatScriptedStudyHuman(result, subject) {
|
|
|
167
186
|
}
|
|
168
187
|
/** `subject` is the study's declared subject, which names it when the result has no URL. */
|
|
169
188
|
export function formatCuaStudyHuman(result, subject) {
|
|
189
|
+
const participants = result.lanes ?? [];
|
|
170
190
|
return withError(result.error, [
|
|
171
191
|
...runHeader(result, "computer-use"),
|
|
192
|
+
...suspensionNextStep(result, participants.length),
|
|
172
193
|
`run: ${result.runId}`,
|
|
173
194
|
`actor: ${result.actor}`,
|
|
174
195
|
`subject: ${subjectLine(result.appUrl, subject)}`,
|
|
@@ -181,8 +202,8 @@ export function formatCuaStudyHuman(result, subject) {
|
|
|
181
202
|
? [`rerun: ${result.rerun.selectedLaneIds.join(", ")} from ${result.rerun.sourceRunId}`]
|
|
182
203
|
: []),
|
|
183
204
|
...(result.diagnostics ? [`diagnostic: ${formatCuaDiagnostics(result.diagnostics)}`] : []),
|
|
184
|
-
...participantLines(
|
|
185
|
-
...(result.session &&
|
|
205
|
+
...participantLines(participants),
|
|
206
|
+
...(result.session && participants.length <= 1
|
|
186
207
|
? [
|
|
187
208
|
`session: ${participantStatus(result.session.status)} (${result.session.completionReason})${result.session.stopCause ? ` · ${formatCuaStopCause(result.session.stopCause)}` : ""} · ${result.session.reason}`,
|
|
188
209
|
`screenshots: ${result.session.screenshots}`,
|
|
@@ -2,6 +2,7 @@ import type { CuaActorDescriptor } from "../../actors/registry.js";
|
|
|
2
2
|
import type { ComputerUsePlan } from "../../study/plan-types.js";
|
|
3
3
|
import type { BundleRun, RunBundle, RunRerunLineage, RunSubjectProvenance } from "../../run/bundle.js";
|
|
4
4
|
import { type Judgment } from "../../run/judge.js";
|
|
5
|
+
import type { HostSuspensionReading } from "../../run/host-suspension.js";
|
|
5
6
|
import type { CuaParticipantPlan, DesktopParticipantRun, ParticipantRunOutcome } from "./types.js";
|
|
6
7
|
/** What every bundle of one run shares, in progress or final. */
|
|
7
8
|
export interface CuaRunBundleBase {
|
|
@@ -35,6 +36,8 @@ interface CuaRunBundleState {
|
|
|
35
36
|
subjects: RunSubjectProvenance[];
|
|
36
37
|
aggregateSubject: RunSubjectProvenance;
|
|
37
38
|
failFastReason?: string;
|
|
39
|
+
/** The final bundle's reading of the run's host suspensions. */
|
|
40
|
+
hostSuspension?: HostSuspensionReading;
|
|
38
41
|
inProgress?: true;
|
|
39
42
|
}
|
|
40
43
|
/**
|
|
@@ -61,6 +61,7 @@ export function buildCuaRunBundle(base, state) {
|
|
|
61
61
|
participantPlan: base.participantPlan,
|
|
62
62
|
...(base.rerun === undefined ? {} : { rerun: base.rerun }),
|
|
63
63
|
...(state.failFastReason === undefined ? {} : { failFastReason: state.failFastReason }),
|
|
64
|
+
...(state.hostSuspension === undefined ? {} : { hostSuspension: state.hostSuspension }),
|
|
64
65
|
...(base.publicRepo === undefined ? {} : { publicRepo: base.publicRepo }),
|
|
65
66
|
...inProgress,
|
|
66
67
|
});
|
|
@@ -117,6 +118,7 @@ function singleParticipantArgs(base, state) {
|
|
|
117
118
|
}),
|
|
118
119
|
source: base.source,
|
|
119
120
|
...(state.inProgress === undefined ? {} : { inProgress: state.inProgress }),
|
|
121
|
+
...(state.hostSuspension === undefined ? {} : { hostSuspension: state.hostSuspension }),
|
|
120
122
|
subject: state.aggregateSubject,
|
|
121
123
|
...(desktopTemplate === undefined ? {} : { desktopTemplate }),
|
|
122
124
|
...(outcome?.desktopBrowser === undefined ? {} : { desktopBrowser: outcome.desktopBrowser }),
|
|
@@ -49,7 +49,25 @@ function fanoutSummary(args, facts) {
|
|
|
49
49
|
return `${rerun}Dry run: ${people} would each use their own copy of ${subjectName(publicSafeAppUrlLabel(args.appUrl), args.aggregateSubject)}. No desktops were launched and $0 was spent.`;
|
|
50
50
|
}
|
|
51
51
|
const tasks = facts.tasks === undefined ? "" : ` Tasks: ${facts.tasks}.`;
|
|
52
|
-
|
|
52
|
+
// A host suspension comes first: it may explain everything after it.
|
|
53
|
+
const host = args.hostSuspension === undefined ? "" : `${args.hostSuspension.summary} `;
|
|
54
|
+
return `${host}${rerun}${people} took part, each in their own copy of the app: ${facts.outcomes}${gapsListClause(count - facts.passed, count)}.${tasks}`;
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* The review's gaps: one line for the participants a host suspension likely failed, then a line
|
|
58
|
+
* for each other participant that did not pass.
|
|
59
|
+
*/
|
|
60
|
+
function fanoutGaps(args, judgment) {
|
|
61
|
+
const suspended = new Set(args.hostSuspension?.participantIds ?? []);
|
|
62
|
+
const others = judgment.participants.flatMap((participant, index) => participant.gapLine === undefined || suspended.has(args.specs[index].planned.id)
|
|
63
|
+
? []
|
|
64
|
+
: [participant.gapLine]);
|
|
65
|
+
if (suspended.size === 0)
|
|
66
|
+
return others;
|
|
67
|
+
return [
|
|
68
|
+
`${[...suspended].join(", ")}: failed after the host was suspended, likely its effect. Each participant's record keeps its own error.`,
|
|
69
|
+
...others,
|
|
70
|
+
];
|
|
53
71
|
}
|
|
54
72
|
function fanoutReview(args, streams, judgment) {
|
|
55
73
|
const { outcomes } = args;
|
|
@@ -91,7 +109,7 @@ function fanoutReview(args, streams, judgment) {
|
|
|
91
109
|
? ["Live fan-out session is still running."]
|
|
92
110
|
: args.dryRun
|
|
93
111
|
? ["Live fan-out session not yet run (dry run only)."]
|
|
94
|
-
:
|
|
112
|
+
: fanoutGaps(args, judgment),
|
|
95
113
|
}, streams);
|
|
96
114
|
return review;
|
|
97
115
|
}
|
|
@@ -38,6 +38,7 @@ export declare function runStudyParticipants(setup: CuaRunSetup, participants: C
|
|
|
38
38
|
rerun?: import("../../run/bundle.js").RunRerunLineage;
|
|
39
39
|
observer?: import("../../index.js").ObserverResult;
|
|
40
40
|
diagnostics?: import("./diagnostics.js").CuaDiagnostics;
|
|
41
|
+
hostSuspension?: import("../../run/host-suspension.js").HostSuspensionReading;
|
|
41
42
|
warnings: string[];
|
|
42
43
|
error?: {
|
|
43
44
|
code: import("./types.js").CuaActorStudyErrorCode;
|
|
@@ -3,7 +3,7 @@ import type { StudyDeps } from "../../study/study-deps.js";
|
|
|
3
3
|
import type { LiveTraceFlush } from "./live-flush.js";
|
|
4
4
|
import type { CuaParticipantDeps } from "./types.js";
|
|
5
5
|
/** The deps a desktop route builds once per run. Each participant adds the app URL it opens. */
|
|
6
|
-
export type DesktopParticipantDeps = Omit<CuaParticipantDeps, "
|
|
6
|
+
export type DesktopParticipantDeps = Omit<CuaParticipantDeps, "signalReady" | "appUrl" | "onObservedUrl">;
|
|
7
7
|
/** What every participant of a desktop run reads. Each route supplies all of it. */
|
|
8
8
|
interface DesktopRunFacts {
|
|
9
9
|
plan: Pick<ComputerUsePlan | SharedWorldPlan, "studyId" | "residual" | "caps" | "dryRun">;
|
|
@@ -37,7 +37,9 @@ export function desktopParticipantDeps(run, overrides) {
|
|
|
37
37
|
redactScreenshots: run.redactScreenshots,
|
|
38
38
|
scrubKnownValues: run.scrubKnownValues,
|
|
39
39
|
runSession: run.runSession,
|
|
40
|
-
|
|
40
|
+
// The run's own clock unless a test sets another, so a participant's times and the run's host
|
|
41
|
+
// suspensions read one clock.
|
|
42
|
+
now: seams.now ?? seams.hostClock?.now ?? Date.now,
|
|
41
43
|
onStream: run.onStream,
|
|
42
44
|
reportSubjectPhase: run.reportSubjectPhase,
|
|
43
45
|
};
|
|
@@ -12,17 +12,18 @@ export declare function makeParticipantWriteScreenshot(artifactRoot: PreparedOut
|
|
|
12
12
|
export declare function runCuaParticipant(spec: DesktopParticipantRun, deps: CuaParticipantDeps): Promise<ParticipantRunOutcome>;
|
|
13
13
|
/**
|
|
14
14
|
* Run N>1 E2B participants on the study's schedule (runOnSchedule: each at its start offset, at
|
|
15
|
-
* most `concurrency` at once), behind
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
* by id; nothing here
|
|
15
|
+
* most `concurrency` at once), behind the pipeline gate when the subject is set up in each sandbox,
|
|
16
|
+
* and session fail-fast on harness errors only (queued participants become `blocked` with a pinned
|
|
17
|
+
* reason + a fail-fast event; mission verdicts never trip it). A participant that stops before its
|
|
18
|
+
* session has failed on its own and trips no fail-fast. A closed gate or a fail-fast ends every
|
|
19
|
+
* wait for a later start at once. Each participant tears down its own sandbox by id; nothing here
|
|
20
|
+
* ever enumerates.
|
|
20
21
|
*
|
|
21
22
|
* Exported for the total-runner tests: the injectable runner lets a test make one participant
|
|
22
23
|
* throw (the exact class the guard exists for) without a live sandbox. Production always uses
|
|
23
24
|
* the default.
|
|
24
25
|
*/
|
|
25
|
-
export declare function runCuaParticipants(runs: DesktopParticipantRun[], deps: Omit<CuaParticipantDeps, "
|
|
26
|
+
export declare function runCuaParticipants(runs: DesktopParticipantRun[], deps: Omit<CuaParticipantDeps, "signalReady">, concurrency: number, runParticipant?: typeof runCuaParticipant): Promise<{
|
|
26
27
|
outcomes: ParticipantRunOutcome[];
|
|
27
28
|
failFastReason?: string;
|
|
28
29
|
}>;
|
|
@@ -32,7 +33,7 @@ export declare function toParticipantResult(spec: DesktopParticipantRun, outcome
|
|
|
32
33
|
* Run every participant of a live run. The in-process route drives its single participant in this
|
|
33
34
|
* process; one hosted participant runs alone; a fan-out runs at the plan's concurrency and may stop early.
|
|
34
35
|
*/
|
|
35
|
-
export declare function runAllCuaParticipants(runs: readonly DesktopParticipantRun[], deps: Omit<CuaParticipantDeps, "
|
|
36
|
+
export declare function runAllCuaParticipants(runs: readonly DesktopParticipantRun[], deps: Omit<CuaParticipantDeps, "signalReady">, participantPlan: CuaParticipantPlan, inProcess: boolean): Promise<{
|
|
36
37
|
outcomes: ParticipantRunOutcome[];
|
|
37
38
|
failFastReason: string | undefined;
|
|
38
39
|
}>;
|
|
@@ -56,7 +56,6 @@ export async function runCuaParticipant(spec, deps) {
|
|
|
56
56
|
// model turn only to stop at its first one, so it is skipped before its desktop exists.
|
|
57
57
|
const crossed = deps.runBudget?.crossed();
|
|
58
58
|
if (crossed !== undefined) {
|
|
59
|
-
deps.signalProvisioned?.(false);
|
|
60
59
|
return skippedOutcome(spec, `skipped: study budget reached before this participant started: ${crossed}; no desktop was created`);
|
|
61
60
|
}
|
|
62
61
|
let model = {};
|
|
@@ -67,23 +66,13 @@ export async function runCuaParticipant(spec, deps) {
|
|
|
67
66
|
let sessionError;
|
|
68
67
|
let providerCleanupError;
|
|
69
68
|
let providerPolicyError;
|
|
70
|
-
let provisioned = false;
|
|
71
|
-
let signaled = false;
|
|
72
|
-
const signal = (ok) => {
|
|
73
|
-
if (!signaled && deps.signalProvisioned) {
|
|
74
|
-
signaled = true;
|
|
75
|
-
deps.signalProvisioned(ok);
|
|
76
|
-
}
|
|
77
|
-
};
|
|
78
69
|
const desktop = deps.createDesktop?.(spec, warnings, deps.artifactRoot) ??
|
|
79
70
|
createE2BParticipantDesktop(spec, deps, warnings);
|
|
80
71
|
try {
|
|
81
72
|
await desktop.prepare();
|
|
82
73
|
const ready = await desktop.openSession();
|
|
83
74
|
model = await startParticipantModel(spec, deps, ready.executor);
|
|
84
|
-
|
|
85
|
-
provisioned = true;
|
|
86
|
-
signal(true);
|
|
75
|
+
deps.signalReady?.();
|
|
87
76
|
session = await deps.runSession(participantSessionOptions(spec, deps, ready, model.provider, writeScreenshot));
|
|
88
77
|
}
|
|
89
78
|
catch (error) {
|
|
@@ -102,18 +91,12 @@ export async function runCuaParticipant(spec, deps) {
|
|
|
102
91
|
: closed.refusal.startsWith("HUMANISH_CLAUDE_PARTICIPANT_")
|
|
103
92
|
? `Claude Code's stream showed a call the participant may not make after its last turn (${closed.refusal}); the participant was stopped.`
|
|
104
93
|
: `Codex output after the participant's last request could not be checked against the item policy (${closed.refusal}).`));
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
failed: sessionError !== undefined ||
|
|
112
|
-
providerCleanupError !== undefined ||
|
|
113
|
-
providerPolicyError !== undefined ||
|
|
114
|
-
session === undefined,
|
|
115
|
-
});
|
|
116
|
-
}
|
|
94
|
+
await desktop.finalize({
|
|
95
|
+
failed: sessionError !== undefined ||
|
|
96
|
+
providerCleanupError !== undefined ||
|
|
97
|
+
providerPolicyError !== undefined ||
|
|
98
|
+
session === undefined,
|
|
99
|
+
});
|
|
117
100
|
}
|
|
118
101
|
const { released, desktopFailure, ...desktopEvidence } = desktop.snapshot();
|
|
119
102
|
// A desktop that died under the participant explains the failure better than the transport
|
|
@@ -147,15 +130,73 @@ export async function runCuaParticipant(spec, deps) {
|
|
|
147
130
|
selfReportedBlocker,
|
|
148
131
|
reportedFriction,
|
|
149
132
|
harnessError,
|
|
133
|
+
endedAt: deps.now(),
|
|
134
|
+
};
|
|
135
|
+
}
|
|
136
|
+
/**
|
|
137
|
+
* Holds a fan-out's participants back until one of them is ready, when each sets up the subject in
|
|
138
|
+
* its own sandbox the same way (a clone or local tree built and served, a desktop CLI installed),
|
|
139
|
+
* so a subject that fails to set up fails once. The first participant to enter holds the gate and
|
|
140
|
+
* the others wait. Any participant that becomes ready opens it. A holder that never got a desktop
|
|
141
|
+
* learned nothing about the subject, so the next waiting participant holds it instead. A holder
|
|
142
|
+
* whose desktop came up and then failed before its session closes it, and `onClose` runs.
|
|
143
|
+
*/
|
|
144
|
+
function pipelineGate(open, onClose) {
|
|
145
|
+
let state = open ? "open" : "vacant";
|
|
146
|
+
let holder;
|
|
147
|
+
const waiting = [];
|
|
148
|
+
const settle = (turn) => {
|
|
149
|
+
for (const participant of waiting.splice(0))
|
|
150
|
+
participant.turn(turn);
|
|
151
|
+
};
|
|
152
|
+
return {
|
|
153
|
+
/** The participant that holds the gate, or held it when it closed. */
|
|
154
|
+
holder: () => holder,
|
|
155
|
+
enter(id) {
|
|
156
|
+
if (state === "open")
|
|
157
|
+
return Promise.resolve("start");
|
|
158
|
+
if (state === "closed")
|
|
159
|
+
return Promise.resolve("closed");
|
|
160
|
+
if (state === "vacant") {
|
|
161
|
+
state = "held";
|
|
162
|
+
holder = id;
|
|
163
|
+
return Promise.resolve("hold");
|
|
164
|
+
}
|
|
165
|
+
return new Promise((turn) => waiting.push({ id, turn }));
|
|
166
|
+
},
|
|
167
|
+
ready() {
|
|
168
|
+
if (state !== "held")
|
|
169
|
+
return;
|
|
170
|
+
state = "open";
|
|
171
|
+
settle("start");
|
|
172
|
+
},
|
|
173
|
+
failed(id, { handOver }) {
|
|
174
|
+
if (state !== "held" || holder !== id)
|
|
175
|
+
return;
|
|
176
|
+
if (handOver) {
|
|
177
|
+
const next = waiting.shift();
|
|
178
|
+
if (next === undefined)
|
|
179
|
+
state = "vacant";
|
|
180
|
+
else {
|
|
181
|
+
holder = next.id;
|
|
182
|
+
next.turn("hold");
|
|
183
|
+
}
|
|
184
|
+
return;
|
|
185
|
+
}
|
|
186
|
+
state = "closed";
|
|
187
|
+
settle("closed");
|
|
188
|
+
onClose();
|
|
189
|
+
},
|
|
150
190
|
};
|
|
151
191
|
}
|
|
152
192
|
/**
|
|
153
193
|
* Run N>1 E2B participants on the study's schedule (runOnSchedule: each at its start offset, at
|
|
154
|
-
* most `concurrency` at once), behind
|
|
155
|
-
*
|
|
156
|
-
*
|
|
157
|
-
*
|
|
158
|
-
* by id; nothing here
|
|
194
|
+
* most `concurrency` at once), behind the pipeline gate when the subject is set up in each sandbox,
|
|
195
|
+
* and session fail-fast on harness errors only (queued participants become `blocked` with a pinned
|
|
196
|
+
* reason + a fail-fast event; mission verdicts never trip it). A participant that stops before its
|
|
197
|
+
* session has failed on its own and trips no fail-fast. A closed gate or a fail-fast ends every
|
|
198
|
+
* wait for a later start at once. Each participant tears down its own sandbox by id; nothing here
|
|
199
|
+
* ever enumerates.
|
|
159
200
|
*
|
|
160
201
|
* Exported for the total-runner tests: the injectable runner lets a test make one participant
|
|
161
202
|
* throw (the exact class the guard exists for) without a live sandbox. Production always uses
|
|
@@ -164,40 +205,26 @@ export async function runCuaParticipant(spec, deps) {
|
|
|
164
205
|
export async function runCuaParticipants(runs, deps, concurrency, runParticipant = runCuaParticipant) {
|
|
165
206
|
const failFast = { tripped: false, reason: "" };
|
|
166
207
|
const stopping = new AbortController();
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
let rejectGate;
|
|
170
|
-
const gate = new Promise((resolve, reject) => {
|
|
171
|
-
resolveGate = resolve;
|
|
172
|
-
rejectGate = () => {
|
|
173
|
-
reject(new Error("gate"));
|
|
174
|
-
stopping.abort();
|
|
175
|
-
};
|
|
176
|
-
});
|
|
177
|
-
// The gate is rejected on the first participant's provisioning failure; swallow the unhandled
|
|
178
|
-
// rejection if no later participant ever awaits it (concurrency could let the first finish
|
|
179
|
-
// alone).
|
|
180
|
-
gate.catch(() => undefined);
|
|
208
|
+
// An app-url subject is only opened, so one participant's start says nothing about another's.
|
|
209
|
+
const gate = pipelineGate(deps.subject.kind === "app-url", () => stopping.abort());
|
|
181
210
|
const outcomes = await runOnSchedule(runs, {
|
|
182
211
|
startAfterMs: (spec) => spec.planned.startAfterMs,
|
|
183
212
|
slots: concurrency,
|
|
184
213
|
now: deps.now,
|
|
185
214
|
signal: stopping.signal,
|
|
186
|
-
}, async (spec, _index, {
|
|
187
|
-
if (order === 0)
|
|
188
|
-
gateHolder = spec.planned.id;
|
|
189
|
-
if (order > 0) {
|
|
190
|
-
try {
|
|
191
|
-
await gate;
|
|
192
|
-
}
|
|
193
|
-
catch {
|
|
194
|
-
return skippedOutcome(spec, `skipped: participant ${gateHolder} failed to provision its world (pipeline gate)`, { scheduledAt });
|
|
195
|
-
}
|
|
196
|
-
}
|
|
215
|
+
}, async (spec, _index, { scheduledAt }) => {
|
|
197
216
|
if (failFast.tripped) {
|
|
198
217
|
return skippedOutcome(spec, `skipped: ${failFast.reason}`, { scheduledAt });
|
|
199
218
|
}
|
|
219
|
+
// A participant trips fail-fast only after the gate opened, or as it closes the gate, so no
|
|
220
|
+
// participant is still waiting at the gate when fail-fast skips the rest.
|
|
221
|
+
const id = spec.planned.id;
|
|
222
|
+
if ((await gate.enter(id)) === "closed") {
|
|
223
|
+
return skippedOutcome(spec, `skipped: participant ${gate.holder()} failed to provision its world (pipeline gate)`, { scheduledAt });
|
|
224
|
+
}
|
|
200
225
|
const arrival = { scheduledAt, startedAt: deps.now() };
|
|
226
|
+
let ready = false;
|
|
227
|
+
let threw = false;
|
|
201
228
|
// The participant runner is total: every exit path returns a recorded outcome. Without this
|
|
202
229
|
// guard, one participant's late throw (e.g. its trace write hitting ENOSPC after its own sandbox was
|
|
203
230
|
// already torn down) rejected the whole map while sibling workers kept launching sandboxes
|
|
@@ -206,25 +233,14 @@ export async function runCuaParticipants(runs, deps, concurrency, runParticipant
|
|
|
206
233
|
try {
|
|
207
234
|
outcome = await runParticipant(spec, {
|
|
208
235
|
...deps,
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
resolveGate?.();
|
|
214
|
-
}
|
|
215
|
-
else {
|
|
216
|
-
rejectGate?.();
|
|
217
|
-
}
|
|
218
|
-
},
|
|
219
|
-
}
|
|
220
|
-
: {}),
|
|
236
|
+
signalReady: () => {
|
|
237
|
+
ready = true;
|
|
238
|
+
gate.ready();
|
|
239
|
+
},
|
|
221
240
|
});
|
|
222
241
|
}
|
|
223
242
|
catch (error) {
|
|
224
|
-
|
|
225
|
-
// blocked rather than leaving them awaiting a gate that will never settle.
|
|
226
|
-
if (order === 0)
|
|
227
|
-
rejectGate?.();
|
|
243
|
+
threw = true;
|
|
228
244
|
const detail = redactText(toErrorMessage(error));
|
|
229
245
|
outcome = {
|
|
230
246
|
spec,
|
|
@@ -239,9 +255,15 @@ export async function runCuaParticipants(runs, deps, concurrency, runParticipant
|
|
|
239
255
|
reportedFriction: false,
|
|
240
256
|
harnessError: true,
|
|
241
257
|
sessionError: `participant runner threw outside the session guard: ${detail}`,
|
|
258
|
+
endedAt: deps.now(),
|
|
242
259
|
};
|
|
243
260
|
}
|
|
244
|
-
|
|
261
|
+
// A participant that stopped before its session failed on its own and trips no fail-fast.
|
|
262
|
+
// Without a desktop it says nothing about the subject, so the next participant starts first.
|
|
263
|
+
// A runner that threw is a harness defect: it closes the gate and trips fail-fast.
|
|
264
|
+
if (!ready)
|
|
265
|
+
gate.failed(id, { handOver: !threw && outcome.sandboxId === undefined });
|
|
266
|
+
if (outcome.harnessError && (ready || threw) && !failFast.tripped) {
|
|
245
267
|
failFast.tripped = true;
|
|
246
268
|
failFast.reason = `a prior participant (${outcome.spec.planned.id}) ended in a harness error (fail-fast)`;
|
|
247
269
|
stopping.abort();
|