humanish 0.120.0 → 0.121.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +27 -0
- package/dist/analysis/provider.d.ts +6 -1
- package/dist/analysis/provider.js +4 -0
- package/dist/cli/commands/study-format.js +25 -4
- package/dist/routes/computer-use/bundle.d.ts +3 -0
- package/dist/routes/computer-use/bundle.js +2 -0
- package/dist/routes/computer-use/fanout-bundle.js +20 -2
- package/dist/routes/computer-use/live-phase.d.ts +1 -0
- package/dist/routes/computer-use/participant-deps.d.ts +1 -1
- package/dist/routes/computer-use/participant-deps.js +3 -1
- package/dist/routes/computer-use/participant-execution.d.ts +8 -7
- package/dist/routes/computer-use/participant-execution.js +93 -71
- package/dist/routes/computer-use/result.js +56 -7
- package/dist/routes/computer-use/setup.d.ts +1 -1
- package/dist/routes/computer-use/single-bundle.d.ts +3 -0
- package/dist/routes/computer-use/single-bundle.js +3 -2
- package/dist/routes/computer-use/types.d.ts +16 -2
- package/dist/run/host-suspension.d.ts +56 -0
- package/dist/run/host-suspension.js +106 -0
- package/dist/run/outcomes.js +3 -1
- package/dist/run/run.d.ts +15 -2
- package/dist/run/run.js +15 -8
- package/dist/run/status.d.ts +8 -2
- package/dist/run/status.js +11 -8
- package/dist/study/study-deps.d.ts +6 -0
- package/dist/tui-app.js +35 -35
- package/docs/contracts/run-bundle.md +19 -0
- package/docs/contracts/schemas.md +2 -1
- package/package.json +1 -1
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { adapterScoreFailureMessage, applyBrowserScorer } from "../../study/adapter-extension.js";
|
|
2
2
|
import { redactText } from "../../evidence/redaction.js";
|
|
3
|
-
import { foldScorerFailures, judgeExecution, judgeParticipantRecords, OUTCOME_POLICIES, resultOk, } from "../../run/judge.js";
|
|
3
|
+
import { foldScorerFailures, judgeExecution, judgeParticipantRecords, OUTCOME_POLICIES, participantHarnessFailed, resultOk, } from "../../run/judge.js";
|
|
4
|
+
import { readHostSuspensions, } from "../../run/host-suspension.js";
|
|
4
5
|
import { validatePreparedRunArtifactPaths } from "../../run/paths.js";
|
|
5
6
|
import { participantExecutionFailures, participantOutcomeOk, participantFactsOf, } from "./participant-facts.js";
|
|
6
7
|
import { participantCapWarning } from "./participant-model.js";
|
|
@@ -53,13 +54,39 @@ function buildParticipantSummary(outcomes, participantCount, participantPlan, dr
|
|
|
53
54
|
waves: participantPlan.waves,
|
|
54
55
|
};
|
|
55
56
|
}
|
|
57
|
+
/**
|
|
58
|
+
* Each participant as the host-suspension reading reads it: a harness error, its session's time
|
|
59
|
+
* limit or a skip, when it started, and when its runner returned, which a setup failure with no
|
|
60
|
+
* session also records.
|
|
61
|
+
*/
|
|
62
|
+
function suspendedParticipants(outcomes) {
|
|
63
|
+
return outcomes.map((outcome) => {
|
|
64
|
+
const facts = participantFactsOf(outcome);
|
|
65
|
+
const trace = outcome.session?.trace;
|
|
66
|
+
const failure = facts.skipped
|
|
67
|
+
? "skipped"
|
|
68
|
+
: participantHarnessFailed(facts)
|
|
69
|
+
? "harness"
|
|
70
|
+
: trace?.stopCause === "time_limit"
|
|
71
|
+
? "time-limit"
|
|
72
|
+
: undefined;
|
|
73
|
+
const startedAtMs = outcome.arrival?.startedAt ?? (trace === undefined ? undefined : Date.parse(trace.startedAt));
|
|
74
|
+
const endedAtMs = outcome.endedAt ?? (trace === undefined ? undefined : Date.parse(trace.completedAt));
|
|
75
|
+
return {
|
|
76
|
+
id: outcome.spec.planned.id,
|
|
77
|
+
...(failure === undefined ? {} : { failure }),
|
|
78
|
+
...(startedAtMs === undefined ? {} : { startedAtMs }),
|
|
79
|
+
...(endedAtMs === undefined ? {} : { endedAtMs }),
|
|
80
|
+
};
|
|
81
|
+
});
|
|
82
|
+
}
|
|
56
83
|
/**
|
|
57
84
|
* The computer-use study result for a finished run. The run passes only when the Observer rendered,
|
|
58
85
|
* every participant passed (dry-run participants pass as contracts), and no adapter score or declared scorer
|
|
59
86
|
* verdict failed; otherwise the error names the first reason.
|
|
60
87
|
*/
|
|
61
88
|
function cuaStudyResult(args) {
|
|
62
|
-
const { studyId, cwd, runId, appUrl, dryRun, participantRuns, outcomes, subjects, aggregateSubject, participantPlan, rerunLineage, bundle, observer, receivingWarnings, aggregateWarnings, adapterWarnings, } = args;
|
|
89
|
+
const { studyId, cwd, runId, appUrl, dryRun, participantRuns, outcomes, subjects, aggregateSubject, participantPlan, rerunLineage, bundle, observer, hostSuspension, receivingWarnings, aggregateWarnings, adapterWarnings, } = args;
|
|
63
90
|
const participantCount = participantRuns.length;
|
|
64
91
|
// Participant-level pass: dry-run participants are contract-ok; live ones need a passed,
|
|
65
92
|
// engaged session.
|
|
@@ -77,6 +104,10 @@ function cuaStudyResult(args) {
|
|
|
77
104
|
const participantResults = participantRuns.map((spec, index) => toParticipantResult(spec, outcomes?.[index], subjects[index], dryRun));
|
|
78
105
|
const summary = buildParticipantSummary(outcomes, participantCount, participantPlan, dryRun);
|
|
79
106
|
const firstOutcome = outcomes?.[0];
|
|
107
|
+
// A failure the host suspension likely caused names the suspension first.
|
|
108
|
+
const suspended = hostSuspension !== undefined && hostSuspension.participantIds.length > 0
|
|
109
|
+
? hostSuspension.summary
|
|
110
|
+
: undefined;
|
|
80
111
|
const errorResult = (() => {
|
|
81
112
|
if (ok)
|
|
82
113
|
return undefined;
|
|
@@ -87,24 +118,31 @@ function cuaStudyResult(args) {
|
|
|
87
118
|
const outcome = firstOutcome;
|
|
88
119
|
return {
|
|
89
120
|
code: outcome?.failureCode ?? "HUMANISH_COMPUTER_USE_FAILED",
|
|
90
|
-
message: outcome?.sessionError ??
|
|
121
|
+
message: `${suspended === undefined ? "" : `${suspended} `}${outcome?.sessionError ??
|
|
91
122
|
outcome?.providerCleanupError ??
|
|
92
123
|
outcome?.providerPolicyError ??
|
|
93
124
|
(outcome?.noEngagement || outcome?.selfReportedBlocker || observer.ok
|
|
94
125
|
? (participantResults[0]?.error?.message ??
|
|
95
126
|
judgeParticipantRecords([participantFactsOf(outcome)]).participants[0]
|
|
96
127
|
.notPassedMessage)
|
|
97
|
-
: (observer.error?.message ?? "Observer failed for the computer-use run."))
|
|
128
|
+
: (observer.error?.message ?? "Observer failed for the computer-use run."))}`,
|
|
98
129
|
};
|
|
99
130
|
}
|
|
100
131
|
const failing = (outcomes ?? []).find((outcome) => !participantOk(outcome));
|
|
132
|
+
const firstFailure = suspended !== undefined
|
|
133
|
+
? `; ${suspended.replace(/\.$/, "")}`
|
|
134
|
+
: failing?.sessionError !== undefined
|
|
135
|
+
? `; first failure: ${failing.sessionError}`
|
|
136
|
+
: failing === undefined && args.outcome.execution.failures[0] !== undefined
|
|
137
|
+
? `; ${args.outcome.execution.failures[0].message}`
|
|
138
|
+
: "";
|
|
101
139
|
// A participant whose desktop failed for a known cause gives the run that cause's code.
|
|
102
140
|
const codedFailure = (outcomes ?? []).find((outcome) => outcome.failureCode !== undefined);
|
|
103
141
|
const code = codedFailure?.failureCode ?? "HUMANISH_COMPUTER_USE_FAILED";
|
|
104
142
|
return {
|
|
105
143
|
code,
|
|
106
144
|
message: observer.ok
|
|
107
|
-
? `Fan-out run failed: ${summary.passed}/${plural(participantCount, "participant")} passed (${summary.skipped} skipped, ${plural(summary.harnessErrors, "harness error")}, ${summary.hollow} without engagement)${
|
|
145
|
+
? `Fan-out run failed: ${summary.passed}/${plural(participantCount, "participant")} passed (${summary.skipped} skipped, ${plural(summary.harnessErrors, "harness error")}, ${summary.hollow} without engagement)${firstFailure}.`
|
|
108
146
|
: (observer.error?.message ?? "Observer failed for the computer-use fan-out run."),
|
|
109
147
|
};
|
|
110
148
|
})();
|
|
@@ -149,6 +187,7 @@ function cuaStudyResult(args) {
|
|
|
149
187
|
laneSummary: summary,
|
|
150
188
|
...(rerunLineage === undefined ? {} : { rerun: rerunLineage }),
|
|
151
189
|
observer,
|
|
190
|
+
...(hostSuspension === undefined ? {} : { hostSuspension }),
|
|
152
191
|
warnings,
|
|
153
192
|
...(errorResult === undefined ? {} : { error: errorResult }),
|
|
154
193
|
};
|
|
@@ -175,6 +214,8 @@ export async function finishCuaRun(setup, finish, ran) {
|
|
|
175
214
|
];
|
|
176
215
|
// One judgment for the whole run: the bundle's verdict and the result's ok both read it.
|
|
177
216
|
const judgment = judgeComputerUseRun(bundleBase, { dryRun, outcomes });
|
|
217
|
+
// One reading of the host suspensions: the review, the outcome and the result all name it.
|
|
218
|
+
const hostSuspension = readHostSuspensions(run.hostSuspensions(), suspendedParticipants(outcomes ?? []), Date.parse(run.createdAt));
|
|
178
219
|
const bundle = buildCuaRunBundle(bundleBase, {
|
|
179
220
|
judgment,
|
|
180
221
|
dryRun,
|
|
@@ -182,6 +223,7 @@ export async function finishCuaRun(setup, finish, ran) {
|
|
|
182
223
|
subjects,
|
|
183
224
|
aggregateSubject,
|
|
184
225
|
...(failFastReason === undefined ? {} : { failFastReason }),
|
|
226
|
+
...(hostSuspension === undefined ? {} : { hostSuspension }),
|
|
185
227
|
});
|
|
186
228
|
const adapterWarnings = [];
|
|
187
229
|
const scorerResult = await applyBrowserScorer({
|
|
@@ -206,8 +248,14 @@ export async function finishCuaRun(setup, finish, ran) {
|
|
|
206
248
|
if (receiving)
|
|
207
249
|
bundle.commsReceiving = receiving.snapshot();
|
|
208
250
|
const policy = OUTCOME_POLICIES["computer-use"];
|
|
209
|
-
// FinishedRun.renderObserver adds an Observer that did not render.
|
|
210
|
-
|
|
251
|
+
// FinishedRun.renderObserver adds an Observer that did not render. A suspension that likely
|
|
252
|
+
// caused participant failures is the run's first failure, ahead of each one it explains.
|
|
253
|
+
const execution = judgeExecution([
|
|
254
|
+
...(hostSuspension !== undefined && hostSuspension.participantIds.length > 0
|
|
255
|
+
? [{ kind: "run", message: hostSuspension.summary }]
|
|
256
|
+
: []),
|
|
257
|
+
...participantExecutionFailures(outcomes, runId),
|
|
258
|
+
], policy);
|
|
211
259
|
const finished = await run.finish(bundle, {
|
|
212
260
|
ok: resultOk({ judgment, execution, scorerFailures: scorerResult.failures, policy }),
|
|
213
261
|
execution,
|
|
@@ -236,6 +284,7 @@ export async function finishCuaRun(setup, finish, ran) {
|
|
|
236
284
|
bundle,
|
|
237
285
|
outcome: finished.outcome,
|
|
238
286
|
observer,
|
|
287
|
+
hostSuspension,
|
|
239
288
|
receivingWarnings,
|
|
240
289
|
aggregateWarnings,
|
|
241
290
|
adapterWarnings,
|
|
@@ -38,7 +38,7 @@ export interface CuaRunSetup {
|
|
|
38
38
|
/** What only runStudyParticipants reads. */
|
|
39
39
|
export interface CuaParticipantsSetup {
|
|
40
40
|
env: Record<string, string | undefined>;
|
|
41
|
-
deps: Omit<CuaParticipantDeps, "
|
|
41
|
+
deps: Omit<CuaParticipantDeps, "signalReady">;
|
|
42
42
|
/** Filled by runStudyParticipants on a live run; deps.onTrace reads it. */
|
|
43
43
|
liveTrace: {
|
|
44
44
|
flush?: LiveTraceFlush["flush"];
|
|
@@ -7,6 +7,7 @@ import { type BundleRun, type RunBundle, type RunFeedbackCandidate, type RunProv
|
|
|
7
7
|
import { type RunDesktopGeometry, type RunStream } from "../../run/streams.js";
|
|
8
8
|
import { type Verdict } from "../../run/judge.js";
|
|
9
9
|
import { type DesktopUsage } from "../../run/cost-summary.js";
|
|
10
|
+
import type { HostSuspensionReading } from "../../run/host-suspension.js";
|
|
10
11
|
/**
|
|
11
12
|
* The bundle of a single-participant run: one participant and no rerun. buildCuaRunBundle
|
|
12
13
|
* (bundle.ts) maps the run's base and state into these arguments.
|
|
@@ -92,4 +93,6 @@ export declare function buildSingleParticipantBundle(args: {
|
|
|
92
93
|
* desktopDurationMs). Absent when no sandbox ran (in-process/dry-run) → no desktop cost line. */
|
|
93
94
|
desktopMinutes?: number;
|
|
94
95
|
desktopUsage?: DesktopUsage;
|
|
96
|
+
/** The run's host suspensions and the failure they likely caused. */
|
|
97
|
+
hostSuspension?: HostSuspensionReading;
|
|
95
98
|
}): RunBundle;
|
|
@@ -332,10 +332,11 @@ function singleReview(args, view, stream) {
|
|
|
332
332
|
}
|
|
333
333
|
: {}),
|
|
334
334
|
...(singleRunTasks === undefined ? {} : { tasks: singleRunTasks }),
|
|
335
|
-
|
|
335
|
+
// A host suspension comes first: it may explain everything after it.
|
|
336
|
+
summary: `${args.hostSuspension === undefined ? "" : `${args.hostSuspension.summary} `}${singleSummary(args, view, {
|
|
336
337
|
...(singleRunTasks === undefined ? {} : { tasks: singleRunTasks }),
|
|
337
338
|
...(credibilityNote === undefined ? {} : { credibilityNote }),
|
|
338
|
-
})
|
|
339
|
+
})}`,
|
|
339
340
|
gaps: args.session || args.sessionError !== undefined
|
|
340
341
|
? []
|
|
341
342
|
: args.inProgress === true
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { Verdict } from "../../run/judge.js";
|
|
2
|
+
import type { HostSuspensionReading } from "../../run/host-suspension.js";
|
|
2
3
|
import type { LocalCapacity } from "../../substrates/local/capacity.js";
|
|
3
4
|
import type { RunDesktopRecording } from "../../evidence/desktop-recording-types.js";
|
|
4
5
|
import type { CuaExecutor, CuaLiveMetadata, CuaLoopResult, CuaProvider } from "../../actors/computer-use/loop.js";
|
|
@@ -233,6 +234,11 @@ export interface CuaActorStudyResult extends AutomaticAnalysisResult, StudyResul
|
|
|
233
234
|
rerun?: RunRerunLineage;
|
|
234
235
|
observer?: ObserverResult;
|
|
235
236
|
diagnostics?: CuaDiagnostics;
|
|
237
|
+
/**
|
|
238
|
+
* Present when the host was suspended during the run (a laptop that slept): when, and which
|
|
239
|
+
* participant failures it likely caused. Each participant's own result is unchanged.
|
|
240
|
+
*/
|
|
241
|
+
hostSuspension?: HostSuspensionReading;
|
|
236
242
|
warnings: string[];
|
|
237
243
|
error?: {
|
|
238
244
|
code: CuaActorStudyErrorCode;
|
|
@@ -391,8 +397,8 @@ export type CuaParticipantDeps = E2BDesktopDeps & ParticipantModelDeps & {
|
|
|
391
397
|
/** The caller's inProcess executor with the run's config bound. Read by the in-process desktop. */
|
|
392
398
|
inProcessExecutor?: (appUrl: string) => Promise<CuaExecutor>;
|
|
393
399
|
runSession: (options: CuaActorSessionOptions) => Promise<CuaLoopResult>;
|
|
394
|
-
/**
|
|
395
|
-
|
|
400
|
+
/** Called once, when the participant's desktop and model are ready and its session starts. */
|
|
401
|
+
signalReady?: () => void;
|
|
396
402
|
};
|
|
397
403
|
/** Why a sandbox was not confirmed released. */
|
|
398
404
|
export interface SandboxReleaseFact {
|
|
@@ -439,6 +445,12 @@ export interface ParticipantRunOutcome {
|
|
|
439
445
|
skippedReason?: string;
|
|
440
446
|
/** When a fan-out participant was due and when its desktop was requested. */
|
|
441
447
|
arrival?: ParticipantArrival;
|
|
448
|
+
/**
|
|
449
|
+
* When the participant's runner returned this outcome, on the deps clock (epoch ms): after its
|
|
450
|
+
* session, or after a setup failure that left it none. Absent on a skipped participant, which
|
|
451
|
+
* never ran, and on outcomes built outside the runner.
|
|
452
|
+
*/
|
|
453
|
+
endedAt?: number;
|
|
442
454
|
noEngagement: boolean;
|
|
443
455
|
selfReportedBlocker: boolean;
|
|
444
456
|
/** The inclusive friction read: blocker-shaped narration incl. self-resolved arcs.
|
|
@@ -470,6 +482,8 @@ export interface CuaFanoutBundleArgs {
|
|
|
470
482
|
participantPlan: CuaParticipantPlan;
|
|
471
483
|
rerun?: RunRerunLineage;
|
|
472
484
|
failFastReason?: string;
|
|
485
|
+
/** The run's host suspensions and the participant failures they likely caused. */
|
|
486
|
+
hostSuspension?: HostSuspensionReading;
|
|
473
487
|
publicRepo?: string;
|
|
474
488
|
inProgress?: boolean;
|
|
475
489
|
}
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import type { RunEvent } from "./bundle.js";
|
|
2
|
+
/** The run's clock and heartbeat timer. Tests pass one whose wall clock they move. */
|
|
3
|
+
export interface HostClock {
|
|
4
|
+
/** Wall-clock time in epoch milliseconds. */
|
|
5
|
+
now(): number;
|
|
6
|
+
/** Call `tick` every `intervalMs` until the returned function runs. Never keeps the process alive. */
|
|
7
|
+
every(intervalMs: number, tick: () => void): () => void;
|
|
8
|
+
}
|
|
9
|
+
export declare const systemHostClock: HostClock;
|
|
10
|
+
/** A time the run's heartbeat did not run: from its last tick before the gap to its first after. */
|
|
11
|
+
export interface HostSuspension {
|
|
12
|
+
startedAt: string;
|
|
13
|
+
endedAt: string;
|
|
14
|
+
durationMs: number;
|
|
15
|
+
}
|
|
16
|
+
export interface Heartbeat {
|
|
17
|
+
/** The suspensions so far. A read also checks the time since the last tick. */
|
|
18
|
+
suspensions(): readonly HostSuspension[];
|
|
19
|
+
/** Stop the timer. Later reads return the suspensions recorded until then. */
|
|
20
|
+
stop(): void;
|
|
21
|
+
}
|
|
22
|
+
/** Run `onTick` every `intervalMs` on `clock`, and record each gap of 30 s or more. */
|
|
23
|
+
export declare function startHeartbeat(clock: HostClock, intervalMs: number, onTick: () => void): Heartbeat;
|
|
24
|
+
/** One warn event per suspension, at its start, naming its start, end, length and offset. */
|
|
25
|
+
export declare function hostSuspensionEvents(suspensions: readonly HostSuspension[], runStartMs: number): RunEvent[];
|
|
26
|
+
/** One participant as the attribution reads it. */
|
|
27
|
+
export interface SuspendedParticipant {
|
|
28
|
+
id: string;
|
|
29
|
+
/**
|
|
30
|
+
* How it did not pass, when a suspension can explain that: an error in the harness, its
|
|
31
|
+
* session's time limit, or a skip after another participant's harness error. Absent when it
|
|
32
|
+
* passed or its session ended on its own report.
|
|
33
|
+
*/
|
|
34
|
+
failure?: "harness" | "time-limit" | "skipped";
|
|
35
|
+
/** When it started; absent when no record says. */
|
|
36
|
+
startedAtMs?: number;
|
|
37
|
+
/** When it stopped, after its session or after a setup failure; absent when no record says. */
|
|
38
|
+
endedAtMs?: number;
|
|
39
|
+
}
|
|
40
|
+
/** What a run says about its host suspensions. */
|
|
41
|
+
export interface HostSuspensionReading {
|
|
42
|
+
/** One sentence for the run outcome, the CLI and the review. */
|
|
43
|
+
summary: string;
|
|
44
|
+
/** The participants whose failure is likely a suspension's effect, in the order given. */
|
|
45
|
+
participantIds: string[];
|
|
46
|
+
suspensions: HostSuspension[];
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* Which failures the run's suspensions likely caused, and the sentence that says so; undefined
|
|
50
|
+
* without a suspension. A frozen loop's failures surface at the wake (macOS fires every overdue
|
|
51
|
+
* timer at once) or up to a turn timeout later (Linux timers do not count suspended time), so the
|
|
52
|
+
* rule reads who was in flight across the suspension. A skip follows another participant's
|
|
53
|
+
* harness error (the pipeline gate, fail-fast), so skips count only when the run had harness
|
|
54
|
+
* errors and every one was in flight across a suspension.
|
|
55
|
+
*/
|
|
56
|
+
export declare function readHostSuspensions(suspensions: readonly HostSuspension[], participants: readonly SuspendedParticipant[], runStartMs: number): HostSuspensionReading | undefined;
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
// Host suspensions: when the machine running a study sleeps, every participant loop on it stops,
|
|
2
|
+
// and on wake its timers and connections fail together. This module notices the gap and says
|
|
3
|
+
// which participant failures it likely caused, so the run names the suspension as their cause.
|
|
4
|
+
//
|
|
5
|
+
// Detection reads the wall clock across a heartbeat. Date.now counts suspended time on macOS and
|
|
6
|
+
// Linux. performance.now and Node's timers run on libuv's clock, which counts sleep on macOS
|
|
7
|
+
// (mach_continuous_time), so the tick after a sleep fires at wake, and stops during a Linux
|
|
8
|
+
// suspend (CLOCK_MONOTONIC), so the tick fires within one interval of resume. Either way the wall
|
|
9
|
+
// clock between two ticks shows the gap.
|
|
10
|
+
import { formatDuration } from "./projection.js";
|
|
11
|
+
import { plural } from "./text.js";
|
|
12
|
+
export const systemHostClock = {
|
|
13
|
+
now: () => Date.now(),
|
|
14
|
+
every(intervalMs, tick) {
|
|
15
|
+
const timer = setInterval(tick, intervalMs);
|
|
16
|
+
timer.unref?.();
|
|
17
|
+
return () => clearInterval(timer);
|
|
18
|
+
},
|
|
19
|
+
};
|
|
20
|
+
/**
|
|
21
|
+
* A wall-clock gap between two heartbeats this long or longer is a suspension. A shorter one is
|
|
22
|
+
* most likely a busy event loop.
|
|
23
|
+
*/
|
|
24
|
+
const HOST_SUSPENSION_MIN_MS = 30_000;
|
|
25
|
+
/** Run `onTick` every `intervalMs` on `clock`, and record each gap of 30 s or more. */
|
|
26
|
+
export function startHeartbeat(clock, intervalMs, onTick) {
|
|
27
|
+
const found = [];
|
|
28
|
+
let lastMs = clock.now();
|
|
29
|
+
let stopped = false;
|
|
30
|
+
const observe = () => {
|
|
31
|
+
if (stopped)
|
|
32
|
+
return;
|
|
33
|
+
const nowMs = clock.now();
|
|
34
|
+
if (nowMs - lastMs >= HOST_SUSPENSION_MIN_MS)
|
|
35
|
+
found.push({
|
|
36
|
+
startedAt: new Date(lastMs).toISOString(),
|
|
37
|
+
endedAt: new Date(nowMs).toISOString(),
|
|
38
|
+
durationMs: nowMs - lastMs,
|
|
39
|
+
});
|
|
40
|
+
lastMs = nowMs;
|
|
41
|
+
};
|
|
42
|
+
const stopTimer = clock.every(intervalMs, () => {
|
|
43
|
+
observe();
|
|
44
|
+
onTick();
|
|
45
|
+
});
|
|
46
|
+
return {
|
|
47
|
+
suspensions() {
|
|
48
|
+
observe();
|
|
49
|
+
return [...found];
|
|
50
|
+
},
|
|
51
|
+
stop() {
|
|
52
|
+
stopped = true;
|
|
53
|
+
stopTimer();
|
|
54
|
+
},
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
/** `6m 5s at +1m`: how long, and how far into the run it started. */
|
|
58
|
+
function span(suspension, runStartMs) {
|
|
59
|
+
const offset = Date.parse(suspension.startedAt) - runStartMs;
|
|
60
|
+
return `${formatDuration(suspension.durationMs)} at +${formatDuration(offset)}`;
|
|
61
|
+
}
|
|
62
|
+
/** One warn event per suspension, at its start, naming its start, end, length and offset. */
|
|
63
|
+
export function hostSuspensionEvents(suspensions, runStartMs) {
|
|
64
|
+
return suspensions.map((suspension, index) => ({
|
|
65
|
+
id: `event-host-suspended-${String(index + 1).padStart(3, "0")}`,
|
|
66
|
+
at: suspension.startedAt,
|
|
67
|
+
level: "warn",
|
|
68
|
+
type: "host.suspended",
|
|
69
|
+
message: `The host was suspended for ${formatDuration(suspension.durationMs)}, from ${suspension.startedAt} to ${suspension.endedAt} (+${formatDuration(Date.parse(suspension.startedAt) - runStartMs)} into the run): the run's heartbeat did not run in that time. A process that was stopped, or a clock set forward, reads the same way.`,
|
|
70
|
+
}));
|
|
71
|
+
}
|
|
72
|
+
/** The participant was running at some moment of the suspension. */
|
|
73
|
+
function inFlight(participant, suspension) {
|
|
74
|
+
return ((participant.startedAtMs ?? -Infinity) <= Date.parse(suspension.endedAt) &&
|
|
75
|
+
(participant.endedAtMs ?? Infinity) >= Date.parse(suspension.startedAt));
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* Which failures the run's suspensions likely caused, and the sentence that says so; undefined
|
|
79
|
+
* without a suspension. A frozen loop's failures surface at the wake (macOS fires every overdue
|
|
80
|
+
* timer at once) or up to a turn timeout later (Linux timers do not count suspended time), so the
|
|
81
|
+
* rule reads who was in flight across the suspension. A skip follows another participant's
|
|
82
|
+
* harness error (the pipeline gate, fail-fast), so skips count only when the run had harness
|
|
83
|
+
* errors and every one was in flight across a suspension.
|
|
84
|
+
*/
|
|
85
|
+
export function readHostSuspensions(suspensions, participants, runStartMs) {
|
|
86
|
+
if (suspensions.length === 0)
|
|
87
|
+
return undefined;
|
|
88
|
+
const explained = new Set(participants
|
|
89
|
+
.filter((participant) => (participant.failure === "harness" || participant.failure === "time-limit") &&
|
|
90
|
+
suspensions.some((suspension) => inFlight(participant, suspension)))
|
|
91
|
+
.map((participant) => participant.id));
|
|
92
|
+
const harness = participants.filter((participant) => participant.failure === "harness");
|
|
93
|
+
const skipsFollow = harness.length > 0 && harness.every((participant) => explained.has(participant.id));
|
|
94
|
+
const participantIds = participants
|
|
95
|
+
.filter((participant) => explained.has(participant.id) || (participant.failure === "skipped" && skipsFollow))
|
|
96
|
+
.map((participant) => participant.id);
|
|
97
|
+
const one = suspensions.length === 1;
|
|
98
|
+
const what = one
|
|
99
|
+
? `The host was suspended for ${span(suspensions[0], runStartMs)}`
|
|
100
|
+
: `The host was suspended ${suspensions.length} times (${suspensions.map((suspension) => span(suspension, runStartMs)).join(", ")})`;
|
|
101
|
+
const count = participantIds.length;
|
|
102
|
+
const effect = count === 0
|
|
103
|
+
? "no participant failure followed"
|
|
104
|
+
: `the ${plural(count, "participant failure")} after ${one ? "it" : "them"} ${count === 1 ? "is" : "are"} likely ${one ? "its" : "their"} effect`;
|
|
105
|
+
return { summary: `${what}; ${effect}.`, participantIds, suspensions: [...suspensions] };
|
|
106
|
+
}
|
package/dist/run/outcomes.js
CHANGED
|
@@ -193,7 +193,9 @@ export function formatParticipantOutcomes(outcomes, terminalCauses = []) {
|
|
|
193
193
|
const description = entry.label === undefined ? fallback : `interrupted (${entry.label})`;
|
|
194
194
|
counts.set(description, (counts.get(description) ?? 0) + 1);
|
|
195
195
|
}
|
|
196
|
-
|
|
196
|
+
// Participants finish in any order, so the causes are sorted: most first, then by name.
|
|
197
|
+
const sorted = [...counts].sort(([a, m], [b, n]) => n - m || a.localeCompare(b));
|
|
198
|
+
for (const [description, n] of sorted)
|
|
197
199
|
parts.push(`${n} ${description}`);
|
|
198
200
|
};
|
|
199
201
|
append(outcomes.abandoned, ["abandoned"], "gave up");
|
package/dist/run/run.d.ts
CHANGED
|
@@ -3,6 +3,7 @@ import { type RunBundle } from "./bundle.js";
|
|
|
3
3
|
import { type ExecutionFailure, type ExecutionOutcome, type OutcomePolicy } from "./judge.js";
|
|
4
4
|
import { type PreparedRunArtifactPaths, type RunIdInUse } from "./paths.js";
|
|
5
5
|
import { RunSecrets } from "./secrets.js";
|
|
6
|
+
import { type HostClock, type HostSuspension } from "./host-suspension.js";
|
|
6
7
|
import type { RunStudyProvenance } from "./study-provenance.js";
|
|
7
8
|
interface StartRunOptions {
|
|
8
9
|
/** Project directory, in the form the route resolved it. */
|
|
@@ -21,8 +22,10 @@ interface StartRunOptions {
|
|
|
21
22
|
open: boolean;
|
|
22
23
|
render?: typeof renderObserver | undefined;
|
|
23
24
|
};
|
|
24
|
-
/** Clock for `createdAt` and the latest pointer. */
|
|
25
|
+
/** Clock for `createdAt` and the latest pointer. Defaults to the clock's `now`. */
|
|
25
26
|
now?: (() => number) | undefined;
|
|
27
|
+
/** The run's wall clock and heartbeat timer. Defaults to systemHostClock. */
|
|
28
|
+
clock?: HostClock | undefined;
|
|
26
29
|
/** Warnings about the study's own fields. Every bundle write records each as a warn event. */
|
|
27
30
|
warnings?: readonly string[] | undefined;
|
|
28
31
|
/** The route's known values, created before the run so a refusal before it is scrubbed too. */
|
|
@@ -42,9 +45,13 @@ interface StartRunInput {
|
|
|
42
45
|
readonly runId?: string | undefined;
|
|
43
46
|
/** Open the Observer page once it renders. */
|
|
44
47
|
readonly open?: boolean | undefined;
|
|
45
|
-
/**
|
|
48
|
+
/**
|
|
49
|
+
* `renderObserver` is the seam FinishedRun.renderObserver calls; `hostClock` is the run's
|
|
50
|
+
* clock and heartbeat timer.
|
|
51
|
+
*/
|
|
46
52
|
readonly deps?: {
|
|
47
53
|
readonly renderObserver?: typeof renderObserver | undefined;
|
|
54
|
+
readonly hostClock?: HostClock | undefined;
|
|
48
55
|
} | undefined;
|
|
49
56
|
}
|
|
50
57
|
/** What a recorded route passes to startRun besides its plan and input. */
|
|
@@ -100,6 +107,12 @@ interface Run {
|
|
|
100
107
|
* and a run that failed before any session (a refused E2B key) does not.
|
|
101
108
|
*/
|
|
102
109
|
participantStarted(): void;
|
|
110
|
+
/**
|
|
111
|
+
* The times this process did not run while the run was alive, most likely because the host
|
|
112
|
+
* slept, as the run's heartbeat saw them up to this call. Every bundle write records each as a
|
|
113
|
+
* `host.suspended` event; a route reads them to say which participant failures they caused.
|
|
114
|
+
*/
|
|
115
|
+
hostSuspensions(): readonly HostSuspension[];
|
|
103
116
|
/**
|
|
104
117
|
* The one final publication: run.json with `outcome` set from `outcome`, then the status
|
|
105
118
|
* outcome copied from it, then review.json, review.md, events.ndjson,
|
package/dist/run/run.js
CHANGED
|
@@ -16,6 +16,7 @@ import { scrubRunSandboxIds, withPublicSandboxIds } from "./sandbox-ids.js";
|
|
|
16
16
|
import { RunSecrets } from "./secrets.js";
|
|
17
17
|
import { RUN_STATUS_FILE, beginRunStatus, runStatusOutcome, } from "./status.js";
|
|
18
18
|
import { readRunJsonIfExists, runJsonValue } from "./locate.js";
|
|
19
|
+
import { hostSuspensionEvents, systemHostClock, } from "./host-suspension.js";
|
|
19
20
|
/** An entry point called in a state that does not admit it. */
|
|
20
21
|
class RunLifecycleError extends Error {
|
|
21
22
|
name = "RunLifecycleError";
|
|
@@ -127,10 +128,14 @@ function runPublisher(args) {
|
|
|
127
128
|
type: "study.warning",
|
|
128
129
|
message: redactText(message),
|
|
129
130
|
}));
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
131
|
+
// So do the host suspensions the heartbeat has seen by the time of the write.
|
|
132
|
+
const withRunEvents = (events) => {
|
|
133
|
+
const owned = [
|
|
134
|
+
...studyWarnings,
|
|
135
|
+
...hostSuspensionEvents(runStatus.suspensions(), Date.parse(createdAt)),
|
|
136
|
+
];
|
|
137
|
+
return [...events.filter((event) => !owned.some((mine) => mine.id === event.id)), ...owned];
|
|
138
|
+
};
|
|
134
139
|
let pointerWritten = false;
|
|
135
140
|
// The bundle of the last write asked for, and the outcome every later write carries once a
|
|
136
141
|
// signal stopped the run, so no write after the interrupt can drop it.
|
|
@@ -162,7 +167,7 @@ function runPublisher(args) {
|
|
|
162
167
|
const recorded = interrupted ?? outcome;
|
|
163
168
|
const publicBundle = await withPublicSandboxIds(paths, {
|
|
164
169
|
...evidence,
|
|
165
|
-
events:
|
|
170
|
+
events: withRunEvents(evidence.events),
|
|
166
171
|
cwd: PUBLIC_TARGET_CWD,
|
|
167
172
|
...(recorded === undefined ? {} : { outcome: recorded }),
|
|
168
173
|
});
|
|
@@ -259,8 +264,8 @@ export async function runScope(fn) {
|
|
|
259
264
|
const created = await createRunArtifactPaths(options.cwd, runId);
|
|
260
265
|
if (!created.ok)
|
|
261
266
|
return created;
|
|
262
|
-
// beginRunStatus reads only the mode, the study provenance
|
|
263
|
-
// the run's options.
|
|
267
|
+
// beginRunStatus reads only the mode, the study provenance, the sandboxes field and the
|
|
268
|
+
// clock from the run's options.
|
|
264
269
|
const runStatus = beginRunStatus(created.paths, { ...options, runId });
|
|
265
270
|
status = runStatus;
|
|
266
271
|
let interruptBundle;
|
|
@@ -290,7 +295,7 @@ export async function runScope(fn) {
|
|
|
290
295
|
return args.length === 1 ? startRunWith(args[0]) : startRecordedRun(startRunWith, ...args);
|
|
291
296
|
}
|
|
292
297
|
const openRun = (options, runId, paths, runStatus) => {
|
|
293
|
-
const now = options.now ??
|
|
298
|
+
const now = options.now ?? (options.clock ?? systemHostClock).now;
|
|
294
299
|
const createdAt = new Date(now()).toISOString();
|
|
295
300
|
let participantsRan = false;
|
|
296
301
|
const publisher = runPublisher({
|
|
@@ -317,6 +322,7 @@ export async function runScope(fn) {
|
|
|
317
322
|
participantStarted() {
|
|
318
323
|
participantsRan = true;
|
|
319
324
|
},
|
|
325
|
+
hostSuspensions: () => runStatus.suspensions(),
|
|
320
326
|
writeSnapshot(bundle) {
|
|
321
327
|
if (closed)
|
|
322
328
|
return refuse("The run scope has closed.");
|
|
@@ -374,6 +380,7 @@ async function startRecordedRun(start, plan, input, route) {
|
|
|
374
380
|
renderReview: route.renderReview,
|
|
375
381
|
observer: { open: input.open === true, render: input.deps?.renderObserver },
|
|
376
382
|
now: route.now,
|
|
383
|
+
clock: input.deps?.hostClock,
|
|
377
384
|
secrets: route.secrets,
|
|
378
385
|
});
|
|
379
386
|
if (!started.ok)
|
package/dist/run/status.d.ts
CHANGED
|
@@ -2,6 +2,7 @@ import type { RunBundle } from "./bundle.js";
|
|
|
2
2
|
import { type RunStudyProvenance } from "./study-provenance.js";
|
|
3
3
|
import { type PreparedOutputRoot } from "./contained-output.js";
|
|
4
4
|
import type { ExecutionOutcome } from "./judge.js";
|
|
5
|
+
import { type HostClock, type HostSuspension } from "./host-suspension.js";
|
|
5
6
|
export declare const RUN_STATUS_SCHEMA = "humanish.run-status.v1";
|
|
6
7
|
/** The file, relative to the run directory. */
|
|
7
8
|
export declare const RUN_STATUS_FILE = "status.json";
|
|
@@ -87,17 +88,22 @@ export interface RunStatusHandle {
|
|
|
87
88
|
interrupt(signal: RunInterruptSignal): Promise<boolean>;
|
|
88
89
|
/** True from the moment `interrupt` accepts a signal, before its write lands. */
|
|
89
90
|
readonly interrupted: boolean;
|
|
91
|
+
/** The host suspensions the cadence saw until now, or until it stopped. */
|
|
92
|
+
suspensions(): readonly HostSuspension[];
|
|
90
93
|
}
|
|
91
94
|
export interface BeginRunStatusOptions {
|
|
92
95
|
runId: string;
|
|
93
96
|
mode: "dry-run" | "live";
|
|
94
97
|
study?: RunStudyProvenance | undefined;
|
|
95
98
|
sandboxes?: "none" | undefined;
|
|
99
|
+
/** The clock its timestamps and cadence read. Defaults to systemHostClock. */
|
|
100
|
+
clock?: HostClock | undefined;
|
|
96
101
|
}
|
|
97
102
|
/**
|
|
98
103
|
* Start a run's status record and keep it fresh. Fire-and-forget by design: a status write that
|
|
99
|
-
* fails must never fail the run it describes, so every write swallows its error. The
|
|
100
|
-
*
|
|
104
|
+
* fails must never fail the run it describes, so every write swallows its error. The cadence is
|
|
105
|
+
* the run's heartbeat, which also records each host suspension, and its timer never keeps the
|
|
106
|
+
* process alive.
|
|
101
107
|
*/
|
|
102
108
|
export declare function beginRunStatus(runPaths: PreparedOutputRoot, options: BeginRunStatusOptions): RunStatusHandle;
|
|
103
109
|
/** The three ways a run reads from disk. `interrupted` is a record its stopped process wrote, or a
|
package/dist/run/status.js
CHANGED
|
@@ -12,6 +12,7 @@
|
|
|
12
12
|
// operator may share the run directory, so a share-safety gate must have nothing to strip here.
|
|
13
13
|
import { isProvenanceField, studyFields } from "./study-provenance.js";
|
|
14
14
|
import { writeContainedOutputFile } from "./contained-output.js";
|
|
15
|
+
import { startHeartbeat, systemHostClock, } from "./host-suspension.js";
|
|
15
16
|
export const RUN_STATUS_SCHEMA = "humanish.run-status.v1";
|
|
16
17
|
/** The file, relative to the run directory. */
|
|
17
18
|
export const RUN_STATUS_FILE = "status.json";
|
|
@@ -52,11 +53,13 @@ export function runStatusOutcome(bundle) {
|
|
|
52
53
|
}
|
|
53
54
|
/**
|
|
54
55
|
* Start a run's status record and keep it fresh. Fire-and-forget by design: a status write that
|
|
55
|
-
* fails must never fail the run it describes, so every write swallows its error. The
|
|
56
|
-
*
|
|
56
|
+
* fails must never fail the run it describes, so every write swallows its error. The cadence is
|
|
57
|
+
* the run's heartbeat, which also records each host suspension, and its timer never keeps the
|
|
58
|
+
* process alive.
|
|
57
59
|
*/
|
|
58
60
|
export function beginRunStatus(runPaths, options) {
|
|
59
|
-
const
|
|
61
|
+
const clock = options.clock ?? systemHostClock;
|
|
62
|
+
const iso = () => new Date(clock.now()).toISOString();
|
|
60
63
|
const startedAt = iso();
|
|
61
64
|
const base = {
|
|
62
65
|
schema: RUN_STATUS_SCHEMA,
|
|
@@ -84,19 +87,18 @@ export function beginRunStatus(runPaths, options) {
|
|
|
84
87
|
return writing;
|
|
85
88
|
};
|
|
86
89
|
const started = write(base);
|
|
87
|
-
const
|
|
90
|
+
const heartbeat = startHeartbeat(clock, RUN_STATUS_TOUCH_MS, () => {
|
|
88
91
|
if (finished)
|
|
89
92
|
return;
|
|
90
93
|
void write({ ...base, updatedAt: iso() });
|
|
91
|
-
}
|
|
92
|
-
timer.unref?.();
|
|
94
|
+
});
|
|
93
95
|
const handle = {
|
|
94
96
|
started,
|
|
95
97
|
async finish(outcome) {
|
|
96
98
|
if (finished)
|
|
97
99
|
return;
|
|
98
100
|
finished = true;
|
|
99
|
-
|
|
101
|
+
heartbeat.stop();
|
|
100
102
|
const completedAt = iso();
|
|
101
103
|
finishedRecord = {
|
|
102
104
|
...base,
|
|
@@ -122,7 +124,7 @@ export function beginRunStatus(runPaths, options) {
|
|
|
122
124
|
}
|
|
123
125
|
finished = true;
|
|
124
126
|
interrupted = true;
|
|
125
|
-
|
|
127
|
+
heartbeat.stop();
|
|
126
128
|
const completedAt = iso();
|
|
127
129
|
await write({ ...base, state: "interrupted", updatedAt: completedAt, completedAt, signal });
|
|
128
130
|
return true;
|
|
@@ -130,6 +132,7 @@ export function beginRunStatus(runPaths, options) {
|
|
|
130
132
|
get interrupted() {
|
|
131
133
|
return interrupted;
|
|
132
134
|
},
|
|
135
|
+
suspensions: () => heartbeat.suspensions(),
|
|
133
136
|
};
|
|
134
137
|
return handle;
|
|
135
138
|
}
|
|
@@ -4,6 +4,7 @@ import type { CuaActorSessionOptions } from "../actors/computer-use/actor.js";
|
|
|
4
4
|
import type { AutomaticAnalysisDeps, runAutomaticAnalysis } from "../analysis/automatic.js";
|
|
5
5
|
import type { CuaLoopResult } from "../actors/computer-use/loop.js";
|
|
6
6
|
import type { renderObserver } from "../observer/render.js";
|
|
7
|
+
import type { HostClock } from "../run/host-suspension.js";
|
|
7
8
|
import type { TerminalCostProbe } from "../routes/terminal/types.js";
|
|
8
9
|
import type { DetachedTimers } from "../substrates/detached.js";
|
|
9
10
|
import type { E2BDesktopModule } from "../substrates/e2b/sdk.js";
|
|
@@ -17,6 +18,11 @@ export interface StudyDeps {
|
|
|
17
18
|
readonly renderObserver?: typeof renderObserver;
|
|
18
19
|
/** The clock, in epoch milliseconds. Defaults to Date.now. */
|
|
19
20
|
readonly now?: () => number;
|
|
21
|
+
/**
|
|
22
|
+
* Every recorded route: the run's wall clock and heartbeat timer, which time the run and notice
|
|
23
|
+
* a host suspension. Defaults to systemHostClock.
|
|
24
|
+
*/
|
|
25
|
+
readonly hostClock?: HostClock;
|
|
20
26
|
/** Clock and sleep for detached provisioning steps. */
|
|
21
27
|
readonly detachedTimers?: DetachedTimers;
|
|
22
28
|
/**
|