humanish 0.121.0 → 0.122.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/dist/actors/computer-use/actor.d.ts +3 -0
- package/dist/actors/computer-use/actor.js +1 -0
- package/dist/actors/computer-use/loop/backstop.d.ts +6 -1
- package/dist/actors/computer-use/loop/backstop.js +13 -1
- package/dist/actors/computer-use/loop/clock.d.ts +20 -0
- package/dist/actors/computer-use/loop/clock.js +64 -0
- package/dist/actors/computer-use/loop/ending.d.ts +1 -1
- package/dist/actors/computer-use/loop/session.d.ts +1 -1
- package/dist/actors/computer-use/loop/session.js +2 -1
- package/dist/actors/computer-use/loop/types.d.ts +16 -0
- package/dist/actors/computer-use/loop.d.ts +1 -1
- package/dist/actors/computer-use/loop.js +43 -8
- package/dist/actors/computer-use/wait.d.ts +3 -2
- package/dist/actors/computer-use/wait.js +5 -4
- package/dist/actors/contract.d.ts +3 -3
- package/dist/actors/contract.js +3 -0
- package/dist/actors/stop-cause.js +1 -0
- package/dist/routes/computer-use/bundle.d.ts +0 -1
- package/dist/routes/computer-use/bundle.js +0 -1
- package/dist/routes/computer-use/diagnostics.d.ts +1 -1
- package/dist/routes/computer-use/diagnostics.js +4 -0
- package/dist/routes/computer-use/fanout-bundle.d.ts +2 -3
- package/dist/routes/computer-use/fanout-bundle.js +2 -12
- package/dist/routes/computer-use/live-phase.d.ts +0 -2
- package/dist/routes/computer-use/live-phase.js +2 -4
- package/dist/routes/computer-use/participant-execution.d.ts +9 -15
- package/dist/routes/computer-use/participant-execution.js +22 -31
- package/dist/routes/computer-use/participant-model.js +17 -0
- package/dist/routes/computer-use/participant-runs.d.ts +11 -1
- package/dist/routes/computer-use/participant-runs.js +31 -4
- package/dist/routes/computer-use/plan.js +8 -1
- package/dist/routes/computer-use/result.js +1 -2
- package/dist/routes/computer-use/types.d.ts +17 -5
- package/dist/routes/shared-world/provisioned.js +2 -2
- package/dist/run/host-suspension.d.ts +2 -2
- package/dist/run/host-suspension.js +2 -2
- package/dist/run/judge.d.ts +1 -1
- package/dist/study/arrivals.d.ts +52 -5
- package/dist/study/arrivals.js +105 -15
- package/dist/study/keys.d.ts +6 -0
- package/dist/study/keys.js +4 -0
- package/dist/study/migrate/v2.js +8 -5
- package/dist/study/parse/actors.d.ts +1 -1
- package/dist/study/parse/actors.js +33 -4
- package/dist/study/plan-participants.d.ts +10 -1
- package/dist/study/plan-participants.js +10 -1
- package/dist/study/preflight.js +11 -1
- package/dist/study/types.d.ts +26 -0
- package/dist/study/validation.d.ts +3 -2
- package/dist/study/validation.js +22 -4
- package/docs/contracts/run-bundle.md +2 -2
- package/docs/contracts/schemas.md +24 -4
- package/package.json +1 -1
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { scheduledParticipantDeps } from "./participant-runs.js";
|
|
1
2
|
import { judgeParticipantRecords } from "../../run/judge.js";
|
|
2
3
|
import { createE2BParticipantDesktop } from "./e2b-desktop/desktop.js";
|
|
3
4
|
import { createInProcessDesktop } from "./in-process-desktop.js";
|
|
@@ -31,7 +32,7 @@ export function makeParticipantWriteScreenshot(artifactRoot, spec, screenshots)
|
|
|
31
32
|
return rel;
|
|
32
33
|
};
|
|
33
34
|
}
|
|
34
|
-
/** A blocked participant outcome
|
|
35
|
+
/** A blocked participant outcome: the pipeline gate or the study budget skipped it. */
|
|
35
36
|
function skippedOutcome(spec, reason, arrival) {
|
|
36
37
|
return {
|
|
37
38
|
spec,
|
|
@@ -110,8 +111,8 @@ export async function runCuaParticipant(spec, deps) {
|
|
|
110
111
|
if (session)
|
|
111
112
|
await recordParticipantTrace(spec, deps, session, warnings);
|
|
112
113
|
const { noEngagement, selfReportedBlocker, reportedFriction } = judgeParticipantSession(session, deps, warnings);
|
|
113
|
-
// A provider-cleanup or provider-policy failure
|
|
114
|
-
//
|
|
114
|
+
// A provider-cleanup or provider-policy failure counts in the participant summary's harness
|
|
115
|
+
// errors.
|
|
115
116
|
const harnessError = sessionError !== undefined ||
|
|
116
117
|
providerCleanupError !== undefined ||
|
|
117
118
|
providerPolicyError !== undefined ||
|
|
@@ -191,33 +192,26 @@ function pipelineGate(open, onClose) {
|
|
|
191
192
|
}
|
|
192
193
|
/**
|
|
193
194
|
* Run N>1 E2B participants on the study's schedule (runOnSchedule: each at its start offset, at
|
|
194
|
-
* most `concurrency` at once), behind the pipeline gate when the subject is set up in each sandbox
|
|
195
|
-
*
|
|
196
|
-
*
|
|
197
|
-
*
|
|
198
|
-
*
|
|
199
|
-
* ever enumerates.
|
|
195
|
+
* most `concurrency` at once), behind the pipeline gate when the subject is set up in each sandbox.
|
|
196
|
+
* A harness error ends only its own participant: one desktop's stall or one provider failure does
|
|
197
|
+
* not predict the next participant's, so the schedule goes on. Only a closed gate ends every wait
|
|
198
|
+
* for a later start at once, and the participants it skips are `blocked` with its reason. Each
|
|
199
|
+
* participant tears down its own sandbox by id; nothing here ever enumerates.
|
|
200
200
|
*
|
|
201
201
|
* Exported for the total-runner tests: the injectable runner lets a test make one participant
|
|
202
202
|
* throw (the exact class the guard exists for) without a live sandbox. Production always uses
|
|
203
203
|
* the default.
|
|
204
204
|
*/
|
|
205
205
|
export async function runCuaParticipants(runs, deps, concurrency, runParticipant = runCuaParticipant) {
|
|
206
|
-
const failFast = { tripped: false, reason: "" };
|
|
207
206
|
const stopping = new AbortController();
|
|
208
207
|
// An app-url subject is only opened, so one participant's start says nothing about another's.
|
|
209
208
|
const gate = pipelineGate(deps.subject.kind === "app-url", () => stopping.abort());
|
|
210
|
-
|
|
209
|
+
return runOnSchedule(runs, {
|
|
211
210
|
startAfterMs: (spec) => spec.planned.startAfterMs,
|
|
212
211
|
slots: concurrency,
|
|
213
212
|
now: deps.now,
|
|
214
213
|
signal: stopping.signal,
|
|
215
214
|
}, async (spec, _index, { scheduledAt }) => {
|
|
216
|
-
if (failFast.tripped) {
|
|
217
|
-
return skippedOutcome(spec, `skipped: ${failFast.reason}`, { scheduledAt });
|
|
218
|
-
}
|
|
219
|
-
// A participant trips fail-fast only after the gate opened, or as it closes the gate, so no
|
|
220
|
-
// participant is still waiting at the gate when fail-fast skips the rest.
|
|
221
215
|
const id = spec.planned.id;
|
|
222
216
|
if ((await gate.enter(id)) === "closed") {
|
|
223
217
|
return skippedOutcome(spec, `skipped: participant ${gate.holder()} failed to provision its world (pipeline gate)`, { scheduledAt });
|
|
@@ -233,6 +227,7 @@ export async function runCuaParticipants(runs, deps, concurrency, runParticipant
|
|
|
233
227
|
try {
|
|
234
228
|
outcome = await runParticipant(spec, {
|
|
235
229
|
...deps,
|
|
230
|
+
...scheduledParticipantDeps(spec.planned, deps, scheduledAt),
|
|
236
231
|
signalReady: () => {
|
|
237
232
|
ready = true;
|
|
238
233
|
gate.ready();
|
|
@@ -258,23 +253,17 @@ export async function runCuaParticipants(runs, deps, concurrency, runParticipant
|
|
|
258
253
|
endedAt: deps.now(),
|
|
259
254
|
};
|
|
260
255
|
}
|
|
261
|
-
// A participant that stopped before its session
|
|
262
|
-
//
|
|
263
|
-
//
|
|
256
|
+
// A participant that stopped before its session without a desktop says nothing about the
|
|
257
|
+
// subject, so the next participant starts first. A runner that threw is a harness defect: it
|
|
258
|
+
// closes the gate.
|
|
264
259
|
if (!ready)
|
|
265
260
|
gate.failed(id, { handOver: !threw && outcome.sandboxId === undefined });
|
|
266
|
-
if (outcome.harnessError && (ready || threw) && !failFast.tripped) {
|
|
267
|
-
failFast.tripped = true;
|
|
268
|
-
failFast.reason = `a prior participant (${outcome.spec.planned.id}) ended in a harness error (fail-fast)`;
|
|
269
|
-
stopping.abort();
|
|
270
|
-
}
|
|
271
261
|
// A participant the study budget skipped never requested its desktop.
|
|
272
262
|
return {
|
|
273
263
|
...outcome,
|
|
274
264
|
arrival: outcome.skippedReason === undefined ? arrival : { scheduledAt },
|
|
275
265
|
};
|
|
276
266
|
});
|
|
277
|
-
return { outcomes, ...(failFast.tripped ? { failFastReason: failFast.reason } : {}) };
|
|
278
267
|
}
|
|
279
268
|
/** Project one participant outcome (or a dry-run contract spec) into the public CuaParticipantResult. */
|
|
280
269
|
export function toParticipantResult(spec, outcome, subject, dryRun) {
|
|
@@ -369,19 +358,21 @@ export function toParticipantResult(spec, outcome, subject, dryRun) {
|
|
|
369
358
|
}
|
|
370
359
|
/**
|
|
371
360
|
* Run every participant of a live run. The in-process route drives its single participant in this
|
|
372
|
-
* process; one hosted participant runs alone; a fan-out runs at the plan's concurrency and
|
|
361
|
+
* process; one hosted participant runs alone; a fan-out runs at the plan's concurrency, and stops
|
|
362
|
+
* early only when the pipeline gate closes.
|
|
373
363
|
*/
|
|
374
364
|
export async function runAllCuaParticipants(runs, deps, participantPlan, inProcess) {
|
|
365
|
+
// A participant that runs alone starts with the run.
|
|
366
|
+
const alone = { ...deps, ...scheduledParticipantDeps(runs[0].planned, deps, deps.now()) };
|
|
375
367
|
if (inProcess) {
|
|
376
368
|
// The caller's executor stands in for a desktop, and the shared runner drives the participant.
|
|
377
369
|
const outcome = await runCuaParticipant(runs[0], {
|
|
378
|
-
...
|
|
370
|
+
...alone,
|
|
379
371
|
createDesktop: () => createInProcessDesktop(deps),
|
|
380
372
|
});
|
|
381
|
-
return
|
|
373
|
+
return [outcome];
|
|
382
374
|
}
|
|
383
375
|
if (runs.length === 1)
|
|
384
|
-
return
|
|
385
|
-
|
|
386
|
-
return { outcomes: ran.outcomes, failFastReason: ran.failFastReason };
|
|
376
|
+
return [await runCuaParticipant(runs[0], alone)];
|
|
377
|
+
return runCuaParticipants([...runs], deps, participantPlan.concurrency);
|
|
387
378
|
}
|
|
@@ -99,6 +99,7 @@ export function participantSessionOptions(spec, deps, ready, provider, writeScre
|
|
|
99
99
|
: spec.instructions,
|
|
100
100
|
persona: spec.persona,
|
|
101
101
|
timeoutMs: deps.timeoutMs,
|
|
102
|
+
...participantClock(spec.planned.limits.timeZone, deps),
|
|
102
103
|
// The brain is either a keyed API client or a CLI the operator is already signed in to.
|
|
103
104
|
// Everything below this line (loop, executor, trace, affordances) is identical either
|
|
104
105
|
// way, which is what makes a local-agent run comparable to an API one.
|
|
@@ -305,3 +306,19 @@ export function makeCuaRunBudget(maxTotalUsd) {
|
|
|
305
306
|
},
|
|
306
307
|
};
|
|
307
308
|
}
|
|
309
|
+
/**
|
|
310
|
+
* The participant's clock: its time zone, and its scheduled end on the run's clock. A participant
|
|
311
|
+
* with an end runs its session on the run's clock, so the end and the schedule read one clock.
|
|
312
|
+
*/
|
|
313
|
+
function participantClock(timeZone, deps) {
|
|
314
|
+
const { scheduledEnd } = deps;
|
|
315
|
+
if (timeZone === undefined && scheduledEnd === undefined)
|
|
316
|
+
return {};
|
|
317
|
+
return {
|
|
318
|
+
clock: {
|
|
319
|
+
...(timeZone === undefined ? {} : { timeZone }),
|
|
320
|
+
...(scheduledEnd === undefined ? {} : { end: scheduledEnd }),
|
|
321
|
+
},
|
|
322
|
+
...(scheduledEnd === undefined ? {} : { now: deps.now }),
|
|
323
|
+
};
|
|
324
|
+
}
|
|
@@ -3,7 +3,8 @@ import { type StudyConfig } from "../../study/types.js";
|
|
|
3
3
|
import { type ResolvedPersona } from "../../study/persona.js";
|
|
4
4
|
import { type RunRerunLineage } from "../../run/bundle.js";
|
|
5
5
|
import type { PreparedSelectedOutputDirectory } from "../../run/contained-output.js";
|
|
6
|
-
import { type CuaActorStudyErrorCode, type CuaParticipantPlan, type CuaParticipantPlanEntry, type DesktopParticipantRun, type ParticipantRunsAndPlan } from "./types.js";
|
|
6
|
+
import { type CuaActorStudyErrorCode, type CuaParticipantDeps, type CuaParticipantPlan, type CuaParticipantPlanEntry, type DesktopParticipantRun, type ParticipantRunsAndPlan } from "./types.js";
|
|
7
|
+
import { type ParticipantWindow } from "../../study/arrivals.js";
|
|
7
8
|
/**
|
|
8
9
|
* The session budget of a study that declares no execution.timeoutMs. On a provisioned route it is
|
|
9
10
|
* what the `ceilingMs` sandbox ceiling leaves after provisioning, seeding and the teardown buffer.
|
|
@@ -16,6 +17,15 @@ export declare function defaultSessionTimeoutMs(config: StudyConfig, ceilingMs:
|
|
|
16
17
|
* git clone for an upload+extract, but the shared install/build/state/start/probe pipeline
|
|
17
18
|
* costs the same wall-clock room either way. */
|
|
18
19
|
export declare function resolveParticipantSandboxMs(config: StudyConfig, ceilingMs: number): number;
|
|
20
|
+
/**
|
|
21
|
+
* The deps one participant runs with once its start comes: its own session budget and sandbox
|
|
22
|
+
* deadline, and with a declared end, that end on the run's clock (`scheduledAt` is when it was
|
|
23
|
+
* due, from runOnSchedule).
|
|
24
|
+
*/
|
|
25
|
+
export declare function scheduledParticipantDeps(participant: ParticipantWindow, deps: {
|
|
26
|
+
readonly timeoutMs: number;
|
|
27
|
+
readonly sandboxMs: number;
|
|
28
|
+
}, scheduledAt: number): Pick<CuaParticipantDeps, "timeoutMs" | "sandboxMs" | "scheduledEnd">;
|
|
19
29
|
/**
|
|
20
30
|
* The in-flight participant bound a study declares. Defaults to the participant count: every
|
|
21
31
|
* declared participant runs at once, because a throttle nobody asked for silently turns "N actors
|
|
@@ -18,7 +18,7 @@ import { composeParticipantInstructions, DEFAULT_MISSION } from "./participant-p
|
|
|
18
18
|
import { resolveCuaRerunSelection } from "./rerun-selection.js";
|
|
19
19
|
import { plural } from "../../run/text.js";
|
|
20
20
|
import { formatDuration } from "../../run/projection.js";
|
|
21
|
-
import { describeArrivals, planArrivals } from "../../study/arrivals.js";
|
|
21
|
+
import { describeArrivals, participantSessionMs, planArrivals, scheduledEndAt, } from "../../study/arrivals.js";
|
|
22
22
|
/**
|
|
23
23
|
* The session budget of a study that declares no execution.timeoutMs. On a provisioned route it is
|
|
24
24
|
* what the `ceilingMs` sandbox ceiling leaves after provisioning, seeding and the teardown buffer.
|
|
@@ -45,6 +45,30 @@ export function resolveParticipantSandboxMs(config, ceilingMs) {
|
|
|
45
45
|
timeoutMs +
|
|
46
46
|
sandboxHeadroomMs(provisionedRoute ? { seed: config.subject.state?.seed ?? [] } : undefined));
|
|
47
47
|
}
|
|
48
|
+
/**
|
|
49
|
+
* A participant's own session budget and sandbox deadline: the study's, or with a declared end its
|
|
50
|
+
* window plus room for the closing report (participantSessionMs), with the same setup and
|
|
51
|
+
* teardown headroom the study's sandbox has over its session.
|
|
52
|
+
*/
|
|
53
|
+
function participantWindow(participant, study) {
|
|
54
|
+
const timeoutMs = participantSessionMs(participant, study.sessionBudgetMs);
|
|
55
|
+
return { timeoutMs, sandboxMs: study.sandboxMs - study.sessionBudgetMs + timeoutMs };
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* The deps one participant runs with once its start comes: its own session budget and sandbox
|
|
59
|
+
* deadline, and with a declared end, that end on the run's clock (`scheduledAt` is when it was
|
|
60
|
+
* due, from runOnSchedule).
|
|
61
|
+
*/
|
|
62
|
+
export function scheduledParticipantDeps(participant, deps, scheduledAt) {
|
|
63
|
+
const end = scheduledEndAt(participant, scheduledAt);
|
|
64
|
+
return {
|
|
65
|
+
...participantWindow(participant, {
|
|
66
|
+
sessionBudgetMs: deps.timeoutMs,
|
|
67
|
+
sandboxMs: deps.sandboxMs,
|
|
68
|
+
}),
|
|
69
|
+
...(end === undefined ? {} : { scheduledEnd: end }),
|
|
70
|
+
};
|
|
71
|
+
}
|
|
48
72
|
/**
|
|
49
73
|
* The in-flight participant bound a study declares. Defaults to the participant count: every
|
|
50
74
|
* declared participant runs at once, because a throttle nobody asked for silently turns "N actors
|
|
@@ -114,10 +138,11 @@ export function participantRunsAndPlan(plan, opts = {}) {
|
|
|
114
138
|
});
|
|
115
139
|
const resolved = envLoweredConcurrency(plan.concurrency, participantCount, env);
|
|
116
140
|
const concurrency = resolved.bound;
|
|
117
|
-
const { sessionBudgetMs
|
|
141
|
+
const { sessionBudgetMs } = plan;
|
|
118
142
|
const schedule = resolved.envLoweredFrom === undefined
|
|
119
143
|
? plan.arrivals
|
|
120
|
-
: planArrivals(participants.map((participant) => participant.startAfterMs), { slots: concurrency, sessionMs: sessionBudgetMs });
|
|
144
|
+
: planArrivals(participants.map((participant) => participant.startAfterMs), { slots: concurrency, sessionMs: sessionBudgetMs }, participants.map((participant) => participantSessionMs(participant, sessionBudgetMs)));
|
|
145
|
+
const sandboxesMs = participants.reduce((sum, participant) => sum + participantWindow(participant, plan).sandboxMs, 0);
|
|
121
146
|
const participantPlan = {
|
|
122
147
|
strategy: CUA_FANOUT_STRATEGY,
|
|
123
148
|
laneCount: participantCount,
|
|
@@ -127,7 +152,7 @@ export function participantRunsAndPlan(plan, opts = {}) {
|
|
|
127
152
|
: { envLoweredConcurrencyFrom: resolved.envLoweredFrom }),
|
|
128
153
|
waves: Math.ceil(participantCount / concurrency),
|
|
129
154
|
perLaneSessionBudgetMs: sessionBudgetMs,
|
|
130
|
-
worstCaseSandboxMinutes: Math.round(
|
|
155
|
+
worstCaseSandboxMinutes: Math.round(sandboxesMs / 60_000),
|
|
131
156
|
...(schedule.declared ? { schedule } : {}),
|
|
132
157
|
dryRun: plan.dryRun,
|
|
133
158
|
lanes: runs.map((spec) => ({
|
|
@@ -158,6 +183,7 @@ export function participantRunsAndPlan(plan, opts = {}) {
|
|
|
158
183
|
...(spec.planned.startAfterMs === undefined
|
|
159
184
|
? {}
|
|
160
185
|
: { startAfterMs: spec.planned.startAfterMs }),
|
|
186
|
+
...(spec.planned.endAfterMs === undefined ? {} : { endAfterMs: spec.planned.endAfterMs }),
|
|
161
187
|
})),
|
|
162
188
|
};
|
|
163
189
|
return { runs, participantPlan };
|
|
@@ -182,6 +208,7 @@ export function formatParticipantPlanEntry(entry) {
|
|
|
182
208
|
entry.caseGroup ? `case=${entry.caseGroup}` : undefined,
|
|
183
209
|
entry.reasoningEffort ? `effort=${entry.reasoningEffort}` : undefined,
|
|
184
210
|
entry.startAfterMs === undefined ? undefined : `start=+${formatDuration(entry.startAfterMs)}`,
|
|
211
|
+
entry.endAfterMs === undefined ? undefined : `end=+${formatDuration(entry.endAfterMs)}`,
|
|
185
212
|
].filter((part) => part !== undefined);
|
|
186
213
|
return `${entry.id}: persona=${entry.persona}${taxonomy.length > 0 ? ` ${taxonomy.join(" ")}` : ""} device=${entry.device} ${entry.resolution[0]}x${entry.resolution[1]} prompt#${entry.instructionDigest}${entry.targetDigest ? ` target#${entry.targetDigest}` : ""}`;
|
|
187
214
|
}
|
|
@@ -9,7 +9,7 @@ import { studyUrlCredentialReason } from "../../study/url-credentials.js";
|
|
|
9
9
|
import { subjectStateInvalidReason } from "../../study/parse/subject-state.js";
|
|
10
10
|
import { brainOf, callerBrainOf, planCaps, desktopRequirements, planBase, provisionedSubject, } from "../../study/plan-base.js";
|
|
11
11
|
import { computerUseParticipants, declaredTargets } from "../../study/plan-participants.js";
|
|
12
|
-
import { planStarts } from "../../study/arrivals.js";
|
|
12
|
+
import { planStarts, windowDeadlineRefusal } from "../../study/arrivals.js";
|
|
13
13
|
import { computerUseParticipantCount, participantCountReason } from "../../study/routing.js";
|
|
14
14
|
import { cloneTargetValidationReason, computerUseValidationReason, desktopMediaValidationReason, outputTokenLimitValidationReason, receivingEmailValidationReason, taskProtocolValidationReason, waitLimitValidationReason, } from "../../study/validation.js";
|
|
15
15
|
import { desktopCliProductReason } from "../../study/composition-rules.js";
|
|
@@ -209,6 +209,13 @@ function startsOf(config, runner, limit, ceilingMs) {
|
|
|
209
209
|
if (!atOnce.ok)
|
|
210
210
|
return atOnce;
|
|
211
211
|
const sessionMs = config.execution?.timeoutMs ?? defaultSessionTimeoutMs(config, ceilingMs);
|
|
212
|
+
// A participant with a declared end gets a sandbox sized from its own window.
|
|
213
|
+
if (runner.desktop === "e2b-desktop") {
|
|
214
|
+
const headroomMs = resolveParticipantSandboxMs(config, ceilingMs) - sessionMs;
|
|
215
|
+
const window = windowDeadlineRefusal(runner.participants, { sessionMs, headroomMs }, ceilingMs);
|
|
216
|
+
if (window !== undefined)
|
|
217
|
+
return { ok: false, message: window };
|
|
218
|
+
}
|
|
212
219
|
const slots = { slots: atOnce.count, sessionMs };
|
|
213
220
|
return {
|
|
214
221
|
ok: true,
|
|
@@ -201,7 +201,7 @@ export async function finishCuaRun(setup, finish, ran) {
|
|
|
201
201
|
const appUrl = plannedAppUrl(plan.runner.subject);
|
|
202
202
|
const { runId } = run;
|
|
203
203
|
const participantCount = participantRuns.length;
|
|
204
|
-
const { outcomes,
|
|
204
|
+
const { outcomes, receiving, receivingWarnings, externalCommsWarnings } = ran;
|
|
205
205
|
// Per-participant subject projections.
|
|
206
206
|
const subjects = projectParticipantSubjects({ ...subjectArgs, outcomes, dryRun });
|
|
207
207
|
const aggregate = aggregateCuaSubject({ subjects, outcomes, participantCount, dryRun });
|
|
@@ -222,7 +222,6 @@ export async function finishCuaRun(setup, finish, ran) {
|
|
|
222
222
|
outcomes,
|
|
223
223
|
subjects,
|
|
224
224
|
aggregateSubject,
|
|
225
|
-
...(failFastReason === undefined ? {} : { failFastReason }),
|
|
226
225
|
...(hostSuspension === undefined ? {} : { hostSuspension }),
|
|
227
226
|
});
|
|
228
227
|
const adapterWarnings = [];
|
|
@@ -112,6 +112,8 @@ export interface CuaParticipantPlanEntry {
|
|
|
112
112
|
targetDigest?: string;
|
|
113
113
|
/** The participant's declared start, in ms after the run starts its participants. */
|
|
114
114
|
startAfterMs?: number;
|
|
115
|
+
/** The participant's declared end, on the same clock. */
|
|
116
|
+
endAfterMs?: number;
|
|
115
117
|
}
|
|
116
118
|
/** The pre-flight spend and participant plan (pure; printed to stderr + recorded as a bundle event before
|
|
117
119
|
* any sandbox or provider call; identical in dry-run, marked $0). */
|
|
@@ -138,7 +140,8 @@ export interface CuaParticipantPlan {
|
|
|
138
140
|
lanes: CuaParticipantPlanEntry[];
|
|
139
141
|
}
|
|
140
142
|
/** One participant's outcome in the result projection. Always present in `result.lanes` (length
|
|
141
|
-
* 1 at N=1). A `blocked` participant is one the pipeline
|
|
143
|
+
* 1 at N=1). A `blocked` participant is one the pipeline gate or the study budget skipped before
|
|
144
|
+
* it ran. */
|
|
142
145
|
export interface CuaParticipantResult {
|
|
143
146
|
id: string;
|
|
144
147
|
actorType?: string;
|
|
@@ -149,7 +152,7 @@ export interface CuaParticipantResult {
|
|
|
149
152
|
device: string;
|
|
150
153
|
/** Requested E2B/X screen resolution. See the run stream's desktopGeometry for measurements. */
|
|
151
154
|
resolution: [number, number];
|
|
152
|
-
/** Terminal participant status; "blocked" = skipped (gate
|
|
155
|
+
/** Terminal participant status; "blocked" = skipped (gate or study budget); "contract_proof_only" = dry-run. */
|
|
153
156
|
status: ActorStatus | "blocked" | "contract_proof_only";
|
|
154
157
|
ok: boolean;
|
|
155
158
|
session?: {
|
|
@@ -183,7 +186,7 @@ export interface CuaParticipantSummary {
|
|
|
183
186
|
total: number;
|
|
184
187
|
/** Participants whose own verdict is ok (terminal, engaged, no harness error). */
|
|
185
188
|
passed: number;
|
|
186
|
-
/** Participants skipped by the pipeline gate
|
|
189
|
+
/** Participants skipped by the pipeline gate or the study budget. */
|
|
187
190
|
skipped: number;
|
|
188
191
|
/** Participants that ended in a harness error. */
|
|
189
192
|
harnessErrors: number;
|
|
@@ -363,6 +366,16 @@ export interface ParticipantModelDeps {
|
|
|
363
366
|
env: Record<string, string | undefined>;
|
|
364
367
|
openaiApiKey: string;
|
|
365
368
|
timeoutMs: number;
|
|
369
|
+
/**
|
|
370
|
+
* The participant's scheduled end on `now`'s clock, when it declares participants[].endAfterMs
|
|
371
|
+
* (scheduledParticipantDeps). Its session ends there with stop cause `scheduled_end`.
|
|
372
|
+
*/
|
|
373
|
+
scheduledEnd?: {
|
|
374
|
+
readonly atMs: number;
|
|
375
|
+
readonly windDownMs: number;
|
|
376
|
+
};
|
|
377
|
+
/** The run's clock. A participant with a scheduled end runs its session on it. */
|
|
378
|
+
now: () => number;
|
|
366
379
|
participantCount: number;
|
|
367
380
|
artifactRoot: PreparedOutputRoot;
|
|
368
381
|
redactScreenshots: boolean;
|
|
@@ -441,7 +454,7 @@ export interface ParticipantRunOutcome {
|
|
|
441
454
|
* folded into bundle.events at build time. Empty on the in-process route (no provisioning). */
|
|
442
455
|
phaseRecords: SubjectPhaseEvent[];
|
|
443
456
|
warnings: string[];
|
|
444
|
-
/** Set when the
|
|
457
|
+
/** Set when the pipeline gate or the study budget skipped the participant (a pinned reason). */
|
|
445
458
|
skippedReason?: string;
|
|
446
459
|
/** When a fan-out participant was due and when its desktop was requested. */
|
|
447
460
|
arrival?: ParticipantArrival;
|
|
@@ -481,7 +494,6 @@ export interface CuaFanoutBundleArgs {
|
|
|
481
494
|
source: RunBundle["source"];
|
|
482
495
|
participantPlan: CuaParticipantPlan;
|
|
483
496
|
rerun?: RunRerunLineage;
|
|
484
|
-
failFastReason?: string;
|
|
485
497
|
/** The run's host suspensions and the participant failures they likely caused. */
|
|
486
498
|
hostSuspension?: HostSuspensionReading;
|
|
487
499
|
publicRepo?: string;
|
|
@@ -383,8 +383,8 @@ async function publishInProgress(plane, ctx, live, setup) {
|
|
|
383
383
|
startParticipantFlush(ctx, live, inProgressBundle);
|
|
384
384
|
}
|
|
385
385
|
// Launch N actor sandboxes independently on the study's schedule (runCuaParticipant +
|
|
386
|
-
// runOnSchedule; runCuaParticipants would add a pipeline gate
|
|
387
|
-
//
|
|
386
|
+
// runOnSchedule; runCuaParticipants would add a pipeline gate). Each actor's window is measured on
|
|
387
|
+
// the one orchestrator clock.
|
|
388
388
|
function runParticipants(plane, ctx, live, setup) {
|
|
389
389
|
const { now } = ctx;
|
|
390
390
|
const participants = ctx.plan.plane.participants;
|
|
@@ -50,7 +50,7 @@ export interface HostSuspensionReading {
|
|
|
50
50
|
* without a suspension. A frozen loop's failures surface at the wake (macOS fires every overdue
|
|
51
51
|
* timer at once) or up to a turn timeout later (Linux timers do not count suspended time), so the
|
|
52
52
|
* rule reads who was in flight across the suspension. A skip follows another participant's
|
|
53
|
-
* harness error (the pipeline gate
|
|
54
|
-
*
|
|
53
|
+
* harness error (the pipeline gate), so skips count only when the run had harness errors and
|
|
54
|
+
* every one was in flight across a suspension.
|
|
55
55
|
*/
|
|
56
56
|
export declare function readHostSuspensions(suspensions: readonly HostSuspension[], participants: readonly SuspendedParticipant[], runStartMs: number): HostSuspensionReading | undefined;
|
|
@@ -79,8 +79,8 @@ function inFlight(participant, suspension) {
|
|
|
79
79
|
* without a suspension. A frozen loop's failures surface at the wake (macOS fires every overdue
|
|
80
80
|
* timer at once) or up to a turn timeout later (Linux timers do not count suspended time), so the
|
|
81
81
|
* rule reads who was in flight across the suspension. A skip follows another participant's
|
|
82
|
-
* harness error (the pipeline gate
|
|
83
|
-
*
|
|
82
|
+
* harness error (the pipeline gate), so skips count only when the run had harness errors and
|
|
83
|
+
* every one was in flight across a suspension.
|
|
84
84
|
*/
|
|
85
85
|
export function readHostSuspensions(suspensions, participants, runStartMs) {
|
|
86
86
|
if (suspensions.length === 0)
|
package/dist/run/judge.d.ts
CHANGED
|
@@ -34,7 +34,7 @@ export interface ParticipantFacts {
|
|
|
34
34
|
completionReason?: ActorCompletionReason;
|
|
35
35
|
/** The harness failed before the session reached a terminal status. */
|
|
36
36
|
sessionError?: string;
|
|
37
|
-
/** The pipeline gate or
|
|
37
|
+
/** The pipeline gate or the study budget skipped the participant. */
|
|
38
38
|
skipped: boolean;
|
|
39
39
|
/** hollowCompletion held for its session. */
|
|
40
40
|
noEngagement: boolean;
|
package/dist/study/arrivals.d.ts
CHANGED
|
@@ -6,6 +6,54 @@ import type { RunParticipantArrival } from "../run/bundle.js";
|
|
|
6
6
|
export declare const LATEST_START_AFTER_MS: number;
|
|
7
7
|
/** Whether `value` is a start offset humanish runs: whole milliseconds from 0 to 24 hours. */
|
|
8
8
|
export declare function isStartAfterMs(value: unknown): value is number;
|
|
9
|
+
/** How long before its end a participant is told the time left, unless it declares `windDownMs`. */
|
|
10
|
+
export declare const DEFAULT_WIND_DOWN_MS: number;
|
|
11
|
+
/** A participant's declared window: when it starts and when it ends, on the run's clock. */
|
|
12
|
+
export interface ParticipantWindow {
|
|
13
|
+
readonly id?: string;
|
|
14
|
+
readonly startAfterMs?: number;
|
|
15
|
+
readonly endAfterMs?: number;
|
|
16
|
+
readonly windDownMs?: number;
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Why a participant's declared end cannot run, or undefined: an end that is not a whole number of
|
|
20
|
+
* milliseconds up to 24 hours, one at or before its start, or a wind-down that is not a whole
|
|
21
|
+
* number of milliseconds or has no end. `path` names the entry, as in "participants[1]".
|
|
22
|
+
*/
|
|
23
|
+
export declare function windowReason(entry: {
|
|
24
|
+
readonly id?: string;
|
|
25
|
+
readonly startAfterMs?: number;
|
|
26
|
+
readonly endAfterMs?: unknown;
|
|
27
|
+
readonly windDownMs?: unknown;
|
|
28
|
+
}, path: string): string | undefined;
|
|
29
|
+
/**
|
|
30
|
+
* A participant's session budget: with a declared end, its window plus SCHEDULED_END_ROOM_MS in
|
|
31
|
+
* place of the study's `sessionMs` (execution.timeoutMs or its default); else `sessionMs`.
|
|
32
|
+
*/
|
|
33
|
+
export declare function participantSessionMs(participant: ParticipantWindow, sessionMs: number): number;
|
|
34
|
+
/**
|
|
35
|
+
* When a participant that was due at `scheduledAt` (runOnSchedule's ScheduledStart) ends, on the
|
|
36
|
+
* same clock, or undefined when it declares no end. A participant that waited for a slot still ends
|
|
37
|
+
* at its declared time.
|
|
38
|
+
*/
|
|
39
|
+
export declare function scheduledEndAt(participant: ParticipantWindow, scheduledAt: number): {
|
|
40
|
+
readonly atMs: number;
|
|
41
|
+
readonly windDownMs: number;
|
|
42
|
+
} | undefined;
|
|
43
|
+
/**
|
|
44
|
+
* Why the sandbox of the first participant whose window passes `ceilingMs` cannot live that long,
|
|
45
|
+
* or undefined. Its deadline is its session (participantSessionMs) plus `headroomMs`, what the
|
|
46
|
+
* route adds to a session for setup and teardown (sandboxHeadroomMs).
|
|
47
|
+
*/
|
|
48
|
+
export declare function windowDeadlineRefusal(participants: readonly ParticipantWindow[], sandbox: {
|
|
49
|
+
readonly sessionMs: number;
|
|
50
|
+
readonly headroomMs: number;
|
|
51
|
+
}, ceilingMs: number): string | undefined;
|
|
52
|
+
/**
|
|
53
|
+
* Each participant's window for `study check`, when any participant declares an end: when it starts
|
|
54
|
+
* and ends and when it is first told the time left, or when it starts and its session budget.
|
|
55
|
+
*/
|
|
56
|
+
export declare function describeWindows(participants: readonly ParticipantWindow[], sessionMs: number): string | undefined;
|
|
9
57
|
/** The refusal of a start offset that is not one, naming its `field`. */
|
|
10
58
|
export declare function startAfterMessage(field: string): string;
|
|
11
59
|
/**
|
|
@@ -49,9 +97,10 @@ export interface StartQueue {
|
|
|
49
97
|
* The schedule a roster's declared offsets give under the queue (`undefined` is 0: the participant
|
|
50
98
|
* starts with the run). Participants start in schedule order, by offset and then roster position,
|
|
51
99
|
* each at its time or when a slot frees, whichever is later; this is the rule runOnSchedule
|
|
52
|
-
* follows.
|
|
100
|
+
* follows. `sessionsMs` gives a participant with a declared end its own session budget
|
|
101
|
+
* (participantSessionMs); the others hold their slot for `queue.sessionMs`.
|
|
53
102
|
*/
|
|
54
|
-
export declare function planArrivals(offsets: readonly (number | undefined)[], queue: StartQueue): ArrivalPlan;
|
|
103
|
+
export declare function planArrivals(offsets: readonly (number | undefined)[], queue: StartQueue, sessionsMs?: readonly number[]): ArrivalPlan;
|
|
55
104
|
/**
|
|
56
105
|
* The plan's schedule, after a "schedule:" label: the first and last start, and how many run at
|
|
57
106
|
* once at the most.
|
|
@@ -64,9 +113,7 @@ export declare function describeArrivals(arrivals: ArrivalPlan): string;
|
|
|
64
113
|
* sandbox with servedScheduleRefusal. `lowered` is the warning that the E2B plan's limit runs the
|
|
65
114
|
* roster in waves.
|
|
66
115
|
*/
|
|
67
|
-
export declare function planStarts(participants: readonly {
|
|
68
|
-
readonly startAfterMs?: number;
|
|
69
|
-
}[], queue: StartQueue, lowered: string | undefined): {
|
|
116
|
+
export declare function planStarts(participants: readonly ParticipantWindow[], queue: StartQueue, lowered: string | undefined): {
|
|
70
117
|
arrivals: ArrivalPlan;
|
|
71
118
|
warnings: string[];
|
|
72
119
|
};
|