humanish 0.117.0 → 0.119.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +42 -0
- package/dist/analysis/admission.d.ts +23 -4
- package/dist/analysis/admission.js +47 -9
- package/dist/analysis/analysis-limits.d.ts +32 -0
- package/dist/analysis/analysis-limits.js +41 -3
- package/dist/analysis/automatic-config.d.ts +11 -1
- package/dist/analysis/automatic-config.js +26 -11
- package/dist/analysis/cohorts.d.ts +103 -0
- package/dist/analysis/cohorts.js +80 -0
- package/dist/analysis/evidence.d.ts +5 -10
- package/dist/analysis/evidence.js +37 -34
- package/dist/analysis/execute.d.ts +24 -7
- package/dist/analysis/execute.js +290 -307
- package/dist/analysis/responses.d.ts +19 -0
- package/dist/analysis/responses.js +201 -0
- package/dist/analysis/store.js +2 -1
- package/dist/analysis/validation.js +13 -7
- package/dist/cli/commands/analyze.js +10 -4
- package/dist/cli/commands/study-format.js +8 -11
- package/dist/cli/commands/study-route-shared-world.js +3 -1
- package/dist/cli/commands/study-run.js +1 -1
- package/dist/cli/doctor.js +11 -1
- package/dist/comms/lease-store.d.ts +1 -1
- package/dist/comms/lease-store.js +3 -2
- package/dist/comms/receiving-types.d.ts +2 -0
- package/dist/comms/receiving-types.js +2 -0
- package/dist/observer-app.html +2 -2
- package/dist/routes/computer-use/fanout-bundle.js +4 -2
- package/dist/routes/computer-use/fanout-records.js +6 -1
- package/dist/routes/computer-use/participant-execution.d.ts +6 -4
- package/dist/routes/computer-use/participant-execution.js +42 -14
- package/dist/routes/computer-use/participant-model.js +18 -8
- package/dist/routes/computer-use/participant-runs.js +15 -9
- package/dist/routes/computer-use/plan.d.ts +3 -1
- package/dist/routes/computer-use/plan.js +38 -12
- package/dist/routes/computer-use/rerun-selection.js +14 -3
- package/dist/routes/computer-use/single-bundle.js +7 -4
- package/dist/routes/computer-use/subject-projection.js +2 -0
- package/dist/routes/computer-use/types.d.ts +13 -1
- package/dist/routes/scripted/plan.d.ts +4 -1
- package/dist/routes/scripted/plan.js +25 -2
- package/dist/routes/scripted/setup.js +1 -4
- package/dist/routes/scripted/types.d.ts +1 -1
- package/dist/routes/shared-world/bundle-records.js +14 -0
- package/dist/routes/shared-world/bundle.js +2 -1
- package/dist/routes/shared-world/external-public.js +15 -3
- package/dist/routes/shared-world/handoff.d.ts +2 -2
- package/dist/routes/shared-world/participant-specs.js +2 -2
- package/dist/routes/shared-world/plan.d.ts +10 -1
- package/dist/routes/shared-world/plan.js +86 -10
- package/dist/routes/shared-world/provisioned.js +15 -8
- package/dist/routes/shared-world/route.js +7 -4
- package/dist/routes/shared-world/types.d.ts +2 -0
- package/dist/run/bundle-shape.js +12 -2
- package/dist/run/bundle.d.ts +20 -3
- package/dist/run/participant-records.js +1 -0
- package/dist/run/projection.js +7 -2
- package/dist/run/run.js +4 -2
- package/dist/run/subject-name.d.ts +11 -0
- package/dist/run/subject-name.js +11 -0
- package/dist/study/arrivals.d.ts +114 -0
- package/dist/study/arrivals.js +204 -0
- package/dist/study/composition-rules.js +1 -1
- package/dist/study/config.js +6 -20
- package/dist/study/doctor.js +2 -1
- package/dist/study/keys.d.ts +1 -0
- package/dist/study/keys.js +2 -0
- package/dist/study/migrate/v2.js +3 -2
- package/dist/study/parse/actors.d.ts +2 -2
- package/dist/study/parse/actors.js +10 -4
- package/dist/study/parse/front.d.ts +1 -1
- package/dist/study/parse/front.js +36 -2
- package/dist/study/plan-base.d.ts +1 -1
- package/dist/study/plan-participants.d.ts +2 -0
- package/dist/study/plan-participants.js +3 -5
- package/dist/study/plan-types.d.ts +13 -3
- package/dist/study/plan.d.ts +4 -4
- package/dist/study/plan.js +21 -12
- package/dist/study/preflight.d.ts +2 -1
- package/dist/study/preflight.js +24 -3
- package/dist/study/routing.d.ts +5 -4
- package/dist/study/routing.js +16 -7
- package/dist/study/types.d.ts +14 -0
- package/dist/study/validation.d.ts +2 -2
- package/dist/study/validation.js +27 -14
- package/dist/study/warnings.js +11 -3
- package/dist/substrates/e2b/lifetime.d.ts +75 -4
- package/dist/substrates/e2b/lifetime.js +134 -10
- package/dist/substrates/e2b/subject-sandbox.js +2 -6
- package/dist/tui-app.js +112 -112
- package/dist/verify/artifacts.d.ts +19 -4
- package/dist/verify/artifacts.js +39 -8
- package/dist/verify/verify.js +6 -15
- package/docs/contracts/run-bundle.md +8 -7
- package/docs/contracts/schemas.md +33 -9
- package/docs/contracts/study-analysis.md +61 -8
- package/docs/principles/invariants-and-defaults.md +14 -14
- package/docs/product/automatic-analysis.md +10 -3
- package/package.json +1 -1
|
@@ -5,6 +5,7 @@ import { redactText } from "../../evidence/redaction.js";
|
|
|
5
5
|
import { REVIEW_SCHEMA, bundleArtifacts, bundleHead, } from "../../run/bundle.js";
|
|
6
6
|
import { aggregateTaskFunnels, formatParticipantOutcomes, formatRunTaskFunnel, gapsListClause, withCuaReviewProvenance, } from "../../run/outcomes.js";
|
|
7
7
|
import { providerResourcesForOutcome, publicSafeAppUrlLabel } from "./bundle-parts.js";
|
|
8
|
+
import { subjectName } from "../../run/subject-name.js";
|
|
8
9
|
import { participantFeedbackCandidates } from "./participant-feedback.js";
|
|
9
10
|
import { participantFactsOf } from "./participant-facts.js";
|
|
10
11
|
import { judgeParticipantRecords } from "../../run/judge.js";
|
|
@@ -12,6 +13,7 @@ import { buildRunCostSummary, desktopSpanToMinutes } from "../../run/cost-summar
|
|
|
12
13
|
import { formatParticipantPlanEntry } from "./participant-runs.js";
|
|
13
14
|
import { fanoutParticipantRecords } from "./fanout-records.js";
|
|
14
15
|
import { plural } from "../../run/text.js";
|
|
16
|
+
import { describeArrivals } from "../../study/arrivals.js";
|
|
15
17
|
/** The run's first two events: its creation and the fan-out plan. */
|
|
16
18
|
function fanoutPlanEvents(args) {
|
|
17
19
|
const { specs, plan } = args;
|
|
@@ -28,7 +30,7 @@ function fanoutPlanEvents(args) {
|
|
|
28
30
|
at: args.run.createdAt,
|
|
29
31
|
level: "info",
|
|
30
32
|
type: "cua-lab.fanout.plan",
|
|
31
|
-
message: `Fan-out plan: ${plural(args.participantPlan.laneCount, "participant")} (${args.participantPlan.strategy}), concurrency ${args.participantPlan.concurrency}, ${plural(args.participantPlan.waves, "wave")}; session budget ${Math.round(args.participantPlan.perLaneSessionBudgetMs / 1000)}s per participant; worst-case ~${args.participantPlan.worstCaseSandboxMinutes} sandbox-minutes${args.dryRun ? " (dry-run: $0)" : ""}. Participants: ${args.participantPlan.lanes.map(formatParticipantPlanEntry).join(", ")}.`,
|
|
33
|
+
message: `Fan-out plan: ${plural(args.participantPlan.laneCount, "participant")} (${args.participantPlan.strategy}), concurrency ${args.participantPlan.concurrency}, ${plural(args.participantPlan.waves, "wave")}; session budget ${Math.round(args.participantPlan.perLaneSessionBudgetMs / 1000)}s per participant; worst-case ~${args.participantPlan.worstCaseSandboxMinutes} sandbox-minutes${args.dryRun ? " (dry-run: $0)" : ""}.${args.participantPlan.schedule === undefined ? "" : ` Schedule: ${describeArrivals(args.participantPlan.schedule)}`} Participants: ${args.participantPlan.lanes.map(formatParticipantPlanEntry).join(", ")}.`,
|
|
32
34
|
});
|
|
33
35
|
return events;
|
|
34
36
|
}
|
|
@@ -44,7 +46,7 @@ function fanoutSummary(args, facts) {
|
|
|
44
46
|
return `${people} ${count === 1 ? "is" : "are"} each using their own copy of the app. The run is still going, so nothing here is final.`;
|
|
45
47
|
}
|
|
46
48
|
if (args.dryRun) {
|
|
47
|
-
return `${rerun}Dry run: ${people} would each use their own copy of ${publicSafeAppUrlLabel(args.appUrl)}. No desktops were launched and $0 was spent.`;
|
|
49
|
+
return `${rerun}Dry run: ${people} would each use their own copy of ${subjectName(publicSafeAppUrlLabel(args.appUrl), args.aggregateSubject)}. No desktops were launched and $0 was spent.`;
|
|
48
50
|
}
|
|
49
51
|
const tasks = facts.tasks === undefined ? "" : ` Tasks: ${facts.tasks}.`;
|
|
50
52
|
return `${rerun}${people} took part, each in their own copy of the app: ${facts.outcomes}${gapsListClause(count - facts.passed, count)}.${tasks}`;
|
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
// Each participant's records in a computer-use fan-out bundle: its simulation, its stream, its
|
|
2
2
|
// subject provenance event and the events of its outcome (session, geometry warnings, phase trail).
|
|
3
3
|
import { participantAssignment } from "../../study/participant-assignment.js";
|
|
4
|
+
import { arrivalRecord } from "../../study/arrivals.js";
|
|
4
5
|
import { participantEvent, participantRecord, participantStream, } from "../../run/participant-records.js";
|
|
5
6
|
import { participantCaption } from "../../run/participant-caption.js";
|
|
6
7
|
import { participantSummary } from "../../run/outcomes.js";
|
|
8
|
+
import { subjectName } from "../../run/subject-name.js";
|
|
7
9
|
import { declaredScreenForRender } from "../../substrates/e2b/desktop-geometry.js";
|
|
8
10
|
import { describeSubjectState, phaseEventIdSuffix, publicSafeAppUrlLabel } from "./bundle-parts.js";
|
|
9
11
|
import { participantSubjectEnv } from "./types.js";
|
|
@@ -60,7 +62,7 @@ function fanoutParticipantRecord(args, spec, index, view) {
|
|
|
60
62
|
mode: "browser-sim",
|
|
61
63
|
progress: args.inProgress === true && outcome === undefined ? 20 : 100,
|
|
62
64
|
currentStep: reason,
|
|
63
|
-
summary: participantSummary(name, { name: "the app", target: publicTargetUrl }, {
|
|
65
|
+
summary: participantSummary(name, { name: "the app", target: subjectName(publicTargetUrl, view.subject) }, {
|
|
64
66
|
trace: session?.trace,
|
|
65
67
|
skippedReason: outcome?.skippedReason,
|
|
66
68
|
sessionError: outcome?.sessionError,
|
|
@@ -68,6 +70,9 @@ function fanoutParticipantRecord(args, spec, index, view) {
|
|
|
68
70
|
}),
|
|
69
71
|
startedAt: args.run.createdAt,
|
|
70
72
|
updatedAt: args.run.createdAt,
|
|
73
|
+
...(args.participantPlan.schedule === undefined
|
|
74
|
+
? {}
|
|
75
|
+
: { arrival: arrivalRecord(spec.planned.startAfterMs, outcome?.arrival) }),
|
|
71
76
|
});
|
|
72
77
|
}
|
|
73
78
|
function fanoutParticipantStream(args, spec, view) {
|
|
@@ -11,10 +11,12 @@ export declare function makeParticipantWriteScreenshot(artifactRoot: PreparedOut
|
|
|
11
11
|
* evidence and cleanup; this runner owns the model, trace and participant outcome. */
|
|
12
12
|
export declare function runCuaParticipant(spec: DesktopParticipantRun, deps: CuaParticipantDeps): Promise<ParticipantRunOutcome>;
|
|
13
13
|
/**
|
|
14
|
-
* Run N>1 E2B participants
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
*
|
|
14
|
+
* Run N>1 E2B participants on the study's schedule (runOnSchedule: each at its start offset, at
|
|
15
|
+
* most `concurrency` at once), behind a pipeline gate (the first to start provisions before the
|
|
16
|
+
* rest start) and session fail-fast on harness errors only (queued participants become `blocked`
|
|
17
|
+
* with a pinned reason + a fail-fast event; mission verdicts never trip it). A gate failure or a
|
|
18
|
+
* fail-fast ends every wait for a later start at once. Each participant tears down its own sandbox
|
|
19
|
+
* by id; nothing here ever enumerates.
|
|
18
20
|
*
|
|
19
21
|
* Exported for the total-runner tests: the injectable runner lets a test make one participant
|
|
20
22
|
* throw (the exact class the guard exists for) without a live sandbox. Production always uses
|
|
@@ -3,7 +3,7 @@ import { createE2BParticipantDesktop } from "./e2b-desktop/desktop.js";
|
|
|
3
3
|
import { createInProcessDesktop } from "./in-process-desktop.js";
|
|
4
4
|
import path from "node:path";
|
|
5
5
|
import { cuaParticipantDiagnostics } from "./diagnostics.js";
|
|
6
|
-
import {
|
|
6
|
+
import { runOnSchedule } from "../../study/arrivals.js";
|
|
7
7
|
import { assertScreenshotEvidence, stripPngMetadataChunks } from "../../evidence/image.js";
|
|
8
8
|
import { redactText, toErrorMessage } from "../../evidence/redaction.js";
|
|
9
9
|
import { assertSafeOutputPathSegment, writeContainedOutputFile, } from "../../run/contained-output.js";
|
|
@@ -32,9 +32,10 @@ export function makeParticipantWriteScreenshot(artifactRoot, spec, screenshots)
|
|
|
32
32
|
};
|
|
33
33
|
}
|
|
34
34
|
/** A blocked participant outcome (pipeline gate / fail-fast skipped it before it ran). */
|
|
35
|
-
function skippedOutcome(spec, reason) {
|
|
35
|
+
function skippedOutcome(spec, reason, arrival) {
|
|
36
36
|
return {
|
|
37
37
|
spec,
|
|
38
|
+
...(arrival === undefined ? {} : { arrival }),
|
|
38
39
|
killed: false,
|
|
39
40
|
streamUrlPresent: false,
|
|
40
41
|
screenshots: [],
|
|
@@ -51,6 +52,13 @@ function skippedOutcome(spec, reason) {
|
|
|
51
52
|
/** Run one participant against a prepared desktop. The adapter owns provisioning, final
|
|
52
53
|
* evidence and cleanup; this runner owns the model, trace and participant outcome. */
|
|
53
54
|
export async function runCuaParticipant(spec, deps) {
|
|
55
|
+
// A participant whose start comes after the study budget is spent would pay for a desktop and a
|
|
56
|
+
// model turn only to stop at its first one, so it is skipped before its desktop exists.
|
|
57
|
+
const crossed = deps.runBudget?.crossed();
|
|
58
|
+
if (crossed !== undefined) {
|
|
59
|
+
deps.signalProvisioned?.(false);
|
|
60
|
+
return skippedOutcome(spec, `skipped: study budget reached before this participant started: ${crossed}; no desktop was created`);
|
|
61
|
+
}
|
|
54
62
|
let model = {};
|
|
55
63
|
const warnings = [];
|
|
56
64
|
const screenshots = [];
|
|
@@ -142,10 +150,12 @@ export async function runCuaParticipant(spec, deps) {
|
|
|
142
150
|
};
|
|
143
151
|
}
|
|
144
152
|
/**
|
|
145
|
-
* Run N>1 E2B participants
|
|
146
|
-
*
|
|
147
|
-
*
|
|
148
|
-
*
|
|
153
|
+
* Run N>1 E2B participants on the study's schedule (runOnSchedule: each at its start offset, at
|
|
154
|
+
* most `concurrency` at once), behind a pipeline gate (the first to start provisions before the
|
|
155
|
+
* rest start) and session fail-fast on harness errors only (queued participants become `blocked`
|
|
156
|
+
* with a pinned reason + a fail-fast event; mission verdicts never trip it). A gate failure or a
|
|
157
|
+
* fail-fast ends every wait for a later start at once. Each participant tears down its own sandbox
|
|
158
|
+
* by id; nothing here ever enumerates.
|
|
149
159
|
*
|
|
150
160
|
* Exported for the total-runner tests: the injectable runner lets a test make one participant
|
|
151
161
|
* throw (the exact class the guard exists for) without a live sandbox. Production always uses
|
|
@@ -153,28 +163,41 @@ export async function runCuaParticipant(spec, deps) {
|
|
|
153
163
|
*/
|
|
154
164
|
export async function runCuaParticipants(runs, deps, concurrency, runParticipant = runCuaParticipant) {
|
|
155
165
|
const failFast = { tripped: false, reason: "" };
|
|
166
|
+
const stopping = new AbortController();
|
|
167
|
+
let gateHolder = runs[0]?.planned.id ?? "lane-01";
|
|
156
168
|
let resolveGate;
|
|
157
169
|
let rejectGate;
|
|
158
170
|
const gate = new Promise((resolve, reject) => {
|
|
159
171
|
resolveGate = resolve;
|
|
160
|
-
rejectGate = () =>
|
|
172
|
+
rejectGate = () => {
|
|
173
|
+
reject(new Error("gate"));
|
|
174
|
+
stopping.abort();
|
|
175
|
+
};
|
|
161
176
|
});
|
|
162
177
|
// The gate is rejected on the first participant's provisioning failure; swallow the unhandled
|
|
163
178
|
// rejection if no later participant ever awaits it (concurrency could let the first finish
|
|
164
179
|
// alone).
|
|
165
180
|
gate.catch(() => undefined);
|
|
166
|
-
const outcomes = await
|
|
167
|
-
|
|
181
|
+
const outcomes = await runOnSchedule(runs, {
|
|
182
|
+
startAfterMs: (spec) => spec.planned.startAfterMs,
|
|
183
|
+
slots: concurrency,
|
|
184
|
+
now: deps.now,
|
|
185
|
+
signal: stopping.signal,
|
|
186
|
+
}, async (spec, _index, { order, scheduledAt }) => {
|
|
187
|
+
if (order === 0)
|
|
188
|
+
gateHolder = spec.planned.id;
|
|
189
|
+
if (order > 0) {
|
|
168
190
|
try {
|
|
169
191
|
await gate;
|
|
170
192
|
}
|
|
171
193
|
catch {
|
|
172
|
-
return skippedOutcome(spec, `skipped: participant ${
|
|
194
|
+
return skippedOutcome(spec, `skipped: participant ${gateHolder} failed to provision its world (pipeline gate)`, { scheduledAt });
|
|
173
195
|
}
|
|
174
196
|
}
|
|
175
197
|
if (failFast.tripped) {
|
|
176
|
-
return skippedOutcome(spec, `skipped: ${failFast.reason}
|
|
198
|
+
return skippedOutcome(spec, `skipped: ${failFast.reason}`, { scheduledAt });
|
|
177
199
|
}
|
|
200
|
+
const arrival = { scheduledAt, startedAt: deps.now() };
|
|
178
201
|
// The participant runner is total: every exit path returns a recorded outcome. Without this
|
|
179
202
|
// guard, one participant's late throw (e.g. its trace write hitting ENOSPC after its own sandbox was
|
|
180
203
|
// already torn down) rejected the whole map while sibling workers kept launching sandboxes
|
|
@@ -183,7 +206,7 @@ export async function runCuaParticipants(runs, deps, concurrency, runParticipant
|
|
|
183
206
|
try {
|
|
184
207
|
outcome = await runParticipant(spec, {
|
|
185
208
|
...deps,
|
|
186
|
-
...(
|
|
209
|
+
...(order === 0
|
|
187
210
|
? {
|
|
188
211
|
signalProvisioned: (ok) => {
|
|
189
212
|
if (ok) {
|
|
@@ -200,7 +223,7 @@ export async function runCuaParticipants(runs, deps, concurrency, runParticipant
|
|
|
200
223
|
catch (error) {
|
|
201
224
|
// The first participant may have thrown before signaling the provisioning gate; release the followers as
|
|
202
225
|
// blocked rather than leaving them awaiting a gate that will never settle.
|
|
203
|
-
if (
|
|
226
|
+
if (order === 0)
|
|
204
227
|
rejectGate?.();
|
|
205
228
|
const detail = redactText(toErrorMessage(error));
|
|
206
229
|
outcome = {
|
|
@@ -221,8 +244,13 @@ export async function runCuaParticipants(runs, deps, concurrency, runParticipant
|
|
|
221
244
|
if (outcome.harnessError && !failFast.tripped) {
|
|
222
245
|
failFast.tripped = true;
|
|
223
246
|
failFast.reason = `a prior participant (${outcome.spec.planned.id}) ended in a harness error (fail-fast)`;
|
|
247
|
+
stopping.abort();
|
|
224
248
|
}
|
|
225
|
-
|
|
249
|
+
// A participant the study budget skipped never requested its desktop.
|
|
250
|
+
return {
|
|
251
|
+
...outcome,
|
|
252
|
+
arrival: outcome.skippedReason === undefined ? arrival : { scheduledAt },
|
|
253
|
+
};
|
|
226
254
|
});
|
|
227
255
|
return { outcomes, ...(failFast.tripped ? { failFastReason: failFast.reason } : {}) };
|
|
228
256
|
}
|
|
@@ -148,10 +148,11 @@ export function participantSessionOptions(spec, deps, ready, provider, writeScre
|
|
|
148
148
|
: {
|
|
149
149
|
overRunBudget: (usage) => {
|
|
150
150
|
const estimate = estimateActorCostForExecution(usage, provider?.version ?? capModelId, provider?.executionProfile).estimatedCostUsd;
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
151
|
+
deps.runBudget.note(spec.planned.id, estimate);
|
|
152
|
+
const crossed = deps.runBudget.crossed();
|
|
153
|
+
return crossed === undefined
|
|
154
|
+
? null
|
|
155
|
+
: `study budget reached: ${crossed}; this lane stops here and sibling lanes stop at their next turn`;
|
|
155
156
|
},
|
|
156
157
|
}),
|
|
157
158
|
...(deps.onObservedUrl === undefined ? {} : { onObservedUrl: deps.onObservedUrl }),
|
|
@@ -283,15 +284,24 @@ export function participantCapWarning(caps, participantCount, capped = "each par
|
|
|
283
284
|
}
|
|
284
285
|
export function makeCuaRunBudget(maxTotalUsd) {
|
|
285
286
|
const participantEstimates = new Map();
|
|
287
|
+
const total = () => {
|
|
288
|
+
let sum = 0;
|
|
289
|
+
for (const value of participantEstimates.values())
|
|
290
|
+
sum += value;
|
|
291
|
+
return sum;
|
|
292
|
+
};
|
|
286
293
|
return {
|
|
287
294
|
maxTotalUsd,
|
|
288
295
|
note(participantId, estimateUsd) {
|
|
289
296
|
if (estimateUsd !== null)
|
|
290
297
|
participantEstimates.set(participantId, estimateUsd);
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
298
|
+
return total();
|
|
299
|
+
},
|
|
300
|
+
crossed() {
|
|
301
|
+
const totalUsd = total();
|
|
302
|
+
return totalUsd > maxTotalUsd
|
|
303
|
+
? `the run's estimated model spend $${round6(totalUsd)} crossed caps.maxTotalUsd=$${maxTotalUsd}`
|
|
304
|
+
: undefined;
|
|
295
305
|
},
|
|
296
306
|
};
|
|
297
307
|
}
|
|
@@ -3,7 +3,6 @@
|
|
|
3
3
|
// paths, and the public plan printed before any sandbox or provider call. loadCuaParticipants
|
|
4
4
|
// compiles the committed personas first and narrows a rerun to its selected participants.
|
|
5
5
|
import { isLocalBrowserStudy, LOCAL_BROWSER_LIFETIME_MS, } from "../../substrates/local/runtime-config.js";
|
|
6
|
-
import { DEFAULT_STATE_STEP_TIMEOUT_MS } from "../../subject/state.js";
|
|
7
6
|
import { resolveParticipant } from "../../run/participant.js";
|
|
8
7
|
import {} from "../../study/types.js";
|
|
9
8
|
import { scrubPersonaBrief } from "../../study/persona.js";
|
|
@@ -12,12 +11,14 @@ import {} from "../../run/bundle.js";
|
|
|
12
11
|
import { participantAssignment } from "../../study/participant-assignment.js";
|
|
13
12
|
import { resolveCommittedPersonas } from "../../study/persona-resolve.js";
|
|
14
13
|
import { CUA_FANOUT_STRATEGY, CUA_MAX_CONCURRENCY_ENV, DEFAULT_APP_URL_SESSION_TIMEOUT_MS, MIN_DERIVED_SESSION_TIMEOUT_MS, } from "./types.js";
|
|
15
|
-
import {
|
|
14
|
+
import { sandboxHeadroomMs } from "../../substrates/e2b/lifetime.js";
|
|
16
15
|
import { readPositiveInt } from "../../study/parse/values.js";
|
|
17
16
|
import { digestUrl } from "./bundle-parts.js";
|
|
18
17
|
import { composeParticipantInstructions, DEFAULT_MISSION } from "./participant-prompt.js";
|
|
19
18
|
import { resolveCuaRerunSelection } from "./rerun-selection.js";
|
|
20
19
|
import { plural } from "../../run/text.js";
|
|
20
|
+
import { formatDuration } from "../../run/projection.js";
|
|
21
|
+
import { describeArrivals, planArrivals } from "../../study/arrivals.js";
|
|
21
22
|
/**
|
|
22
23
|
* The session budget of a study that declares no execution.timeoutMs. On a provisioned route it is
|
|
23
24
|
* what the `ceilingMs` sandbox ceiling leaves after provisioning, seeding and the teardown buffer.
|
|
@@ -26,8 +27,7 @@ export function defaultSessionTimeoutMs(config, ceilingMs) {
|
|
|
26
27
|
const provisionedRoute = config.subject.source === "clone" || config.subject.source === "local-tree";
|
|
27
28
|
if (!provisionedRoute)
|
|
28
29
|
return DEFAULT_APP_URL_SESSION_TIMEOUT_MS;
|
|
29
|
-
const
|
|
30
|
-
const room = ceilingMs - SUBJECT_PROVISION_BUDGET_MS - stateBudgetMs - SANDBOX_TIMEOUT_BUFFER_MS;
|
|
30
|
+
const room = ceilingMs - sandboxHeadroomMs({ seed: config.subject.state?.seed ?? [] });
|
|
31
31
|
return Math.max(MIN_DERIVED_SESSION_TIMEOUT_MS, Math.min(DEFAULT_APP_URL_SESSION_TIMEOUT_MS, room));
|
|
32
32
|
}
|
|
33
33
|
/** Per-participant sandbox deadline (each one owns its own desktop). Mirrors the
|
|
@@ -41,13 +41,9 @@ export function resolveParticipantSandboxMs(config, ceilingMs) {
|
|
|
41
41
|
return LOCAL_BROWSER_LIFETIME_MS;
|
|
42
42
|
const timeoutMs = config.execution?.timeoutMs ?? defaultSessionTimeoutMs(config, ceilingMs);
|
|
43
43
|
const provisionedRoute = config.subject.source === "clone" || config.subject.source === "local-tree";
|
|
44
|
-
const stateBudgetMs = provisionedRoute
|
|
45
|
-
? (config.subject.state?.seed ?? []).reduce((sum, step) => sum + (step.timeoutMs ?? DEFAULT_STATE_STEP_TIMEOUT_MS), 0)
|
|
46
|
-
: 0;
|
|
47
44
|
return (config.execution?.desktop?.sandboxTimeoutMs ??
|
|
48
45
|
timeoutMs +
|
|
49
|
-
(provisionedRoute ?
|
|
50
|
-
SANDBOX_TIMEOUT_BUFFER_MS);
|
|
46
|
+
sandboxHeadroomMs(provisionedRoute ? { seed: config.subject.state?.seed ?? [] } : undefined));
|
|
51
47
|
}
|
|
52
48
|
/**
|
|
53
49
|
* The in-flight participant bound a study declares. Defaults to the participant count: every
|
|
@@ -119,6 +115,9 @@ export function participantRunsAndPlan(plan, opts = {}) {
|
|
|
119
115
|
const resolved = envLoweredConcurrency(plan.concurrency, participantCount, env);
|
|
120
116
|
const concurrency = resolved.bound;
|
|
121
117
|
const { sessionBudgetMs, sandboxMs } = plan;
|
|
118
|
+
const schedule = resolved.envLoweredFrom === undefined
|
|
119
|
+
? plan.arrivals
|
|
120
|
+
: planArrivals(participants.map((participant) => participant.startAfterMs), { slots: concurrency, sessionMs: sessionBudgetMs });
|
|
122
121
|
const participantPlan = {
|
|
123
122
|
strategy: CUA_FANOUT_STRATEGY,
|
|
124
123
|
laneCount: participantCount,
|
|
@@ -129,6 +128,7 @@ export function participantRunsAndPlan(plan, opts = {}) {
|
|
|
129
128
|
waves: Math.ceil(participantCount / concurrency),
|
|
130
129
|
perLaneSessionBudgetMs: sessionBudgetMs,
|
|
131
130
|
worstCaseSandboxMinutes: Math.round((participantCount * sandboxMs) / 60_000),
|
|
131
|
+
...(schedule.declared ? { schedule } : {}),
|
|
132
132
|
dryRun: plan.dryRun,
|
|
133
133
|
lanes: runs.map((spec) => ({
|
|
134
134
|
id: spec.planned.id,
|
|
@@ -155,6 +155,9 @@ export function participantRunsAndPlan(plan, opts = {}) {
|
|
|
155
155
|
...(spec.planned.targetUrl === undefined
|
|
156
156
|
? {}
|
|
157
157
|
: { targetDigest: digestUrl(spec.planned.targetUrl) }),
|
|
158
|
+
...(spec.planned.startAfterMs === undefined
|
|
159
|
+
? {}
|
|
160
|
+
: { startAfterMs: spec.planned.startAfterMs }),
|
|
158
161
|
})),
|
|
159
162
|
};
|
|
160
163
|
return { runs, participantPlan };
|
|
@@ -165,6 +168,8 @@ export function emitPreflightPlan(participantPlan, studyId) {
|
|
|
165
168
|
const lines = [];
|
|
166
169
|
lines.push(`humanish computer-use fan-out plan (${studyId}): ${plural(participantPlan.laneCount, "participant")}, strategy ${participantPlan.strategy}, concurrency ${participantPlan.concurrency}${participantPlan.envLoweredConcurrencyFrom === undefined ? "" : ` (lowered from ${participantPlan.envLoweredConcurrencyFrom} by ${CUA_MAX_CONCURRENCY_ENV})`}, ${plural(participantPlan.waves, "wave")}.`);
|
|
167
170
|
lines.push(` session budget ${Math.round(participantPlan.perLaneSessionBudgetMs / 1000)}s per participant; worst-case ~${participantPlan.worstCaseSandboxMinutes} sandbox-minutes total${participantPlan.dryRun ? " (dry-run: $0)" : ""}.`);
|
|
171
|
+
if (participantPlan.schedule)
|
|
172
|
+
lines.push(` schedule: ${describeArrivals(participantPlan.schedule)}`);
|
|
168
173
|
for (const entry of participantPlan.lanes) {
|
|
169
174
|
lines.push(` - ${formatParticipantPlanEntry(entry)}`);
|
|
170
175
|
}
|
|
@@ -176,6 +181,7 @@ export function formatParticipantPlanEntry(entry) {
|
|
|
176
181
|
entry.surface ? `surface=${entry.surface}` : undefined,
|
|
177
182
|
entry.caseGroup ? `case=${entry.caseGroup}` : undefined,
|
|
178
183
|
entry.reasoningEffort ? `effort=${entry.reasoningEffort}` : undefined,
|
|
184
|
+
entry.startAfterMs === undefined ? undefined : `start=+${formatDuration(entry.startAfterMs)}`,
|
|
179
185
|
].filter((part) => part !== undefined);
|
|
180
186
|
return `${entry.id}: persona=${entry.persona}${taxonomy.length > 0 ? ` ${taxonomy.join(" ")}` : ""} device=${entry.device} ${entry.resolution[0]}x${entry.resolution[1]} prompt#${entry.instructionDigest}${entry.targetDigest ? ` target#${entry.targetDigest}` : ""}`;
|
|
181
187
|
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { type CuaActorDescriptor } from "../../actors/registry.js";
|
|
2
2
|
import type { ComputerUsePlan, ComputerUseRunner, RoutePlanResult, RouteRefusal } from "../../study/plan-types.js";
|
|
3
3
|
import type { StudyConfig, StudySubjectState } from "../../study/types.js";
|
|
4
|
-
import { type SandboxCeiling } from "../../substrates/e2b/lifetime.js";
|
|
4
|
+
import { type ConcurrentSandboxes, type SandboxCeiling } from "../../substrates/e2b/lifetime.js";
|
|
5
5
|
import { type ComputerUseRunInput, type CuaActorStudyErrorCode } from "./types.js";
|
|
6
6
|
/** The error a computer-use study returns before a run starts. */
|
|
7
7
|
export interface ComputerUseRefusal extends RouteRefusal<"computer-use", CuaActorStudyErrorCode> {
|
|
@@ -48,4 +48,6 @@ export declare function planComputerUseStudy(config: StudyConfig, input: {
|
|
|
48
48
|
readonly rerun?: ComputerUseRunInput["rerun"];
|
|
49
49
|
/** The longest sandbox lifetime the operator's E2B plan allows (sandboxCeiling). */
|
|
50
50
|
readonly sandboxCeiling: SandboxCeiling;
|
|
51
|
+
/** How many sandboxes the operator's E2B plan runs at once (concurrentSandboxes). */
|
|
52
|
+
readonly concurrentSandboxes: ConcurrentSandboxes;
|
|
51
53
|
}): ComputerUsePlanResult;
|
|
@@ -9,12 +9,13 @@ import { studyUrlCredentialReason } from "../../study/url-credentials.js";
|
|
|
9
9
|
import { subjectStateInvalidReason } from "../../study/parse/subject-state.js";
|
|
10
10
|
import { brainOf, callerBrainOf, planCaps, desktopRequirements, planBase, provisionedSubject, } from "../../study/plan-base.js";
|
|
11
11
|
import { computerUseParticipants, declaredTargets } from "../../study/plan-participants.js";
|
|
12
|
-
import {
|
|
12
|
+
import { planStarts } from "../../study/arrivals.js";
|
|
13
|
+
import { computerUseParticipantCount, participantCountReason } from "../../study/routing.js";
|
|
13
14
|
import { cloneTargetValidationReason, computerUseValidationReason, desktopMediaValidationReason, outputTokenLimitValidationReason, receivingEmailValidationReason, taskProtocolValidationReason, waitLimitValidationReason, } from "../../study/validation.js";
|
|
14
15
|
import { desktopCliProductReason } from "../../study/composition-rules.js";
|
|
15
16
|
import { isLocalBrowserStudy } from "../../substrates/local/runtime-config.js";
|
|
16
17
|
import { boundedConcurrency, defaultSessionTimeoutMs, resolveParticipantSandboxMs, } from "./participant-runs.js";
|
|
17
|
-
import { ceilingAdvice } from "../../substrates/e2b/lifetime.js";
|
|
18
|
+
import { ceilingAdvice, participantsAtOnce, } from "../../substrates/e2b/lifetime.js";
|
|
18
19
|
import {} from "./types.js";
|
|
19
20
|
/** The registered descriptor for a planned actor id; planComputerUseStudy checked the registry. */
|
|
20
21
|
export function cuaDescriptorOf(actor) {
|
|
@@ -192,6 +193,29 @@ function rosterShapeReason(config, ceilingMs) {
|
|
|
192
193
|
const headroomMs = derivedSandboxMs - sessionMs;
|
|
193
194
|
return invalid(`execution.timeoutMs ${Math.round(sessionMs / 60_000)}m derives a ${Math.round(derivedSandboxMs / 60_000)}m sandbox deadline, and a sandbox may not live longer than ${ceilingMs / 60_000}m. The deadline is the session budget plus ${Math.round(headroomMs / 60_000)}m of provisioning and teardown headroom${provisionedRoute ? " (this route clones, installs, builds and serves the subject before the actor starts)" : ""}. Lower execution.timeoutMs to at most ${Math.round((ceilingMs - headroomMs) / 60_000)}m, or set execution.desktop.sandboxTimeoutMs explicitly.${ceilingAdvice(derivedSandboxMs)}`);
|
|
194
195
|
}
|
|
196
|
+
/**
|
|
197
|
+
* How many of the runner's participants run at once, and when each one starts, with the session
|
|
198
|
+
* budget the `ceilingMs` sandbox ceiling leaves. Hosted desktops share the E2B plan's concurrent
|
|
199
|
+
* sandboxes; the local VM has its own capacity rules and the in-process route runs one
|
|
200
|
+
* participant.
|
|
201
|
+
*/
|
|
202
|
+
function startsOf(config, runner, limit, ceilingMs) {
|
|
203
|
+
const declared = config.execution?.concurrency;
|
|
204
|
+
const n = runner.participants.length;
|
|
205
|
+
const concurrency = boundedConcurrency(declared, n);
|
|
206
|
+
const atOnce = runner.desktop === "e2b-desktop"
|
|
207
|
+
? participantsAtOnce({ concurrency, declared: declared !== undefined, participants: n }, limit)
|
|
208
|
+
: { ok: true, count: concurrency };
|
|
209
|
+
if (!atOnce.ok)
|
|
210
|
+
return atOnce;
|
|
211
|
+
const sessionMs = config.execution?.timeoutMs ?? defaultSessionTimeoutMs(config, ceilingMs);
|
|
212
|
+
const slots = { slots: atOnce.count, sessionMs };
|
|
213
|
+
return {
|
|
214
|
+
ok: true,
|
|
215
|
+
count: atOnce.count,
|
|
216
|
+
...planStarts(runner.participants, slots, atOnce.lowered),
|
|
217
|
+
};
|
|
218
|
+
}
|
|
195
219
|
/** The plan's rerun: the source run and, when given, the participants to rerun. */
|
|
196
220
|
function rerunPlan({ sourceRunId, participantIds, }) {
|
|
197
221
|
return { sourceRunId, ...(participantIds === undefined ? {} : { participantIds }) };
|
|
@@ -246,9 +270,11 @@ export function planComputerUseStudy(config, input) {
|
|
|
246
270
|
const productReason = desktopCliProductReason(config);
|
|
247
271
|
if (productReason)
|
|
248
272
|
return refuse("in-scope", "HUMANISH_COMPUTER_USE_SUBJECT_INVALID", productReason, actor);
|
|
273
|
+
// Counted before any participant is built, so a large --count is refused without building it.
|
|
274
|
+
const countReason = participantCountReason(computerUseParticipantCount(config, input.countOverride));
|
|
275
|
+
if (countReason)
|
|
276
|
+
return refuse("after-personas", "HUMANISH_COMPUTER_USE_FANOUT_INVALID", countReason, actor);
|
|
249
277
|
const participants = computerUseParticipants(config, input.countOverride);
|
|
250
|
-
if (participants.length > MAX_COMPUTER_USE_PARTICIPANTS)
|
|
251
|
-
return refuse("after-personas", "HUMANISH_COMPUTER_USE_FANOUT_INVALID", `Computer-use fan-out is capped at ${MAX_COMPUTER_USE_PARTICIPANTS} participants (resolved ${participants.length}); N concurrent paid desktops is real spend.`, actor);
|
|
252
278
|
const [first, ...rest] = participants;
|
|
253
279
|
if (driving.inProcess && rest.length > 0)
|
|
254
280
|
return refuse("after-personas", "HUMANISH_COMPUTER_USE_FANOUT_INVALID", "Fan-out to more than one participant is not supported on the in-process route (RunStudyOptions.inProcess): fan-out provisions one independent E2B desktop per participant, which the in-process route deliberately skips. Run a single in-process participant, or fan out on the E2B route.", actor);
|
|
@@ -278,22 +304,22 @@ export function planComputerUseStudy(config, input) {
|
|
|
278
304
|
: isLocalBrowserStudy(config)
|
|
279
305
|
? { desktop: "local-vm", brain, participants: [first, ...rest], subject: appUrlSubject }
|
|
280
306
|
: { desktop: "e2b-desktop", brain, participants: [first, ...rest], subject: hosted };
|
|
281
|
-
const
|
|
282
|
-
|
|
307
|
+
const starts = startsOf(config, runner, input.concurrentSandboxes, ceiling.ms);
|
|
308
|
+
if (!starts.ok)
|
|
309
|
+
return refuse("after-personas", "HUMANISH_COMPUTER_USE_FANOUT_INVALID", starts.message, actor);
|
|
283
310
|
const provisioned = source === "clone" || source === "local-tree";
|
|
284
|
-
const base = planBase(config, {
|
|
285
|
-
dryRun: input.dryRun,
|
|
286
|
-
analysis,
|
|
287
|
-
});
|
|
311
|
+
const base = planBase(config, { dryRun: input.dryRun, analysis });
|
|
288
312
|
return {
|
|
289
313
|
ok: true,
|
|
290
314
|
plan: {
|
|
291
315
|
...base,
|
|
316
|
+
...(starts.warnings.length === 0 ? {} : { warnings: starts.warnings }),
|
|
292
317
|
route: "computer-use",
|
|
293
318
|
actor,
|
|
294
319
|
runner,
|
|
295
|
-
concurrency:
|
|
296
|
-
|
|
320
|
+
concurrency: starts.count,
|
|
321
|
+
arrivals: starts.arrivals,
|
|
322
|
+
sessionBudgetMs: starts.arrivals.sessionMs,
|
|
297
323
|
sandboxMs: resolveParticipantSandboxMs(config, ceiling.ms),
|
|
298
324
|
caps: planCaps(config),
|
|
299
325
|
...(input.rerun === undefined ? {} : { rerun: rerunPlan(input.rerun) }),
|
|
@@ -57,12 +57,17 @@ export async function resolveCuaRerunSelection(args) {
|
|
|
57
57
|
message: `${plural(missingCurrent.length, "selected participant id")} ${missingCurrent.length === 1 ? "is" : "are"} not present in the current study file ${args.studyId}: ${missingCurrent.join(", ")}`,
|
|
58
58
|
};
|
|
59
59
|
}
|
|
60
|
-
|
|
60
|
+
// A rerun runs the selected participants again now, all together: the study's start offsets
|
|
61
|
+
// placed them in the first run's day, and that day is over.
|
|
62
|
+
const selectedSpecs = selectedIds.map((id) => startingWithTheRun(specsById.get(id)));
|
|
61
63
|
const selectedPlanIds = new Set(selectedIds);
|
|
62
|
-
const selectedPlanEntries = args.participantPlan.lanes
|
|
64
|
+
const selectedPlanEntries = args.participantPlan.lanes
|
|
65
|
+
.filter((entry) => selectedPlanIds.has(entry.id))
|
|
66
|
+
.map(({ startAfterMs: _offset, ...entry }) => entry);
|
|
63
67
|
const concurrency = Math.max(1, Math.min(args.participantPlan.concurrency, selectedSpecs.length));
|
|
68
|
+
const { schedule: _schedule, ...unscheduled } = args.participantPlan;
|
|
64
69
|
const participantPlan = {
|
|
65
|
-
...
|
|
70
|
+
...unscheduled,
|
|
66
71
|
laneCount: selectedSpecs.length,
|
|
67
72
|
concurrency,
|
|
68
73
|
waves: Math.ceil(selectedSpecs.length / concurrency),
|
|
@@ -81,6 +86,12 @@ export async function resolveCuaRerunSelection(args) {
|
|
|
81
86
|
},
|
|
82
87
|
};
|
|
83
88
|
}
|
|
89
|
+
function startingWithTheRun(spec) {
|
|
90
|
+
if (spec.planned.startAfterMs === undefined)
|
|
91
|
+
return spec;
|
|
92
|
+
const { startAfterMs: _offset, ...planned } = spec.planned;
|
|
93
|
+
return { ...spec, planned };
|
|
94
|
+
}
|
|
84
95
|
function uniqueIds(values) {
|
|
85
96
|
const seen = new Set();
|
|
86
97
|
const result = [];
|
|
@@ -10,6 +10,7 @@ import { participantFeedbackCandidates } from "./participant-feedback.js";
|
|
|
10
10
|
import { judgeParticipantRecords } from "../../run/judge.js";
|
|
11
11
|
import { buildRunCostSummary } from "../../run/cost-summary.js";
|
|
12
12
|
import { participantEvent, participantIds, participantRecord, participantStream, } from "../../run/participant-records.js";
|
|
13
|
+
import { subjectName } from "../../run/subject-name.js";
|
|
13
14
|
import { plural } from "../../run/text.js";
|
|
14
15
|
/** The one participant's record and stream ids: sim-001 and stream-001. */
|
|
15
16
|
const SINGLE = participantIds(0);
|
|
@@ -79,6 +80,8 @@ function participantView(args, publicAppUrl) {
|
|
|
79
80
|
: (args.captureRedaction ?? "raw");
|
|
80
81
|
return {
|
|
81
82
|
publicAppUrl,
|
|
83
|
+
/** How the participant's sentence and the summary name the subject. */
|
|
84
|
+
subjectName: subjectName(publicAppUrl, args.subject),
|
|
82
85
|
participant,
|
|
83
86
|
tally: judgment.tally,
|
|
84
87
|
status,
|
|
@@ -90,7 +93,7 @@ function participantView(args, publicAppUrl) {
|
|
|
90
93
|
};
|
|
91
94
|
}
|
|
92
95
|
function singleSimulation(args, view) {
|
|
93
|
-
const {
|
|
96
|
+
const { status, reason } = view;
|
|
94
97
|
const name = participantCaption({
|
|
95
98
|
id: args.participantId ?? "lane-01",
|
|
96
99
|
personaId: args.persona.id,
|
|
@@ -103,7 +106,7 @@ function singleSimulation(args, view) {
|
|
|
103
106
|
mode: "browser-sim",
|
|
104
107
|
progress: args.inProgress === true ? 20 : 100,
|
|
105
108
|
currentStep: reason,
|
|
106
|
-
summary: participantSummary(name, { name: "the app", place: browserPlace(args), target:
|
|
109
|
+
summary: participantSummary(name, { name: "the app", place: browserPlace(args), target: view.subjectName }, {
|
|
107
110
|
trace: args.session?.trace,
|
|
108
111
|
sessionError: args.sessionError,
|
|
109
112
|
inProgress: args.inProgress === true,
|
|
@@ -285,11 +288,11 @@ function singleSummary(args, view, facts) {
|
|
|
285
288
|
return "1 participant is using the app. The run is still going, so nothing here is final.";
|
|
286
289
|
}
|
|
287
290
|
if (args.dryRun) {
|
|
288
|
-
return `Dry run: 1 participant would use ${view.
|
|
291
|
+
return `Dry run: 1 participant would use ${view.subjectName}. Nothing was launched and $0 was spent.`;
|
|
289
292
|
}
|
|
290
293
|
if (args.session === undefined) {
|
|
291
294
|
return args.sessionError === undefined
|
|
292
|
-
? `1 participant was planned against ${view.
|
|
295
|
+
? `1 participant was planned against ${view.subjectName}, but no session ran.`
|
|
293
296
|
: `1 participant was planned, but the run failed before their session finished: ${view.reason}`;
|
|
294
297
|
}
|
|
295
298
|
const ending = actorEnding(args.session.trace);
|
|
@@ -103,6 +103,8 @@ function participantSubjectProjection(args) {
|
|
|
103
103
|
if (args.subject.kind === "desktop-cli") {
|
|
104
104
|
return { source: "desktop-cli", product: args.subject.product.name, state: args.subjectState };
|
|
105
105
|
}
|
|
106
|
+
if (args.subject.kind === "local-app")
|
|
107
|
+
return { source: "local-app", state: args.subjectState };
|
|
106
108
|
return { source: "app-url", state: args.subjectState };
|
|
107
109
|
}
|
|
108
110
|
/**
|
|
@@ -12,6 +12,7 @@ import { type CuaDiagnostics } from "./diagnostics.js";
|
|
|
12
12
|
import type { ActorCompletionReason, ActorStatus, ActorStopCause, ActorTokenUsage, ActorTraceItem } from "../../actors/contract.js";
|
|
13
13
|
import { type CuaActorDescriptor } from "../../actors/registry.js";
|
|
14
14
|
import type { StudyDeps } from "../../study/study-deps.js";
|
|
15
|
+
import type { ArrivalPlan, ParticipantArrival } from "../../study/arrivals.js";
|
|
15
16
|
import type { StudyEvent, ParticipantRef } from "../../study/run-study-events.js";
|
|
16
17
|
import type { InProcessDriver, ProviderFactory, RunStudyHomes } from "../../study/run-study-homes.js";
|
|
17
18
|
import { type BrowserScorer } from "../../study/adapter-extension.js";
|
|
@@ -108,6 +109,8 @@ export interface CuaParticipantPlanEntry {
|
|
|
108
109
|
maxOutputTokens?: number;
|
|
109
110
|
/** Present only when a participant overrides subject.appUrl; digest avoids leaking preview hosts in plan logs. */
|
|
110
111
|
targetDigest?: string;
|
|
112
|
+
/** The participant's declared start, in ms after the run starts its participants. */
|
|
113
|
+
startAfterMs?: number;
|
|
111
114
|
}
|
|
112
115
|
/** The pre-flight spend and participant plan (pure; printed to stderr + recorded as a bundle event before
|
|
113
116
|
* any sandbox or provider call; identical in dry-run, marked $0). */
|
|
@@ -126,6 +129,9 @@ export interface CuaParticipantPlan {
|
|
|
126
129
|
perLaneSessionBudgetMs: number;
|
|
127
130
|
/** Worst-case total sandbox-minutes across all participants (each one's full sandbox deadline). */
|
|
128
131
|
worstCaseSandboxMinutes: number;
|
|
132
|
+
/** The schedule, when the study declares `participants[].startAfterMs`; a rerun starts its
|
|
133
|
+
* participants together and has none. */
|
|
134
|
+
schedule?: ArrivalPlan;
|
|
129
135
|
/** True for a dry-run plan (no spend); the same table appears live. */
|
|
130
136
|
dryRun: boolean;
|
|
131
137
|
lanes: CuaParticipantPlanEntry[];
|
|
@@ -261,7 +267,8 @@ export interface ParticipantRunsAndPlan {
|
|
|
261
267
|
/**
|
|
262
268
|
* The study's shared spend ledger: one counter across every participant. Each one notes its
|
|
263
269
|
* own latest running model-spend estimate (monotone per participant: an estimate can only grow)
|
|
264
|
-
* and reads back the run total; the loop stops the participant the moment the total crosses the study budget
|
|
270
|
+
* and reads back the run total; the loop stops the participant the moment the total crosses the study budget,
|
|
271
|
+
* and a participant whose start comes after that is skipped before its desktop is created.
|
|
265
272
|
* Estimated model spend only: desktop-minutes ride the cost summary, not this ledger.
|
|
266
273
|
*/
|
|
267
274
|
export interface CuaRunBudget {
|
|
@@ -269,6 +276,9 @@ export interface CuaRunBudget {
|
|
|
269
276
|
/** Record this participant's latest running estimate (null = unpriceable, ignored) and return
|
|
270
277
|
* the run's current total across all participants. */
|
|
271
278
|
note(participantId: string, estimateUsd: number | null): number;
|
|
279
|
+
/** The run total against caps.maxTotalUsd once the total has crossed it, in words; undefined
|
|
280
|
+
* while it has not. */
|
|
281
|
+
crossed(): string | undefined;
|
|
272
282
|
}
|
|
273
283
|
/**
|
|
274
284
|
* The subject as a participant's desktop meets it. A computer-use participant gets the plan's
|
|
@@ -427,6 +437,8 @@ export interface ParticipantRunOutcome {
|
|
|
427
437
|
warnings: string[];
|
|
428
438
|
/** Set when the participant was skipped by the pipeline gate / fail-fast (a pinned reason). */
|
|
429
439
|
skippedReason?: string;
|
|
440
|
+
/** When a fan-out participant was due and when its desktop was requested. */
|
|
441
|
+
arrival?: ParticipantArrival;
|
|
430
442
|
noEngagement: boolean;
|
|
431
443
|
selfReportedBlocker: boolean;
|
|
432
444
|
/** The inclusive friction read: blocker-shaped narration incl. self-resolved arcs.
|
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import type { StudyDeps } from "../../study/study-deps.js";
|
|
2
2
|
import type { RoutePlanResult, RouteRefusal, ScriptedPlan } from "../../study/plan-types.js";
|
|
3
3
|
import type { StudyConfig } from "../../study/types.js";
|
|
4
|
+
import { type SandboxCeiling } from "../../substrates/e2b/lifetime.js";
|
|
4
5
|
/** The error a scripted study returns before a run starts. */
|
|
5
|
-
export interface ScriptedRefusal extends RouteRefusal<"scripted", "HUMANISH_STUDY_ANALYSIS_INVALID" | "HUMANISH_STUDY_TASKS_UNSUPPORTED" | "HUMANISH_SCRIPTED_ACTOR_UNSUPPORTED" | "HUMANISH_SCRIPTED_SCENARIO_INVALID" | "HUMANISH_SCRIPTED_SUBJECT_UNSAFE"> {
|
|
6
|
+
export interface ScriptedRefusal extends RouteRefusal<"scripted", "HUMANISH_STUDY_ANALYSIS_INVALID" | "HUMANISH_STUDY_TASKS_UNSUPPORTED" | "HUMANISH_SCRIPTED_ACTOR_UNSUPPORTED" | "HUMANISH_SCRIPTED_SCENARIO_INVALID" | "HUMANISH_SCRIPTED_SUBJECT_UNSAFE" | "HUMANISH_SCRIPTED_SUBJECT_INVALID"> {
|
|
6
7
|
/**
|
|
7
8
|
* Set on the receiving, analysis and tasks refusals. The route returns those before it opens its
|
|
8
9
|
* run scope, so their result has its own field order and no automatic-analysis record.
|
|
@@ -25,4 +26,6 @@ export declare function planScriptedStudy(config: StudyConfig, input: {
|
|
|
25
26
|
readonly dryRun: boolean;
|
|
26
27
|
/** A test's injected browser means a live run needs no host browser. */
|
|
27
28
|
readonly injectedBrowser?: boolean;
|
|
29
|
+
/** The longest sandbox lifetime the operator's E2B plan allows (sandboxCeiling). */
|
|
30
|
+
readonly sandboxCeiling: SandboxCeiling;
|
|
28
31
|
}): ScriptedPlanResult;
|