humanish 0.118.0 → 0.120.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +39 -0
- package/dist/analysis/admission.d.ts +16 -2
- package/dist/analysis/admission.js +35 -6
- package/dist/analysis/analysis-limits.d.ts +32 -0
- package/dist/analysis/analysis-limits.js +41 -3
- package/dist/analysis/automatic-config.d.ts +5 -0
- package/dist/analysis/automatic-config.js +15 -1
- package/dist/analysis/cohorts.d.ts +103 -0
- package/dist/analysis/cohorts.js +80 -0
- package/dist/analysis/evidence.d.ts +5 -10
- package/dist/analysis/evidence.js +37 -34
- package/dist/analysis/execute.d.ts +7 -2
- package/dist/analysis/execute.js +242 -297
- package/dist/analysis/provider.js +8 -1
- package/dist/analysis/responses.d.ts +19 -0
- package/dist/analysis/responses.js +201 -0
- package/dist/analysis/store.js +2 -1
- package/dist/analysis/validation.js +13 -7
- package/dist/cli/commands/analyze.js +8 -3
- package/dist/cli/commands/study-format.js +46 -7
- package/dist/cli/commands/study-route-computer-use.js +2 -2
- package/dist/cli/commands/study-route-scripted.js +2 -2
- package/dist/cli/commands/study-route-shared-world.js +2 -2
- package/dist/cli/commands/study-route-terminal.js +2 -2
- package/dist/cli/commands/study-run.js +3 -3
- package/dist/cli/commands/study.js +2 -2
- package/dist/cli/io.js +2 -3
- package/dist/observer-app.html +2 -2
- package/dist/routes/computer-use/fanout-bundle.js +2 -1
- package/dist/routes/computer-use/fanout-records.js +4 -0
- package/dist/routes/computer-use/participant-execution.d.ts +6 -4
- package/dist/routes/computer-use/participant-execution.js +42 -14
- package/dist/routes/computer-use/participant-model.js +18 -8
- package/dist/routes/computer-use/participant-runs.js +12 -0
- package/dist/routes/computer-use/plan.js +23 -15
- package/dist/routes/computer-use/rerun-selection.js +14 -3
- package/dist/routes/computer-use/types.d.ts +13 -1
- package/dist/routes/shared-world/bundle-records.js +14 -0
- package/dist/routes/shared-world/bundle.js +2 -1
- package/dist/routes/shared-world/external-public.js +15 -3
- package/dist/routes/shared-world/handoff.d.ts +2 -2
- package/dist/routes/shared-world/plan.d.ts +7 -0
- package/dist/routes/shared-world/plan.js +79 -34
- package/dist/routes/shared-world/provisioned.js +15 -8
- package/dist/routes/shared-world/route.js +5 -2
- package/dist/routes/shared-world/types.d.ts +2 -0
- package/dist/run/bundle-shape.js +10 -1
- package/dist/run/bundle.d.ts +15 -0
- package/dist/run/participant-records.js +1 -0
- package/dist/run/route-shell.js +5 -10
- package/dist/run/run.js +4 -2
- package/dist/study/arrivals.d.ts +114 -0
- package/dist/study/arrivals.js +204 -0
- package/dist/study/doctor.js +2 -15
- package/dist/study/keys.d.ts +1 -0
- package/dist/study/keys.js +2 -0
- package/dist/study/migrate/v2.js +3 -2
- package/dist/study/parse/actors.d.ts +1 -1
- package/dist/study/parse/actors.js +9 -3
- package/dist/study/parse/front.js +29 -1
- package/dist/study/plan-base.d.ts +1 -3
- package/dist/study/plan-base.js +0 -3
- package/dist/study/plan-participants.d.ts +2 -0
- package/dist/study/plan-participants.js +1 -0
- package/dist/study/plan-types.d.ts +8 -3
- package/dist/study/preflight-result.js +2 -2
- package/dist/study/preflight.js +12 -0
- package/dist/study/summary.d.ts +3 -3
- package/dist/study/summary.js +5 -3
- package/dist/study/types.d.ts +14 -0
- package/dist/study/validation.js +5 -0
- package/dist/study/warnings.js +11 -3
- package/docs/contracts/run-bundle.md +1 -1
- package/docs/contracts/schemas.md +19 -1
- package/docs/contracts/study-analysis.md +59 -6
- package/docs/principles/invariants-and-defaults.md +14 -14
- package/docs/product/automatic-analysis.md +7 -1
- package/package.json +1 -1
- package/dist/study/automatic-analysis-plan.d.ts +0 -28
- package/dist/study/automatic-analysis-plan.js +0 -57
|
@@ -13,6 +13,7 @@ import { buildRunCostSummary, desktopSpanToMinutes } from "../../run/cost-summar
|
|
|
13
13
|
import { formatParticipantPlanEntry } from "./participant-runs.js";
|
|
14
14
|
import { fanoutParticipantRecords } from "./fanout-records.js";
|
|
15
15
|
import { plural } from "../../run/text.js";
|
|
16
|
+
import { describeArrivals } from "../../study/arrivals.js";
|
|
16
17
|
/** The run's first two events: its creation and the fan-out plan. */
|
|
17
18
|
function fanoutPlanEvents(args) {
|
|
18
19
|
const { specs, plan } = args;
|
|
@@ -29,7 +30,7 @@ function fanoutPlanEvents(args) {
|
|
|
29
30
|
at: args.run.createdAt,
|
|
30
31
|
level: "info",
|
|
31
32
|
type: "cua-lab.fanout.plan",
|
|
32
|
-
message: `Fan-out plan: ${plural(args.participantPlan.laneCount, "participant")} (${args.participantPlan.strategy}), concurrency ${args.participantPlan.concurrency}, ${plural(args.participantPlan.waves, "wave")}; session budget ${Math.round(args.participantPlan.perLaneSessionBudgetMs / 1000)}s per participant; worst-case ~${args.participantPlan.worstCaseSandboxMinutes} sandbox-minutes${args.dryRun ? " (dry-run: $0)" : ""}. Participants: ${args.participantPlan.lanes.map(formatParticipantPlanEntry).join(", ")}.`,
|
|
33
|
+
message: `Fan-out plan: ${plural(args.participantPlan.laneCount, "participant")} (${args.participantPlan.strategy}), concurrency ${args.participantPlan.concurrency}, ${plural(args.participantPlan.waves, "wave")}; session budget ${Math.round(args.participantPlan.perLaneSessionBudgetMs / 1000)}s per participant; worst-case ~${args.participantPlan.worstCaseSandboxMinutes} sandbox-minutes${args.dryRun ? " (dry-run: $0)" : ""}.${args.participantPlan.schedule === undefined ? "" : ` Schedule: ${describeArrivals(args.participantPlan.schedule)}`} Participants: ${args.participantPlan.lanes.map(formatParticipantPlanEntry).join(", ")}.`,
|
|
33
34
|
});
|
|
34
35
|
return events;
|
|
35
36
|
}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
// Each participant's records in a computer-use fan-out bundle: its simulation, its stream, its
|
|
2
2
|
// subject provenance event and the events of its outcome (session, geometry warnings, phase trail).
|
|
3
3
|
import { participantAssignment } from "../../study/participant-assignment.js";
|
|
4
|
+
import { arrivalRecord } from "../../study/arrivals.js";
|
|
4
5
|
import { participantEvent, participantRecord, participantStream, } from "../../run/participant-records.js";
|
|
5
6
|
import { participantCaption } from "../../run/participant-caption.js";
|
|
6
7
|
import { participantSummary } from "../../run/outcomes.js";
|
|
@@ -69,6 +70,9 @@ function fanoutParticipantRecord(args, spec, index, view) {
|
|
|
69
70
|
}),
|
|
70
71
|
startedAt: args.run.createdAt,
|
|
71
72
|
updatedAt: args.run.createdAt,
|
|
73
|
+
...(args.participantPlan.schedule === undefined
|
|
74
|
+
? {}
|
|
75
|
+
: { arrival: arrivalRecord(spec.planned.startAfterMs, outcome?.arrival) }),
|
|
72
76
|
});
|
|
73
77
|
}
|
|
74
78
|
function fanoutParticipantStream(args, spec, view) {
|
|
@@ -11,10 +11,12 @@ export declare function makeParticipantWriteScreenshot(artifactRoot: PreparedOut
|
|
|
11
11
|
* evidence and cleanup; this runner owns the model, trace and participant outcome. */
|
|
12
12
|
export declare function runCuaParticipant(spec: DesktopParticipantRun, deps: CuaParticipantDeps): Promise<ParticipantRunOutcome>;
|
|
13
13
|
/**
|
|
14
|
-
* Run N>1 E2B participants
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
*
|
|
14
|
+
* Run N>1 E2B participants on the study's schedule (runOnSchedule: each at its start offset, at
|
|
15
|
+
* most `concurrency` at once), behind a pipeline gate (the first to start provisions before the
|
|
16
|
+
* rest start) and session fail-fast on harness errors only (queued participants become `blocked`
|
|
17
|
+
* with a pinned reason + a fail-fast event; mission verdicts never trip it). A gate failure or a
|
|
18
|
+
* fail-fast ends every wait for a later start at once. Each participant tears down its own sandbox
|
|
19
|
+
* by id; nothing here ever enumerates.
|
|
18
20
|
*
|
|
19
21
|
* Exported for the total-runner tests: the injectable runner lets a test make one participant
|
|
20
22
|
* throw (the exact class the guard exists for) without a live sandbox. Production always uses
|
|
@@ -3,7 +3,7 @@ import { createE2BParticipantDesktop } from "./e2b-desktop/desktop.js";
|
|
|
3
3
|
import { createInProcessDesktop } from "./in-process-desktop.js";
|
|
4
4
|
import path from "node:path";
|
|
5
5
|
import { cuaParticipantDiagnostics } from "./diagnostics.js";
|
|
6
|
-
import {
|
|
6
|
+
import { runOnSchedule } from "../../study/arrivals.js";
|
|
7
7
|
import { assertScreenshotEvidence, stripPngMetadataChunks } from "../../evidence/image.js";
|
|
8
8
|
import { redactText, toErrorMessage } from "../../evidence/redaction.js";
|
|
9
9
|
import { assertSafeOutputPathSegment, writeContainedOutputFile, } from "../../run/contained-output.js";
|
|
@@ -32,9 +32,10 @@ export function makeParticipantWriteScreenshot(artifactRoot, spec, screenshots)
|
|
|
32
32
|
};
|
|
33
33
|
}
|
|
34
34
|
/** A blocked participant outcome (pipeline gate / fail-fast skipped it before it ran). */
|
|
35
|
-
function skippedOutcome(spec, reason) {
|
|
35
|
+
function skippedOutcome(spec, reason, arrival) {
|
|
36
36
|
return {
|
|
37
37
|
spec,
|
|
38
|
+
...(arrival === undefined ? {} : { arrival }),
|
|
38
39
|
killed: false,
|
|
39
40
|
streamUrlPresent: false,
|
|
40
41
|
screenshots: [],
|
|
@@ -51,6 +52,13 @@ function skippedOutcome(spec, reason) {
|
|
|
51
52
|
/** Run one participant against a prepared desktop. The adapter owns provisioning, final
|
|
52
53
|
* evidence and cleanup; this runner owns the model, trace and participant outcome. */
|
|
53
54
|
export async function runCuaParticipant(spec, deps) {
|
|
55
|
+
// A participant whose start comes after the study budget is spent would pay for a desktop and a
|
|
56
|
+
// model turn only to stop at its first one, so it is skipped before its desktop exists.
|
|
57
|
+
const crossed = deps.runBudget?.crossed();
|
|
58
|
+
if (crossed !== undefined) {
|
|
59
|
+
deps.signalProvisioned?.(false);
|
|
60
|
+
return skippedOutcome(spec, `skipped: study budget reached before this participant started: ${crossed}; no desktop was created`);
|
|
61
|
+
}
|
|
54
62
|
let model = {};
|
|
55
63
|
const warnings = [];
|
|
56
64
|
const screenshots = [];
|
|
@@ -142,10 +150,12 @@ export async function runCuaParticipant(spec, deps) {
|
|
|
142
150
|
};
|
|
143
151
|
}
|
|
144
152
|
/**
|
|
145
|
-
* Run N>1 E2B participants
|
|
146
|
-
*
|
|
147
|
-
*
|
|
148
|
-
*
|
|
153
|
+
* Run N>1 E2B participants on the study's schedule (runOnSchedule: each at its start offset, at
|
|
154
|
+
* most `concurrency` at once), behind a pipeline gate (the first to start provisions before the
|
|
155
|
+
* rest start) and session fail-fast on harness errors only (queued participants become `blocked`
|
|
156
|
+
* with a pinned reason + a fail-fast event; mission verdicts never trip it). A gate failure or a
|
|
157
|
+
* fail-fast ends every wait for a later start at once. Each participant tears down its own sandbox
|
|
158
|
+
* by id; nothing here ever enumerates.
|
|
149
159
|
*
|
|
150
160
|
* Exported for the total-runner tests: the injectable runner lets a test make one participant
|
|
151
161
|
* throw (the exact class the guard exists for) without a live sandbox. Production always uses
|
|
@@ -153,28 +163,41 @@ export async function runCuaParticipant(spec, deps) {
|
|
|
153
163
|
*/
|
|
154
164
|
export async function runCuaParticipants(runs, deps, concurrency, runParticipant = runCuaParticipant) {
|
|
155
165
|
const failFast = { tripped: false, reason: "" };
|
|
166
|
+
const stopping = new AbortController();
|
|
167
|
+
let gateHolder = runs[0]?.planned.id ?? "lane-01";
|
|
156
168
|
let resolveGate;
|
|
157
169
|
let rejectGate;
|
|
158
170
|
const gate = new Promise((resolve, reject) => {
|
|
159
171
|
resolveGate = resolve;
|
|
160
|
-
rejectGate = () =>
|
|
172
|
+
rejectGate = () => {
|
|
173
|
+
reject(new Error("gate"));
|
|
174
|
+
stopping.abort();
|
|
175
|
+
};
|
|
161
176
|
});
|
|
162
177
|
// The gate is rejected on the first participant's provisioning failure; swallow the unhandled
|
|
163
178
|
// rejection if no later participant ever awaits it (concurrency could let the first finish
|
|
164
179
|
// alone).
|
|
165
180
|
gate.catch(() => undefined);
|
|
166
|
-
const outcomes = await
|
|
167
|
-
|
|
181
|
+
const outcomes = await runOnSchedule(runs, {
|
|
182
|
+
startAfterMs: (spec) => spec.planned.startAfterMs,
|
|
183
|
+
slots: concurrency,
|
|
184
|
+
now: deps.now,
|
|
185
|
+
signal: stopping.signal,
|
|
186
|
+
}, async (spec, _index, { order, scheduledAt }) => {
|
|
187
|
+
if (order === 0)
|
|
188
|
+
gateHolder = spec.planned.id;
|
|
189
|
+
if (order > 0) {
|
|
168
190
|
try {
|
|
169
191
|
await gate;
|
|
170
192
|
}
|
|
171
193
|
catch {
|
|
172
|
-
return skippedOutcome(spec, `skipped: participant ${
|
|
194
|
+
return skippedOutcome(spec, `skipped: participant ${gateHolder} failed to provision its world (pipeline gate)`, { scheduledAt });
|
|
173
195
|
}
|
|
174
196
|
}
|
|
175
197
|
if (failFast.tripped) {
|
|
176
|
-
return skippedOutcome(spec, `skipped: ${failFast.reason}
|
|
198
|
+
return skippedOutcome(spec, `skipped: ${failFast.reason}`, { scheduledAt });
|
|
177
199
|
}
|
|
200
|
+
const arrival = { scheduledAt, startedAt: deps.now() };
|
|
178
201
|
// The participant runner is total: every exit path returns a recorded outcome. Without this
|
|
179
202
|
// guard, one participant's late throw (e.g. its trace write hitting ENOSPC after its own sandbox was
|
|
180
203
|
// already torn down) rejected the whole map while sibling workers kept launching sandboxes
|
|
@@ -183,7 +206,7 @@ export async function runCuaParticipants(runs, deps, concurrency, runParticipant
|
|
|
183
206
|
try {
|
|
184
207
|
outcome = await runParticipant(spec, {
|
|
185
208
|
...deps,
|
|
186
|
-
...(
|
|
209
|
+
...(order === 0
|
|
187
210
|
? {
|
|
188
211
|
signalProvisioned: (ok) => {
|
|
189
212
|
if (ok) {
|
|
@@ -200,7 +223,7 @@ export async function runCuaParticipants(runs, deps, concurrency, runParticipant
|
|
|
200
223
|
catch (error) {
|
|
201
224
|
// The first participant may have thrown before signaling the provisioning gate; release the followers as
|
|
202
225
|
// blocked rather than leaving them awaiting a gate that will never settle.
|
|
203
|
-
if (
|
|
226
|
+
if (order === 0)
|
|
204
227
|
rejectGate?.();
|
|
205
228
|
const detail = redactText(toErrorMessage(error));
|
|
206
229
|
outcome = {
|
|
@@ -221,8 +244,13 @@ export async function runCuaParticipants(runs, deps, concurrency, runParticipant
|
|
|
221
244
|
if (outcome.harnessError && !failFast.tripped) {
|
|
222
245
|
failFast.tripped = true;
|
|
223
246
|
failFast.reason = `a prior participant (${outcome.spec.planned.id}) ended in a harness error (fail-fast)`;
|
|
247
|
+
stopping.abort();
|
|
224
248
|
}
|
|
225
|
-
|
|
249
|
+
// A participant the study budget skipped never requested its desktop.
|
|
250
|
+
return {
|
|
251
|
+
...outcome,
|
|
252
|
+
arrival: outcome.skippedReason === undefined ? arrival : { scheduledAt },
|
|
253
|
+
};
|
|
226
254
|
});
|
|
227
255
|
return { outcomes, ...(failFast.tripped ? { failFastReason: failFast.reason } : {}) };
|
|
228
256
|
}
|
|
@@ -148,10 +148,11 @@ export function participantSessionOptions(spec, deps, ready, provider, writeScre
|
|
|
148
148
|
: {
|
|
149
149
|
overRunBudget: (usage) => {
|
|
150
150
|
const estimate = estimateActorCostForExecution(usage, provider?.version ?? capModelId, provider?.executionProfile).estimatedCostUsd;
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
151
|
+
deps.runBudget.note(spec.planned.id, estimate);
|
|
152
|
+
const crossed = deps.runBudget.crossed();
|
|
153
|
+
return crossed === undefined
|
|
154
|
+
? null
|
|
155
|
+
: `study budget reached: ${crossed}; this lane stops here and sibling lanes stop at their next turn`;
|
|
155
156
|
},
|
|
156
157
|
}),
|
|
157
158
|
...(deps.onObservedUrl === undefined ? {} : { onObservedUrl: deps.onObservedUrl }),
|
|
@@ -283,15 +284,24 @@ export function participantCapWarning(caps, participantCount, capped = "each par
|
|
|
283
284
|
}
|
|
284
285
|
export function makeCuaRunBudget(maxTotalUsd) {
|
|
285
286
|
const participantEstimates = new Map();
|
|
287
|
+
const total = () => {
|
|
288
|
+
let sum = 0;
|
|
289
|
+
for (const value of participantEstimates.values())
|
|
290
|
+
sum += value;
|
|
291
|
+
return sum;
|
|
292
|
+
};
|
|
286
293
|
return {
|
|
287
294
|
maxTotalUsd,
|
|
288
295
|
note(participantId, estimateUsd) {
|
|
289
296
|
if (estimateUsd !== null)
|
|
290
297
|
participantEstimates.set(participantId, estimateUsd);
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
298
|
+
return total();
|
|
299
|
+
},
|
|
300
|
+
crossed() {
|
|
301
|
+
const totalUsd = total();
|
|
302
|
+
return totalUsd > maxTotalUsd
|
|
303
|
+
? `the run's estimated model spend $${round6(totalUsd)} crossed caps.maxTotalUsd=$${maxTotalUsd}`
|
|
304
|
+
: undefined;
|
|
295
305
|
},
|
|
296
306
|
};
|
|
297
307
|
}
|
|
@@ -17,6 +17,8 @@ import { digestUrl } from "./bundle-parts.js";
|
|
|
17
17
|
import { composeParticipantInstructions, DEFAULT_MISSION } from "./participant-prompt.js";
|
|
18
18
|
import { resolveCuaRerunSelection } from "./rerun-selection.js";
|
|
19
19
|
import { plural } from "../../run/text.js";
|
|
20
|
+
import { formatDuration } from "../../run/projection.js";
|
|
21
|
+
import { describeArrivals, planArrivals } from "../../study/arrivals.js";
|
|
20
22
|
/**
|
|
21
23
|
* The session budget of a study that declares no execution.timeoutMs. On a provisioned route it is
|
|
22
24
|
* what the `ceilingMs` sandbox ceiling leaves after provisioning, seeding and the teardown buffer.
|
|
@@ -113,6 +115,9 @@ export function participantRunsAndPlan(plan, opts = {}) {
|
|
|
113
115
|
const resolved = envLoweredConcurrency(plan.concurrency, participantCount, env);
|
|
114
116
|
const concurrency = resolved.bound;
|
|
115
117
|
const { sessionBudgetMs, sandboxMs } = plan;
|
|
118
|
+
const schedule = resolved.envLoweredFrom === undefined
|
|
119
|
+
? plan.arrivals
|
|
120
|
+
: planArrivals(participants.map((participant) => participant.startAfterMs), { slots: concurrency, sessionMs: sessionBudgetMs });
|
|
116
121
|
const participantPlan = {
|
|
117
122
|
strategy: CUA_FANOUT_STRATEGY,
|
|
118
123
|
laneCount: participantCount,
|
|
@@ -123,6 +128,7 @@ export function participantRunsAndPlan(plan, opts = {}) {
|
|
|
123
128
|
waves: Math.ceil(participantCount / concurrency),
|
|
124
129
|
perLaneSessionBudgetMs: sessionBudgetMs,
|
|
125
130
|
worstCaseSandboxMinutes: Math.round((participantCount * sandboxMs) / 60_000),
|
|
131
|
+
...(schedule.declared ? { schedule } : {}),
|
|
126
132
|
dryRun: plan.dryRun,
|
|
127
133
|
lanes: runs.map((spec) => ({
|
|
128
134
|
id: spec.planned.id,
|
|
@@ -149,6 +155,9 @@ export function participantRunsAndPlan(plan, opts = {}) {
|
|
|
149
155
|
...(spec.planned.targetUrl === undefined
|
|
150
156
|
? {}
|
|
151
157
|
: { targetDigest: digestUrl(spec.planned.targetUrl) }),
|
|
158
|
+
...(spec.planned.startAfterMs === undefined
|
|
159
|
+
? {}
|
|
160
|
+
: { startAfterMs: spec.planned.startAfterMs }),
|
|
152
161
|
})),
|
|
153
162
|
};
|
|
154
163
|
return { runs, participantPlan };
|
|
@@ -159,6 +168,8 @@ export function emitPreflightPlan(participantPlan, studyId) {
|
|
|
159
168
|
const lines = [];
|
|
160
169
|
lines.push(`humanish computer-use fan-out plan (${studyId}): ${plural(participantPlan.laneCount, "participant")}, strategy ${participantPlan.strategy}, concurrency ${participantPlan.concurrency}${participantPlan.envLoweredConcurrencyFrom === undefined ? "" : ` (lowered from ${participantPlan.envLoweredConcurrencyFrom} by ${CUA_MAX_CONCURRENCY_ENV})`}, ${plural(participantPlan.waves, "wave")}.`);
|
|
161
170
|
lines.push(` session budget ${Math.round(participantPlan.perLaneSessionBudgetMs / 1000)}s per participant; worst-case ~${participantPlan.worstCaseSandboxMinutes} sandbox-minutes total${participantPlan.dryRun ? " (dry-run: $0)" : ""}.`);
|
|
171
|
+
if (participantPlan.schedule)
|
|
172
|
+
lines.push(` schedule: ${describeArrivals(participantPlan.schedule)}`);
|
|
162
173
|
for (const entry of participantPlan.lanes) {
|
|
163
174
|
lines.push(` - ${formatParticipantPlanEntry(entry)}`);
|
|
164
175
|
}
|
|
@@ -170,6 +181,7 @@ export function formatParticipantPlanEntry(entry) {
|
|
|
170
181
|
entry.surface ? `surface=${entry.surface}` : undefined,
|
|
171
182
|
entry.caseGroup ? `case=${entry.caseGroup}` : undefined,
|
|
172
183
|
entry.reasoningEffort ? `effort=${entry.reasoningEffort}` : undefined,
|
|
184
|
+
entry.startAfterMs === undefined ? undefined : `start=+${formatDuration(entry.startAfterMs)}`,
|
|
173
185
|
].filter((part) => part !== undefined);
|
|
174
186
|
return `${entry.id}: persona=${entry.persona}${taxonomy.length > 0 ? ` ${taxonomy.join(" ")}` : ""} device=${entry.device} ${entry.resolution[0]}x${entry.resolution[1]} prompt#${entry.instructionDigest}${entry.targetDigest ? ` target#${entry.targetDigest}` : ""}`;
|
|
175
187
|
}
|
|
@@ -9,6 +9,7 @@ import { studyUrlCredentialReason } from "../../study/url-credentials.js";
|
|
|
9
9
|
import { subjectStateInvalidReason } from "../../study/parse/subject-state.js";
|
|
10
10
|
import { brainOf, callerBrainOf, planCaps, desktopRequirements, planBase, provisionedSubject, } from "../../study/plan-base.js";
|
|
11
11
|
import { computerUseParticipants, declaredTargets } from "../../study/plan-participants.js";
|
|
12
|
+
import { planStarts } from "../../study/arrivals.js";
|
|
12
13
|
import { computerUseParticipantCount, participantCountReason } from "../../study/routing.js";
|
|
13
14
|
import { cloneTargetValidationReason, computerUseValidationReason, desktopMediaValidationReason, outputTokenLimitValidationReason, receivingEmailValidationReason, taskProtocolValidationReason, waitLimitValidationReason, } from "../../study/validation.js";
|
|
14
15
|
import { desktopCliProductReason } from "../../study/composition-rules.js";
|
|
@@ -193,17 +194,27 @@ function rosterShapeReason(config, ceilingMs) {
|
|
|
193
194
|
return invalid(`execution.timeoutMs ${Math.round(sessionMs / 60_000)}m derives a ${Math.round(derivedSandboxMs / 60_000)}m sandbox deadline, and a sandbox may not live longer than ${ceilingMs / 60_000}m. The deadline is the session budget plus ${Math.round(headroomMs / 60_000)}m of provisioning and teardown headroom${provisionedRoute ? " (this route clones, installs, builds and serves the subject before the actor starts)" : ""}. Lower execution.timeoutMs to at most ${Math.round((ceilingMs - headroomMs) / 60_000)}m, or set execution.desktop.sandboxTimeoutMs explicitly.${ceilingAdvice(derivedSandboxMs)}`);
|
|
194
195
|
}
|
|
195
196
|
/**
|
|
196
|
-
* How many of the runner's participants run at once
|
|
197
|
-
*
|
|
197
|
+
* How many of the runner's participants run at once, and when each one starts, with the session
|
|
198
|
+
* budget the `ceilingMs` sandbox ceiling leaves. Hosted desktops share the E2B plan's concurrent
|
|
199
|
+
* sandboxes; the local VM has its own capacity rules and the in-process route runs one
|
|
198
200
|
* participant.
|
|
199
201
|
*/
|
|
200
|
-
function
|
|
202
|
+
function startsOf(config, runner, limit, ceilingMs) {
|
|
201
203
|
const declared = config.execution?.concurrency;
|
|
202
204
|
const n = runner.participants.length;
|
|
203
205
|
const concurrency = boundedConcurrency(declared, n);
|
|
204
|
-
|
|
206
|
+
const atOnce = runner.desktop === "e2b-desktop"
|
|
205
207
|
? participantsAtOnce({ concurrency, declared: declared !== undefined, participants: n }, limit)
|
|
206
208
|
: { ok: true, count: concurrency };
|
|
209
|
+
if (!atOnce.ok)
|
|
210
|
+
return atOnce;
|
|
211
|
+
const sessionMs = config.execution?.timeoutMs ?? defaultSessionTimeoutMs(config, ceilingMs);
|
|
212
|
+
const slots = { slots: atOnce.count, sessionMs };
|
|
213
|
+
return {
|
|
214
|
+
ok: true,
|
|
215
|
+
count: atOnce.count,
|
|
216
|
+
...planStarts(runner.participants, slots, atOnce.lowered),
|
|
217
|
+
};
|
|
207
218
|
}
|
|
208
219
|
/** The plan's rerun: the source run and, when given, the participants to rerun. */
|
|
209
220
|
function rerunPlan({ sourceRunId, participantIds, }) {
|
|
@@ -293,25 +304,22 @@ export function planComputerUseStudy(config, input) {
|
|
|
293
304
|
: isLocalBrowserStudy(config)
|
|
294
305
|
? { desktop: "local-vm", brain, participants: [first, ...rest], subject: appUrlSubject }
|
|
295
306
|
: { desktop: "e2b-desktop", brain, participants: [first, ...rest], subject: hosted };
|
|
296
|
-
const
|
|
297
|
-
if (!
|
|
298
|
-
return refuse("after-personas", "HUMANISH_COMPUTER_USE_FANOUT_INVALID",
|
|
307
|
+
const starts = startsOf(config, runner, input.concurrentSandboxes, ceiling.ms);
|
|
308
|
+
if (!starts.ok)
|
|
309
|
+
return refuse("after-personas", "HUMANISH_COMPUTER_USE_FANOUT_INVALID", starts.message, actor);
|
|
299
310
|
const provisioned = source === "clone" || source === "local-tree";
|
|
300
|
-
const base = planBase(config, {
|
|
301
|
-
dryRun: input.dryRun,
|
|
302
|
-
analysis,
|
|
303
|
-
participants: participants.length,
|
|
304
|
-
});
|
|
311
|
+
const base = planBase(config, { dryRun: input.dryRun, analysis });
|
|
305
312
|
return {
|
|
306
313
|
ok: true,
|
|
307
314
|
plan: {
|
|
308
315
|
...base,
|
|
309
|
-
...(
|
|
316
|
+
...(starts.warnings.length === 0 ? {} : { warnings: starts.warnings }),
|
|
310
317
|
route: "computer-use",
|
|
311
318
|
actor,
|
|
312
319
|
runner,
|
|
313
|
-
concurrency:
|
|
314
|
-
|
|
320
|
+
concurrency: starts.count,
|
|
321
|
+
arrivals: starts.arrivals,
|
|
322
|
+
sessionBudgetMs: starts.arrivals.sessionMs,
|
|
315
323
|
sandboxMs: resolveParticipantSandboxMs(config, ceiling.ms),
|
|
316
324
|
caps: planCaps(config),
|
|
317
325
|
...(input.rerun === undefined ? {} : { rerun: rerunPlan(input.rerun) }),
|
|
@@ -57,12 +57,17 @@ export async function resolveCuaRerunSelection(args) {
|
|
|
57
57
|
message: `${plural(missingCurrent.length, "selected participant id")} ${missingCurrent.length === 1 ? "is" : "are"} not present in the current study file ${args.studyId}: ${missingCurrent.join(", ")}`,
|
|
58
58
|
};
|
|
59
59
|
}
|
|
60
|
-
|
|
60
|
+
// A rerun runs the selected participants again now, all together: the study's start offsets
|
|
61
|
+
// placed them in the first run's day, and that day is over.
|
|
62
|
+
const selectedSpecs = selectedIds.map((id) => startingWithTheRun(specsById.get(id)));
|
|
61
63
|
const selectedPlanIds = new Set(selectedIds);
|
|
62
|
-
const selectedPlanEntries = args.participantPlan.lanes
|
|
64
|
+
const selectedPlanEntries = args.participantPlan.lanes
|
|
65
|
+
.filter((entry) => selectedPlanIds.has(entry.id))
|
|
66
|
+
.map(({ startAfterMs: _offset, ...entry }) => entry);
|
|
63
67
|
const concurrency = Math.max(1, Math.min(args.participantPlan.concurrency, selectedSpecs.length));
|
|
68
|
+
const { schedule: _schedule, ...unscheduled } = args.participantPlan;
|
|
64
69
|
const participantPlan = {
|
|
65
|
-
...
|
|
70
|
+
...unscheduled,
|
|
66
71
|
laneCount: selectedSpecs.length,
|
|
67
72
|
concurrency,
|
|
68
73
|
waves: Math.ceil(selectedSpecs.length / concurrency),
|
|
@@ -81,6 +86,12 @@ export async function resolveCuaRerunSelection(args) {
|
|
|
81
86
|
},
|
|
82
87
|
};
|
|
83
88
|
}
|
|
89
|
+
function startingWithTheRun(spec) {
|
|
90
|
+
if (spec.planned.startAfterMs === undefined)
|
|
91
|
+
return spec;
|
|
92
|
+
const { startAfterMs: _offset, ...planned } = spec.planned;
|
|
93
|
+
return { ...spec, planned };
|
|
94
|
+
}
|
|
84
95
|
function uniqueIds(values) {
|
|
85
96
|
const seen = new Set();
|
|
86
97
|
const result = [];
|
|
@@ -12,6 +12,7 @@ import { type CuaDiagnostics } from "./diagnostics.js";
|
|
|
12
12
|
import type { ActorCompletionReason, ActorStatus, ActorStopCause, ActorTokenUsage, ActorTraceItem } from "../../actors/contract.js";
|
|
13
13
|
import { type CuaActorDescriptor } from "../../actors/registry.js";
|
|
14
14
|
import type { StudyDeps } from "../../study/study-deps.js";
|
|
15
|
+
import type { ArrivalPlan, ParticipantArrival } from "../../study/arrivals.js";
|
|
15
16
|
import type { StudyEvent, ParticipantRef } from "../../study/run-study-events.js";
|
|
16
17
|
import type { InProcessDriver, ProviderFactory, RunStudyHomes } from "../../study/run-study-homes.js";
|
|
17
18
|
import { type BrowserScorer } from "../../study/adapter-extension.js";
|
|
@@ -108,6 +109,8 @@ export interface CuaParticipantPlanEntry {
|
|
|
108
109
|
maxOutputTokens?: number;
|
|
109
110
|
/** Present only when a participant overrides subject.appUrl; digest avoids leaking preview hosts in plan logs. */
|
|
110
111
|
targetDigest?: string;
|
|
112
|
+
/** The participant's declared start, in ms after the run starts its participants. */
|
|
113
|
+
startAfterMs?: number;
|
|
111
114
|
}
|
|
112
115
|
/** The pre-flight spend and participant plan (pure; printed to stderr + recorded as a bundle event before
|
|
113
116
|
* any sandbox or provider call; identical in dry-run, marked $0). */
|
|
@@ -126,6 +129,9 @@ export interface CuaParticipantPlan {
|
|
|
126
129
|
perLaneSessionBudgetMs: number;
|
|
127
130
|
/** Worst-case total sandbox-minutes across all participants (each one's full sandbox deadline). */
|
|
128
131
|
worstCaseSandboxMinutes: number;
|
|
132
|
+
/** The schedule, when the study declares `participants[].startAfterMs`; a rerun starts its
|
|
133
|
+
* participants together and has none. */
|
|
134
|
+
schedule?: ArrivalPlan;
|
|
129
135
|
/** True for a dry-run plan (no spend); the same table appears live. */
|
|
130
136
|
dryRun: boolean;
|
|
131
137
|
lanes: CuaParticipantPlanEntry[];
|
|
@@ -261,7 +267,8 @@ export interface ParticipantRunsAndPlan {
|
|
|
261
267
|
/**
|
|
262
268
|
* The study's shared spend ledger: one counter across every participant. Each one notes its
|
|
263
269
|
* own latest running model-spend estimate (monotone per participant: an estimate can only grow)
|
|
264
|
-
* and reads back the run total; the loop stops the participant the moment the total crosses the study budget
|
|
270
|
+
* and reads back the run total; the loop stops the participant the moment the total crosses the study budget,
|
|
271
|
+
* and a participant whose start comes after that is skipped before its desktop is created.
|
|
265
272
|
* Estimated model spend only: desktop-minutes ride the cost summary, not this ledger.
|
|
266
273
|
*/
|
|
267
274
|
export interface CuaRunBudget {
|
|
@@ -269,6 +276,9 @@ export interface CuaRunBudget {
|
|
|
269
276
|
/** Record this participant's latest running estimate (null = unpriceable, ignored) and return
|
|
270
277
|
* the run's current total across all participants. */
|
|
271
278
|
note(participantId: string, estimateUsd: number | null): number;
|
|
279
|
+
/** The run total against caps.maxTotalUsd once the total has crossed it, in words; undefined
|
|
280
|
+
* while it has not. */
|
|
281
|
+
crossed(): string | undefined;
|
|
272
282
|
}
|
|
273
283
|
/**
|
|
274
284
|
* The subject as a participant's desktop meets it. A computer-use participant gets the plan's
|
|
@@ -427,6 +437,8 @@ export interface ParticipantRunOutcome {
|
|
|
427
437
|
warnings: string[];
|
|
428
438
|
/** Set when the participant was skipped by the pipeline gate / fail-fast (a pinned reason). */
|
|
429
439
|
skippedReason?: string;
|
|
440
|
+
/** When a fan-out participant was due and when its desktop was requested. */
|
|
441
|
+
arrival?: ParticipantArrival;
|
|
430
442
|
noEngagement: boolean;
|
|
431
443
|
selfReportedBlocker: boolean;
|
|
432
444
|
/** The inclusive friction read: blocker-shaped narration incl. self-resolved arcs.
|
|
@@ -7,6 +7,7 @@ import { declaredScreenForRender } from "../../substrates/e2b/desktop-geometry.j
|
|
|
7
7
|
import { publicSafeRouteLabel } from "./provenance.js";
|
|
8
8
|
import { participantCaption } from "../../run/participant-caption.js";
|
|
9
9
|
import { participantSummary } from "../../run/outcomes.js";
|
|
10
|
+
import { arrivalRecord } from "../../study/arrivals.js";
|
|
10
11
|
function participantView(ctx, spec, index) {
|
|
11
12
|
const { args, external } = ctx;
|
|
12
13
|
const name = participantCaption({ id: spec.planned.id, personaId: spec.persona.id });
|
|
@@ -69,8 +70,21 @@ function sharedWorldSimulation(ctx, spec, index, view) {
|
|
|
69
70
|
summary: participantSummary(name, { name: "the shared app" }, { trace: session?.trace, sessionError: outcome?.sessionError, inProgress }),
|
|
70
71
|
startedAt: args.run.createdAt,
|
|
71
72
|
updatedAt: args.run.createdAt,
|
|
73
|
+
...(args.plan.arrivals.declared
|
|
74
|
+
? {
|
|
75
|
+
arrival: arrivalRecord(spec.planned.startAfterMs, actorArrival(args.actorResults[index])),
|
|
76
|
+
}
|
|
77
|
+
: {}),
|
|
72
78
|
});
|
|
73
79
|
}
|
|
80
|
+
// A follower the handoff stopped before it opened the app never started its desktop.
|
|
81
|
+
function actorArrival(result) {
|
|
82
|
+
if (result === undefined)
|
|
83
|
+
return undefined;
|
|
84
|
+
return result.outcome.skippedReason === undefined
|
|
85
|
+
? { scheduledAt: result.scheduledAt, startedAt: result.startedAt }
|
|
86
|
+
: { scheduledAt: result.scheduledAt };
|
|
87
|
+
}
|
|
74
88
|
function sharedWorldStream(ctx, spec, index, view) {
|
|
75
89
|
const { args } = ctx;
|
|
76
90
|
const { name, session, screenshots, lastScreenshot, desktopGeometry, screenshotMode } = view;
|
|
@@ -18,6 +18,7 @@ import { sharedWorldParticipantRecords } from "./bundle-records.js";
|
|
|
18
18
|
import { CONCURRENT_ATTRIBUTION_LIMITS, EXTERNAL_PUBLIC_ATTRIBUTION_LIMITS, } from "./types.js";
|
|
19
19
|
import { gapsListClause } from "../../run/outcomes.js";
|
|
20
20
|
import { plural } from "../../run/text.js";
|
|
21
|
+
import { describeArrivals } from "../../study/arrivals.js";
|
|
21
22
|
/** Max windows live at the same instant (sweep over start/end points). The simultaneity
|
|
22
23
|
* count: the participant count says how many existed; this says how many ever ran at once. */
|
|
23
24
|
export function maxSimultaneousWindows(windows) {
|
|
@@ -107,7 +108,7 @@ function planeEvents(args, external) {
|
|
|
107
108
|
at: createdAt,
|
|
108
109
|
level: "info",
|
|
109
110
|
type: "concurrent-shared-world.run.created",
|
|
110
|
-
message: `Created a concurrent shared-world run for ${plan.studyId} (actor ${descriptor.id}, ${plural(actorSpecs.length, "persona")} on one shared app, at most ${plan.concurrency} at once)
|
|
111
|
+
message: `Created a concurrent shared-world run for ${plan.studyId} (actor ${descriptor.id}, ${plural(actorSpecs.length, "persona")} on one shared app, at most ${plan.concurrency} at once).${plan.arrivals.declared ? ` Schedule: ${describeArrivals(plan.arrivals)}` : ""}`,
|
|
111
112
|
});
|
|
112
113
|
// Human-readable plane label, byte-stable for the clone route. local-tree has no repo slug: it
|
|
113
114
|
// labels the packed archive instead (archiveSha256 + dirty/clean when the packed root was a git
|
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
// deadline.
|
|
7
7
|
import { liveObserverResult } from "../../observer/live.js";
|
|
8
8
|
import { redactText, toErrorMessage } from "../../evidence/redaction.js";
|
|
9
|
-
import {
|
|
9
|
+
import { runOnSchedule } from "../../study/arrivals.js";
|
|
10
10
|
import { buildConcurrentSharedWorldBundle, judgeSharedWorldRun } from "./bundle.js";
|
|
11
11
|
import { drainExternalComms } from "./comms.js";
|
|
12
12
|
import { LobbyHandoff, runFollower, runHost } from "./handoff.js";
|
|
@@ -95,10 +95,22 @@ export async function runExternalPublicPlane(ctx, live, inbox) {
|
|
|
95
95
|
.filter(({ index }) => index !== hostIndex);
|
|
96
96
|
const orderedResults = new Array(actorSpecs.length);
|
|
97
97
|
try {
|
|
98
|
+
// The host starts with the run (parse refuses a later start for it); the followers start on
|
|
99
|
+
// the study's schedule.
|
|
98
100
|
const hostPromise = hostIndex >= 0 && actorSpecs[hostIndex] !== undefined
|
|
99
|
-
? runHost(handoff, deps, actorSpecs[hostIndex], hostIndex)
|
|
101
|
+
? runHost(handoff, deps, actorSpecs[hostIndex], hostIndex).then((result) => ({
|
|
102
|
+
...result,
|
|
103
|
+
scheduledAt: result.startedAt,
|
|
104
|
+
}))
|
|
100
105
|
: undefined;
|
|
101
|
-
const followerResultsPromise =
|
|
106
|
+
const followerResultsPromise = runOnSchedule(followerEntries, {
|
|
107
|
+
startAfterMs: ({ spec }) => spec.planned.startAfterMs,
|
|
108
|
+
slots: Math.max(1, concurrency - 1),
|
|
109
|
+
now: ctx.now,
|
|
110
|
+
}, async ({ spec, index }, _position, { scheduledAt }) => ({
|
|
111
|
+
...(await runFollower(handoff, deps, spec, index)),
|
|
112
|
+
scheduledAt,
|
|
113
|
+
}));
|
|
102
114
|
const [hostResult, followerResults] = await Promise.all([hostPromise, followerResultsPromise]);
|
|
103
115
|
if (hostResult !== undefined && hostIndex >= 0) {
|
|
104
116
|
orderedResults[hostIndex] = hostResult;
|
|
@@ -59,5 +59,5 @@ export declare class LobbyHandoff {
|
|
|
59
59
|
lobbyConvergenceDigest: string | undefined;
|
|
60
60
|
};
|
|
61
61
|
}
|
|
62
|
-
export declare function runHost(handoff: LobbyHandoff, deps: HandoffParticipantDeps, spec: DesktopParticipantRun, participantIndex: number): Promise<ActorRunResult
|
|
63
|
-
export declare function runFollower(handoff: LobbyHandoff, deps: HandoffParticipantDeps, spec: DesktopParticipantRun, participantIndex: number): Promise<ActorRunResult
|
|
62
|
+
export declare function runHost(handoff: LobbyHandoff, deps: HandoffParticipantDeps, spec: DesktopParticipantRun, participantIndex: number): Promise<Omit<ActorRunResult, "scheduledAt">>;
|
|
63
|
+
export declare function runFollower(handoff: LobbyHandoff, deps: HandoffParticipantDeps, spec: DesktopParticipantRun, participantIndex: number): Promise<Omit<ActorRunResult, "scheduledAt">>;
|
|
@@ -24,5 +24,12 @@ export declare function planSharedWorldStudy(config: StudyConfig, input: {
|
|
|
24
24
|
/** How many sandboxes the operator's E2B plan runs at once (concurrentSandboxes). */
|
|
25
25
|
readonly concurrentSandboxes: ConcurrentSandboxes;
|
|
26
26
|
}): SharedWorldPlanResult;
|
|
27
|
+
/**
|
|
28
|
+
* The schedule lines a shared-world run prints before it creates anything, when the study declares
|
|
29
|
+
* `participants[].startAfterMs`: the schedule, and the worst case in sandbox-minutes, which counts
|
|
30
|
+
* every participant's sandbox (its session plus the teardown buffer) and on the provisioned plane
|
|
31
|
+
* the app's sandbox until the last participant ends.
|
|
32
|
+
*/
|
|
33
|
+
export declare function sharedWorldScheduleLines(plan: SharedWorldPlan): string;
|
|
27
34
|
/** The provisioned plane's declared subject state. The external-public plane declares none. */
|
|
28
35
|
export declare function planeStateOf(plan: SharedWorldPlan): StudySubjectState | undefined;
|