humanish 0.126.0 → 0.127.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +13 -0
- package/dist/actors/computer-use/loop/ending.d.ts +3 -1
- package/dist/actors/computer-use/loop/ending.js +5 -3
- package/dist/actors/computer-use/openai-provider.js +1 -1
- package/dist/actors/goal-source.d.ts +32 -2
- package/dist/actors/goal-source.js +41 -2
- package/dist/analysis/evidence-sources.js +2 -2
- package/dist/analysis/types.d.ts +2 -2
- package/dist/analysis/validation.js +2 -1
- package/dist/observer/data.js +8 -5
- package/dist/observer-app.html +1 -1
- package/dist/routes/computer-use/fanout-bundle.js +3 -0
- package/dist/routes/computer-use/fanout-records.js +1 -1
- package/dist/routes/computer-use/participant-execution.js +6 -2
- package/dist/routes/computer-use/participant-facts.js +4 -1
- package/dist/routes/computer-use/participant-runs.d.ts +9 -9
- package/dist/routes/computer-use/participant-runs.js +52 -86
- package/dist/routes/computer-use/plan.d.ts +2 -0
- package/dist/routes/computer-use/plan.js +26 -16
- package/dist/routes/computer-use/rerun-selection.js +8 -5
- package/dist/routes/computer-use/setup.js +1 -1
- package/dist/routes/computer-use/single-bundle.js +9 -1
- package/dist/routes/computer-use/types.d.ts +2 -3
- package/dist/routes/computer-use/types.js +0 -3
- package/dist/routes/shared-world/bundle-records.js +1 -1
- package/dist/routes/shared-world/external-public.js +1 -1
- package/dist/routes/shared-world/plan.d.ts +2 -0
- package/dist/routes/shared-world/plan.js +3 -3
- package/dist/routes/shared-world/provisioned.js +1 -1
- package/dist/run/bundle.d.ts +11 -1
- package/dist/run/judge.d.ts +7 -0
- package/dist/run/judge.js +4 -4
- package/dist/run/notes.js +1 -1
- package/dist/run/outcomes.d.ts +8 -2
- package/dist/run/outcomes.js +56 -14
- package/dist/study/arrivals.d.ts +84 -31
- package/dist/study/arrivals.js +165 -86
- package/dist/study/bundle-estimate.d.ts +8 -7
- package/dist/study/bundle-estimate.js +131 -70
- package/dist/study/desktop-files.js +4 -4
- package/dist/study/plan-participants.d.ts +4 -9
- package/dist/study/plan-participants.js +2 -14
- package/dist/study/plan-types.d.ts +6 -1
- package/dist/study/plan.d.ts +2 -1
- package/dist/study/plan.js +6 -4
- package/dist/study/types.d.ts +2 -1
- package/dist/substrates/e2b/lifetime.d.ts +22 -8
- package/dist/substrates/e2b/lifetime.js +54 -10
- package/docs/contracts/run-bundle.md +1 -1
- package/docs/contracts/schemas.md +24 -11
- package/package.json +1 -1
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { e2bDesktopTemplate } from "../../substrates/e2b/sandbox.js";
|
|
2
2
|
import { receivingPublication } from "../../comms/receiving-runtime.js";
|
|
3
3
|
import { actorEnding } from "../../actors/stop-cause.js";
|
|
4
|
+
import { cuaGoalSource } from "../../actors/goal-source.js";
|
|
4
5
|
import { redactText } from "../../evidence/redaction.js";
|
|
5
6
|
import { REVIEW_SCHEMA, bundleArtifacts, bundleHead, } from "../../run/bundle.js";
|
|
6
7
|
import { aggregateTaskFunnels, formatParticipantOutcomes, formatRunTaskFunnel, gapsListClause, withCuaReviewProvenance, } from "../../run/outcomes.js";
|
|
@@ -86,9 +87,11 @@ function fanoutReview(args, streams, judgment) {
|
|
|
86
87
|
const runTasks = args.inProgress === true ? undefined : aggregateTaskFunnels(participantFunnels);
|
|
87
88
|
const participantEndings = terminalOutcomes.map((outcome) => {
|
|
88
89
|
const ending = actorEnding(outcome.session.trace);
|
|
90
|
+
const goalSource = cuaGoalSource(outcome.session.trace);
|
|
89
91
|
return {
|
|
90
92
|
status: outcome.session.status,
|
|
91
93
|
...(ending === undefined ? {} : { label: ending.label }),
|
|
94
|
+
...(goalSource === undefined ? {} : { goalSource }),
|
|
92
95
|
};
|
|
93
96
|
});
|
|
94
97
|
const review = withCuaReviewProvenance({
|
|
@@ -72,7 +72,7 @@ function fanoutParticipantRecord(args, spec, index, view) {
|
|
|
72
72
|
updatedAt: args.run.createdAt,
|
|
73
73
|
...(args.participantPlan.schedule === undefined
|
|
74
74
|
? {}
|
|
75
|
-
: { arrival: arrivalRecord(spec.planned.startAfterMs, outcome?.arrival) }),
|
|
75
|
+
: { arrival: arrivalRecord(spec.planned.schedule.startAfterMs, outcome?.arrival) }),
|
|
76
76
|
});
|
|
77
77
|
}
|
|
78
78
|
function fanoutParticipantStream(args, spec, view) {
|
|
@@ -4,7 +4,7 @@ import { createE2BParticipantDesktop } from "./e2b-desktop/desktop.js";
|
|
|
4
4
|
import { createInProcessDesktop } from "./in-process-desktop.js";
|
|
5
5
|
import path from "node:path";
|
|
6
6
|
import { cuaParticipantDiagnostics } from "./diagnostics.js";
|
|
7
|
-
import { runOnSchedule } from "../../study/arrivals.js";
|
|
7
|
+
import { pastEndReason, runOnSchedule } from "../../study/arrivals.js";
|
|
8
8
|
import { assertScreenshotEvidence, stripPngMetadataChunks } from "../../evidence/image.js";
|
|
9
9
|
import { redactText, toErrorMessage } from "../../evidence/redaction.js";
|
|
10
10
|
import { assertSafeOutputPathSegment, writeContainedOutputFile, } from "../../run/contained-output.js";
|
|
@@ -59,6 +59,10 @@ export async function runCuaParticipant(spec, deps) {
|
|
|
59
59
|
if (crossed !== undefined) {
|
|
60
60
|
return skippedOutcome(spec, `skipped: study budget reached before this participant started: ${crossed}; no desktop was created`);
|
|
61
61
|
}
|
|
62
|
+
// One that waited for a slot until its scheduled end would only be asked for its closing report.
|
|
63
|
+
const pastEnd = pastEndReason(deps.end, deps.now());
|
|
64
|
+
if (pastEnd !== undefined)
|
|
65
|
+
return skippedOutcome(spec, pastEnd);
|
|
62
66
|
let model = {};
|
|
63
67
|
const warnings = [];
|
|
64
68
|
const screenshots = [];
|
|
@@ -207,7 +211,7 @@ export async function runCuaParticipants(runs, deps, concurrency, runParticipant
|
|
|
207
211
|
// An app-url subject is only opened, so one participant's start says nothing about another's.
|
|
208
212
|
const gate = pipelineGate(deps.subject.kind === "app-url", () => stopping.abort());
|
|
209
213
|
return runOnSchedule(runs, {
|
|
210
|
-
startAfterMs: (spec) => spec.planned.startAfterMs,
|
|
214
|
+
startAfterMs: (spec) => spec.planned.schedule.startAfterMs,
|
|
211
215
|
slots: concurrency,
|
|
212
216
|
now: deps.now,
|
|
213
217
|
signal: stopping.signal,
|
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
// A participant run outcome's facts for the judge (src/run/judge.ts), and the pass check built on
|
|
2
2
|
// them. The bundle builders, the result, the participant runner and shared-world all read these,
|
|
3
|
-
// so they sit in a file that imports only the judge and the types.
|
|
3
|
+
// so they sit in a file that imports only the judge, the clock-end rule and the types.
|
|
4
|
+
import { ranUntilClockEnd } from "../../actors/goal-source.js";
|
|
4
5
|
import { participantHarnessFailed, participantPassed, sandboxCleanupFailure, } from "../../run/judge.js";
|
|
5
6
|
/** A participant outcome's facts for the judge. */
|
|
6
7
|
export function participantFactsOf(outcome) {
|
|
7
8
|
if (outcome === undefined)
|
|
8
9
|
return { skipped: false, noEngagement: false, selfReportedBlocker: false };
|
|
10
|
+
const ranUntil = outcome.session === undefined ? undefined : ranUntilClockEnd(outcome.session.trace);
|
|
9
11
|
return {
|
|
10
12
|
id: outcome.spec.planned.id,
|
|
11
13
|
...(outcome.skippedReason === undefined ? {} : { skippedReason: outcome.skippedReason }),
|
|
@@ -21,6 +23,7 @@ export function participantFactsOf(outcome) {
|
|
|
21
23
|
skipped: outcome.skippedReason !== undefined,
|
|
22
24
|
noEngagement: outcome.noEngagement === true,
|
|
23
25
|
selfReportedBlocker: outcome.selfReportedBlocker === true,
|
|
26
|
+
...(ranUntil === undefined ? {} : { ranUntil }),
|
|
24
27
|
};
|
|
25
28
|
}
|
|
26
29
|
/** Whether a participant passed. A dry-run participant passes as a contract. */
|
|
@@ -4,7 +4,7 @@ import { type ResolvedPersona } from "../../study/persona.js";
|
|
|
4
4
|
import { type RunRerunLineage } from "../../run/bundle.js";
|
|
5
5
|
import type { PreparedSelectedOutputDirectory } from "../../run/contained-output.js";
|
|
6
6
|
import { type CuaActorStudyErrorCode, type CuaParticipantDeps, type CuaParticipantPlan, type CuaParticipantPlanEntry, type DesktopParticipantRun, type ParticipantRunsAndPlan } from "./types.js";
|
|
7
|
-
import { type
|
|
7
|
+
import { type ParticipantSchedule } from "../../study/arrivals.js";
|
|
8
8
|
/**
|
|
9
9
|
* The session budget of a study that declares no execution.timeoutMs. On a provisioned route it is
|
|
10
10
|
* what the `ceilingMs` sandbox ceiling leaves after provisioning, seeding and the teardown buffer.
|
|
@@ -19,10 +19,12 @@ export declare function defaultSessionTimeoutMs(config: StudyConfig, ceilingMs:
|
|
|
19
19
|
export declare function resolveParticipantSandboxMs(config: StudyConfig, ceilingMs: number): number;
|
|
20
20
|
/**
|
|
21
21
|
* The deps one participant runs with once its start comes: its own session budget and sandbox
|
|
22
|
-
* deadline, and with a declared end or cap, how its session ends
|
|
23
|
-
* due, from runOnSchedule).
|
|
22
|
+
* deadline (participantSession), and with a declared end or cap, how its session ends
|
|
23
|
+
* (`scheduledAt` is when it was due, from runOnSchedule).
|
|
24
24
|
*/
|
|
25
|
-
export declare function scheduledParticipantDeps(participant:
|
|
25
|
+
export declare function scheduledParticipantDeps(participant: {
|
|
26
|
+
readonly schedule: ParticipantSchedule;
|
|
27
|
+
}, deps: {
|
|
26
28
|
readonly timeoutMs: number;
|
|
27
29
|
readonly sandboxMs: number;
|
|
28
30
|
}, scheduledAt: number): Pick<CuaParticipantDeps, "timeoutMs" | "sandboxMs" | "end">;
|
|
@@ -31,16 +33,15 @@ export declare function scheduledParticipantDeps(participant: ParticipantWindow,
|
|
|
31
33
|
* declared participant runs at once, because a throttle nobody asked for silently turns "N actors
|
|
32
34
|
* live" into waves; total session count and spend are the same either way, only wall-clock
|
|
33
35
|
* and simultaneity differ. A declared execution.concurrency is a cap, clamped to [1, participants].
|
|
34
|
-
*
|
|
36
|
+
* participantsAtOnce then lowers it to what the E2B plan and HUMANISH_CUA_MAX_CONCURRENCY allow.
|
|
35
37
|
*/
|
|
36
38
|
export declare function boundedConcurrency(declared: number | undefined, participantCount: number): number;
|
|
37
39
|
/**
|
|
38
40
|
* Build the participant runs and the public plan from a computer-use plan (pure). The same plan
|
|
39
|
-
* appears in dry-run and live, marked $0 for a dry run. `
|
|
40
|
-
*
|
|
41
|
+
* appears in dry-run and live, marked $0 for a dry run. `personas` are the compiled committed
|
|
42
|
+
* personas the prompts carry.
|
|
41
43
|
*/
|
|
42
44
|
export declare function participantRunsAndPlan(plan: ComputerUsePlan, opts?: {
|
|
43
|
-
env?: Record<string, string | undefined>;
|
|
44
45
|
personas?: Map<string, ResolvedPersona>;
|
|
45
46
|
}): ParticipantRunsAndPlan;
|
|
46
47
|
/** Print the participant plan to stderr before any sandbox/provider call (public-safe: ids, devices,
|
|
@@ -63,7 +64,6 @@ export declare function loadCuaParticipants(args: {
|
|
|
63
64
|
plan: ComputerUsePlan;
|
|
64
65
|
cwd: string;
|
|
65
66
|
projectRoot: PreparedSelectedOutputDirectory;
|
|
66
|
-
env: Record<string, string | undefined>;
|
|
67
67
|
}): Promise<{
|
|
68
68
|
ok: true;
|
|
69
69
|
participantRuns: DesktopParticipantRun[];
|
|
@@ -10,15 +10,14 @@ import { redactText } from "../../evidence/redaction.js";
|
|
|
10
10
|
import {} from "../../run/bundle.js";
|
|
11
11
|
import { participantAssignment } from "../../study/participant-assignment.js";
|
|
12
12
|
import { resolveCommittedPersonas } from "../../study/persona-resolve.js";
|
|
13
|
-
import { CUA_FANOUT_STRATEGY,
|
|
14
|
-
import { sandboxHeadroomMs } from "../../substrates/e2b/lifetime.js";
|
|
15
|
-
import { readPositiveInt } from "../../study/parse/values.js";
|
|
13
|
+
import { CUA_FANOUT_STRATEGY, DEFAULT_APP_URL_SESSION_TIMEOUT_MS, MIN_DERIVED_SESSION_TIMEOUT_MS, } from "./types.js";
|
|
14
|
+
import { CUA_MAX_CONCURRENCY_ENV, sandboxHeadroomMs } from "../../substrates/e2b/lifetime.js";
|
|
16
15
|
import { digestUrl } from "./bundle-parts.js";
|
|
17
16
|
import { composeParticipantInstructions, DEFAULT_MISSION } from "./participant-prompt.js";
|
|
18
17
|
import { resolveCuaRerunSelection } from "./rerun-selection.js";
|
|
19
18
|
import { plural } from "../../run/text.js";
|
|
20
19
|
import { formatDuration } from "../../run/projection.js";
|
|
21
|
-
import { describeArrivals, participantEnd,
|
|
20
|
+
import { describeArrivals, participantEnd, participantSession, } from "../../study/arrivals.js";
|
|
22
21
|
/**
|
|
23
22
|
* The session budget of a study that declares no execution.timeoutMs. On a provisioned route it is
|
|
24
23
|
* what the `ceilingMs` sandbox ceiling leaves after provisioning, seeding and the teardown buffer.
|
|
@@ -45,27 +44,20 @@ export function resolveParticipantSandboxMs(config, ceilingMs) {
|
|
|
45
44
|
timeoutMs +
|
|
46
45
|
sandboxHeadroomMs(provisionedRoute ? { seed: config.subject.state?.seed ?? [] } : undefined));
|
|
47
46
|
}
|
|
48
|
-
/**
|
|
49
|
-
* A participant's own session budget and sandbox deadline: the study's, or with a declared end or
|
|
50
|
-
* cap its window or cap plus room for the closing report (participantSessionMs), with the same
|
|
51
|
-
* setup and teardown headroom the study's sandbox has over its session.
|
|
52
|
-
*/
|
|
53
|
-
function participantWindow(participant, study) {
|
|
54
|
-
const timeoutMs = participantSessionMs(participant, study.sessionBudgetMs);
|
|
55
|
-
return { timeoutMs, sandboxMs: study.sandboxMs - study.sessionBudgetMs + timeoutMs };
|
|
56
|
-
}
|
|
57
47
|
/**
|
|
58
48
|
* The deps one participant runs with once its start comes: its own session budget and sandbox
|
|
59
|
-
* deadline, and with a declared end or cap, how its session ends
|
|
60
|
-
* due, from runOnSchedule).
|
|
49
|
+
* deadline (participantSession), and with a declared end or cap, how its session ends
|
|
50
|
+
* (`scheduledAt` is when it was due, from runOnSchedule).
|
|
61
51
|
*/
|
|
62
52
|
export function scheduledParticipantDeps(participant, deps, scheduledAt) {
|
|
63
|
-
const
|
|
53
|
+
const own = participantSession(participant.schedule, {
|
|
54
|
+
sessionMs: deps.timeoutMs,
|
|
55
|
+
sandboxMs: deps.sandboxMs,
|
|
56
|
+
});
|
|
57
|
+
const end = participantEnd(participant.schedule, scheduledAt);
|
|
64
58
|
return {
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
sandboxMs: deps.sandboxMs,
|
|
68
|
-
}),
|
|
59
|
+
timeoutMs: own.sessionMs,
|
|
60
|
+
sandboxMs: own.sandboxMs,
|
|
69
61
|
...(end === undefined ? {} : { end }),
|
|
70
62
|
};
|
|
71
63
|
}
|
|
@@ -74,33 +66,17 @@ export function scheduledParticipantDeps(participant, deps, scheduledAt) {
|
|
|
74
66
|
* declared participant runs at once, because a throttle nobody asked for silently turns "N actors
|
|
75
67
|
* live" into waves; total session count and spend are the same either way, only wall-clock
|
|
76
68
|
* and simultaneity differ. A declared execution.concurrency is a cap, clamped to [1, participants].
|
|
77
|
-
*
|
|
69
|
+
* participantsAtOnce then lowers it to what the E2B plan and HUMANISH_CUA_MAX_CONCURRENCY allow.
|
|
78
70
|
*/
|
|
79
71
|
export function boundedConcurrency(declared, participantCount) {
|
|
80
72
|
return Math.max(1, declared === undefined ? participantCount : Math.min(Math.max(1, declared), participantCount));
|
|
81
73
|
}
|
|
82
|
-
/**
|
|
83
|
-
* The planned bound, lowered by the env override. The override may only lower it (never raise
|
|
84
|
-
* concurrent paid desktops), and a lowering is reported via envLoweredFrom so the
|
|
85
|
-
* plan never silently disagrees with the manifest.
|
|
86
|
-
*/
|
|
87
|
-
function envLoweredConcurrency(planned, participantCount, env) {
|
|
88
|
-
const envLower = readPositiveInt(env[CUA_MAX_CONCURRENCY_ENV], 0);
|
|
89
|
-
if (envLower > 0 && envLower < planned) {
|
|
90
|
-
return {
|
|
91
|
-
bound: Math.max(1, Math.min(planned, envLower, participantCount)),
|
|
92
|
-
envLoweredFrom: planned,
|
|
93
|
-
};
|
|
94
|
-
}
|
|
95
|
-
return { bound: planned };
|
|
96
|
-
}
|
|
97
74
|
/**
|
|
98
75
|
* Build the participant runs and the public plan from a computer-use plan (pure). The same plan
|
|
99
|
-
* appears in dry-run and live, marked $0 for a dry run. `
|
|
100
|
-
*
|
|
76
|
+
* appears in dry-run and live, marked $0 for a dry run. `personas` are the compiled committed
|
|
77
|
+
* personas the prompts carry.
|
|
101
78
|
*/
|
|
102
79
|
export function participantRunsAndPlan(plan, opts = {}) {
|
|
103
|
-
const env = opts.env ?? {};
|
|
104
80
|
// Who each participant is (id, persona, focus, device, limits) comes from the plan; this adds
|
|
105
81
|
// what the route derives from it: the prompt, bundle ids and artifact paths.
|
|
106
82
|
const { participants } = plan.runner;
|
|
@@ -136,61 +112,54 @@ export function participantRunsAndPlan(plan, opts = {}) {
|
|
|
136
112
|
traceArtifactPath: participantCount === 1 ? "actor.json" : `actors/${run.streamId}.json`,
|
|
137
113
|
};
|
|
138
114
|
});
|
|
139
|
-
const
|
|
140
|
-
const
|
|
141
|
-
const
|
|
142
|
-
const schedule = resolved.envLoweredFrom === undefined
|
|
143
|
-
? plan.arrivals
|
|
144
|
-
: planArrivals(participants.map((participant) => participant.startAfterMs), { slots: concurrency, sessionMs: sessionBudgetMs }, participants.map((participant) => participantSessionMs(participant, sessionBudgetMs)));
|
|
145
|
-
const sandboxesMs = participants.reduce((sum, participant) => sum + participantWindow(participant, plan).sandboxMs, 0);
|
|
115
|
+
const { concurrency, sessionBudgetMs, arrivals: schedule, envLoweredConcurrencyFrom } = plan;
|
|
116
|
+
const study = { sessionMs: sessionBudgetMs, sandboxMs: plan.sandboxMs };
|
|
117
|
+
const sandboxesMs = participants.reduce((sum, participant) => sum + participantSession(participant.schedule, study).sandboxMs, 0);
|
|
146
118
|
const participantPlan = {
|
|
147
119
|
strategy: CUA_FANOUT_STRATEGY,
|
|
148
120
|
laneCount: participantCount,
|
|
149
121
|
concurrency,
|
|
150
|
-
...(
|
|
151
|
-
? {}
|
|
152
|
-
: { envLoweredConcurrencyFrom: resolved.envLoweredFrom }),
|
|
122
|
+
...(envLoweredConcurrencyFrom === undefined ? {} : { envLoweredConcurrencyFrom }),
|
|
153
123
|
waves: Math.ceil(participantCount / concurrency),
|
|
154
124
|
perLaneSessionBudgetMs: sessionBudgetMs,
|
|
155
125
|
worstCaseSandboxMinutes: Math.round(sandboxesMs / 60_000),
|
|
156
126
|
...(schedule.declared ? { schedule } : {}),
|
|
157
127
|
dryRun: plan.dryRun,
|
|
158
|
-
lanes: runs.map((spec) => (
|
|
159
|
-
id: spec.planned.id,
|
|
160
|
-
...(spec.planned.labels.actorType === undefined
|
|
161
|
-
? {}
|
|
162
|
-
: { actorType: spec.planned.labels.actorType }),
|
|
163
|
-
...(spec.planned.labels.surface === undefined
|
|
164
|
-
? {}
|
|
165
|
-
: { surface: spec.planned.labels.surface }),
|
|
166
|
-
...(spec.planned.labels.caseGroup === undefined
|
|
167
|
-
? {}
|
|
168
|
-
: { caseGroup: spec.planned.labels.caseGroup }),
|
|
169
|
-
index: spec.planned.index + 1,
|
|
170
|
-
persona: spec.persona.id,
|
|
171
|
-
device: spec.planned.device.name,
|
|
172
|
-
resolution: spec.planned.device.resolution,
|
|
173
|
-
instructionDigest: spec.persona.promptDigest,
|
|
174
|
-
...(spec.planned.limits.reasoningEffort === undefined
|
|
175
|
-
? {}
|
|
176
|
-
: { reasoningEffort: spec.planned.limits.reasoningEffort }),
|
|
177
|
-
...(spec.planned.limits.maxOutputTokens === undefined
|
|
178
|
-
? {}
|
|
179
|
-
: { maxOutputTokens: spec.planned.limits.maxOutputTokens }),
|
|
180
|
-
...(spec.planned.targetUrl === undefined
|
|
181
|
-
? {}
|
|
182
|
-
: { targetDigest: digestUrl(spec.planned.targetUrl) }),
|
|
183
|
-
...(spec.planned.startAfterMs === undefined
|
|
184
|
-
? {}
|
|
185
|
-
: { startAfterMs: spec.planned.startAfterMs }),
|
|
186
|
-
...(spec.planned.endAfterMs === undefined ? {} : { endAfterMs: spec.planned.endAfterMs }),
|
|
187
|
-
...(spec.planned.maxSessionMs === undefined
|
|
188
|
-
? {}
|
|
189
|
-
: { maxSessionMs: spec.planned.maxSessionMs }),
|
|
190
|
-
})),
|
|
128
|
+
lanes: runs.map((spec) => planEntry(spec)),
|
|
191
129
|
};
|
|
192
130
|
return { runs, participantPlan };
|
|
193
131
|
}
|
|
132
|
+
/** One participant's row in the plan: ids, labels, device, digests and its declared times. */
|
|
133
|
+
function planEntry(spec) {
|
|
134
|
+
const { startAfterMs, end } = spec.planned.schedule;
|
|
135
|
+
return {
|
|
136
|
+
id: spec.planned.id,
|
|
137
|
+
...(spec.planned.labels.actorType === undefined
|
|
138
|
+
? {}
|
|
139
|
+
: { actorType: spec.planned.labels.actorType }),
|
|
140
|
+
...(spec.planned.labels.surface === undefined ? {} : { surface: spec.planned.labels.surface }),
|
|
141
|
+
...(spec.planned.labels.caseGroup === undefined
|
|
142
|
+
? {}
|
|
143
|
+
: { caseGroup: spec.planned.labels.caseGroup }),
|
|
144
|
+
index: spec.planned.index + 1,
|
|
145
|
+
persona: spec.persona.id,
|
|
146
|
+
device: spec.planned.device.name,
|
|
147
|
+
resolution: spec.planned.device.resolution,
|
|
148
|
+
instructionDigest: spec.persona.promptDigest,
|
|
149
|
+
...(spec.planned.limits.reasoningEffort === undefined
|
|
150
|
+
? {}
|
|
151
|
+
: { reasoningEffort: spec.planned.limits.reasoningEffort }),
|
|
152
|
+
...(spec.planned.limits.maxOutputTokens === undefined
|
|
153
|
+
? {}
|
|
154
|
+
: { maxOutputTokens: spec.planned.limits.maxOutputTokens }),
|
|
155
|
+
...(spec.planned.targetUrl === undefined
|
|
156
|
+
? {}
|
|
157
|
+
: { targetDigest: digestUrl(spec.planned.targetUrl) }),
|
|
158
|
+
...(startAfterMs === undefined ? {} : { startAfterMs }),
|
|
159
|
+
...(end?.kind === "scheduled_end" ? { endAfterMs: end.endAfterMs } : {}),
|
|
160
|
+
...(end?.kind === "session_cap" ? { maxSessionMs: end.windowMs } : {}),
|
|
161
|
+
};
|
|
162
|
+
}
|
|
194
163
|
/** Print the participant plan to stderr before any sandbox/provider call (public-safe: ids, devices,
|
|
195
164
|
* digests, and budgets only; no prompt text, no secrets). */
|
|
196
165
|
export function emitPreflightPlan(participantPlan, studyId) {
|
|
@@ -240,10 +209,7 @@ export async function loadCuaParticipants(args) {
|
|
|
240
209
|
// Compile any committed personas before planning, so the plan builder stays pure and each participant's
|
|
241
210
|
// prompt carries real behavioral directives rather than a bare `Persona: <id>.` label.
|
|
242
211
|
const personas = await compileParticipantPersonas(args.projectRoot, participants.map((participant) => participant.personaId));
|
|
243
|
-
const { runs: participantRuns, participantPlan } = participantRunsAndPlan(plan, {
|
|
244
|
-
env: args.env,
|
|
245
|
-
personas,
|
|
246
|
-
});
|
|
212
|
+
const { runs: participantRuns, participantPlan } = participantRunsAndPlan(plan, { personas });
|
|
247
213
|
const rerun = plan.rerun;
|
|
248
214
|
if (!rerun)
|
|
249
215
|
return { ok: true, participantRuns, participantPlan };
|
|
@@ -50,4 +50,6 @@ export declare function planComputerUseStudy(config: StudyConfig, input: {
|
|
|
50
50
|
readonly sandboxCeiling: SandboxCeiling;
|
|
51
51
|
/** How many sandboxes the operator's E2B plan runs at once (concurrentSandboxes). */
|
|
52
52
|
readonly concurrentSandboxes: ConcurrentSandboxes;
|
|
53
|
+
/** HUMANISH_CUA_MAX_CONCURRENCY (concurrencyCap), which may only lower the participants at once. */
|
|
54
|
+
readonly concurrencyCap?: number | undefined;
|
|
53
55
|
}): ComputerUsePlanResult;
|
|
@@ -198,30 +198,42 @@ function rosterShapeReason(config, ceilingMs) {
|
|
|
198
198
|
* How many of the runner's participants run at once, and when each one starts, with the session
|
|
199
199
|
* budget the `ceilingMs` sandbox ceiling leaves. Hosted desktops share the E2B plan's concurrent
|
|
200
200
|
* sandboxes; the local VM has its own capacity rules and the in-process route runs one
|
|
201
|
-
* participant.
|
|
201
|
+
* participant. HUMANISH_CUA_MAX_CONCURRENCY (`concurrencyCap`) lowers the count on every desktop.
|
|
202
|
+
* `starts` holds the plan's fields for them.
|
|
202
203
|
*/
|
|
203
|
-
function startsOf(config, runner,
|
|
204
|
+
function startsOf(config, runner, limits, ceilingMs) {
|
|
204
205
|
const declared = config.execution?.concurrency;
|
|
205
206
|
const n = runner.participants.length;
|
|
206
|
-
const
|
|
207
|
-
const atOnce =
|
|
208
|
-
|
|
209
|
-
:
|
|
207
|
+
const hosted = runner.desktop === "e2b-desktop";
|
|
208
|
+
const atOnce = participantsAtOnce({
|
|
209
|
+
concurrency: boundedConcurrency(declared, n),
|
|
210
|
+
declared: declared !== undefined,
|
|
211
|
+
participants: n,
|
|
212
|
+
}, {
|
|
213
|
+
...(hosted ? { sandboxes: limits.concurrentSandboxes } : {}),
|
|
214
|
+
cap: limits.concurrencyCap,
|
|
215
|
+
});
|
|
210
216
|
if (!atOnce.ok)
|
|
211
217
|
return atOnce;
|
|
212
218
|
const sessionMs = config.execution?.timeoutMs ?? defaultSessionTimeoutMs(config, ceilingMs);
|
|
213
219
|
// A participant with a declared end gets a sandbox sized from its own window.
|
|
214
|
-
if (
|
|
215
|
-
const
|
|
216
|
-
const window = windowDeadlineRefusal(runner.participants, { sessionMs,
|
|
220
|
+
if (hosted) {
|
|
221
|
+
const sandboxMs = resolveParticipantSandboxMs(config, ceilingMs);
|
|
222
|
+
const window = windowDeadlineRefusal(runner.participants, { sessionMs, sandboxMs }, ceilingMs);
|
|
217
223
|
if (window !== undefined)
|
|
218
224
|
return { ok: false, message: window };
|
|
219
225
|
}
|
|
220
|
-
const
|
|
226
|
+
const { arrivals, warnings } = planStarts(runner.participants, { slots: atOnce.count, sessionMs }, atOnce.lowered);
|
|
227
|
+
const lowered = atOnce.loweredFrom;
|
|
221
228
|
return {
|
|
222
229
|
ok: true,
|
|
223
|
-
|
|
224
|
-
|
|
230
|
+
starts: {
|
|
231
|
+
concurrency: atOnce.count,
|
|
232
|
+
...(lowered === undefined ? {} : { envLoweredConcurrencyFrom: lowered }),
|
|
233
|
+
arrivals,
|
|
234
|
+
sessionBudgetMs: sessionMs,
|
|
235
|
+
},
|
|
236
|
+
warnings,
|
|
225
237
|
};
|
|
226
238
|
}
|
|
227
239
|
/** The plan's rerun: the source run and, when given, the participants to rerun. */
|
|
@@ -349,7 +361,7 @@ export function planComputerUseStudy(config, input) {
|
|
|
349
361
|
const unsupported = runnerRefusal(runner.desktop, participants);
|
|
350
362
|
if (unsupported)
|
|
351
363
|
return refuse("in-scope", unsupported.code, unsupported.message, actor);
|
|
352
|
-
const starts = startsOf(config, runner, input
|
|
364
|
+
const starts = startsOf(config, runner, input, ceiling.ms);
|
|
353
365
|
if (!starts.ok)
|
|
354
366
|
return refuse("after-personas", "HUMANISH_COMPUTER_USE_FANOUT_INVALID", starts.message, actor);
|
|
355
367
|
const provisioned = source === "clone" || source === "local-tree";
|
|
@@ -362,9 +374,7 @@ export function planComputerUseStudy(config, input) {
|
|
|
362
374
|
route: "computer-use",
|
|
363
375
|
actor,
|
|
364
376
|
runner,
|
|
365
|
-
|
|
366
|
-
arrivals: starts.arrivals,
|
|
367
|
-
sessionBudgetMs: starts.arrivals.sessionMs,
|
|
377
|
+
...starts.starts,
|
|
368
378
|
sandboxMs: resolveParticipantSandboxMs(config, ceiling.ms),
|
|
369
379
|
caps: planCaps(config),
|
|
370
380
|
...(input.rerun === undefined ? {} : { rerun: rerunPlan(input.rerun) }),
|
|
@@ -7,6 +7,7 @@ import {} from "../../run/streams.js";
|
|
|
7
7
|
import { plural } from "../../run/text.js";
|
|
8
8
|
import { access } from "node:fs/promises";
|
|
9
9
|
import { cli } from "../../cli/invocation.js";
|
|
10
|
+
import { startingWithTheRun } from "../../study/arrivals.js";
|
|
10
11
|
export async function resolveCuaRerunSelection(args) {
|
|
11
12
|
const { studyId, sourceRunId } = args;
|
|
12
13
|
// Every refusal ends with the command that runs the whole study, which is what a user who
|
|
@@ -59,7 +60,7 @@ export async function resolveCuaRerunSelection(args) {
|
|
|
59
60
|
}
|
|
60
61
|
// A rerun runs the selected participants again now, all together: the study's start offsets
|
|
61
62
|
// placed them in the first run's day, and that day is over.
|
|
62
|
-
const selectedSpecs = selectedIds.map((id) =>
|
|
63
|
+
const selectedSpecs = selectedIds.map((id) => startingNow(specsById.get(id)));
|
|
63
64
|
const selectedPlanIds = new Set(selectedIds);
|
|
64
65
|
const selectedPlanEntries = args.participantPlan.lanes
|
|
65
66
|
.filter((entry) => selectedPlanIds.has(entry.id))
|
|
@@ -86,11 +87,13 @@ export async function resolveCuaRerunSelection(args) {
|
|
|
86
87
|
},
|
|
87
88
|
};
|
|
88
89
|
}
|
|
89
|
-
function
|
|
90
|
-
if (spec.planned.startAfterMs === undefined)
|
|
90
|
+
function startingNow(spec) {
|
|
91
|
+
if (spec.planned.schedule.startAfterMs === undefined)
|
|
91
92
|
return spec;
|
|
92
|
-
|
|
93
|
-
|
|
93
|
+
return {
|
|
94
|
+
...spec,
|
|
95
|
+
planned: { ...spec.planned, schedule: startingWithTheRun(spec.planned.schedule) },
|
|
96
|
+
};
|
|
94
97
|
}
|
|
95
98
|
function uniqueIds(values) {
|
|
96
99
|
const seen = new Set();
|
|
@@ -108,7 +108,7 @@ export async function admitCuaRun(plan, input, config) {
|
|
|
108
108
|
? comms?.email?.external
|
|
109
109
|
: undefined;
|
|
110
110
|
const externalCommsEmail = externalCommsConfig ? comms?.email : undefined;
|
|
111
|
-
const participants = await loadCuaParticipants({ plan, cwd, projectRoot
|
|
111
|
+
const participants = await loadCuaParticipants({ plan, cwd, projectRoot });
|
|
112
112
|
if (!participants.ok) {
|
|
113
113
|
return refuse(participants.code, participants.message, descriptor.id);
|
|
114
114
|
}
|
|
@@ -5,6 +5,7 @@ import {} from "../../run/streams.js";
|
|
|
5
5
|
import { aggregateTaskFunnels, formatParticipantOutcomes, formatRunTaskFunnel, participantSummary, withCuaReviewProvenance, } from "../../run/outcomes.js";
|
|
6
6
|
import { participantCaption } from "../../run/participant-caption.js";
|
|
7
7
|
import { actorEnding } from "../../actors/stop-cause.js";
|
|
8
|
+
import { cuaGoalSource, ranUntilClockEnd } from "../../actors/goal-source.js";
|
|
8
9
|
import { describeSubjectState, phaseEventIdSuffix, publicSafeAppUrlLabel, subjectProvenanceMessage, } from "./bundle-parts.js";
|
|
9
10
|
import { participantFeedbackCandidates } from "./participant-feedback.js";
|
|
10
11
|
import { judgeParticipantRecords } from "../../run/judge.js";
|
|
@@ -46,6 +47,7 @@ function runCost(args) {
|
|
|
46
47
|
}
|
|
47
48
|
/** The participant's status, reason, last frame, geometry and screenshot mode, shared by its records. */
|
|
48
49
|
function participantView(args, publicAppUrl) {
|
|
50
|
+
const ranUntil = args.session === undefined ? undefined : ranUntilClockEnd(args.session.trace);
|
|
49
51
|
const judgment = judgeParticipantRecords([
|
|
50
52
|
{
|
|
51
53
|
...(args.session === undefined
|
|
@@ -55,6 +57,7 @@ function participantView(args, publicAppUrl) {
|
|
|
55
57
|
completionReason: args.session.completionReason,
|
|
56
58
|
reason: args.session.reason,
|
|
57
59
|
}),
|
|
60
|
+
...(ranUntil === undefined ? {} : { ranUntil }),
|
|
58
61
|
...(args.sessionError === undefined ? {} : { sessionError: args.sessionError }),
|
|
59
62
|
inProgress: args.inProgress === true,
|
|
60
63
|
skipped: false,
|
|
@@ -298,8 +301,13 @@ function singleSummary(args, view, facts) {
|
|
|
298
301
|
: `1 participant was planned, but the run failed before their session finished: ${view.reason}`;
|
|
299
302
|
}
|
|
300
303
|
const ending = actorEnding(args.session.trace);
|
|
304
|
+
const goalSource = cuaGoalSource(args.session.trace);
|
|
301
305
|
const outcomes = formatParticipantOutcomes(view.tally, [
|
|
302
|
-
{
|
|
306
|
+
{
|
|
307
|
+
status: args.session.status,
|
|
308
|
+
...(ending === undefined ? {} : { label: ending.label }),
|
|
309
|
+
...(goalSource === undefined ? {} : { goalSource }),
|
|
310
|
+
},
|
|
303
311
|
]);
|
|
304
312
|
return [
|
|
305
313
|
`1 participant took part: ${outcomes}.`,
|
|
@@ -32,7 +32,6 @@ import type { ComputerUseParticipant, SharedWorldParticipant } from "../../study
|
|
|
32
32
|
import type { ResolvedParticipant } from "../../run/participant.js";
|
|
33
33
|
import { type StudyResultErrorCode, type StudyResultIdentity } from "../../run/study-result.js";
|
|
34
34
|
export declare const CUA_FANOUT_STRATEGY: "per-lane-worlds";
|
|
35
|
-
export declare const CUA_MAX_CONCURRENCY_ENV = "HUMANISH_CUA_MAX_CONCURRENCY";
|
|
36
35
|
export declare const DEFAULT_APP_URL_SESSION_TIMEOUT_MS: number;
|
|
37
36
|
export declare const MIN_DERIVED_SESSION_TIMEOUT_MS: number;
|
|
38
37
|
/**
|
|
@@ -126,8 +125,8 @@ export interface CuaParticipantPlan {
|
|
|
126
125
|
/** Effective in-flight bound (defaults to laneCount, all participants live; a declared
|
|
127
126
|
* execution.concurrency is a cap; the env override may only lower it). */
|
|
128
127
|
concurrency: number;
|
|
129
|
-
/** Present when
|
|
130
|
-
* plan never silently disagrees with the manifest. */
|
|
128
|
+
/** Present when HUMANISH_CUA_MAX_CONCURRENCY lowered the bound (the plan's
|
|
129
|
+
* envLoweredConcurrencyFrom), recorded so the plan never silently disagrees with the manifest. */
|
|
131
130
|
envLoweredConcurrencyFrom?: number;
|
|
132
131
|
/** ceil(laneCount / concurrency). */
|
|
133
132
|
waves: number;
|
|
@@ -15,9 +15,6 @@ import {} from "../../run/study-result.js";
|
|
|
15
15
|
// The fan-out topology of this route: N participants = N independent E2B desktop sandboxes,
|
|
16
16
|
// each its own world (clone/serve + subject.state per participant). Shared-world has its own route.
|
|
17
17
|
export const CUA_FANOUT_STRATEGY = "per-lane-worlds";
|
|
18
|
-
// Env override that may only lower the effective concurrency (never raise concurrent paid
|
|
19
|
-
// desktops). Read names-only into a local; the value never persists.
|
|
20
|
-
export const CUA_MAX_CONCURRENCY_ENV = "HUMANISH_CUA_MAX_CONCURRENCY";
|
|
21
18
|
// The default session budget, sized so a study can finish (docs/principles/three-roles.md: a
|
|
22
19
|
// session ends because the participant is done, not because a timer fired; the time-box is a
|
|
23
20
|
// session-level cap a researcher sets generously; spend protection is the dollar caps' job).
|
|
@@ -72,7 +72,7 @@ function sharedWorldSimulation(ctx, spec, index, view) {
|
|
|
72
72
|
updatedAt: args.run.createdAt,
|
|
73
73
|
...(args.plan.arrivals.declared
|
|
74
74
|
? {
|
|
75
|
-
arrival: arrivalRecord(spec.planned.startAfterMs, actorArrival(args.actorResults[index])),
|
|
75
|
+
arrival: arrivalRecord(spec.planned.schedule.startAfterMs, actorArrival(args.actorResults[index])),
|
|
76
76
|
}
|
|
77
77
|
: {}),
|
|
78
78
|
});
|
|
@@ -104,7 +104,7 @@ export async function runExternalPublicPlane(ctx, live, inbox) {
|
|
|
104
104
|
}))
|
|
105
105
|
: undefined;
|
|
106
106
|
const followerResultsPromise = runOnSchedule(followerEntries, {
|
|
107
|
-
startAfterMs: ({ spec }) => spec.planned.startAfterMs,
|
|
107
|
+
startAfterMs: ({ spec }) => spec.planned.schedule.startAfterMs,
|
|
108
108
|
slots: Math.max(1, concurrency - 1),
|
|
109
109
|
now: ctx.now,
|
|
110
110
|
}, async ({ spec, index }, _position, { scheduledAt }) => ({
|
|
@@ -23,6 +23,8 @@ export declare function planSharedWorldStudy(config: StudyConfig, input: {
|
|
|
23
23
|
readonly sandboxCeiling: SandboxCeiling;
|
|
24
24
|
/** How many sandboxes the operator's E2B plan runs at once (concurrentSandboxes). */
|
|
25
25
|
readonly concurrentSandboxes: ConcurrentSandboxes;
|
|
26
|
+
/** HUMANISH_CUA_MAX_CONCURRENCY (concurrencyCap), which may only lower the participants at once. */
|
|
27
|
+
readonly concurrencyCap?: number | undefined;
|
|
26
28
|
}): SharedWorldPlanResult;
|
|
27
29
|
/**
|
|
28
30
|
* The schedule lines a shared-world run prints before it creates anything, when the study declares
|
|
@@ -103,7 +103,7 @@ export function planSharedWorldStudy(config, input) {
|
|
|
103
103
|
if (plane === undefined)
|
|
104
104
|
throw new Error("shared-world validation admitted a plane it cannot plan");
|
|
105
105
|
const sessionTimeoutMs = config.execution?.timeoutMs ?? defaultSessionTimeoutMs(plane, ceiling.ms);
|
|
106
|
-
const starts = startsOf(plane, config.execution?.concurrency, input.concurrentSandboxes, sessionTimeoutMs);
|
|
106
|
+
const starts = startsOf(plane, config.execution?.concurrency, { sandboxes: input.concurrentSandboxes, cap: input.concurrencyCap }, sessionTimeoutMs);
|
|
107
107
|
if (!starts.ok)
|
|
108
108
|
return refuse(invalid, starts.message, actor);
|
|
109
109
|
const { arrivals, warnings } = starts;
|
|
@@ -154,7 +154,7 @@ export function planSharedWorldStudy(config, input) {
|
|
|
154
154
|
* shared world needs two participants live together. On the external-public plane the host runs
|
|
155
155
|
* on a slot of its own and the others queue for the rest (external-public.ts).
|
|
156
156
|
*/
|
|
157
|
-
function startsOf(plane, declared,
|
|
157
|
+
function startsOf(plane, declared, limits, sessionMs) {
|
|
158
158
|
const n = plane.participants.length;
|
|
159
159
|
const atOnce = participantsAtOnce({
|
|
160
160
|
concurrency: boundedConcurrency(declared, n),
|
|
@@ -162,7 +162,7 @@ function startsOf(plane, declared, limit, sessionMs) {
|
|
|
162
162
|
participants: n,
|
|
163
163
|
alongside: plane.kind === "provisioned" ? 1 : 0,
|
|
164
164
|
minimum: 2,
|
|
165
|
-
},
|
|
165
|
+
}, limits);
|
|
166
166
|
if (!atOnce.ok)
|
|
167
167
|
return atOnce;
|
|
168
168
|
const hostIndex = plane.participants.findIndex((participant) => participant.host === true);
|
|
@@ -391,7 +391,7 @@ function runParticipants(plane, ctx, live, setup) {
|
|
|
391
391
|
const { commsEmail } = setup;
|
|
392
392
|
const baseActorDeps = participantRunDeps(ctx, live, ctx.scrubKnownValues);
|
|
393
393
|
const schedule = {
|
|
394
|
-
startAfterMs: (spec) => spec.planned.startAfterMs,
|
|
394
|
+
startAfterMs: (spec) => spec.planned.schedule.startAfterMs,
|
|
395
395
|
slots: Math.max(1, ctx.concurrency),
|
|
396
396
|
now,
|
|
397
397
|
};
|
package/dist/run/bundle.d.ts
CHANGED
|
@@ -511,8 +511,18 @@ export interface RunRerunLineage {
|
|
|
511
511
|
export interface ParticipantOutcomes {
|
|
512
512
|
/** Participants whose sessions reached a terminal state: the denominator for every count below. */
|
|
513
513
|
total: number;
|
|
514
|
-
/**
|
|
514
|
+
/**
|
|
515
|
+
* Recorded successful sessions; completion provenance depends on the actor and its evidence. A
|
|
516
|
+
* computer-use session that passed at its scheduled end or session cap counts here only when
|
|
517
|
+
* its participant declared that it reached the goal; otherwise it counts in `ranUntilEnd`.
|
|
518
|
+
*/
|
|
515
519
|
reachedGoal: number;
|
|
520
|
+
/**
|
|
521
|
+
* Passed at a scheduled end or a session cap without declaring that they reached the goal: the
|
|
522
|
+
* closing report said `DID NOT REACH THE GOAL.` or declared nothing. Present when above 0.
|
|
523
|
+
* Bundles written before this count existed tallied them under `reachedGoal`.
|
|
524
|
+
*/
|
|
525
|
+
ranUntilEnd?: number;
|
|
516
526
|
/** Stopped trying. A finding about the product. */
|
|
517
527
|
abandoned: number;
|
|
518
528
|
/** Interrupted before reaching the goal, including session, spend and provider limits. */
|