humanish 0.121.0 → 0.123.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +31 -0
- package/CONTEXT.md +1 -1
- package/README.md +33 -6
- package/dist/actors/computer-use/actor.d.ts +3 -0
- package/dist/actors/computer-use/actor.js +1 -0
- package/dist/actors/computer-use/loop/backstop.d.ts +6 -1
- package/dist/actors/computer-use/loop/backstop.js +13 -1
- package/dist/actors/computer-use/loop/clock.d.ts +20 -0
- package/dist/actors/computer-use/loop/clock.js +64 -0
- package/dist/actors/computer-use/loop/ending.d.ts +1 -1
- package/dist/actors/computer-use/loop/session.d.ts +1 -1
- package/dist/actors/computer-use/loop/session.js +2 -1
- package/dist/actors/computer-use/loop/types.d.ts +16 -0
- package/dist/actors/computer-use/loop.d.ts +1 -1
- package/dist/actors/computer-use/loop.js +43 -8
- package/dist/actors/computer-use/wait.d.ts +3 -2
- package/dist/actors/computer-use/wait.js +5 -4
- package/dist/actors/contract.d.ts +3 -3
- package/dist/actors/contract.js +3 -0
- package/dist/actors/stop-cause.js +1 -0
- package/dist/analysis/automatic.js +2 -1
- package/dist/analysis/evidence.js +41 -21
- package/dist/cli/commands/study-run.js +9 -3
- package/dist/feedback/derivative-text.d.ts +12 -0
- package/dist/feedback/derivative-text.js +130 -0
- package/dist/feedback/export-bundle.js +38 -131
- package/dist/feedback/export.js +9 -1
- package/dist/observer/run-routes.js +61 -18
- package/dist/routes/computer-use/bundle.d.ts +0 -1
- package/dist/routes/computer-use/bundle.js +0 -1
- package/dist/routes/computer-use/diagnostics.d.ts +1 -1
- package/dist/routes/computer-use/diagnostics.js +4 -0
- package/dist/routes/computer-use/fanout-bundle.d.ts +2 -3
- package/dist/routes/computer-use/fanout-bundle.js +2 -12
- package/dist/routes/computer-use/live-phase.d.ts +0 -2
- package/dist/routes/computer-use/live-phase.js +2 -4
- package/dist/routes/computer-use/participant-execution.d.ts +9 -15
- package/dist/routes/computer-use/participant-execution.js +22 -31
- package/dist/routes/computer-use/participant-model.js +17 -0
- package/dist/routes/computer-use/participant-runs.d.ts +11 -1
- package/dist/routes/computer-use/participant-runs.js +31 -4
- package/dist/routes/computer-use/plan.js +8 -1
- package/dist/routes/computer-use/result.js +1 -2
- package/dist/routes/computer-use/types.d.ts +17 -5
- package/dist/routes/shared-world/provisioned.js +2 -2
- package/dist/run/bundle-shape.js +2 -2
- package/dist/run/bundle.d.ts +5 -2
- package/dist/run/bundle.js +4 -1
- package/dist/run/detail.d.ts +6 -0
- package/dist/run/detail.js +14 -1
- package/dist/run/host-suspension.d.ts +2 -2
- package/dist/run/host-suspension.js +2 -2
- package/dist/run/judge.d.ts +1 -1
- package/dist/run/locate.js +3 -1
- package/dist/run/projection.d.ts +7 -0
- package/dist/run/projection.js +10 -0
- package/dist/run/run.js +7 -1
- package/dist/run/stored-runs.js +10 -4
- package/dist/run/stream-shape.d.ts +11 -1
- package/dist/run/stream-shape.js +26 -2
- package/dist/run/stream-trace.d.ts +68 -0
- package/dist/run/stream-trace.js +260 -0
- package/dist/run/streams.d.ts +20 -0
- package/dist/study/arrivals.d.ts +52 -5
- package/dist/study/arrivals.js +105 -15
- package/dist/study/bundle-estimate.d.ts +48 -0
- package/dist/study/bundle-estimate.js +159 -0
- package/dist/study/doctor.js +4 -0
- package/dist/study/keys.d.ts +6 -0
- package/dist/study/keys.js +4 -0
- package/dist/study/migrate/v2.js +8 -5
- package/dist/study/parse/actors.d.ts +1 -1
- package/dist/study/parse/actors.js +33 -4
- package/dist/study/plan-participants.d.ts +10 -1
- package/dist/study/plan-participants.js +10 -1
- package/dist/study/plan-types.d.ts +3 -0
- package/dist/study/plan.js +22 -2
- package/dist/study/preflight.js +19 -1
- package/dist/study/types.d.ts +26 -0
- package/dist/study/validation.d.ts +3 -2
- package/dist/study/validation.js +22 -4
- package/dist/tui-app.js +87 -87
- package/dist/verify/artifacts.d.ts +17 -10
- package/dist/verify/artifacts.js +22 -13
- package/dist/verify/scan-limit.d.ts +6 -0
- package/dist/verify/scan-limit.js +25 -0
- package/dist/verify/verify.d.ts +1 -1
- package/dist/verify/verify.js +68 -18
- package/docs/architecture/github-feedback-loop.md +1 -1
- package/docs/contracts/README.md +1 -1
- package/docs/contracts/run-bundle.md +89 -8
- package/docs/contracts/schemas.md +38 -11
- package/package.json +6 -3
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
// HTTP handlers for one run's Observer: the page, its data with live status and runtime stream
|
|
2
2
|
// URLs, its analysis, and its media with byte ranges. serveObserver (render.ts) and the library
|
|
3
3
|
// server (serve.ts) both route requests through them.
|
|
4
|
+
import { lstat } from "node:fs/promises";
|
|
4
5
|
import path from "node:path";
|
|
5
6
|
import { isAnalysisRecordPath, projectShareCheckedAnalysis } from "../analysis/sharing.js";
|
|
6
7
|
import { loadAnalysis } from "../analysis/load.js";
|
|
@@ -9,6 +10,8 @@ import { listRuns } from "../run/stored-runs.js";
|
|
|
9
10
|
import { readAnalysisAccounting } from "../run/costs.js";
|
|
10
11
|
import { bindExistingRunArtifactPaths, isPathInside, isSafeRunIdSegment, } from "../run/paths.js";
|
|
11
12
|
import { isRunOutcome } from "../run/bundle-shape.js";
|
|
13
|
+
import { displayedTraces, loadStreamTraces } from "../run/stream-trace.js";
|
|
14
|
+
import { isNodeError, isRecord } from "../run/type-guards.js";
|
|
12
15
|
import { MAX_NOTE_FILE_BYTES, readRunNotes, RUN_NOTES_DIR, } from "../run/note-files.js";
|
|
13
16
|
import { analysisCostOf, runCost, runCostLabel } from "../run/run-cost.js";
|
|
14
17
|
import { RUN_STATUS_FILE, RUN_STATUS_STALE_MS, isRunStatusRecord, } from "../run/status.js";
|
|
@@ -99,9 +102,9 @@ export async function serveRunPath(runRoot, relativePath, response, runtimeStrea
|
|
|
99
102
|
return;
|
|
100
103
|
}
|
|
101
104
|
if (cleanedRelativePath === "observer/index.html") {
|
|
102
|
-
const observerData = await
|
|
103
|
-
if (
|
|
104
|
-
writeResponse(response, 404,
|
|
105
|
+
const observerData = await observerDataOrReason(runRoot, runtimeStreamUrls);
|
|
106
|
+
if ("unavailable" in observerData) {
|
|
107
|
+
writeResponse(response, 404, `Observer data not found: ${observerData.unavailable}`, "text/plain; charset=utf-8");
|
|
105
108
|
return;
|
|
106
109
|
}
|
|
107
110
|
const analysis = await readObserverAnalysis(runRoot);
|
|
@@ -114,9 +117,9 @@ export async function serveRunPath(runRoot, relativePath, response, runtimeStrea
|
|
|
114
117
|
return;
|
|
115
118
|
}
|
|
116
119
|
if (cleanedRelativePath === "observer/observer-data.json") {
|
|
117
|
-
const observerData = await
|
|
118
|
-
if (
|
|
119
|
-
writeResponse(response, 404,
|
|
120
|
+
const observerData = await observerDataOrReason(runRoot, runtimeStreamUrls);
|
|
121
|
+
if ("unavailable" in observerData) {
|
|
122
|
+
writeResponse(response, 404, `Observer data not found: ${observerData.unavailable}`, "text/plain; charset=utf-8");
|
|
120
123
|
return;
|
|
121
124
|
}
|
|
122
125
|
writeResponse(response, 200, JSON.stringify(observerData, null, 2), "application/json; charset=utf-8");
|
|
@@ -153,29 +156,69 @@ export async function serveRunPath(runRoot, relativePath, response, runtimeStrea
|
|
|
153
156
|
writeResponse(response, 404, "Not found", "text/plain; charset=utf-8");
|
|
154
157
|
}
|
|
155
158
|
}
|
|
159
|
+
/** The run's trace files, read through its pinned folder. */
|
|
160
|
+
function pinnedTraceReader(runRoot) {
|
|
161
|
+
return async (relativePath, maxBytes) => {
|
|
162
|
+
const bytes = await readContainedFile(runRoot, path.join(runRoot.physicalPath, relativePath), {
|
|
163
|
+
maxBytes,
|
|
164
|
+
}).catch(() => null);
|
|
165
|
+
return bytes === null ? { refused: `${relativePath} could not be read` } : { bytes };
|
|
166
|
+
};
|
|
167
|
+
}
|
|
156
168
|
async function readObserverData(runRoot, runtimeStreamUrls = []) {
|
|
169
|
+
const read = await observerDataOrReason(runRoot, runtimeStreamUrls);
|
|
170
|
+
return "unavailable" in read ? null : read;
|
|
171
|
+
}
|
|
172
|
+
/**
|
|
173
|
+
* The run's Observer data, or why there is none. run.json is the source: each trace is loaded
|
|
174
|
+
* through the loader, and a participant whose trace it refuses shows as unavailable, with no
|
|
175
|
+
* fallback. The saved observer-data.json is read only while the run has no run.json at all, or in
|
|
176
|
+
* safe serving, where only its admitted bytes can be read. Otherwise a run.json that is there and
|
|
177
|
+
* cannot be read, parsed or used as a bundle (which a browser poll can meet mid-write) gives the
|
|
178
|
+
* reason, a 404 the poller retries, never the saved projection.
|
|
179
|
+
*/
|
|
180
|
+
async function observerDataOrReason(runRoot, runtimeStreamUrls) {
|
|
157
181
|
const record = await readLocalRunStatus(runRoot);
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
//
|
|
182
|
+
const runJson = path.join(runRoot.physicalPath, "run.json");
|
|
183
|
+
const bundleBytes = await readContainedFile(runRoot, runJson).catch(() => null);
|
|
184
|
+
const absent = bundleBytes === null &&
|
|
185
|
+
(await lstat(runJson).then(() => false, (error) => isNodeError(error) && error.code === "ENOENT"));
|
|
186
|
+
// A root admitted for safe serving checks every read against the identity and digest admitted
|
|
187
|
+
// with the run's share_ready verification, so a run.json changed since then is refused, and the
|
|
188
|
+
// saved projection it may serve is the one admitted with the verified run.
|
|
189
|
+
if (absent || (bundleBytes === null && runRoot.admitsContent !== undefined))
|
|
190
|
+
return savedObserverData(runRoot, record, runtimeStreamUrls);
|
|
191
|
+
if (bundleBytes === null)
|
|
192
|
+
return { unavailable: "run.json is there but could not be read: it is a link or changed." };
|
|
193
|
+
let stored;
|
|
194
|
+
try {
|
|
195
|
+
const parsed = JSON.parse(bundleBytes.toString("utf8"));
|
|
196
|
+
if (!isRecord(parsed) || !Array.isArray(parsed.streams))
|
|
197
|
+
throw new Error("not a run bundle");
|
|
198
|
+
stored = parsed;
|
|
199
|
+
}
|
|
200
|
+
catch {
|
|
201
|
+
return { unavailable: "run.json is not a run bundle; it may be mid-write, so try again." };
|
|
202
|
+
}
|
|
163
203
|
try {
|
|
164
|
-
const
|
|
165
|
-
if (!bundleBytes)
|
|
166
|
-
throw new Error("run.json unavailable");
|
|
167
|
-
const bundle = JSON.parse(bundleBytes.toString("utf8"));
|
|
204
|
+
const bundle = displayedTraces(await loadStreamTraces(pinnedTraceReader(runRoot), stored));
|
|
168
205
|
return withRuntimeStreamUrls(withLocalRunStatus(record, buildObserverData(bundle, undefined, record), bundle.outcome), runtimeStreamUrls);
|
|
169
206
|
}
|
|
170
|
-
catch {
|
|
207
|
+
catch {
|
|
208
|
+
return { unavailable: "run.json could not be shown as a run." };
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
/** The saved observer-data.json of a run that has written no run.json yet. */
|
|
212
|
+
async function savedObserverData(runRoot, record, runtimeStreamUrls) {
|
|
171
213
|
try {
|
|
172
214
|
const observerBytes = await readContainedFile(runRoot, path.join(runRoot.physicalPath, "observer", "observer-data.json"));
|
|
173
215
|
if (!observerBytes)
|
|
174
216
|
throw new Error("observer-data.json unavailable");
|
|
175
217
|
return withRuntimeStreamUrls(withLocalRunStatus(record, withObserverEndings(JSON.parse(observerBytes.toString("utf8")))), runtimeStreamUrls);
|
|
176
218
|
}
|
|
177
|
-
catch {
|
|
178
|
-
|
|
219
|
+
catch {
|
|
220
|
+
return { unavailable: "The run has written no run.json or Observer data yet." };
|
|
221
|
+
}
|
|
179
222
|
}
|
|
180
223
|
/** The run directory's own status record, when it is well formed and names that directory. */
|
|
181
224
|
async function readLocalRunStatus(runRoot) {
|
|
@@ -35,7 +35,6 @@ interface CuaRunBundleState {
|
|
|
35
35
|
outcomes: ParticipantRunOutcome[] | undefined;
|
|
36
36
|
subjects: RunSubjectProvenance[];
|
|
37
37
|
aggregateSubject: RunSubjectProvenance;
|
|
38
|
-
failFastReason?: string;
|
|
39
38
|
/** The final bundle's reading of the run's host suspensions. */
|
|
40
39
|
hostSuspension?: HostSuspensionReading;
|
|
41
40
|
inProgress?: true;
|
|
@@ -60,7 +60,6 @@ export function buildCuaRunBundle(base, state) {
|
|
|
60
60
|
source: base.source,
|
|
61
61
|
participantPlan: base.participantPlan,
|
|
62
62
|
...(base.rerun === undefined ? {} : { rerun: base.rerun }),
|
|
63
|
-
...(state.failFastReason === undefined ? {} : { failFastReason: state.failFastReason }),
|
|
64
63
|
...(state.hostSuspension === undefined ? {} : { hostSuspension: state.hostSuspension }),
|
|
65
64
|
...(base.publicRepo === undefined ? {} : { publicRepo: base.publicRepo }),
|
|
66
65
|
...inProgress,
|
|
@@ -3,7 +3,7 @@ import { type ActorCompletionReason, type ActorStatus, type ActorStopCause } fro
|
|
|
3
3
|
declare const CUA_DIAGNOSTIC_CATEGORIES: readonly ["preview", "participant_outcome", "session_interrupted", "execution_error", "evidence_invalid", "mixed", "unknown"];
|
|
4
4
|
export type CuaDiagnosticCategory = (typeof CUA_DIAGNOSTIC_CATEGORIES)[number];
|
|
5
5
|
/** Closed at both projection and telemetry boundaries. No provider messages are classified. */
|
|
6
|
-
declare const CUA_DIAGNOSTIC_STOP_CAUSES: readonly ["provider_output_limit", "provider_token_limit", "time_limit", "spend_limit", "study_spend_limit", "adapter_limit", "provider_incomplete", "provider_status", "provider_refused_prompt", "harness_aborted", "usage_unreported", "unspecified_limit", "mixed", "unknown"];
|
|
6
|
+
declare const CUA_DIAGNOSTIC_STOP_CAUSES: readonly ["provider_output_limit", "provider_token_limit", "time_limit", "spend_limit", "study_spend_limit", "adapter_limit", "provider_incomplete", "provider_status", "provider_refused_prompt", "harness_aborted", "usage_unreported", "scheduled_end", "unspecified_limit", "mixed", "unknown"];
|
|
7
7
|
export type CuaDiagnosticStopCause = (typeof CUA_DIAGNOSTIC_STOP_CAUSES)[number];
|
|
8
8
|
export interface CuaDiagnostics {
|
|
9
9
|
category: CuaDiagnosticCategory;
|
|
@@ -30,6 +30,10 @@ export function cuaParticipantDiagnostics(input) {
|
|
|
30
30
|
const session = input.session;
|
|
31
31
|
if (!session)
|
|
32
32
|
return { category: "unknown" };
|
|
33
|
+
// A scheduled end is the participant's planned ending, so a run whose other participants ended
|
|
34
|
+
// themselves reads as one kind of ending.
|
|
35
|
+
if (session.stopCause === "scheduled_end")
|
|
36
|
+
return { category: "participant_outcome" };
|
|
33
37
|
if (session.stopCause !== undefined) {
|
|
34
38
|
return {
|
|
35
39
|
category: "session_interrupted",
|
|
@@ -3,9 +3,8 @@ import type { CuaFanoutBundleArgs } from "./types.js";
|
|
|
3
3
|
/**
|
|
4
4
|
* Project N>1 fan-out participants into a humanish.run-bundle.v1 (the evidence schema is unchanged; this
|
|
5
5
|
* is a new producer for the multi-stream shape). One sim + one stream per participant; per-participant
|
|
6
|
-
* provenance/session events; a recorded `cua-lab.fanout.plan` event
|
|
7
|
-
*
|
|
8
|
-
* streams. A single-participant run without a rerun never reaches here: buildCuaRunBundle
|
|
6
|
+
* provenance/session events; a recorded `cua-lab.fanout.plan` event. N-ary verify/Observer already
|
|
7
|
+
* handle multiple streams. A single-participant run without a rerun never reaches here: buildCuaRunBundle
|
|
9
8
|
* (bundle.ts) sends it to buildSingleParticipantBundle (single-bundle.ts).
|
|
10
9
|
*/
|
|
11
10
|
export declare function buildCuaFanoutBundle(args: CuaFanoutBundleArgs): RunBundle;
|
|
@@ -181,9 +181,8 @@ function fanoutFeedbackCandidates(args) {
|
|
|
181
181
|
/**
|
|
182
182
|
* Project N>1 fan-out participants into a humanish.run-bundle.v1 (the evidence schema is unchanged; this
|
|
183
183
|
* is a new producer for the multi-stream shape). One sim + one stream per participant; per-participant
|
|
184
|
-
* provenance/session events; a recorded `cua-lab.fanout.plan` event
|
|
185
|
-
*
|
|
186
|
-
* streams. A single-participant run without a rerun never reaches here: buildCuaRunBundle
|
|
184
|
+
* provenance/session events; a recorded `cua-lab.fanout.plan` event. N-ary verify/Observer already
|
|
185
|
+
* handle multiple streams. A single-participant run without a rerun never reaches here: buildCuaRunBundle
|
|
187
186
|
* (bundle.ts) sends it to buildSingleParticipantBundle (single-bundle.ts).
|
|
188
187
|
*/
|
|
189
188
|
export function buildCuaFanoutBundle(args) {
|
|
@@ -215,15 +214,6 @@ export function buildCuaFanoutBundle(args) {
|
|
|
215
214
|
streams.push(records.stream);
|
|
216
215
|
events.push(...records.events);
|
|
217
216
|
});
|
|
218
|
-
if (args.failFastReason) {
|
|
219
|
-
events.push({
|
|
220
|
-
id: nextEventId("fanout-fail-fast"),
|
|
221
|
-
at: args.run.createdAt,
|
|
222
|
-
level: "warn",
|
|
223
|
-
type: "cua-lab.fanout.fail-fast",
|
|
224
|
-
message: `Fan-out fail-fast: ${args.failFastReason}. In-flight participants finished; queued participants were skipped (blocked); completed evidence is retained.`,
|
|
225
|
-
});
|
|
226
|
-
}
|
|
227
217
|
const review = fanoutReview(args, streams, judgment);
|
|
228
218
|
const anyRaw = (outcomes ?? []).some((outcome) => outcome.session?.trace.redaction.screenshots === "raw");
|
|
229
219
|
const ranLive = (outcomes ?? []).some((outcome) => outcome.session !== undefined || outcome.sessionError !== undefined);
|
|
@@ -47,7 +47,6 @@ export declare function runStudyParticipants(setup: CuaRunSetup, participants: C
|
|
|
47
47
|
runId: string;
|
|
48
48
|
};
|
|
49
49
|
outcomes?: never;
|
|
50
|
-
failFastReason?: never;
|
|
51
50
|
receiving?: never;
|
|
52
51
|
receivingWarnings?: never;
|
|
53
52
|
externalCommsWarnings?: never;
|
|
@@ -55,7 +54,6 @@ export declare function runStudyParticipants(setup: CuaRunSetup, participants: C
|
|
|
55
54
|
result?: never;
|
|
56
55
|
ok: true;
|
|
57
56
|
outcomes: ParticipantRunOutcome[] | undefined;
|
|
58
|
-
failFastReason: string | undefined;
|
|
59
57
|
receiving: CommsReceivingRun | undefined;
|
|
60
58
|
receivingWarnings: string[];
|
|
61
59
|
externalCommsWarnings: string[];
|
|
@@ -94,17 +94,16 @@ export async function runStudyParticipants(setup, participants) {
|
|
|
94
94
|
}
|
|
95
95
|
// Run the participants (dry-run runs none). In-process is always one.
|
|
96
96
|
let outcomes;
|
|
97
|
-
let failFastReason;
|
|
98
97
|
try {
|
|
99
98
|
if (!dryRun)
|
|
100
|
-
|
|
99
|
+
outcomes = await runAllCuaParticipants(participantRuns, {
|
|
101
100
|
...deps,
|
|
102
101
|
// A session that throws returns no trace; the run still records that it started.
|
|
103
102
|
runSession: (options) => {
|
|
104
103
|
run.participantStarted();
|
|
105
104
|
return deps.runSession(options);
|
|
106
105
|
},
|
|
107
|
-
}, participantPlan, inProcess)
|
|
106
|
+
}, participantPlan, inProcess);
|
|
108
107
|
}
|
|
109
108
|
finally {
|
|
110
109
|
try {
|
|
@@ -130,7 +129,6 @@ export async function runStudyParticipants(setup, participants) {
|
|
|
130
129
|
return {
|
|
131
130
|
ok: true,
|
|
132
131
|
outcomes,
|
|
133
|
-
failFastReason,
|
|
134
132
|
receiving,
|
|
135
133
|
receivingWarnings,
|
|
136
134
|
externalCommsWarnings,
|
|
@@ -12,28 +12,22 @@ export declare function makeParticipantWriteScreenshot(artifactRoot: PreparedOut
|
|
|
12
12
|
export declare function runCuaParticipant(spec: DesktopParticipantRun, deps: CuaParticipantDeps): Promise<ParticipantRunOutcome>;
|
|
13
13
|
/**
|
|
14
14
|
* Run N>1 E2B participants on the study's schedule (runOnSchedule: each at its start offset, at
|
|
15
|
-
* most `concurrency` at once), behind the pipeline gate when the subject is set up in each sandbox
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
* ever enumerates.
|
|
15
|
+
* most `concurrency` at once), behind the pipeline gate when the subject is set up in each sandbox.
|
|
16
|
+
* A harness error ends only its own participant: one desktop's stall or one provider failure does
|
|
17
|
+
* not predict the next participant's, so the schedule goes on. Only a closed gate ends every wait
|
|
18
|
+
* for a later start at once, and the participants it skips are `blocked` with its reason. Each
|
|
19
|
+
* participant tears down its own sandbox by id; nothing here ever enumerates.
|
|
21
20
|
*
|
|
22
21
|
* Exported for the total-runner tests: the injectable runner lets a test make one participant
|
|
23
22
|
* throw (the exact class the guard exists for) without a live sandbox. Production always uses
|
|
24
23
|
* the default.
|
|
25
24
|
*/
|
|
26
|
-
export declare function runCuaParticipants(runs: DesktopParticipantRun[], deps: Omit<CuaParticipantDeps, "signalReady">, concurrency: number, runParticipant?: typeof runCuaParticipant): Promise<
|
|
27
|
-
outcomes: ParticipantRunOutcome[];
|
|
28
|
-
failFastReason?: string;
|
|
29
|
-
}>;
|
|
25
|
+
export declare function runCuaParticipants(runs: DesktopParticipantRun[], deps: Omit<CuaParticipantDeps, "signalReady">, concurrency: number, runParticipant?: typeof runCuaParticipant): Promise<ParticipantRunOutcome[]>;
|
|
30
26
|
/** Project one participant outcome (or a dry-run contract spec) into the public CuaParticipantResult. */
|
|
31
27
|
export declare function toParticipantResult(spec: DesktopParticipantRun, outcome: ParticipantRunOutcome | undefined, subject: RunSubjectProvenance, dryRun: boolean): CuaParticipantResult;
|
|
32
28
|
/**
|
|
33
29
|
* Run every participant of a live run. The in-process route drives its single participant in this
|
|
34
|
-
* process; one hosted participant runs alone; a fan-out runs at the plan's concurrency and
|
|
30
|
+
* process; one hosted participant runs alone; a fan-out runs at the plan's concurrency, and stops
|
|
31
|
+
* early only when the pipeline gate closes.
|
|
35
32
|
*/
|
|
36
|
-
export declare function runAllCuaParticipants(runs: readonly DesktopParticipantRun[], deps: Omit<CuaParticipantDeps, "signalReady">, participantPlan: CuaParticipantPlan, inProcess: boolean): Promise<
|
|
37
|
-
outcomes: ParticipantRunOutcome[];
|
|
38
|
-
failFastReason: string | undefined;
|
|
39
|
-
}>;
|
|
33
|
+
export declare function runAllCuaParticipants(runs: readonly DesktopParticipantRun[], deps: Omit<CuaParticipantDeps, "signalReady">, participantPlan: CuaParticipantPlan, inProcess: boolean): Promise<ParticipantRunOutcome[]>;
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { scheduledParticipantDeps } from "./participant-runs.js";
|
|
1
2
|
import { judgeParticipantRecords } from "../../run/judge.js";
|
|
2
3
|
import { createE2BParticipantDesktop } from "./e2b-desktop/desktop.js";
|
|
3
4
|
import { createInProcessDesktop } from "./in-process-desktop.js";
|
|
@@ -31,7 +32,7 @@ export function makeParticipantWriteScreenshot(artifactRoot, spec, screenshots)
|
|
|
31
32
|
return rel;
|
|
32
33
|
};
|
|
33
34
|
}
|
|
34
|
-
/** A blocked participant outcome
|
|
35
|
+
/** A blocked participant outcome: the pipeline gate or the study budget skipped it. */
|
|
35
36
|
function skippedOutcome(spec, reason, arrival) {
|
|
36
37
|
return {
|
|
37
38
|
spec,
|
|
@@ -110,8 +111,8 @@ export async function runCuaParticipant(spec, deps) {
|
|
|
110
111
|
if (session)
|
|
111
112
|
await recordParticipantTrace(spec, deps, session, warnings);
|
|
112
113
|
const { noEngagement, selfReportedBlocker, reportedFriction } = judgeParticipantSession(session, deps, warnings);
|
|
113
|
-
// A provider-cleanup or provider-policy failure
|
|
114
|
-
//
|
|
114
|
+
// A provider-cleanup or provider-policy failure counts in the participant summary's harness
|
|
115
|
+
// errors.
|
|
115
116
|
const harnessError = sessionError !== undefined ||
|
|
116
117
|
providerCleanupError !== undefined ||
|
|
117
118
|
providerPolicyError !== undefined ||
|
|
@@ -191,33 +192,26 @@ function pipelineGate(open, onClose) {
|
|
|
191
192
|
}
|
|
192
193
|
/**
|
|
193
194
|
* Run N>1 E2B participants on the study's schedule (runOnSchedule: each at its start offset, at
|
|
194
|
-
* most `concurrency` at once), behind the pipeline gate when the subject is set up in each sandbox
|
|
195
|
-
*
|
|
196
|
-
*
|
|
197
|
-
*
|
|
198
|
-
*
|
|
199
|
-
* ever enumerates.
|
|
195
|
+
* most `concurrency` at once), behind the pipeline gate when the subject is set up in each sandbox.
|
|
196
|
+
* A harness error ends only its own participant: one desktop's stall or one provider failure does
|
|
197
|
+
* not predict the next participant's, so the schedule goes on. Only a closed gate ends every wait
|
|
198
|
+
* for a later start at once, and the participants it skips are `blocked` with its reason. Each
|
|
199
|
+
* participant tears down its own sandbox by id; nothing here ever enumerates.
|
|
200
200
|
*
|
|
201
201
|
* Exported for the total-runner tests: the injectable runner lets a test make one participant
|
|
202
202
|
* throw (the exact class the guard exists for) without a live sandbox. Production always uses
|
|
203
203
|
* the default.
|
|
204
204
|
*/
|
|
205
205
|
export async function runCuaParticipants(runs, deps, concurrency, runParticipant = runCuaParticipant) {
|
|
206
|
-
const failFast = { tripped: false, reason: "" };
|
|
207
206
|
const stopping = new AbortController();
|
|
208
207
|
// An app-url subject is only opened, so one participant's start says nothing about another's.
|
|
209
208
|
const gate = pipelineGate(deps.subject.kind === "app-url", () => stopping.abort());
|
|
210
|
-
|
|
209
|
+
return runOnSchedule(runs, {
|
|
211
210
|
startAfterMs: (spec) => spec.planned.startAfterMs,
|
|
212
211
|
slots: concurrency,
|
|
213
212
|
now: deps.now,
|
|
214
213
|
signal: stopping.signal,
|
|
215
214
|
}, async (spec, _index, { scheduledAt }) => {
|
|
216
|
-
if (failFast.tripped) {
|
|
217
|
-
return skippedOutcome(spec, `skipped: ${failFast.reason}`, { scheduledAt });
|
|
218
|
-
}
|
|
219
|
-
// A participant trips fail-fast only after the gate opened, or as it closes the gate, so no
|
|
220
|
-
// participant is still waiting at the gate when fail-fast skips the rest.
|
|
221
215
|
const id = spec.planned.id;
|
|
222
216
|
if ((await gate.enter(id)) === "closed") {
|
|
223
217
|
return skippedOutcome(spec, `skipped: participant ${gate.holder()} failed to provision its world (pipeline gate)`, { scheduledAt });
|
|
@@ -233,6 +227,7 @@ export async function runCuaParticipants(runs, deps, concurrency, runParticipant
|
|
|
233
227
|
try {
|
|
234
228
|
outcome = await runParticipant(spec, {
|
|
235
229
|
...deps,
|
|
230
|
+
...scheduledParticipantDeps(spec.planned, deps, scheduledAt),
|
|
236
231
|
signalReady: () => {
|
|
237
232
|
ready = true;
|
|
238
233
|
gate.ready();
|
|
@@ -258,23 +253,17 @@ export async function runCuaParticipants(runs, deps, concurrency, runParticipant
|
|
|
258
253
|
endedAt: deps.now(),
|
|
259
254
|
};
|
|
260
255
|
}
|
|
261
|
-
// A participant that stopped before its session
|
|
262
|
-
//
|
|
263
|
-
//
|
|
256
|
+
// A participant that stopped before its session without a desktop says nothing about the
|
|
257
|
+
// subject, so the next participant starts first. A runner that threw is a harness defect: it
|
|
258
|
+
// closes the gate.
|
|
264
259
|
if (!ready)
|
|
265
260
|
gate.failed(id, { handOver: !threw && outcome.sandboxId === undefined });
|
|
266
|
-
if (outcome.harnessError && (ready || threw) && !failFast.tripped) {
|
|
267
|
-
failFast.tripped = true;
|
|
268
|
-
failFast.reason = `a prior participant (${outcome.spec.planned.id}) ended in a harness error (fail-fast)`;
|
|
269
|
-
stopping.abort();
|
|
270
|
-
}
|
|
271
261
|
// A participant the study budget skipped never requested its desktop.
|
|
272
262
|
return {
|
|
273
263
|
...outcome,
|
|
274
264
|
arrival: outcome.skippedReason === undefined ? arrival : { scheduledAt },
|
|
275
265
|
};
|
|
276
266
|
});
|
|
277
|
-
return { outcomes, ...(failFast.tripped ? { failFastReason: failFast.reason } : {}) };
|
|
278
267
|
}
|
|
279
268
|
/** Project one participant outcome (or a dry-run contract spec) into the public CuaParticipantResult. */
|
|
280
269
|
export function toParticipantResult(spec, outcome, subject, dryRun) {
|
|
@@ -369,19 +358,21 @@ export function toParticipantResult(spec, outcome, subject, dryRun) {
|
|
|
369
358
|
}
|
|
370
359
|
/**
|
|
371
360
|
* Run every participant of a live run. The in-process route drives its single participant in this
|
|
372
|
-
* process; one hosted participant runs alone; a fan-out runs at the plan's concurrency and
|
|
361
|
+
* process; one hosted participant runs alone; a fan-out runs at the plan's concurrency, and stops
|
|
362
|
+
* early only when the pipeline gate closes.
|
|
373
363
|
*/
|
|
374
364
|
export async function runAllCuaParticipants(runs, deps, participantPlan, inProcess) {
|
|
365
|
+
// A participant that runs alone starts with the run.
|
|
366
|
+
const alone = { ...deps, ...scheduledParticipantDeps(runs[0].planned, deps, deps.now()) };
|
|
375
367
|
if (inProcess) {
|
|
376
368
|
// The caller's executor stands in for a desktop, and the shared runner drives the participant.
|
|
377
369
|
const outcome = await runCuaParticipant(runs[0], {
|
|
378
|
-
...
|
|
370
|
+
...alone,
|
|
379
371
|
createDesktop: () => createInProcessDesktop(deps),
|
|
380
372
|
});
|
|
381
|
-
return
|
|
373
|
+
return [outcome];
|
|
382
374
|
}
|
|
383
375
|
if (runs.length === 1)
|
|
384
|
-
return
|
|
385
|
-
|
|
386
|
-
return { outcomes: ran.outcomes, failFastReason: ran.failFastReason };
|
|
376
|
+
return [await runCuaParticipant(runs[0], alone)];
|
|
377
|
+
return runCuaParticipants([...runs], deps, participantPlan.concurrency);
|
|
387
378
|
}
|
|
@@ -99,6 +99,7 @@ export function participantSessionOptions(spec, deps, ready, provider, writeScre
|
|
|
99
99
|
: spec.instructions,
|
|
100
100
|
persona: spec.persona,
|
|
101
101
|
timeoutMs: deps.timeoutMs,
|
|
102
|
+
...participantClock(spec.planned.limits.timeZone, deps),
|
|
102
103
|
// The brain is either a keyed API client or a CLI the operator is already signed in to.
|
|
103
104
|
// Everything below this line (loop, executor, trace, affordances) is identical either
|
|
104
105
|
// way, which is what makes a local-agent run comparable to an API one.
|
|
@@ -305,3 +306,19 @@ export function makeCuaRunBudget(maxTotalUsd) {
|
|
|
305
306
|
},
|
|
306
307
|
};
|
|
307
308
|
}
|
|
309
|
+
/**
|
|
310
|
+
* The participant's clock: its time zone, and its scheduled end on the run's clock. A participant
|
|
311
|
+
* with an end runs its session on the run's clock, so the end and the schedule read one clock.
|
|
312
|
+
*/
|
|
313
|
+
function participantClock(timeZone, deps) {
|
|
314
|
+
const { scheduledEnd } = deps;
|
|
315
|
+
if (timeZone === undefined && scheduledEnd === undefined)
|
|
316
|
+
return {};
|
|
317
|
+
return {
|
|
318
|
+
clock: {
|
|
319
|
+
...(timeZone === undefined ? {} : { timeZone }),
|
|
320
|
+
...(scheduledEnd === undefined ? {} : { end: scheduledEnd }),
|
|
321
|
+
},
|
|
322
|
+
...(scheduledEnd === undefined ? {} : { now: deps.now }),
|
|
323
|
+
};
|
|
324
|
+
}
|
|
@@ -3,7 +3,8 @@ import { type StudyConfig } from "../../study/types.js";
|
|
|
3
3
|
import { type ResolvedPersona } from "../../study/persona.js";
|
|
4
4
|
import { type RunRerunLineage } from "../../run/bundle.js";
|
|
5
5
|
import type { PreparedSelectedOutputDirectory } from "../../run/contained-output.js";
|
|
6
|
-
import { type CuaActorStudyErrorCode, type CuaParticipantPlan, type CuaParticipantPlanEntry, type DesktopParticipantRun, type ParticipantRunsAndPlan } from "./types.js";
|
|
6
|
+
import { type CuaActorStudyErrorCode, type CuaParticipantDeps, type CuaParticipantPlan, type CuaParticipantPlanEntry, type DesktopParticipantRun, type ParticipantRunsAndPlan } from "./types.js";
|
|
7
|
+
import { type ParticipantWindow } from "../../study/arrivals.js";
|
|
7
8
|
/**
|
|
8
9
|
* The session budget of a study that declares no execution.timeoutMs. On a provisioned route it is
|
|
9
10
|
* what the `ceilingMs` sandbox ceiling leaves after provisioning, seeding and the teardown buffer.
|
|
@@ -16,6 +17,15 @@ export declare function defaultSessionTimeoutMs(config: StudyConfig, ceilingMs:
|
|
|
16
17
|
* git clone for an upload+extract, but the shared install/build/state/start/probe pipeline
|
|
17
18
|
* costs the same wall-clock room either way. */
|
|
18
19
|
export declare function resolveParticipantSandboxMs(config: StudyConfig, ceilingMs: number): number;
|
|
20
|
+
/**
|
|
21
|
+
* The deps one participant runs with once its start comes: its own session budget and sandbox
|
|
22
|
+
* deadline, and with a declared end, that end on the run's clock (`scheduledAt` is when it was
|
|
23
|
+
* due, from runOnSchedule).
|
|
24
|
+
*/
|
|
25
|
+
export declare function scheduledParticipantDeps(participant: ParticipantWindow, deps: {
|
|
26
|
+
readonly timeoutMs: number;
|
|
27
|
+
readonly sandboxMs: number;
|
|
28
|
+
}, scheduledAt: number): Pick<CuaParticipantDeps, "timeoutMs" | "sandboxMs" | "scheduledEnd">;
|
|
19
29
|
/**
|
|
20
30
|
* The in-flight participant bound a study declares. Defaults to the participant count: every
|
|
21
31
|
* declared participant runs at once, because a throttle nobody asked for silently turns "N actors
|
|
@@ -18,7 +18,7 @@ import { composeParticipantInstructions, DEFAULT_MISSION } from "./participant-p
|
|
|
18
18
|
import { resolveCuaRerunSelection } from "./rerun-selection.js";
|
|
19
19
|
import { plural } from "../../run/text.js";
|
|
20
20
|
import { formatDuration } from "../../run/projection.js";
|
|
21
|
-
import { describeArrivals, planArrivals } from "../../study/arrivals.js";
|
|
21
|
+
import { describeArrivals, participantSessionMs, planArrivals, scheduledEndAt, } from "../../study/arrivals.js";
|
|
22
22
|
/**
|
|
23
23
|
* The session budget of a study that declares no execution.timeoutMs. On a provisioned route it is
|
|
24
24
|
* what the `ceilingMs` sandbox ceiling leaves after provisioning, seeding and the teardown buffer.
|
|
@@ -45,6 +45,30 @@ export function resolveParticipantSandboxMs(config, ceilingMs) {
|
|
|
45
45
|
timeoutMs +
|
|
46
46
|
sandboxHeadroomMs(provisionedRoute ? { seed: config.subject.state?.seed ?? [] } : undefined));
|
|
47
47
|
}
|
|
48
|
+
/**
|
|
49
|
+
* A participant's own session budget and sandbox deadline: the study's, or with a declared end its
|
|
50
|
+
* window plus room for the closing report (participantSessionMs), with the same setup and
|
|
51
|
+
* teardown headroom the study's sandbox has over its session.
|
|
52
|
+
*/
|
|
53
|
+
function participantWindow(participant, study) {
|
|
54
|
+
const timeoutMs = participantSessionMs(participant, study.sessionBudgetMs);
|
|
55
|
+
return { timeoutMs, sandboxMs: study.sandboxMs - study.sessionBudgetMs + timeoutMs };
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* The deps one participant runs with once its start comes: its own session budget and sandbox
|
|
59
|
+
* deadline, and with a declared end, that end on the run's clock (`scheduledAt` is when it was
|
|
60
|
+
* due, from runOnSchedule).
|
|
61
|
+
*/
|
|
62
|
+
export function scheduledParticipantDeps(participant, deps, scheduledAt) {
|
|
63
|
+
const end = scheduledEndAt(participant, scheduledAt);
|
|
64
|
+
return {
|
|
65
|
+
...participantWindow(participant, {
|
|
66
|
+
sessionBudgetMs: deps.timeoutMs,
|
|
67
|
+
sandboxMs: deps.sandboxMs,
|
|
68
|
+
}),
|
|
69
|
+
...(end === undefined ? {} : { scheduledEnd: end }),
|
|
70
|
+
};
|
|
71
|
+
}
|
|
48
72
|
/**
|
|
49
73
|
* The in-flight participant bound a study declares. Defaults to the participant count: every
|
|
50
74
|
* declared participant runs at once, because a throttle nobody asked for silently turns "N actors
|
|
@@ -114,10 +138,11 @@ export function participantRunsAndPlan(plan, opts = {}) {
|
|
|
114
138
|
});
|
|
115
139
|
const resolved = envLoweredConcurrency(plan.concurrency, participantCount, env);
|
|
116
140
|
const concurrency = resolved.bound;
|
|
117
|
-
const { sessionBudgetMs
|
|
141
|
+
const { sessionBudgetMs } = plan;
|
|
118
142
|
const schedule = resolved.envLoweredFrom === undefined
|
|
119
143
|
? plan.arrivals
|
|
120
|
-
: planArrivals(participants.map((participant) => participant.startAfterMs), { slots: concurrency, sessionMs: sessionBudgetMs });
|
|
144
|
+
: planArrivals(participants.map((participant) => participant.startAfterMs), { slots: concurrency, sessionMs: sessionBudgetMs }, participants.map((participant) => participantSessionMs(participant, sessionBudgetMs)));
|
|
145
|
+
const sandboxesMs = participants.reduce((sum, participant) => sum + participantWindow(participant, plan).sandboxMs, 0);
|
|
121
146
|
const participantPlan = {
|
|
122
147
|
strategy: CUA_FANOUT_STRATEGY,
|
|
123
148
|
laneCount: participantCount,
|
|
@@ -127,7 +152,7 @@ export function participantRunsAndPlan(plan, opts = {}) {
|
|
|
127
152
|
: { envLoweredConcurrencyFrom: resolved.envLoweredFrom }),
|
|
128
153
|
waves: Math.ceil(participantCount / concurrency),
|
|
129
154
|
perLaneSessionBudgetMs: sessionBudgetMs,
|
|
130
|
-
worstCaseSandboxMinutes: Math.round(
|
|
155
|
+
worstCaseSandboxMinutes: Math.round(sandboxesMs / 60_000),
|
|
131
156
|
...(schedule.declared ? { schedule } : {}),
|
|
132
157
|
dryRun: plan.dryRun,
|
|
133
158
|
lanes: runs.map((spec) => ({
|
|
@@ -158,6 +183,7 @@ export function participantRunsAndPlan(plan, opts = {}) {
|
|
|
158
183
|
...(spec.planned.startAfterMs === undefined
|
|
159
184
|
? {}
|
|
160
185
|
: { startAfterMs: spec.planned.startAfterMs }),
|
|
186
|
+
...(spec.planned.endAfterMs === undefined ? {} : { endAfterMs: spec.planned.endAfterMs }),
|
|
161
187
|
})),
|
|
162
188
|
};
|
|
163
189
|
return { runs, participantPlan };
|
|
@@ -182,6 +208,7 @@ export function formatParticipantPlanEntry(entry) {
|
|
|
182
208
|
entry.caseGroup ? `case=${entry.caseGroup}` : undefined,
|
|
183
209
|
entry.reasoningEffort ? `effort=${entry.reasoningEffort}` : undefined,
|
|
184
210
|
entry.startAfterMs === undefined ? undefined : `start=+${formatDuration(entry.startAfterMs)}`,
|
|
211
|
+
entry.endAfterMs === undefined ? undefined : `end=+${formatDuration(entry.endAfterMs)}`,
|
|
185
212
|
].filter((part) => part !== undefined);
|
|
186
213
|
return `${entry.id}: persona=${entry.persona}${taxonomy.length > 0 ? ` ${taxonomy.join(" ")}` : ""} device=${entry.device} ${entry.resolution[0]}x${entry.resolution[1]} prompt#${entry.instructionDigest}${entry.targetDigest ? ` target#${entry.targetDigest}` : ""}`;
|
|
187
214
|
}
|
|
@@ -9,7 +9,7 @@ import { studyUrlCredentialReason } from "../../study/url-credentials.js";
|
|
|
9
9
|
import { subjectStateInvalidReason } from "../../study/parse/subject-state.js";
|
|
10
10
|
import { brainOf, callerBrainOf, planCaps, desktopRequirements, planBase, provisionedSubject, } from "../../study/plan-base.js";
|
|
11
11
|
import { computerUseParticipants, declaredTargets } from "../../study/plan-participants.js";
|
|
12
|
-
import { planStarts } from "../../study/arrivals.js";
|
|
12
|
+
import { planStarts, windowDeadlineRefusal } from "../../study/arrivals.js";
|
|
13
13
|
import { computerUseParticipantCount, participantCountReason } from "../../study/routing.js";
|
|
14
14
|
import { cloneTargetValidationReason, computerUseValidationReason, desktopMediaValidationReason, outputTokenLimitValidationReason, receivingEmailValidationReason, taskProtocolValidationReason, waitLimitValidationReason, } from "../../study/validation.js";
|
|
15
15
|
import { desktopCliProductReason } from "../../study/composition-rules.js";
|
|
@@ -209,6 +209,13 @@ function startsOf(config, runner, limit, ceilingMs) {
|
|
|
209
209
|
if (!atOnce.ok)
|
|
210
210
|
return atOnce;
|
|
211
211
|
const sessionMs = config.execution?.timeoutMs ?? defaultSessionTimeoutMs(config, ceilingMs);
|
|
212
|
+
// A participant with a declared end gets a sandbox sized from its own window.
|
|
213
|
+
if (runner.desktop === "e2b-desktop") {
|
|
214
|
+
const headroomMs = resolveParticipantSandboxMs(config, ceilingMs) - sessionMs;
|
|
215
|
+
const window = windowDeadlineRefusal(runner.participants, { sessionMs, headroomMs }, ceilingMs);
|
|
216
|
+
if (window !== undefined)
|
|
217
|
+
return { ok: false, message: window };
|
|
218
|
+
}
|
|
212
219
|
const slots = { slots: atOnce.count, sessionMs };
|
|
213
220
|
return {
|
|
214
221
|
ok: true,
|
|
@@ -201,7 +201,7 @@ export async function finishCuaRun(setup, finish, ran) {
|
|
|
201
201
|
const appUrl = plannedAppUrl(plan.runner.subject);
|
|
202
202
|
const { runId } = run;
|
|
203
203
|
const participantCount = participantRuns.length;
|
|
204
|
-
const { outcomes,
|
|
204
|
+
const { outcomes, receiving, receivingWarnings, externalCommsWarnings } = ran;
|
|
205
205
|
// Per-participant subject projections.
|
|
206
206
|
const subjects = projectParticipantSubjects({ ...subjectArgs, outcomes, dryRun });
|
|
207
207
|
const aggregate = aggregateCuaSubject({ subjects, outcomes, participantCount, dryRun });
|
|
@@ -222,7 +222,6 @@ export async function finishCuaRun(setup, finish, ran) {
|
|
|
222
222
|
outcomes,
|
|
223
223
|
subjects,
|
|
224
224
|
aggregateSubject,
|
|
225
|
-
...(failFastReason === undefined ? {} : { failFastReason }),
|
|
226
225
|
...(hostSuspension === undefined ? {} : { hostSuspension }),
|
|
227
226
|
});
|
|
228
227
|
const adapterWarnings = [];
|