humanish 0.125.0 → 0.127.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +29 -0
- package/dist/actors/computer-use/actor.d.ts +2 -0
- package/dist/actors/computer-use/actor.js +1 -0
- package/dist/actors/computer-use/loop/clock.d.ts +9 -8
- package/dist/actors/computer-use/loop/clock.js +27 -17
- package/dist/actors/computer-use/loop/debrief.d.ts +1 -1
- package/dist/actors/computer-use/loop/debrief.js +15 -5
- package/dist/actors/computer-use/loop/ending.d.ts +10 -5
- package/dist/actors/computer-use/loop/ending.js +12 -7
- package/dist/actors/computer-use/loop/observation.d.ts +6 -0
- package/dist/actors/computer-use/loop/observation.js +19 -1
- package/dist/actors/computer-use/loop/session.d.ts +4 -2
- package/dist/actors/computer-use/loop/session.js +4 -1
- package/dist/actors/computer-use/loop/trace.js +4 -2
- package/dist/actors/computer-use/loop/types.d.ts +22 -8
- package/dist/actors/computer-use/loop.js +5 -5
- package/dist/actors/computer-use/openai-provider.js +1 -1
- package/dist/actors/contract.d.ts +31 -8
- package/dist/actors/contract.js +3 -0
- package/dist/actors/goal-source.d.ts +32 -2
- package/dist/actors/goal-source.js +41 -2
- package/dist/actors/stop-cause.js +1 -0
- package/dist/analysis/evidence-sources.js +2 -2
- package/dist/analysis/types.d.ts +2 -2
- package/dist/analysis/validation.js +2 -1
- package/dist/cli/commands/study.js +8 -0
- package/dist/evidence/redaction.d.ts +1 -0
- package/dist/evidence/redaction.js +1 -0
- package/dist/evidence/trace-page.d.ts +6 -0
- package/dist/evidence/trace-page.js +90 -0
- package/dist/observer/data.js +8 -5
- package/dist/observer/feed-cursors.d.ts +14 -0
- package/dist/observer/feed-cursors.js +61 -0
- package/dist/observer/participant-items.d.ts +7 -0
- package/dist/observer/pinned-files.d.ts +2 -1
- package/dist/observer/pinned-files.js +2 -2
- package/dist/observer/run-routes.d.ts +1 -1
- package/dist/observer/run-routes.js +38 -17
- package/dist/observer-app.html +7 -7
- package/dist/routes/computer-use/bundle.js +4 -0
- package/dist/routes/computer-use/diagnostics.d.ts +1 -1
- package/dist/routes/computer-use/diagnostics.js +3 -3
- package/dist/routes/computer-use/e2b-desktop/desktop.js +2 -1
- package/dist/routes/computer-use/e2b-desktop/prepare.d.ts +6 -0
- package/dist/routes/computer-use/e2b-desktop/prepare.js +47 -1
- package/dist/routes/computer-use/e2b-desktop/start.d.ts +2 -1
- package/dist/routes/computer-use/e2b-desktop/start.js +15 -3
- package/dist/routes/computer-use/e2b-desktop/state.d.ts +3 -1
- package/dist/routes/computer-use/e2b-desktop/state.js +2 -0
- package/dist/routes/computer-use/fanout-bundle.js +3 -0
- package/dist/routes/computer-use/fanout-records.js +5 -1
- package/dist/routes/computer-use/participant-deps.d.ts +1 -1
- package/dist/routes/computer-use/participant-desktop.d.ts +1 -1
- package/dist/routes/computer-use/participant-execution.js +6 -2
- package/dist/routes/computer-use/participant-facts.js +4 -1
- package/dist/routes/computer-use/participant-model.js +16 -6
- package/dist/routes/computer-use/participant-runs.d.ts +10 -10
- package/dist/routes/computer-use/participant-runs.js +54 -84
- package/dist/routes/computer-use/plan.d.ts +2 -0
- package/dist/routes/computer-use/plan.js +65 -17
- package/dist/routes/computer-use/rerun-selection.js +8 -5
- package/dist/routes/computer-use/setup.d.ts +1 -0
- package/dist/routes/computer-use/setup.js +29 -1
- package/dist/routes/computer-use/single-bundle.d.ts +5 -1
- package/dist/routes/computer-use/single-bundle.js +11 -1
- package/dist/routes/computer-use/types.d.ts +18 -13
- package/dist/routes/computer-use/types.js +0 -3
- package/dist/routes/shared-world/bundle-records.js +4 -1
- package/dist/routes/shared-world/external-public.js +1 -1
- package/dist/routes/shared-world/participant-specs.js +3 -2
- package/dist/routes/shared-world/plan.d.ts +2 -0
- package/dist/routes/shared-world/plan.js +8 -3
- package/dist/routes/shared-world/provisioned.js +1 -1
- package/dist/run/bundle.d.ts +11 -1
- package/dist/run/contained-output.d.ts +7 -3
- package/dist/run/contained-output.js +11 -7
- package/dist/run/judge.d.ts +14 -4
- package/dist/run/judge.js +17 -12
- package/dist/run/live-items.d.ts +19 -0
- package/dist/run/live-items.js +28 -8
- package/dist/run/notes.js +1 -1
- package/dist/run/outcomes.d.ts +8 -2
- package/dist/run/outcomes.js +56 -14
- package/dist/run/stream-trace.d.ts +25 -4
- package/dist/run/stream-trace.js +55 -17
- package/dist/run/streams.d.ts +16 -0
- package/dist/study/arrivals.d.ts +105 -34
- package/dist/study/arrivals.js +179 -74
- package/dist/study/bundle-estimate.d.ts +12 -10
- package/dist/study/bundle-estimate.js +138 -72
- package/dist/study/desktop-files.d.ts +37 -0
- package/dist/study/desktop-files.js +174 -0
- package/dist/study/keys.d.ts +8 -0
- package/dist/study/keys.js +3 -0
- package/dist/study/migrate/v2.js +12 -3
- package/dist/study/parse/actors.d.ts +1 -1
- package/dist/study/parse/actors.js +28 -20
- package/dist/study/parse/clock.d.ts +18 -0
- package/dist/study/parse/clock.js +36 -0
- package/dist/study/plan-participants.d.ts +19 -10
- package/dist/study/plan-participants.js +13 -9
- package/dist/study/plan-types.d.ts +6 -1
- package/dist/study/plan.d.ts +2 -1
- package/dist/study/plan.js +28 -15
- package/dist/study/preflight.js +4 -0
- package/dist/study/types.d.ts +35 -5
- package/dist/study/validation.js +3 -0
- package/dist/study/warnings.d.ts +2 -2
- package/dist/study/warnings.js +7 -0
- package/dist/subject/local-tree-archive.d.ts +7 -0
- package/dist/subject/local-tree-archive.js +12 -0
- package/dist/substrates/e2b/desktop-browser.d.ts +4 -1
- package/dist/substrates/e2b/desktop-browser.js +6 -2
- package/dist/substrates/e2b/desktop-clock.d.ts +10 -0
- package/dist/substrates/e2b/desktop-clock.js +29 -0
- package/dist/substrates/e2b/lifetime.d.ts +22 -8
- package/dist/substrates/e2b/lifetime.js +54 -10
- package/dist/substrates/e2b/sdk.d.ts +6 -0
- package/dist/verify/pages.d.ts +7 -0
- package/dist/verify/pages.js +50 -0
- package/dist/verify/verify.js +10 -3
- package/docs/architecture/actor-contract.md +2 -0
- package/docs/architecture/external-public-shared-world.md +4 -1
- package/docs/architecture/observer.md +10 -1
- package/docs/contracts/policy.md +7 -0
- package/docs/contracts/run-bundle.md +31 -1
- package/docs/contracts/schemas.md +93 -11
- package/package.json +2 -2
|
@@ -101,6 +101,10 @@ function singleParticipantArgs(base, state) {
|
|
|
101
101
|
: "e2b-desktop",
|
|
102
102
|
...(outcome?.desktopGeometry === undefined ? {} : { desktopGeometry: outcome.desktopGeometry }),
|
|
103
103
|
...(outcome?.recording === undefined ? {} : { recording: outcome.recording }),
|
|
104
|
+
...(outcome?.desktopFiles === undefined ? {} : { desktopFiles: outcome.desktopFiles }),
|
|
105
|
+
...(spec.planned.limits.timeZone === undefined
|
|
106
|
+
? {}
|
|
107
|
+
: { timeZone: spec.planned.limits.timeZone }),
|
|
104
108
|
isMobile: spec.planned.device.preset.isMobile,
|
|
105
109
|
screenshots: outcome?.screenshots ?? [],
|
|
106
110
|
captureRedaction: base.redactScreenshots ? "blurred" : "raw",
|
|
@@ -3,7 +3,7 @@ import { type ActorCompletionReason, type ActorStatus, type ActorStopCause } fro
|
|
|
3
3
|
declare const CUA_DIAGNOSTIC_CATEGORIES: readonly ["preview", "participant_outcome", "session_interrupted", "execution_error", "evidence_invalid", "mixed", "unknown"];
|
|
4
4
|
export type CuaDiagnosticCategory = (typeof CUA_DIAGNOSTIC_CATEGORIES)[number];
|
|
5
5
|
/** Closed at both projection and telemetry boundaries. No provider messages are classified. */
|
|
6
|
-
declare const CUA_DIAGNOSTIC_STOP_CAUSES: readonly ["provider_output_limit", "provider_token_limit", "time_limit", "spend_limit", "study_spend_limit", "adapter_limit", "provider_incomplete", "provider_status", "provider_refused_prompt", "harness_aborted", "usage_unreported", "scheduled_end", "unspecified_limit", "mixed", "unknown"];
|
|
6
|
+
declare const CUA_DIAGNOSTIC_STOP_CAUSES: readonly ["provider_output_limit", "provider_token_limit", "time_limit", "spend_limit", "study_spend_limit", "adapter_limit", "provider_incomplete", "provider_status", "provider_refused_prompt", "harness_aborted", "usage_unreported", "scheduled_end", "session_cap", "unspecified_limit", "mixed", "unknown"];
|
|
7
7
|
export type CuaDiagnosticStopCause = (typeof CUA_DIAGNOSTIC_STOP_CAUSES)[number];
|
|
8
8
|
export interface CuaDiagnostics {
|
|
9
9
|
category: CuaDiagnosticCategory;
|
|
@@ -30,9 +30,9 @@ export function cuaParticipantDiagnostics(input) {
|
|
|
30
30
|
const session = input.session;
|
|
31
31
|
if (!session)
|
|
32
32
|
return { category: "unknown" };
|
|
33
|
-
// A scheduled end is the participant's planned ending, so a run whose other
|
|
34
|
-
// themselves reads as one kind of ending.
|
|
35
|
-
if (session.stopCause === "scheduled_end")
|
|
33
|
+
// A scheduled end or a session cap is the participant's planned ending, so a run whose other
|
|
34
|
+
// participants ended themselves reads as one kind of ending.
|
|
35
|
+
if (session.stopCause === "scheduled_end" || session.stopCause === "session_cap")
|
|
36
36
|
return { category: "participant_outcome" };
|
|
37
37
|
if (session.stopCause !== undefined) {
|
|
38
38
|
return {
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { createE2BDesktopExecutor } from "../../../substrates/e2b/desktop-executor.js";
|
|
2
2
|
import { participantInbox, planParticipantComms } from "./comms.js";
|
|
3
3
|
import { participantBrowserStateObserver } from "./fidelity.js";
|
|
4
|
-
import { acquireParticipantDesktop, provisionParticipantSubject, verifyParticipantScreen, } from "./prepare.js";
|
|
4
|
+
import { acquireParticipantDesktop, placeParticipantFiles, provisionParticipantSubject, verifyParticipantScreen, } from "./prepare.js";
|
|
5
5
|
import { openParticipantSurface, startParticipantMedia, startParticipantStream } from "./start.js";
|
|
6
6
|
import { desktopEvidenceOf, newParticipantState } from "./state.js";
|
|
7
7
|
import { finishE2BDesktop } from "./teardown.js";
|
|
@@ -40,6 +40,7 @@ export function createE2BParticipantDesktop(spec, deps, warnings) {
|
|
|
40
40
|
const desktop = await acquireParticipantDesktop(ctx, state);
|
|
41
41
|
// The device claim is verified in the sandbox, and fails closed.
|
|
42
42
|
await verifyParticipantScreen(ctx, state, desktop);
|
|
43
|
+
await placeParticipantFiles(ctx, state, desktop);
|
|
43
44
|
await provisionParticipantSubject(ctx, state, desktop);
|
|
44
45
|
await startParticipantMedia(ctx, state, desktop);
|
|
45
46
|
await openParticipantSurface(ctx, state, desktop);
|
|
@@ -14,5 +14,11 @@ export declare function acquireParticipantDesktop(ctx: E2BParticipantContext, st
|
|
|
14
14
|
* requested and verified geometry side by side.
|
|
15
15
|
*/
|
|
16
16
|
export declare function verifyParticipantScreen(ctx: E2BParticipantContext, state: E2BParticipantState, desktop: E2BDesktopSandbox): Promise<void>;
|
|
17
|
+
/**
|
|
18
|
+
* Copy the participant's files (participants[].files) into its home folder and confirm each one's
|
|
19
|
+
* size there, before the subject and the browser start. A file that does not land whole fails the
|
|
20
|
+
* participant before its first turn. The state records each confirmed file as it lands.
|
|
21
|
+
*/
|
|
22
|
+
export declare function placeParticipantFiles(ctx: E2BParticipantContext, state: E2BParticipantState, desktop: E2BDesktopSandbox): Promise<void>;
|
|
17
23
|
/** Install a desktop CLI, or clone or unpack the subject and serve it, in the sandbox. */
|
|
18
24
|
export declare function provisionParticipantSubject(ctx: E2BParticipantContext, state: E2BParticipantState, desktop: E2BDesktopSandbox): Promise<void>;
|
|
@@ -8,11 +8,12 @@ import { provisionLocalTreeSubject } from "../../../subject/local-tree.js";
|
|
|
8
8
|
import { SubjectBuildError } from "../../../subject/serve.js";
|
|
9
9
|
import { inspectDesktopScreenGeometry } from "../../../substrates/e2b/desktop-geometry.js";
|
|
10
10
|
import { observeDesktopResources } from "../../../substrates/e2b/desktop-resources.js";
|
|
11
|
-
import { acquireE2BDesktopSandbox, e2bDesktopTemplate } from "../../../substrates/e2b/sandbox.js";
|
|
11
|
+
import { acquireE2BDesktopSandbox, e2bDesktopTemplate, withOneRetryOnTransientE2BError, } from "../../../substrates/e2b/sandbox.js";
|
|
12
12
|
import { loadE2BDesktopModule } from "../../../substrates/e2b/sdk.js";
|
|
13
13
|
import { e2bShell } from "../../../substrates/e2b/shell.js";
|
|
14
14
|
import { attachReceivingInbox, participantCommsEnv, startCommsCatch } from "./comms.js";
|
|
15
15
|
import { participantSubjectEnv } from "../types.js";
|
|
16
|
+
import { plural } from "../../../run/text.js";
|
|
16
17
|
export const CUA_ACTOR_STUDY_PROVIDER_METADATA = {
|
|
17
18
|
mode: "cua-actor-lab",
|
|
18
19
|
tool: "humanish",
|
|
@@ -151,6 +152,51 @@ export async function verifyParticipantScreen(ctx, state, desktop) {
|
|
|
151
152
|
};
|
|
152
153
|
}
|
|
153
154
|
}
|
|
155
|
+
/**
|
|
156
|
+
* Copy the participant's files (participants[].files) into its home folder and confirm each one's
|
|
157
|
+
* size there, before the subject and the browser start. A file that does not land whole fails the
|
|
158
|
+
* participant before its first turn. The state records each confirmed file as it lands.
|
|
159
|
+
*/
|
|
160
|
+
export async function placeParticipantFiles(ctx, state, desktop) {
|
|
161
|
+
const { spec, deps } = ctx;
|
|
162
|
+
const files = deps.desktopFiles?.get(spec.planned.id) ?? [];
|
|
163
|
+
if (files.length === 0)
|
|
164
|
+
return;
|
|
165
|
+
const getInfo = desktop.files.getInfo?.bind(desktop.files);
|
|
166
|
+
if (getInfo === undefined)
|
|
167
|
+
throw new Error("The installed E2B SDK has no files.getInfo, so humanish cannot confirm that participants[].files landed. Update @e2b/desktop.");
|
|
168
|
+
const requestTimeoutMs = deps.requestTimeoutMs;
|
|
169
|
+
const startedAt = deps.now();
|
|
170
|
+
ctx.onSubjectPhase({
|
|
171
|
+
at: new Date(startedAt).toISOString(),
|
|
172
|
+
type: "cua-lab.desktop.files.started",
|
|
173
|
+
message: `copying ${plural(files.length, "file")} onto the desktop`,
|
|
174
|
+
});
|
|
175
|
+
const filesCompleted = (ok) => ctx.onSubjectPhase({
|
|
176
|
+
at: new Date(deps.now()).toISOString(),
|
|
177
|
+
type: "cua-lab.desktop.files.completed",
|
|
178
|
+
ok,
|
|
179
|
+
durationMs: Math.max(0, deps.now() - startedAt),
|
|
180
|
+
message: ok
|
|
181
|
+
? `${plural(files.length, "file")} on the desktop`
|
|
182
|
+
: `copying files onto the desktop failed after ${plural(state.desktopFiles.length, "file")}`,
|
|
183
|
+
});
|
|
184
|
+
try {
|
|
185
|
+
for (const file of files) {
|
|
186
|
+
// A relative path lands under the sandbox user's home folder, and missing folders are made.
|
|
187
|
+
await withOneRetryOnTransientE2BError(() => desktop.files.write(file.as, file.data, { requestTimeoutMs, useOctetStream: true }));
|
|
188
|
+
const landed = await getInfo(file.as, { requestTimeoutMs });
|
|
189
|
+
if (landed.size !== file.size)
|
|
190
|
+
throw new Error(`participants[].files: ${file.as} measured ${landed.size} bytes on the desktop of participant ${spec.planned.id} after humanish copied ${file.size}, so the participant did not start.`);
|
|
191
|
+
state.desktopFiles.push({ as: file.as, size: file.size, sha256: file.sha256 });
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
catch (error) {
|
|
195
|
+
filesCompleted(false);
|
|
196
|
+
throw error;
|
|
197
|
+
}
|
|
198
|
+
filesCompleted(true);
|
|
199
|
+
}
|
|
154
200
|
/** Install a desktop CLI, or clone or unpack the subject and serve it, in the sandbox. */
|
|
155
201
|
export async function provisionParticipantSubject(ctx, state, desktop) {
|
|
156
202
|
const { deps } = ctx;
|
|
@@ -4,7 +4,8 @@ import type { E2BParticipantContext, E2BParticipantState } from "./state.js";
|
|
|
4
4
|
export declare function startParticipantMedia(ctx: E2BParticipantContext, state: E2BParticipantState, desktop: E2BDesktopSandbox): Promise<void>;
|
|
5
5
|
/**
|
|
6
6
|
* Open what the participant sees: the browser at the target, with any declared camera and mobile
|
|
7
|
-
* fidelity, or a terminal window on the desktop-cli route.
|
|
7
|
+
* fidelity, or a terminal window on the desktop-cli route. A participant with a time zone sees the
|
|
8
|
+
* desktop's panel clock in it; a clock that cannot be set is a warning, and it shows UTC.
|
|
8
9
|
*/
|
|
9
10
|
export declare function openParticipantSurface(ctx: E2BParticipantContext, state: E2BParticipantState, desktop: E2BDesktopSandbox): Promise<void>;
|
|
10
11
|
/**
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
// fills the participant's state.
|
|
4
4
|
import { redactText, toErrorMessage } from "../../../evidence/redaction.js";
|
|
5
5
|
import { CHROME_DEVTOOLS_PORT, DESKTOP_SETTLE_MS, openDesktopBrowserTarget, openDesktopTerminal, startDesktopStream, } from "../../../substrates/e2b/desktop-browser.js";
|
|
6
|
+
import { setPanelClockZone } from "../../../substrates/e2b/desktop-clock.js";
|
|
6
7
|
import { captureDesktopBrowserGeometry } from "../../../substrates/e2b/desktop-geometry.js";
|
|
7
8
|
import { prepareDesktopMedia, startE2BDesktopMedia, } from "../../../substrates/e2b/desktop-media.js";
|
|
8
9
|
import { startE2BDesktopRecording } from "../../../substrates/e2b/desktop-recording.js";
|
|
@@ -39,16 +40,23 @@ export async function startParticipantMedia(ctx, state, desktop) {
|
|
|
39
40
|
}
|
|
40
41
|
/**
|
|
41
42
|
* Open what the participant sees: the browser at the target, with any declared camera and mobile
|
|
42
|
-
* fidelity, or a terminal window on the desktop-cli route.
|
|
43
|
+
* fidelity, or a terminal window on the desktop-cli route. A participant with a time zone sees the
|
|
44
|
+
* desktop's panel clock in it; a clock that cannot be set is a warning, and it shows UTC.
|
|
43
45
|
*/
|
|
44
46
|
export async function openParticipantSurface(ctx, state, desktop) {
|
|
45
47
|
const { spec, deps, warnings, targetUrl } = ctx;
|
|
46
48
|
const { residual, subject } = deps;
|
|
49
|
+
const { timeZone } = spec.planned.limits;
|
|
50
|
+
if (timeZone !== undefined) {
|
|
51
|
+
await setPanelClockZone(desktop, timeZone, deps.requestTimeoutMs).catch((error) => {
|
|
52
|
+
warnings.push(`The desktop's panel clock for participant ${spec.planned.id} could not be set to ${timeZone} and shows UTC: ${redactText(deps.scrubKnownValues(toErrorMessage(error)))}`);
|
|
53
|
+
});
|
|
54
|
+
}
|
|
47
55
|
if (subject.kind === "desktop-cli") {
|
|
48
56
|
// A terminal window, opened the way the browser is opened on every other route: the
|
|
49
57
|
// participant arrives at a desktop with the thing they were asked to use already in front
|
|
50
58
|
// of them. They can still open another from the dock; that is the point of a desktop.
|
|
51
|
-
await openDesktopTerminal(desktop, deps.requestTimeoutMs, subject.product.workdir);
|
|
59
|
+
await openDesktopTerminal(desktop, deps.requestTimeoutMs, subject.product.workdir, timeZone);
|
|
52
60
|
await desktop.wait(DESKTOP_SETTLE_MS).catch(() => undefined);
|
|
53
61
|
return;
|
|
54
62
|
}
|
|
@@ -59,7 +67,11 @@ export async function openParticipantSurface(ctx, state, desktop) {
|
|
|
59
67
|
? undefined
|
|
60
68
|
: await prepareDesktopMedia(desktop, requestedMedia, residual.policies?.mediaPermission ?? "prompt", deps.studyCwd, deps.requestTimeoutMs);
|
|
61
69
|
const emulationFlags = mobileLaunchFlags(residual, spec);
|
|
62
|
-
|
|
70
|
+
// The participant's time zone is TZ on the launch: the browser process and every renderer it
|
|
71
|
+
// starts read it at startup, so every tab runs in the zone from the first page load. A CDP
|
|
72
|
+
// timezone override holds only on the one tab whose DevTools session sets it, from after its load.
|
|
73
|
+
const mediaEnv = state.speech?.env ?? state.recording?.env;
|
|
74
|
+
const browserLaunch = await openDesktopBrowserTarget(desktop, targetUrl, deps.requestTimeoutMs, residual.execution?.desktop?.browser, [...emulationFlags, ...(mediaEvidence?.flags ?? [])], timeZone === undefined ? mediaEnv : { ...mediaEnv, TZ: timeZone });
|
|
63
75
|
state.desktopBrowser =
|
|
64
76
|
mediaEvidence === undefined
|
|
65
77
|
? browserLaunch.evidence
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { RunDesktopRecording } from "../../../evidence/desktop-recording-types.js";
|
|
2
2
|
import type { RunSubjectStateStepRecord } from "../../../run/bundle.js";
|
|
3
|
-
import type { RunDesktopGeometry } from "../../../run/streams.js";
|
|
3
|
+
import type { RunDesktopFile, RunDesktopGeometry } from "../../../run/streams.js";
|
|
4
4
|
import type { SubjectPhaseEvent } from "../../../subject/steps.js";
|
|
5
5
|
import type { OwnedDesktopAllocation } from "../../../substrates/desktop-session.js";
|
|
6
6
|
import type { DesktopBrowserEvidence, DesktopBrowserFamily, DesktopBrowserLaunchIdentity } from "../../../substrates/e2b/desktop-browser.js";
|
|
@@ -45,6 +45,8 @@ export interface E2BParticipantState {
|
|
|
45
45
|
readonly stateStepRecords: RunSubjectStateStepRecord[];
|
|
46
46
|
/** Completed phases only (durationMs and ok are set on completed events); bundle.events keeps these. */
|
|
47
47
|
readonly phaseRecords: SubjectPhaseEvent[];
|
|
48
|
+
/** participants[].files confirmed on the desktop, in the order they landed. */
|
|
49
|
+
readonly desktopFiles: RunDesktopFile[];
|
|
48
50
|
subjectCommit: string | undefined;
|
|
49
51
|
speech: Awaited<ReturnType<typeof startE2BDesktopMedia>> | undefined;
|
|
50
52
|
recording: Awaited<ReturnType<typeof startE2BDesktopRecording>> | undefined;
|
|
@@ -18,6 +18,7 @@ export function newParticipantState(spec) {
|
|
|
18
18
|
receivingInboxUrl: undefined,
|
|
19
19
|
stateStepRecords: [],
|
|
20
20
|
phaseRecords: [],
|
|
21
|
+
desktopFiles: [],
|
|
21
22
|
subjectCommit: undefined,
|
|
22
23
|
speech: undefined,
|
|
23
24
|
recording: undefined,
|
|
@@ -60,6 +61,7 @@ export function desktopEvidenceOf(state) {
|
|
|
60
61
|
...(state.subjectCommit === undefined ? {} : { subjectCommit: state.subjectCommit }),
|
|
61
62
|
...(state.desktopBrowser === undefined ? {} : { desktopBrowser: state.desktopBrowser }),
|
|
62
63
|
...(state.recordingEvidence === undefined ? {} : { recording: state.recordingEvidence }),
|
|
64
|
+
...(state.desktopFiles.length === 0 ? {} : { desktopFiles: state.desktopFiles }),
|
|
63
65
|
desktopGeometry: state.desktopGeometry,
|
|
64
66
|
stateStepRecords: state.stateStepRecords,
|
|
65
67
|
phaseRecords: state.phaseRecords,
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { e2bDesktopTemplate } from "../../substrates/e2b/sandbox.js";
|
|
2
2
|
import { receivingPublication } from "../../comms/receiving-runtime.js";
|
|
3
3
|
import { actorEnding } from "../../actors/stop-cause.js";
|
|
4
|
+
import { cuaGoalSource } from "../../actors/goal-source.js";
|
|
4
5
|
import { redactText } from "../../evidence/redaction.js";
|
|
5
6
|
import { REVIEW_SCHEMA, bundleArtifacts, bundleHead, } from "../../run/bundle.js";
|
|
6
7
|
import { aggregateTaskFunnels, formatParticipantOutcomes, formatRunTaskFunnel, gapsListClause, withCuaReviewProvenance, } from "../../run/outcomes.js";
|
|
@@ -86,9 +87,11 @@ function fanoutReview(args, streams, judgment) {
|
|
|
86
87
|
const runTasks = args.inProgress === true ? undefined : aggregateTaskFunnels(participantFunnels);
|
|
87
88
|
const participantEndings = terminalOutcomes.map((outcome) => {
|
|
88
89
|
const ending = actorEnding(outcome.session.trace);
|
|
90
|
+
const goalSource = cuaGoalSource(outcome.session.trace);
|
|
89
91
|
return {
|
|
90
92
|
status: outcome.session.status,
|
|
91
93
|
...(ending === undefined ? {} : { label: ending.label }),
|
|
94
|
+
...(goalSource === undefined ? {} : { goalSource }),
|
|
92
95
|
};
|
|
93
96
|
});
|
|
94
97
|
const review = withCuaReviewProvenance({
|
|
@@ -72,7 +72,7 @@ function fanoutParticipantRecord(args, spec, index, view) {
|
|
|
72
72
|
updatedAt: args.run.createdAt,
|
|
73
73
|
...(args.participantPlan.schedule === undefined
|
|
74
74
|
? {}
|
|
75
|
-
: { arrival: arrivalRecord(spec.planned.startAfterMs, outcome?.arrival) }),
|
|
75
|
+
: { arrival: arrivalRecord(spec.planned.schedule.startAfterMs, outcome?.arrival) }),
|
|
76
76
|
});
|
|
77
77
|
}
|
|
78
78
|
function fanoutParticipantStream(args, spec, view) {
|
|
@@ -120,6 +120,10 @@ function fanoutParticipantStream(args, spec, view) {
|
|
|
120
120
|
},
|
|
121
121
|
}),
|
|
122
122
|
desktopGeometry,
|
|
123
|
+
...(outcome?.desktopFiles === undefined ? {} : { desktopFiles: outcome.desktopFiles }),
|
|
124
|
+
...(spec.planned.limits.timeZone === undefined
|
|
125
|
+
? {}
|
|
126
|
+
: { timeZone: spec.planned.limits.timeZone }),
|
|
123
127
|
...(outcome?.recording === undefined ? {} : { recording: outcome.recording }),
|
|
124
128
|
ui: {
|
|
125
129
|
route: publicTargetUrl,
|
|
@@ -36,7 +36,7 @@ interface DesktopRunFacts {
|
|
|
36
36
|
}
|
|
37
37
|
/** The fields one desktop route sets and the other does not. Undefined leaves a field out. */
|
|
38
38
|
type DesktopRouteOverrides = {
|
|
39
|
-
[K in "createDesktop" | "createProvider" | "inProcessExecutor" | "localTreeArchiveBuffer" | "externalComms" | "receiving" | "screenMismatchPolicy"]?: CuaParticipantDeps[K] | undefined;
|
|
39
|
+
[K in "createDesktop" | "createProvider" | "inProcessExecutor" | "localTreeArchiveBuffer" | "desktopFiles" | "externalComms" | "receiving" | "screenMismatchPolicy"]?: CuaParticipantDeps[K] | undefined;
|
|
40
40
|
};
|
|
41
41
|
/**
|
|
42
42
|
* One run's participant deps. Call it once per run: the run budget it creates is the one ledger
|
|
@@ -22,7 +22,7 @@ export type ParticipantDesktopEvidence = {
|
|
|
22
22
|
* who did not pass.
|
|
23
23
|
*/
|
|
24
24
|
desktopFailure?: string;
|
|
25
|
-
} & Pick<ParticipantRunOutcome, "sandboxId" | "sandboxRelease" | "desktopDurationMs" | "desktopResources" | "streamUrlPresent" | "subjectCommit" | "desktopBrowser" | "desktopGeometry" | "stateStepRecords" | "phaseRecords" | "failureCode" | "commsArtifactPath" | "recording">;
|
|
25
|
+
} & Pick<ParticipantRunOutcome, "sandboxId" | "sandboxRelease" | "desktopDurationMs" | "desktopResources" | "streamUrlPresent" | "subjectCommit" | "desktopBrowser" | "desktopGeometry" | "stateStepRecords" | "phaseRecords" | "failureCode" | "commsArtifactPath" | "recording" | "desktopFiles">;
|
|
26
26
|
/**
|
|
27
27
|
* The desktop one participant runs on, owned from preparation through final evidence collection.
|
|
28
28
|
* Call methods sequentially: prepare, openSession, then finalize in a finally block.
|
|
@@ -4,7 +4,7 @@ import { createE2BParticipantDesktop } from "./e2b-desktop/desktop.js";
|
|
|
4
4
|
import { createInProcessDesktop } from "./in-process-desktop.js";
|
|
5
5
|
import path from "node:path";
|
|
6
6
|
import { cuaParticipantDiagnostics } from "./diagnostics.js";
|
|
7
|
-
import { runOnSchedule } from "../../study/arrivals.js";
|
|
7
|
+
import { pastEndReason, runOnSchedule } from "../../study/arrivals.js";
|
|
8
8
|
import { assertScreenshotEvidence, stripPngMetadataChunks } from "../../evidence/image.js";
|
|
9
9
|
import { redactText, toErrorMessage } from "../../evidence/redaction.js";
|
|
10
10
|
import { assertSafeOutputPathSegment, writeContainedOutputFile, } from "../../run/contained-output.js";
|
|
@@ -59,6 +59,10 @@ export async function runCuaParticipant(spec, deps) {
|
|
|
59
59
|
if (crossed !== undefined) {
|
|
60
60
|
return skippedOutcome(spec, `skipped: study budget reached before this participant started: ${crossed}; no desktop was created`);
|
|
61
61
|
}
|
|
62
|
+
// One that waited for a slot until its scheduled end would only be asked for its closing report.
|
|
63
|
+
const pastEnd = pastEndReason(deps.end, deps.now());
|
|
64
|
+
if (pastEnd !== undefined)
|
|
65
|
+
return skippedOutcome(spec, pastEnd);
|
|
62
66
|
let model = {};
|
|
63
67
|
const warnings = [];
|
|
64
68
|
const screenshots = [];
|
|
@@ -207,7 +211,7 @@ export async function runCuaParticipants(runs, deps, concurrency, runParticipant
|
|
|
207
211
|
// An app-url subject is only opened, so one participant's start says nothing about another's.
|
|
208
212
|
const gate = pipelineGate(deps.subject.kind === "app-url", () => stopping.abort());
|
|
209
213
|
return runOnSchedule(runs, {
|
|
210
|
-
startAfterMs: (spec) => spec.planned.startAfterMs,
|
|
214
|
+
startAfterMs: (spec) => spec.planned.schedule.startAfterMs,
|
|
211
215
|
slots: concurrency,
|
|
212
216
|
now: deps.now,
|
|
213
217
|
signal: stopping.signal,
|
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
// A participant run outcome's facts for the judge (src/run/judge.ts), and the pass check built on
|
|
2
2
|
// them. The bundle builders, the result, the participant runner and shared-world all read these,
|
|
3
|
-
// so they sit in a file that imports only the judge and the types.
|
|
3
|
+
// so they sit in a file that imports only the judge, the clock-end rule and the types.
|
|
4
|
+
import { ranUntilClockEnd } from "../../actors/goal-source.js";
|
|
4
5
|
import { participantHarnessFailed, participantPassed, sandboxCleanupFailure, } from "../../run/judge.js";
|
|
5
6
|
/** A participant outcome's facts for the judge. */
|
|
6
7
|
export function participantFactsOf(outcome) {
|
|
7
8
|
if (outcome === undefined)
|
|
8
9
|
return { skipped: false, noEngagement: false, selfReportedBlocker: false };
|
|
10
|
+
const ranUntil = outcome.session === undefined ? undefined : ranUntilClockEnd(outcome.session.trace);
|
|
9
11
|
return {
|
|
10
12
|
id: outcome.spec.planned.id,
|
|
11
13
|
...(outcome.skippedReason === undefined ? {} : { skippedReason: outcome.skippedReason }),
|
|
@@ -21,6 +23,7 @@ export function participantFactsOf(outcome) {
|
|
|
21
23
|
skipped: outcome.skippedReason !== undefined,
|
|
22
24
|
noEngagement: outcome.noEngagement === true,
|
|
23
25
|
selfReportedBlocker: outcome.selfReportedBlocker === true,
|
|
26
|
+
...(ranUntil === undefined ? {} : { ranUntil }),
|
|
24
27
|
};
|
|
25
28
|
}
|
|
26
29
|
/** Whether a participant passed. A dry-run participant passes as a contract. */
|
|
@@ -307,18 +307,28 @@ export function makeCuaRunBudget(maxTotalUsd) {
|
|
|
307
307
|
};
|
|
308
308
|
}
|
|
309
309
|
/**
|
|
310
|
-
* The participant's clock: its time zone, and its
|
|
311
|
-
*
|
|
310
|
+
* The participant's clock: its time zone, and its end on the run's clock. A participant with an
|
|
311
|
+
* end runs its session on the run's clock, so the end and the schedule read one clock. A cap
|
|
312
|
+
* counts from here, where the session starts, so the desktop's setup does not use it.
|
|
312
313
|
*/
|
|
313
314
|
function participantClock(timeZone, deps) {
|
|
314
|
-
const
|
|
315
|
-
if (timeZone === undefined &&
|
|
315
|
+
const declared = deps.end;
|
|
316
|
+
if (timeZone === undefined && declared === undefined)
|
|
316
317
|
return {};
|
|
318
|
+
const end = declared === undefined
|
|
319
|
+
? undefined
|
|
320
|
+
: declared.kind === "session_cap"
|
|
321
|
+
? {
|
|
322
|
+
atMs: deps.now() + declared.afterMs,
|
|
323
|
+
windDownMs: declared.windDownMs,
|
|
324
|
+
kind: declared.kind,
|
|
325
|
+
}
|
|
326
|
+
: { atMs: declared.atMs, windDownMs: declared.windDownMs };
|
|
317
327
|
return {
|
|
318
328
|
clock: {
|
|
319
329
|
...(timeZone === undefined ? {} : { timeZone }),
|
|
320
|
-
...(
|
|
330
|
+
...(end === undefined ? {} : { end }),
|
|
321
331
|
},
|
|
322
|
-
...(
|
|
332
|
+
...(end === undefined ? {} : { now: deps.now }),
|
|
323
333
|
};
|
|
324
334
|
}
|
|
@@ -4,7 +4,7 @@ import { type ResolvedPersona } from "../../study/persona.js";
|
|
|
4
4
|
import { type RunRerunLineage } from "../../run/bundle.js";
|
|
5
5
|
import type { PreparedSelectedOutputDirectory } from "../../run/contained-output.js";
|
|
6
6
|
import { type CuaActorStudyErrorCode, type CuaParticipantDeps, type CuaParticipantPlan, type CuaParticipantPlanEntry, type DesktopParticipantRun, type ParticipantRunsAndPlan } from "./types.js";
|
|
7
|
-
import { type
|
|
7
|
+
import { type ParticipantSchedule } from "../../study/arrivals.js";
|
|
8
8
|
/**
|
|
9
9
|
* The session budget of a study that declares no execution.timeoutMs. On a provisioned route it is
|
|
10
10
|
* what the `ceilingMs` sandbox ceiling leaves after provisioning, seeding and the teardown buffer.
|
|
@@ -19,28 +19,29 @@ export declare function defaultSessionTimeoutMs(config: StudyConfig, ceilingMs:
|
|
|
19
19
|
export declare function resolveParticipantSandboxMs(config: StudyConfig, ceilingMs: number): number;
|
|
20
20
|
/**
|
|
21
21
|
* The deps one participant runs with once its start comes: its own session budget and sandbox
|
|
22
|
-
* deadline, and with a declared end
|
|
23
|
-
* due, from runOnSchedule).
|
|
22
|
+
* deadline (participantSession), and with a declared end or cap, how its session ends
|
|
23
|
+
* (`scheduledAt` is when it was due, from runOnSchedule).
|
|
24
24
|
*/
|
|
25
|
-
export declare function scheduledParticipantDeps(participant:
|
|
25
|
+
export declare function scheduledParticipantDeps(participant: {
|
|
26
|
+
readonly schedule: ParticipantSchedule;
|
|
27
|
+
}, deps: {
|
|
26
28
|
readonly timeoutMs: number;
|
|
27
29
|
readonly sandboxMs: number;
|
|
28
|
-
}, scheduledAt: number): Pick<CuaParticipantDeps, "timeoutMs" | "sandboxMs" | "
|
|
30
|
+
}, scheduledAt: number): Pick<CuaParticipantDeps, "timeoutMs" | "sandboxMs" | "end">;
|
|
29
31
|
/**
|
|
30
32
|
* The in-flight participant bound a study declares. Defaults to the participant count: every
|
|
31
33
|
* declared participant runs at once, because a throttle nobody asked for silently turns "N actors
|
|
32
34
|
* live" into waves; total session count and spend are the same either way, only wall-clock
|
|
33
35
|
* and simultaneity differ. A declared execution.concurrency is a cap, clamped to [1, participants].
|
|
34
|
-
*
|
|
36
|
+
* participantsAtOnce then lowers it to what the E2B plan and HUMANISH_CUA_MAX_CONCURRENCY allow.
|
|
35
37
|
*/
|
|
36
38
|
export declare function boundedConcurrency(declared: number | undefined, participantCount: number): number;
|
|
37
39
|
/**
|
|
38
40
|
* Build the participant runs and the public plan from a computer-use plan (pure). The same plan
|
|
39
|
-
* appears in dry-run and live, marked $0 for a dry run. `
|
|
40
|
-
*
|
|
41
|
+
* appears in dry-run and live, marked $0 for a dry run. `personas` are the compiled committed
|
|
42
|
+
* personas the prompts carry.
|
|
41
43
|
*/
|
|
42
44
|
export declare function participantRunsAndPlan(plan: ComputerUsePlan, opts?: {
|
|
43
|
-
env?: Record<string, string | undefined>;
|
|
44
45
|
personas?: Map<string, ResolvedPersona>;
|
|
45
46
|
}): ParticipantRunsAndPlan;
|
|
46
47
|
/** Print the participant plan to stderr before any sandbox/provider call (public-safe: ids, devices,
|
|
@@ -63,7 +64,6 @@ export declare function loadCuaParticipants(args: {
|
|
|
63
64
|
plan: ComputerUsePlan;
|
|
64
65
|
cwd: string;
|
|
65
66
|
projectRoot: PreparedSelectedOutputDirectory;
|
|
66
|
-
env: Record<string, string | undefined>;
|
|
67
67
|
}): Promise<{
|
|
68
68
|
ok: true;
|
|
69
69
|
participantRuns: DesktopParticipantRun[];
|