humanish 0.116.0 → 0.118.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +40 -0
- package/dist/actors/codex/restricted-participant-policy.d.ts +0 -2
- package/dist/actors/codex/restricted-participant-policy.js +4 -32
- package/dist/actors/codex/restricted-participant.d.ts +5 -0
- package/dist/actors/codex/restricted-participant.js +3 -2
- package/dist/actors/computer-use/actor.d.ts +4 -0
- package/dist/actors/computer-use/actor.js +2 -0
- package/dist/actors/computer-use/loop/actions.d.ts +4 -4
- package/dist/actors/computer-use/loop/actions.js +25 -18
- package/dist/actors/computer-use/loop/session.d.ts +3 -0
- package/dist/actors/computer-use/loop/session.js +9 -1
- package/dist/actors/computer-use/loop/trace.js +1 -0
- package/dist/actors/computer-use/loop/types.d.ts +16 -14
- package/dist/actors/computer-use/loop.d.ts +1 -1
- package/dist/actors/computer-use/loop.js +1 -1
- package/dist/actors/computer-use/wait.d.ts +74 -0
- package/dist/actors/computer-use/wait.js +100 -0
- package/dist/actors/contract.d.ts +14 -0
- package/dist/analysis/admission.d.ts +7 -2
- package/dist/analysis/admission.js +12 -3
- package/dist/analysis/automatic-config.d.ts +6 -1
- package/dist/analysis/automatic-config.js +12 -11
- package/dist/analysis/execute.d.ts +19 -7
- package/dist/analysis/execute.js +55 -17
- package/dist/browser-control/protocol.d.ts +86 -6
- package/dist/browser-control/protocol.js +13 -6
- package/dist/cli/commands/analyze.js +2 -1
- package/dist/cli/commands/study-format.js +8 -11
- package/dist/cli/commands/study-route-computer-use.js +2 -2
- package/dist/cli/commands/study-route-scripted.js +2 -2
- package/dist/cli/commands/study-route-shared-world.js +5 -3
- package/dist/cli/commands/study-route-terminal.js +2 -2
- package/dist/cli/commands/study-run.js +3 -3
- package/dist/cli/commands/study.js +2 -2
- package/dist/cli/doctor.js +11 -1
- package/dist/cli/io.js +3 -2
- package/dist/comms/lease-store.d.ts +1 -1
- package/dist/comms/lease-store.js +3 -2
- package/dist/comms/receiving-types.d.ts +2 -0
- package/dist/comms/receiving-types.js +2 -0
- package/dist/guest/desktop-executor.js +3 -1
- package/dist/observer-app.html +7 -7
- package/dist/routes/computer-use/e2b-desktop/prepare.js +40 -31
- package/dist/routes/computer-use/fanout-bundle.js +2 -1
- package/dist/routes/computer-use/fanout-records.js +2 -1
- package/dist/routes/computer-use/local-vm.js +5 -2
- package/dist/routes/computer-use/participant-model.js +9 -0
- package/dist/routes/computer-use/participant-runs.js +3 -9
- package/dist/routes/computer-use/plan.d.ts +3 -1
- package/dist/routes/computer-use/plan.js +29 -9
- package/dist/routes/computer-use/single-bundle.js +7 -4
- package/dist/routes/computer-use/subject-projection.js +2 -0
- package/dist/routes/computer-use/types.d.ts +1 -1
- package/dist/routes/scripted/plan.d.ts +4 -1
- package/dist/routes/scripted/plan.js +25 -2
- package/dist/routes/scripted/setup.js +1 -4
- package/dist/routes/scripted/types.d.ts +1 -1
- package/dist/routes/shared-world/participant-specs.js +2 -2
- package/dist/routes/shared-world/plan.d.ts +3 -1
- package/dist/routes/shared-world/plan.js +39 -7
- package/dist/routes/shared-world/route.js +2 -2
- package/dist/routes/terminal/live-sandbox.d.ts +1 -1
- package/dist/routes/terminal/live-sandbox.js +14 -11
- package/dist/routes/terminal/sandbox.d.ts +4 -3
- package/dist/routes/terminal/sandbox.js +5 -5
- package/dist/run/bundle-shape.js +2 -1
- package/dist/run/bundle.d.ts +5 -3
- package/dist/run/projection.js +7 -2
- package/dist/run/route-shell.js +10 -5
- package/dist/run/subject-name.d.ts +11 -0
- package/dist/run/subject-name.js +11 -0
- package/dist/study/automatic-analysis-plan.d.ts +28 -0
- package/dist/study/automatic-analysis-plan.js +57 -0
- package/dist/study/composition-rules.js +1 -1
- package/dist/study/config.js +10 -21
- package/dist/study/doctor.js +17 -3
- package/dist/study/keys.d.ts +2 -0
- package/dist/study/keys.js +2 -0
- package/dist/study/migrate/v2.js +2 -1
- package/dist/study/parse/actors.d.ts +1 -1
- package/dist/study/parse/actors.js +9 -1
- package/dist/study/parse/front.d.ts +1 -1
- package/dist/study/parse/front.js +7 -1
- package/dist/study/plan-base.d.ts +2 -0
- package/dist/study/plan-base.js +3 -0
- package/dist/study/plan-participants.d.ts +4 -0
- package/dist/study/plan-participants.js +6 -5
- package/dist/study/plan-types.d.ts +8 -3
- package/dist/study/plan.d.ts +4 -4
- package/dist/study/plan.js +21 -12
- package/dist/study/preflight-result.js +2 -2
- package/dist/study/preflight.d.ts +2 -1
- package/dist/study/preflight.js +12 -3
- package/dist/study/routing.d.ts +5 -4
- package/dist/study/routing.js +16 -7
- package/dist/study/summary.d.ts +3 -3
- package/dist/study/summary.js +3 -5
- package/dist/study/types.d.ts +14 -0
- package/dist/study/validation.d.ts +7 -2
- package/dist/study/validation.js +30 -14
- package/dist/study/warnings.js +12 -0
- package/dist/subject/serve.d.ts +3 -0
- package/dist/subject/serve.js +4 -1
- package/dist/substrates/detached.js +14 -7
- package/dist/substrates/e2b/desktop-executor.d.ts +0 -2
- package/dist/substrates/e2b/desktop-executor.js +3 -3
- package/dist/substrates/e2b/lifetime.d.ts +75 -4
- package/dist/substrates/e2b/lifetime.js +134 -10
- package/dist/substrates/e2b/subject-sandbox.js +2 -6
- package/dist/tui-app.js +119 -118
- package/dist/verify/artifacts.d.ts +19 -4
- package/dist/verify/artifacts.js +39 -8
- package/dist/verify/verify.js +6 -15
- package/docs/architecture/browser-control.md +10 -4
- package/docs/contracts/run-bundle.md +7 -6
- package/docs/contracts/schemas.md +33 -19
- package/docs/contracts/study-analysis.md +2 -2
- package/docs/principles/invariants-and-defaults.md +1 -1
- package/docs/product/automatic-analysis.md +4 -3
- package/package.json +2 -2
|
@@ -5,6 +5,7 @@ import { validatePreparedRunArtifactPaths } from "../../../run/paths.js";
|
|
|
5
5
|
import { provisionCloneSubject } from "../../../subject/clone.js";
|
|
6
6
|
import { provisionDesktopCli } from "../../../subject/desktop-cli.js";
|
|
7
7
|
import { provisionLocalTreeSubject } from "../../../subject/local-tree.js";
|
|
8
|
+
import { SubjectBuildError } from "../../../subject/serve.js";
|
|
8
9
|
import { inspectDesktopScreenGeometry } from "../../../substrates/e2b/desktop-geometry.js";
|
|
9
10
|
import { observeDesktopResources } from "../../../substrates/e2b/desktop-resources.js";
|
|
10
11
|
import { acquireE2BDesktopSandbox, e2bDesktopTemplate } from "../../../substrates/e2b/sandbox.js";
|
|
@@ -166,37 +167,45 @@ export async function provisionParticipantSubject(ctx, state, desktop) {
|
|
|
166
167
|
onPhase: ctx.onSubjectPhase,
|
|
167
168
|
});
|
|
168
169
|
}
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
170
|
+
try {
|
|
171
|
+
if (subject.kind === "clone") {
|
|
172
|
+
state.subjectCommit = await provisionCloneSubject(shell, {
|
|
173
|
+
repo: subject.repo,
|
|
174
|
+
depth: residual.subject.clone?.depth ?? 1,
|
|
175
|
+
serve: subject.serve,
|
|
176
|
+
...(subject.state === undefined ? {} : { state: subject.state }),
|
|
177
|
+
hasGithubToken: subject.env.includes("GITHUB_TOKEN"),
|
|
178
|
+
requestTimeoutMs: deps.requestTimeoutMs,
|
|
179
|
+
scrub: deps.scrubKnownValues,
|
|
180
|
+
onCommit: (commit) => {
|
|
181
|
+
state.subjectCommit = commit;
|
|
182
|
+
},
|
|
183
|
+
onStateStep: (record) => {
|
|
184
|
+
state.stateStepRecords.push(record);
|
|
185
|
+
},
|
|
186
|
+
onPhase: ctx.onSubjectPhase,
|
|
187
|
+
...deps.detachedTimers,
|
|
188
|
+
});
|
|
189
|
+
}
|
|
190
|
+
else if (subject.kind === "local-tree" && deps.localTreeArchiveBuffer) {
|
|
191
|
+
await provisionLocalTreeSubject(shell, {
|
|
192
|
+
archiveBuffer: deps.localTreeArchiveBuffer,
|
|
193
|
+
serve: subject.serve,
|
|
194
|
+
...(subject.state === undefined ? {} : { state: subject.state }),
|
|
195
|
+
requestTimeoutMs: deps.requestTimeoutMs,
|
|
196
|
+
scrub: deps.scrubKnownValues,
|
|
197
|
+
onStateStep: (record) => {
|
|
198
|
+
state.stateStepRecords.push(record);
|
|
199
|
+
},
|
|
200
|
+
onPhase: ctx.onSubjectPhase,
|
|
201
|
+
...deps.detachedTimers,
|
|
202
|
+
});
|
|
203
|
+
}
|
|
187
204
|
}
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
requestTimeoutMs: deps.requestTimeoutMs,
|
|
194
|
-
scrub: deps.scrubKnownValues,
|
|
195
|
-
onStateStep: (record) => {
|
|
196
|
-
state.stateStepRecords.push(record);
|
|
197
|
-
},
|
|
198
|
-
onPhase: ctx.onSubjectPhase,
|
|
199
|
-
...deps.detachedTimers,
|
|
200
|
-
});
|
|
205
|
+
catch (error) {
|
|
206
|
+
// A failed serve.build gets its own code, so the run names the step the author has to fix.
|
|
207
|
+
if (error instanceof SubjectBuildError)
|
|
208
|
+
state.failureCode = "HUMANISH_COMPUTER_USE_SUBJECT_BUILD_FAILED";
|
|
209
|
+
throw error;
|
|
201
210
|
}
|
|
202
211
|
}
|
|
@@ -5,6 +5,7 @@ import { redactText } from "../../evidence/redaction.js";
|
|
|
5
5
|
import { REVIEW_SCHEMA, bundleArtifacts, bundleHead, } from "../../run/bundle.js";
|
|
6
6
|
import { aggregateTaskFunnels, formatParticipantOutcomes, formatRunTaskFunnel, gapsListClause, withCuaReviewProvenance, } from "../../run/outcomes.js";
|
|
7
7
|
import { providerResourcesForOutcome, publicSafeAppUrlLabel } from "./bundle-parts.js";
|
|
8
|
+
import { subjectName } from "../../run/subject-name.js";
|
|
8
9
|
import { participantFeedbackCandidates } from "./participant-feedback.js";
|
|
9
10
|
import { participantFactsOf } from "./participant-facts.js";
|
|
10
11
|
import { judgeParticipantRecords } from "../../run/judge.js";
|
|
@@ -44,7 +45,7 @@ function fanoutSummary(args, facts) {
|
|
|
44
45
|
return `${people} ${count === 1 ? "is" : "are"} each using their own copy of the app. The run is still going, so nothing here is final.`;
|
|
45
46
|
}
|
|
46
47
|
if (args.dryRun) {
|
|
47
|
-
return `${rerun}Dry run: ${people} would each use their own copy of ${publicSafeAppUrlLabel(args.appUrl)}. No desktops were launched and $0 was spent.`;
|
|
48
|
+
return `${rerun}Dry run: ${people} would each use their own copy of ${subjectName(publicSafeAppUrlLabel(args.appUrl), args.aggregateSubject)}. No desktops were launched and $0 was spent.`;
|
|
48
49
|
}
|
|
49
50
|
const tasks = facts.tasks === undefined ? "" : ` Tasks: ${facts.tasks}.`;
|
|
50
51
|
return `${rerun}${people} took part, each in their own copy of the app: ${facts.outcomes}${gapsListClause(count - facts.passed, count)}.${tasks}`;
|
|
@@ -4,6 +4,7 @@ import { participantAssignment } from "../../study/participant-assignment.js";
|
|
|
4
4
|
import { participantEvent, participantRecord, participantStream, } from "../../run/participant-records.js";
|
|
5
5
|
import { participantCaption } from "../../run/participant-caption.js";
|
|
6
6
|
import { participantSummary } from "../../run/outcomes.js";
|
|
7
|
+
import { subjectName } from "../../run/subject-name.js";
|
|
7
8
|
import { declaredScreenForRender } from "../../substrates/e2b/desktop-geometry.js";
|
|
8
9
|
import { describeSubjectState, phaseEventIdSuffix, publicSafeAppUrlLabel } from "./bundle-parts.js";
|
|
9
10
|
import { participantSubjectEnv } from "./types.js";
|
|
@@ -60,7 +61,7 @@ function fanoutParticipantRecord(args, spec, index, view) {
|
|
|
60
61
|
mode: "browser-sim",
|
|
61
62
|
progress: args.inProgress === true && outcome === undefined ? 20 : 100,
|
|
62
63
|
currentStep: reason,
|
|
63
|
-
summary: participantSummary(name, { name: "the app", target: publicTargetUrl }, {
|
|
64
|
+
summary: participantSummary(name, { name: "the app", target: subjectName(publicTargetUrl, view.subject) }, {
|
|
64
65
|
trace: session?.trace,
|
|
65
66
|
skippedReason: outcome?.skippedReason,
|
|
66
67
|
sessionError: outcome?.sessionError,
|
|
@@ -139,10 +139,11 @@ function createLocalParticipantDesktop(context, run, warnings, artifactRoot) {
|
|
|
139
139
|
};
|
|
140
140
|
}
|
|
141
141
|
/** The Codex account participant, whose unconfirmed cleanup blocks the study's analysis. */
|
|
142
|
-
function accountProvider(state) {
|
|
142
|
+
function accountProvider(state, maxWaitMs) {
|
|
143
143
|
return async ({ executor }) => {
|
|
144
144
|
const participant = createRestrictedCodexParticipant({
|
|
145
145
|
speechEnabled: executor.speechEnabled === true,
|
|
146
|
+
...(maxWaitMs === undefined ? {} : { maxWaitMs }),
|
|
146
147
|
});
|
|
147
148
|
state.participants.push(participant);
|
|
148
149
|
const provider = Object.assign(participant.provider, {
|
|
@@ -183,7 +184,9 @@ export function prepareLocalVmRun(options) {
|
|
|
183
184
|
};
|
|
184
185
|
const { config: _config, assets: _assets, signal: _signal, ...runOptions } = options;
|
|
185
186
|
return {
|
|
186
|
-
options: account
|
|
187
|
+
options: account
|
|
188
|
+
? { ...runOptions, createProvider: accountProvider(state, config.actor?.maxWaitMs) }
|
|
189
|
+
: runOptions,
|
|
187
190
|
localVm: {
|
|
188
191
|
desktop: (run, warnings, artifactRoot) => createLocalParticipantDesktop(context, run, warnings, artifactRoot),
|
|
189
192
|
capacity: () => localRuntimeCapacity(),
|
|
@@ -44,6 +44,9 @@ export async function startParticipantModel(spec, deps, executor) {
|
|
|
44
44
|
: { reasoningEffort: spec.planned.limits.reasoningEffort }),
|
|
45
45
|
...(brain.declaredModel === undefined ? {} : { model: brain.declaredModel }),
|
|
46
46
|
...(executor.speechEnabled === true ? { speechEnabled: true } : {}),
|
|
47
|
+
...(spec.planned.limits.maxWaitMs === undefined
|
|
48
|
+
? {}
|
|
49
|
+
: { maxWaitMs: spec.planned.limits.maxWaitMs }),
|
|
47
50
|
session: { env },
|
|
48
51
|
});
|
|
49
52
|
return { provider: codexParticipant.provider, codexParticipant };
|
|
@@ -130,6 +133,12 @@ export function participantSessionOptions(spec, deps, ready, provider, writeScre
|
|
|
130
133
|
? {}
|
|
131
134
|
: { stopWhen: spec.planned.limits.stopWhen }),
|
|
132
135
|
...(spec.planned.limits.dwell === undefined ? {} : { dwell: spec.planned.limits.dwell }),
|
|
136
|
+
...(spec.planned.limits.maxWaitMs === undefined
|
|
137
|
+
? {}
|
|
138
|
+
: { maxWaitMs: spec.planned.limits.maxWaitMs }),
|
|
139
|
+
...(spec.planned.limits.idleWaitMs === undefined
|
|
140
|
+
? {}
|
|
141
|
+
: { idleWaitMs: spec.planned.limits.idleWaitMs }),
|
|
133
142
|
...(spec.planned.tasks === undefined ? {} : { tasks: spec.planned.tasks }),
|
|
134
143
|
// The study budget: this participant notes its own running estimate on the shared
|
|
135
144
|
// ledger and stops when the run total crosses the cap, independent of the per-participant
|
|
@@ -3,7 +3,6 @@
|
|
|
3
3
|
// paths, and the public plan printed before any sandbox or provider call. loadCuaParticipants
|
|
4
4
|
// compiles the committed personas first and narrows a rerun to its selected participants.
|
|
5
5
|
import { isLocalBrowserStudy, LOCAL_BROWSER_LIFETIME_MS, } from "../../substrates/local/runtime-config.js";
|
|
6
|
-
import { DEFAULT_STATE_STEP_TIMEOUT_MS } from "../../subject/state.js";
|
|
7
6
|
import { resolveParticipant } from "../../run/participant.js";
|
|
8
7
|
import {} from "../../study/types.js";
|
|
9
8
|
import { scrubPersonaBrief } from "../../study/persona.js";
|
|
@@ -12,7 +11,7 @@ import {} from "../../run/bundle.js";
|
|
|
12
11
|
import { participantAssignment } from "../../study/participant-assignment.js";
|
|
13
12
|
import { resolveCommittedPersonas } from "../../study/persona-resolve.js";
|
|
14
13
|
import { CUA_FANOUT_STRATEGY, CUA_MAX_CONCURRENCY_ENV, DEFAULT_APP_URL_SESSION_TIMEOUT_MS, MIN_DERIVED_SESSION_TIMEOUT_MS, } from "./types.js";
|
|
15
|
-
import {
|
|
14
|
+
import { sandboxHeadroomMs } from "../../substrates/e2b/lifetime.js";
|
|
16
15
|
import { readPositiveInt } from "../../study/parse/values.js";
|
|
17
16
|
import { digestUrl } from "./bundle-parts.js";
|
|
18
17
|
import { composeParticipantInstructions, DEFAULT_MISSION } from "./participant-prompt.js";
|
|
@@ -26,8 +25,7 @@ export function defaultSessionTimeoutMs(config, ceilingMs) {
|
|
|
26
25
|
const provisionedRoute = config.subject.source === "clone" || config.subject.source === "local-tree";
|
|
27
26
|
if (!provisionedRoute)
|
|
28
27
|
return DEFAULT_APP_URL_SESSION_TIMEOUT_MS;
|
|
29
|
-
const
|
|
30
|
-
const room = ceilingMs - SUBJECT_PROVISION_BUDGET_MS - stateBudgetMs - SANDBOX_TIMEOUT_BUFFER_MS;
|
|
28
|
+
const room = ceilingMs - sandboxHeadroomMs({ seed: config.subject.state?.seed ?? [] });
|
|
31
29
|
return Math.max(MIN_DERIVED_SESSION_TIMEOUT_MS, Math.min(DEFAULT_APP_URL_SESSION_TIMEOUT_MS, room));
|
|
32
30
|
}
|
|
33
31
|
/** Per-participant sandbox deadline (each one owns its own desktop). Mirrors the
|
|
@@ -41,13 +39,9 @@ export function resolveParticipantSandboxMs(config, ceilingMs) {
|
|
|
41
39
|
return LOCAL_BROWSER_LIFETIME_MS;
|
|
42
40
|
const timeoutMs = config.execution?.timeoutMs ?? defaultSessionTimeoutMs(config, ceilingMs);
|
|
43
41
|
const provisionedRoute = config.subject.source === "clone" || config.subject.source === "local-tree";
|
|
44
|
-
const stateBudgetMs = provisionedRoute
|
|
45
|
-
? (config.subject.state?.seed ?? []).reduce((sum, step) => sum + (step.timeoutMs ?? DEFAULT_STATE_STEP_TIMEOUT_MS), 0)
|
|
46
|
-
: 0;
|
|
47
42
|
return (config.execution?.desktop?.sandboxTimeoutMs ??
|
|
48
43
|
timeoutMs +
|
|
49
|
-
(provisionedRoute ?
|
|
50
|
-
SANDBOX_TIMEOUT_BUFFER_MS);
|
|
44
|
+
sandboxHeadroomMs(provisionedRoute ? { seed: config.subject.state?.seed ?? [] } : undefined));
|
|
51
45
|
}
|
|
52
46
|
/**
|
|
53
47
|
* The in-flight participant bound a study declares. Defaults to the participant count: every
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { type CuaActorDescriptor } from "../../actors/registry.js";
|
|
2
2
|
import type { ComputerUsePlan, ComputerUseRunner, RoutePlanResult, RouteRefusal } from "../../study/plan-types.js";
|
|
3
3
|
import type { StudyConfig, StudySubjectState } from "../../study/types.js";
|
|
4
|
-
import { type SandboxCeiling } from "../../substrates/e2b/lifetime.js";
|
|
4
|
+
import { type ConcurrentSandboxes, type SandboxCeiling } from "../../substrates/e2b/lifetime.js";
|
|
5
5
|
import { type ComputerUseRunInput, type CuaActorStudyErrorCode } from "./types.js";
|
|
6
6
|
/** The error a computer-use study returns before a run starts. */
|
|
7
7
|
export interface ComputerUseRefusal extends RouteRefusal<"computer-use", CuaActorStudyErrorCode> {
|
|
@@ -48,4 +48,6 @@ export declare function planComputerUseStudy(config: StudyConfig, input: {
|
|
|
48
48
|
readonly rerun?: ComputerUseRunInput["rerun"];
|
|
49
49
|
/** The longest sandbox lifetime the operator's E2B plan allows (sandboxCeiling). */
|
|
50
50
|
readonly sandboxCeiling: SandboxCeiling;
|
|
51
|
+
/** How many sandboxes the operator's E2B plan runs at once (concurrentSandboxes). */
|
|
52
|
+
readonly concurrentSandboxes: ConcurrentSandboxes;
|
|
51
53
|
}): ComputerUsePlanResult;
|
|
@@ -9,12 +9,12 @@ import { studyUrlCredentialReason } from "../../study/url-credentials.js";
|
|
|
9
9
|
import { subjectStateInvalidReason } from "../../study/parse/subject-state.js";
|
|
10
10
|
import { brainOf, callerBrainOf, planCaps, desktopRequirements, planBase, provisionedSubject, } from "../../study/plan-base.js";
|
|
11
11
|
import { computerUseParticipants, declaredTargets } from "../../study/plan-participants.js";
|
|
12
|
-
import {
|
|
13
|
-
import { cloneTargetValidationReason, computerUseValidationReason, desktopMediaValidationReason, outputTokenLimitValidationReason, receivingEmailValidationReason, taskProtocolValidationReason, } from "../../study/validation.js";
|
|
12
|
+
import { computerUseParticipantCount, participantCountReason } from "../../study/routing.js";
|
|
13
|
+
import { cloneTargetValidationReason, computerUseValidationReason, desktopMediaValidationReason, outputTokenLimitValidationReason, receivingEmailValidationReason, taskProtocolValidationReason, waitLimitValidationReason, } from "../../study/validation.js";
|
|
14
14
|
import { desktopCliProductReason } from "../../study/composition-rules.js";
|
|
15
15
|
import { isLocalBrowserStudy } from "../../substrates/local/runtime-config.js";
|
|
16
16
|
import { boundedConcurrency, defaultSessionTimeoutMs, resolveParticipantSandboxMs, } from "./participant-runs.js";
|
|
17
|
-
import { ceilingAdvice } from "../../substrates/e2b/lifetime.js";
|
|
17
|
+
import { ceilingAdvice, participantsAtOnce, } from "../../substrates/e2b/lifetime.js";
|
|
18
18
|
import {} from "./types.js";
|
|
19
19
|
/** The registered descriptor for a planned actor id; planComputerUseStudy checked the registry. */
|
|
20
20
|
export function cuaDescriptorOf(actor) {
|
|
@@ -83,7 +83,9 @@ function cuaStudyRejection(config, hasRunSession, driving, subjectRoute, ceiling
|
|
|
83
83
|
}
|
|
84
84
|
/** A declaration this route cannot honor, for any subject or with the caller's own driver. */
|
|
85
85
|
function unsupportedDeclarationReason(config, hasRunSession, driving, { inProcessRoute }) {
|
|
86
|
-
const reason = desktopMediaValidationReason(config) ||
|
|
86
|
+
const reason = desktopMediaValidationReason(config) ||
|
|
87
|
+
outputTokenLimitValidationReason(config) ||
|
|
88
|
+
waitLimitValidationReason(config);
|
|
87
89
|
if (reason)
|
|
88
90
|
return invalid(reason);
|
|
89
91
|
if (config.actor?.maxOutputTokens !== undefined &&
|
|
@@ -190,6 +192,19 @@ function rosterShapeReason(config, ceilingMs) {
|
|
|
190
192
|
const headroomMs = derivedSandboxMs - sessionMs;
|
|
191
193
|
return invalid(`execution.timeoutMs ${Math.round(sessionMs / 60_000)}m derives a ${Math.round(derivedSandboxMs / 60_000)}m sandbox deadline, and a sandbox may not live longer than ${ceilingMs / 60_000}m. The deadline is the session budget plus ${Math.round(headroomMs / 60_000)}m of provisioning and teardown headroom${provisionedRoute ? " (this route clones, installs, builds and serves the subject before the actor starts)" : ""}. Lower execution.timeoutMs to at most ${Math.round((ceilingMs - headroomMs) / 60_000)}m, or set execution.desktop.sandboxTimeoutMs explicitly.${ceilingAdvice(derivedSandboxMs)}`);
|
|
192
194
|
}
|
|
195
|
+
/**
|
|
196
|
+
* How many of the runner's participants run at once. Hosted desktops share the E2B plan's
|
|
197
|
+
* concurrent sandboxes; the local VM has its own capacity rules and the in-process route runs one
|
|
198
|
+
* participant.
|
|
199
|
+
*/
|
|
200
|
+
function concurrencyOf(config, runner, limit) {
|
|
201
|
+
const declared = config.execution?.concurrency;
|
|
202
|
+
const n = runner.participants.length;
|
|
203
|
+
const concurrency = boundedConcurrency(declared, n);
|
|
204
|
+
return runner.desktop === "e2b-desktop"
|
|
205
|
+
? participantsAtOnce({ concurrency, declared: declared !== undefined, participants: n }, limit)
|
|
206
|
+
: { ok: true, count: concurrency };
|
|
207
|
+
}
|
|
193
208
|
/** The plan's rerun: the source run and, when given, the participants to rerun. */
|
|
194
209
|
function rerunPlan({ sourceRunId, participantIds, }) {
|
|
195
210
|
return { sourceRunId, ...(participantIds === undefined ? {} : { participantIds }) };
|
|
@@ -244,9 +259,11 @@ export function planComputerUseStudy(config, input) {
|
|
|
244
259
|
const productReason = desktopCliProductReason(config);
|
|
245
260
|
if (productReason)
|
|
246
261
|
return refuse("in-scope", "HUMANISH_COMPUTER_USE_SUBJECT_INVALID", productReason, actor);
|
|
262
|
+
// Counted before any participant is built, so a large --count is refused without building it.
|
|
263
|
+
const countReason = participantCountReason(computerUseParticipantCount(config, input.countOverride));
|
|
264
|
+
if (countReason)
|
|
265
|
+
return refuse("after-personas", "HUMANISH_COMPUTER_USE_FANOUT_INVALID", countReason, actor);
|
|
247
266
|
const participants = computerUseParticipants(config, input.countOverride);
|
|
248
|
-
if (participants.length > MAX_COMPUTER_USE_PARTICIPANTS)
|
|
249
|
-
return refuse("after-personas", "HUMANISH_COMPUTER_USE_FANOUT_INVALID", `Computer-use fan-out is capped at ${MAX_COMPUTER_USE_PARTICIPANTS} participants (resolved ${participants.length}); N concurrent paid desktops is real spend.`, actor);
|
|
250
267
|
const [first, ...rest] = participants;
|
|
251
268
|
if (driving.inProcess && rest.length > 0)
|
|
252
269
|
return refuse("after-personas", "HUMANISH_COMPUTER_USE_FANOUT_INVALID", "Fan-out to more than one participant is not supported on the in-process route (RunStudyOptions.inProcess): fan-out provisions one independent E2B desktop per participant, which the in-process route deliberately skips. Run a single in-process participant, or fan out on the E2B route.", actor);
|
|
@@ -276,21 +293,24 @@ export function planComputerUseStudy(config, input) {
|
|
|
276
293
|
: isLocalBrowserStudy(config)
|
|
277
294
|
? { desktop: "local-vm", brain, participants: [first, ...rest], subject: appUrlSubject }
|
|
278
295
|
: { desktop: "e2b-desktop", brain, participants: [first, ...rest], subject: hosted };
|
|
279
|
-
const
|
|
280
|
-
|
|
296
|
+
const atOnce = concurrencyOf(config, runner, input.concurrentSandboxes);
|
|
297
|
+
if (!atOnce.ok)
|
|
298
|
+
return refuse("after-personas", "HUMANISH_COMPUTER_USE_FANOUT_INVALID", atOnce.message, actor);
|
|
281
299
|
const provisioned = source === "clone" || source === "local-tree";
|
|
282
300
|
const base = planBase(config, {
|
|
283
301
|
dryRun: input.dryRun,
|
|
284
302
|
analysis,
|
|
303
|
+
participants: participants.length,
|
|
285
304
|
});
|
|
286
305
|
return {
|
|
287
306
|
ok: true,
|
|
288
307
|
plan: {
|
|
289
308
|
...base,
|
|
309
|
+
...(atOnce.lowered === undefined ? {} : { warnings: [atOnce.lowered] }),
|
|
290
310
|
route: "computer-use",
|
|
291
311
|
actor,
|
|
292
312
|
runner,
|
|
293
|
-
concurrency:
|
|
313
|
+
concurrency: atOnce.count,
|
|
294
314
|
sessionBudgetMs: config.execution?.timeoutMs ?? defaultSessionTimeoutMs(config, ceiling.ms),
|
|
295
315
|
sandboxMs: resolveParticipantSandboxMs(config, ceiling.ms),
|
|
296
316
|
caps: planCaps(config),
|
|
@@ -10,6 +10,7 @@ import { participantFeedbackCandidates } from "./participant-feedback.js";
|
|
|
10
10
|
import { judgeParticipantRecords } from "../../run/judge.js";
|
|
11
11
|
import { buildRunCostSummary } from "../../run/cost-summary.js";
|
|
12
12
|
import { participantEvent, participantIds, participantRecord, participantStream, } from "../../run/participant-records.js";
|
|
13
|
+
import { subjectName } from "../../run/subject-name.js";
|
|
13
14
|
import { plural } from "../../run/text.js";
|
|
14
15
|
/** The one participant's record and stream ids: sim-001 and stream-001. */
|
|
15
16
|
const SINGLE = participantIds(0);
|
|
@@ -79,6 +80,8 @@ function participantView(args, publicAppUrl) {
|
|
|
79
80
|
: (args.captureRedaction ?? "raw");
|
|
80
81
|
return {
|
|
81
82
|
publicAppUrl,
|
|
83
|
+
/** How the participant's sentence and the summary name the subject. */
|
|
84
|
+
subjectName: subjectName(publicAppUrl, args.subject),
|
|
82
85
|
participant,
|
|
83
86
|
tally: judgment.tally,
|
|
84
87
|
status,
|
|
@@ -90,7 +93,7 @@ function participantView(args, publicAppUrl) {
|
|
|
90
93
|
};
|
|
91
94
|
}
|
|
92
95
|
function singleSimulation(args, view) {
|
|
93
|
-
const {
|
|
96
|
+
const { status, reason } = view;
|
|
94
97
|
const name = participantCaption({
|
|
95
98
|
id: args.participantId ?? "lane-01",
|
|
96
99
|
personaId: args.persona.id,
|
|
@@ -103,7 +106,7 @@ function singleSimulation(args, view) {
|
|
|
103
106
|
mode: "browser-sim",
|
|
104
107
|
progress: args.inProgress === true ? 20 : 100,
|
|
105
108
|
currentStep: reason,
|
|
106
|
-
summary: participantSummary(name, { name: "the app", place: browserPlace(args), target:
|
|
109
|
+
summary: participantSummary(name, { name: "the app", place: browserPlace(args), target: view.subjectName }, {
|
|
107
110
|
trace: args.session?.trace,
|
|
108
111
|
sessionError: args.sessionError,
|
|
109
112
|
inProgress: args.inProgress === true,
|
|
@@ -285,11 +288,11 @@ function singleSummary(args, view, facts) {
|
|
|
285
288
|
return "1 participant is using the app. The run is still going, so nothing here is final.";
|
|
286
289
|
}
|
|
287
290
|
if (args.dryRun) {
|
|
288
|
-
return `Dry run: 1 participant would use ${view.
|
|
291
|
+
return `Dry run: 1 participant would use ${view.subjectName}. Nothing was launched and $0 was spent.`;
|
|
289
292
|
}
|
|
290
293
|
if (args.session === undefined) {
|
|
291
294
|
return args.sessionError === undefined
|
|
292
|
-
? `1 participant was planned against ${view.
|
|
295
|
+
? `1 participant was planned against ${view.subjectName}, but no session ran.`
|
|
293
296
|
: `1 participant was planned, but the run failed before their session finished: ${view.reason}`;
|
|
294
297
|
}
|
|
295
298
|
const ending = actorEnding(args.session.trace);
|
|
@@ -103,6 +103,8 @@ function participantSubjectProjection(args) {
|
|
|
103
103
|
if (args.subject.kind === "desktop-cli") {
|
|
104
104
|
return { source: "desktop-cli", product: args.subject.product.name, state: args.subjectState };
|
|
105
105
|
}
|
|
106
|
+
if (args.subject.kind === "local-app")
|
|
107
|
+
return { source: "local-app", state: args.subjectState };
|
|
106
108
|
return { source: "app-url", state: args.subjectState };
|
|
107
109
|
}
|
|
108
110
|
/**
|
|
@@ -185,7 +185,7 @@ export interface CuaParticipantSummary {
|
|
|
185
185
|
concurrency: number;
|
|
186
186
|
waves: number;
|
|
187
187
|
}
|
|
188
|
-
export type CuaActorStudyErrorCode = StudyResultErrorCode | "HUMANISH_COMPUTER_USE_FAILED" | "HUMANISH_COMPUTER_USE_KEYS_MISSING" | "HUMANISH_COMPUTER_USE_AGENT_MISSING" | "HUMANISH_COMPUTER_USE_AGENT_SIGNIN_REQUIRED" | "HUMANISH_COMPUTER_USE_SUBJECT_ENV_MISSING" | "HUMANISH_COMPUTER_USE_ACTOR_UNSUPPORTED" | "HUMANISH_COMPUTER_USE_SUBJECT_INVALID" | "HUMANISH_COMPUTER_USE_SUBJECT_UNSAFE" | "HUMANISH_COMPUTER_USE_EXECUTOR_NO_PROVIDER" | "HUMANISH_COMPUTER_USE_LOCAL_APP_NO_EXECUTOR" | "HUMANISH_COMPUTER_USE_LOCAL_DESKTOP_MISSING" | "HUMANISH_COMPUTER_USE_LOCAL_CAPACITY_EXCEEDED" | "HUMANISH_COMPUTER_USE_DESKTOP_OUT_OF_MEMORY" | "HUMANISH_COMPUTER_USE_FANOUT_INVALID" | "HUMANISH_COMPUTER_USE_RERUN_INVALID" | "HUMANISH_COMPUTER_USE_DEVICE_GEOMETRY" | "HUMANISH_COMPUTER_USE_UNPRICED_CAP" | "HUMANISH_COMPUTER_USE_COMMS_CATCH_UNREACHABLE" | "HUMANISH_COMPUTER_USE_COMMS_TOKEN_INVALID" | "HUMANISH_WATCH_ALLOW_REQUIRES_OAUTH" | "HUMANISH_WATCH_OAUTH_REQUIRES_TUNNEL" | "HUMANISH_WATCH_OPTION_CONFLICT" | "HUMANISH_WATCH_TUNNEL_REQUIRES_EXPOSE" | "HUMANISH_WATCH_EXPOSE_REQUIRES_EDGE_AUTH" | "HUMANISH_WATCH_EXPOSE_REQUIRES_LIVE_FOLLOW" | "HUMANISH_WATCH_SAFE_NOT_APPLICABLE" | "HUMANISH_SERVE_TUNNEL_NOT_FOUND" | "HUMANISH_SERVE_TUNNEL_START_FAILED";
|
|
188
|
+
export type CuaActorStudyErrorCode = StudyResultErrorCode | "HUMANISH_COMPUTER_USE_FAILED" | "HUMANISH_COMPUTER_USE_KEYS_MISSING" | "HUMANISH_COMPUTER_USE_AGENT_MISSING" | "HUMANISH_COMPUTER_USE_AGENT_SIGNIN_REQUIRED" | "HUMANISH_COMPUTER_USE_SUBJECT_ENV_MISSING" | "HUMANISH_COMPUTER_USE_ACTOR_UNSUPPORTED" | "HUMANISH_COMPUTER_USE_SUBJECT_INVALID" | "HUMANISH_COMPUTER_USE_SUBJECT_UNSAFE" | "HUMANISH_COMPUTER_USE_EXECUTOR_NO_PROVIDER" | "HUMANISH_COMPUTER_USE_LOCAL_APP_NO_EXECUTOR" | "HUMANISH_COMPUTER_USE_LOCAL_DESKTOP_MISSING" | "HUMANISH_COMPUTER_USE_LOCAL_CAPACITY_EXCEEDED" | "HUMANISH_COMPUTER_USE_DESKTOP_OUT_OF_MEMORY" | "HUMANISH_COMPUTER_USE_FANOUT_INVALID" | "HUMANISH_COMPUTER_USE_RERUN_INVALID" | "HUMANISH_COMPUTER_USE_DEVICE_GEOMETRY" | "HUMANISH_COMPUTER_USE_SUBJECT_BUILD_FAILED" | "HUMANISH_COMPUTER_USE_UNPRICED_CAP" | "HUMANISH_COMPUTER_USE_COMMS_CATCH_UNREACHABLE" | "HUMANISH_COMPUTER_USE_COMMS_TOKEN_INVALID" | "HUMANISH_WATCH_ALLOW_REQUIRES_OAUTH" | "HUMANISH_WATCH_OAUTH_REQUIRES_TUNNEL" | "HUMANISH_WATCH_OPTION_CONFLICT" | "HUMANISH_WATCH_TUNNEL_REQUIRES_EXPOSE" | "HUMANISH_WATCH_EXPOSE_REQUIRES_EDGE_AUTH" | "HUMANISH_WATCH_EXPOSE_REQUIRES_LIVE_FOLLOW" | "HUMANISH_WATCH_SAFE_NOT_APPLICABLE" | "HUMANISH_SERVE_TUNNEL_NOT_FOUND" | "HUMANISH_SERVE_TUNNEL_START_FAILED";
|
|
189
189
|
export interface CuaActorStudyResult extends AutomaticAnalysisResult, StudyResultIdentity<"computer-use"> {
|
|
190
190
|
/** True when the Observer verified the bundle, all live participants passed credibility checks
|
|
191
191
|
* (or this is a dry-run), and no declared adapter/scorer verdict failed. */
|
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import type { StudyDeps } from "../../study/study-deps.js";
|
|
2
2
|
import type { RoutePlanResult, RouteRefusal, ScriptedPlan } from "../../study/plan-types.js";
|
|
3
3
|
import type { StudyConfig } from "../../study/types.js";
|
|
4
|
+
import { type SandboxCeiling } from "../../substrates/e2b/lifetime.js";
|
|
4
5
|
/** The error a scripted study returns before a run starts. */
|
|
5
|
-
export interface ScriptedRefusal extends RouteRefusal<"scripted", "HUMANISH_STUDY_ANALYSIS_INVALID" | "HUMANISH_STUDY_TASKS_UNSUPPORTED" | "HUMANISH_SCRIPTED_ACTOR_UNSUPPORTED" | "HUMANISH_SCRIPTED_SCENARIO_INVALID" | "HUMANISH_SCRIPTED_SUBJECT_UNSAFE"> {
|
|
6
|
+
export interface ScriptedRefusal extends RouteRefusal<"scripted", "HUMANISH_STUDY_ANALYSIS_INVALID" | "HUMANISH_STUDY_TASKS_UNSUPPORTED" | "HUMANISH_SCRIPTED_ACTOR_UNSUPPORTED" | "HUMANISH_SCRIPTED_SCENARIO_INVALID" | "HUMANISH_SCRIPTED_SUBJECT_UNSAFE" | "HUMANISH_SCRIPTED_SUBJECT_INVALID"> {
|
|
6
7
|
/**
|
|
7
8
|
* Set on the receiving, analysis and tasks refusals. The route returns those before it opens its
|
|
8
9
|
* run scope, so their result has its own field order and no automatic-analysis record.
|
|
@@ -25,4 +26,6 @@ export declare function planScriptedStudy(config: StudyConfig, input: {
|
|
|
25
26
|
readonly dryRun: boolean;
|
|
26
27
|
/** A test's injected browser means a live run needs no host browser. */
|
|
27
28
|
readonly injectedBrowser?: boolean;
|
|
29
|
+
/** The longest sandbox lifetime the operator's E2B plan allows (sandboxCeiling). */
|
|
30
|
+
readonly sandboxCeiling: SandboxCeiling;
|
|
28
31
|
}): ScriptedPlanResult;
|
|
@@ -7,9 +7,13 @@ import { browserSurfaces } from "../../actors/scripted-browser/types.js";
|
|
|
7
7
|
import { resolveAutomaticAnalysis } from "../../analysis/automatic-config.js";
|
|
8
8
|
import { isNonEmpty, planBase, provisionedSubject } from "../../study/plan-base.js";
|
|
9
9
|
import { cloneTargetValidationReason, desktopMediaValidationReason, taskProtocolValidationReason, } from "../../study/validation.js";
|
|
10
|
+
import { sandboxDeadlineRefusal } from "../../substrates/e2b/lifetime.js";
|
|
10
11
|
// Default surface roster is 1 (desktop only): the defaults-table single-participant row governs;
|
|
11
12
|
// `count: 2` is the declared override that adds the mobile surface.
|
|
12
13
|
const DEFAULT_SURFACE_COUNT = 1;
|
|
14
|
+
// Journey wall-clock budget per surface: 5 minutes. A scripted surface has zero model cost and
|
|
15
|
+
// sandbox-seconds are pennies; a short default only truncated slow-loading subjects.
|
|
16
|
+
const DEFAULT_SESSION_TIMEOUT_MS = 300_000;
|
|
13
17
|
/** The app URL a run's evidence names: a provisioned subject's getHost URL is never persisted. */
|
|
14
18
|
export function evidenceAppUrlOf(subject) {
|
|
15
19
|
return subject.kind === "clone" ? "[provisioned-subject]" : subject.appUrl;
|
|
@@ -66,6 +70,26 @@ export function planScriptedStudy(config, input) {
|
|
|
66
70
|
const scenarioRef = config.scenario;
|
|
67
71
|
if (!scenarioRef?.trim())
|
|
68
72
|
return refuse("HUMANISH_SCRIPTED_SCENARIO_INVALID", "A scripted-browser study needs `scenario`: the actor runs the browser steps in that scenario file.", { actor, appUrl: evidenceAppUrlOf(subject) });
|
|
73
|
+
const timeoutMs = config.execution?.timeoutMs;
|
|
74
|
+
const sessionTimeoutMs = timeoutMs ?? DEFAULT_SESSION_TIMEOUT_MS;
|
|
75
|
+
// A clone is served from an E2B sandbox that must outlive the journey; a loopback app runs on
|
|
76
|
+
// this machine and needs no sandbox.
|
|
77
|
+
if (subject.kind === "clone") {
|
|
78
|
+
const ceiling = input.sandboxCeiling;
|
|
79
|
+
const reason = ceiling.ok
|
|
80
|
+
? sandboxDeadlineRefusal({
|
|
81
|
+
name: "the subject sandbox",
|
|
82
|
+
sessionMs: sessionTimeoutMs,
|
|
83
|
+
sessionDeclared: timeoutMs !== undefined,
|
|
84
|
+
servedSubject: { seed: subject.state?.seed ?? [] },
|
|
85
|
+
}, ceiling.ms)
|
|
86
|
+
: ceiling.message;
|
|
87
|
+
if (reason)
|
|
88
|
+
return refuse("HUMANISH_SCRIPTED_SUBJECT_INVALID", reason, {
|
|
89
|
+
actor,
|
|
90
|
+
appUrl: evidenceAppUrlOf(subject),
|
|
91
|
+
});
|
|
92
|
+
}
|
|
69
93
|
const requirements = [];
|
|
70
94
|
if (!input.dryRun && subject.kind === "clone") {
|
|
71
95
|
requirements.push({ kind: "key", name: "E2B_API_KEY" });
|
|
@@ -76,7 +100,6 @@ export function planScriptedStudy(config, input) {
|
|
|
76
100
|
if (!input.dryRun && input.injectedBrowser !== true)
|
|
77
101
|
requirements.push({ kind: "host-browser" });
|
|
78
102
|
const persona = config.actor?.persona;
|
|
79
|
-
const timeoutMs = config.execution?.timeoutMs;
|
|
80
103
|
return {
|
|
81
104
|
ok: true,
|
|
82
105
|
plan: {
|
|
@@ -90,7 +113,7 @@ export function planScriptedStudy(config, input) {
|
|
|
90
113
|
scenarioRef,
|
|
91
114
|
surfaces: browserSurfaces.slice(0, config.surfaces?.length ?? DEFAULT_SURFACE_COUNT),
|
|
92
115
|
...(persona === undefined ? {} : { personaId: persona }),
|
|
93
|
-
|
|
116
|
+
sessionTimeoutMs,
|
|
94
117
|
requirements,
|
|
95
118
|
},
|
|
96
119
|
};
|
|
@@ -12,9 +12,6 @@ import { evidenceAppUrlOf } from "./plan.js";
|
|
|
12
12
|
import { resolveScriptedScenario } from "./scenario.js";
|
|
13
13
|
import {} from "./types.js";
|
|
14
14
|
import { refusedResult } from "../../run/study-result.js";
|
|
15
|
-
// Journey wall-clock budget per surface: 5 minutes. A scripted surface has zero model cost and
|
|
16
|
-
// sandbox-seconds are pennies; a short default only truncated slow-loading subjects.
|
|
17
|
-
const DEFAULT_SESSION_TIMEOUT_MS = 300_000;
|
|
18
15
|
/**
|
|
19
16
|
* Checks what the plan cannot: the scenario file, the E2B key and subject env of a live clone,
|
|
20
17
|
* and a browser to launch. Returns the refusal, or what the run needs.
|
|
@@ -83,7 +80,7 @@ export async function prepareScriptedRun(plan, input) {
|
|
|
83
80
|
if (refusal)
|
|
84
81
|
return { ok: false, result: failed(refusal.code, refusal.message) };
|
|
85
82
|
const surfaces = plan.surfaces;
|
|
86
|
-
const timeoutMs = plan.sessionTimeoutMs
|
|
83
|
+
const timeoutMs = plan.sessionTimeoutMs;
|
|
87
84
|
const persona = {
|
|
88
85
|
id: plan.personaId ?? "scripted-journey",
|
|
89
86
|
traitsApplied: [],
|
|
@@ -58,7 +58,7 @@ export interface ScriptedBrowserStudyResult extends AutomaticAnalysisResult, Stu
|
|
|
58
58
|
observer?: ObserverResult;
|
|
59
59
|
warnings: string[];
|
|
60
60
|
error?: {
|
|
61
|
-
code: StudyResultErrorCode | "HUMANISH_SCRIPTED_FAILED" | "HUMANISH_SCRIPTED_ACTOR_UNSUPPORTED" | "HUMANISH_SCRIPTED_SCENARIO_INVALID" | "HUMANISH_SCRIPTED_SUBJECT_UNSAFE" | "HUMANISH_SCRIPTED_BROWSER_MISSING" | "HUMANISH_SCRIPTED_KEYS_MISSING" | "HUMANISH_SCRIPTED_SUBJECT_ENV_MISSING" | "HUMANISH_SCRIPTED_GETHOST_UNAVAILABLE";
|
|
61
|
+
code: StudyResultErrorCode | "HUMANISH_SCRIPTED_FAILED" | "HUMANISH_SCRIPTED_ACTOR_UNSUPPORTED" | "HUMANISH_SCRIPTED_SCENARIO_INVALID" | "HUMANISH_SCRIPTED_SUBJECT_UNSAFE" | "HUMANISH_SCRIPTED_SUBJECT_INVALID" | "HUMANISH_SCRIPTED_BROWSER_MISSING" | "HUMANISH_SCRIPTED_KEYS_MISSING" | "HUMANISH_SCRIPTED_SUBJECT_ENV_MISSING" | "HUMANISH_SCRIPTED_GETHOST_UNAVAILABLE";
|
|
62
62
|
message: string;
|
|
63
63
|
};
|
|
64
64
|
}
|
|
@@ -11,7 +11,7 @@ import { desktopParticipantDeps, } from "../computer-use/participant-deps.js";
|
|
|
11
11
|
import { resolveCommittedPersonasForCwd } from "../../study/persona-resolve.js";
|
|
12
12
|
import { participantAssignment } from "../../study/participant-assignment.js";
|
|
13
13
|
import { redactText } from "../../evidence/redaction.js";
|
|
14
|
-
import {
|
|
14
|
+
import { sandboxHeadroomMs } from "../../substrates/e2b/lifetime.js";
|
|
15
15
|
// prose-check: model prompt (the participant model reads this, not a person)
|
|
16
16
|
const DEFAULT_MISSION = "You are one of MANY users hitting a shared web application at the same time. The browser is already open at the app. Accomplish your role's task, then stop.";
|
|
17
17
|
/** Resolve an actor's participant URL against the harness-minted getHost base (entry is a
|
|
@@ -134,7 +134,7 @@ export function participantRunDeps(ctx, live, scrubKnownValues) {
|
|
|
134
134
|
openaiApiKey: ctx.openaiApiKey,
|
|
135
135
|
e2bApiKey: ctx.e2bApiKey,
|
|
136
136
|
timeoutMs: ctx.timeoutMs,
|
|
137
|
-
sandboxMs: ctx.timeoutMs +
|
|
137
|
+
sandboxMs: ctx.timeoutMs + sandboxHeadroomMs(),
|
|
138
138
|
participantCount: plan.plane.participants.length,
|
|
139
139
|
artifactRoot: ctx.runPaths,
|
|
140
140
|
studyCwd: ctx.cwd,
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { type CuaActorDescriptor } from "../../actors/registry.js";
|
|
2
2
|
import type { RoutePlanResult, RouteRefusal, SharedWorldPlan } from "../../study/plan-types.js";
|
|
3
3
|
import type { StudyConfig, StudySubjectState } from "../../study/types.js";
|
|
4
|
-
import { type SandboxCeiling } from "../../substrates/e2b/lifetime.js";
|
|
4
|
+
import { type ConcurrentSandboxes, type SandboxCeiling } from "../../substrates/e2b/lifetime.js";
|
|
5
5
|
import type { ConcurrentSharedWorldStudyErrorCode } from "./types.js";
|
|
6
6
|
/** The error a shared-world study returns before a run starts. */
|
|
7
7
|
export interface SharedWorldRefusal extends RouteRefusal<"shared-world", ConcurrentSharedWorldStudyErrorCode> {
|
|
@@ -21,6 +21,8 @@ export declare function planSharedWorldStudy(config: StudyConfig, input: {
|
|
|
21
21
|
readonly hasRunSession?: boolean;
|
|
22
22
|
/** The longest sandbox lifetime the operator's E2B plan allows (sandboxCeiling). */
|
|
23
23
|
readonly sandboxCeiling: SandboxCeiling;
|
|
24
|
+
/** How many sandboxes the operator's E2B plan runs at once (concurrentSandboxes). */
|
|
25
|
+
readonly concurrentSandboxes: ConcurrentSandboxes;
|
|
24
26
|
}): SharedWorldPlanResult;
|
|
25
27
|
/** The provisioned plane's declared subject state. The external-public plane declares none. */
|
|
26
28
|
export declare function planeStateOf(plan: SharedWorldPlan): StudySubjectState | undefined;
|
|
@@ -10,10 +10,10 @@ import { brainOf, planCaps, desktopRequirements, isNonEmpty, planBase, } from ".
|
|
|
10
10
|
import { sharedWorldParticipants } from "../../study/plan-participants.js";
|
|
11
11
|
import { PUBLIC_TARGET_OWNER_PATTERN } from "../../study/parse/subject.js";
|
|
12
12
|
import { REPO_SLUG_PATTERN } from "../../study/parse/values.js";
|
|
13
|
-
import { concurrentSharedWorldValidationReason, desktopMediaValidationReason, externalPublicSharedWorldValidationReason, outputTokenLimitValidationReason, receivingEmailValidationReason, taskProtocolValidationReason, } from "../../study/validation.js";
|
|
13
|
+
import { concurrentSharedWorldValidationReason, desktopMediaValidationReason, externalPublicSharedWorldValidationReason, outputTokenLimitValidationReason, receivingEmailValidationReason, taskProtocolValidationReason, waitLimitValidationReason, } from "../../study/validation.js";
|
|
14
14
|
import { unpricedCapCheck } from "../../study/requirements.js";
|
|
15
|
-
import {
|
|
16
|
-
import {
|
|
15
|
+
import { participantsAtOnce, sandboxDeadlineRefusal, sandboxHeadroomMs, } from "../../substrates/e2b/lifetime.js";
|
|
16
|
+
import { boundedConcurrency } from "../computer-use/participant-runs.js";
|
|
17
17
|
/** The registered descriptor for a planned actor; planSharedWorldStudy checked the registry. */
|
|
18
18
|
export function sharedWorldDescriptorOf(actorType) {
|
|
19
19
|
const descriptor = actorRegistry[actorType];
|
|
@@ -55,6 +55,7 @@ export function planSharedWorldStudy(config, input) {
|
|
|
55
55
|
// A library caller's config skips the parser, which refuses these first.
|
|
56
56
|
studyUrlCredentialReason(config) ??
|
|
57
57
|
outputTokenLimitValidationReason(config) ??
|
|
58
|
+
waitLimitValidationReason(config) ??
|
|
58
59
|
(externalPublic
|
|
59
60
|
? externalPublicSharedWorldValidationReason(config)
|
|
60
61
|
: concurrentSharedWorldValidationReason(config));
|
|
@@ -95,22 +96,54 @@ export function planSharedWorldStudy(config, input) {
|
|
|
95
96
|
const plane = planeOf(config);
|
|
96
97
|
if (plane === undefined)
|
|
97
98
|
throw new Error("shared-world validation admitted a plane it cannot plan");
|
|
99
|
+
// Every participant's desktop and, on the provisioned plane, the app's own sandbox share the E2B
|
|
100
|
+
// plan's concurrent sandboxes. A shared world needs two participants live together.
|
|
101
|
+
const declared = config.execution?.concurrency;
|
|
102
|
+
const n = plane.participants.length;
|
|
103
|
+
const atOnce = participantsAtOnce({
|
|
104
|
+
concurrency: boundedConcurrency(declared, n),
|
|
105
|
+
declared: declared !== undefined,
|
|
106
|
+
participants: n,
|
|
107
|
+
alongside: plane.kind === "provisioned" ? 1 : 0,
|
|
108
|
+
minimum: 2,
|
|
109
|
+
}, input.concurrentSandboxes);
|
|
110
|
+
if (!atOnce.ok)
|
|
111
|
+
return refuse(invalid, atOnce.message, actor);
|
|
112
|
+
const sessionTimeoutMs = config.execution?.timeoutMs ?? defaultSessionTimeoutMs(plane, ceiling.ms);
|
|
113
|
+
// The run's longest deadline: on the provisioned plane the subject sandbox, which serves the app
|
|
114
|
+
// until every participant ends; on the external-public plane each participant's own sandbox.
|
|
115
|
+
const deadlineReason = sandboxDeadlineRefusal(plane.kind === "provisioned"
|
|
116
|
+
? {
|
|
117
|
+
name: "the subject sandbox, which serves the app until every participant ends",
|
|
118
|
+
sessionMs: sessionTimeoutMs,
|
|
119
|
+
sessionDeclared: config.execution?.timeoutMs !== undefined,
|
|
120
|
+
servedSubject: { seed: plane.subject.state.seed ?? [] },
|
|
121
|
+
}
|
|
122
|
+
: {
|
|
123
|
+
name: "each participant's sandbox",
|
|
124
|
+
sessionMs: sessionTimeoutMs,
|
|
125
|
+
sessionDeclared: config.execution?.timeoutMs !== undefined,
|
|
126
|
+
}, ceiling.ms);
|
|
127
|
+
if (deadlineReason)
|
|
128
|
+
return refuse(invalid, deadlineReason, actor);
|
|
98
129
|
const brain = brainOf(config, false);
|
|
99
130
|
if (brain.kind === "caller")
|
|
100
131
|
throw new Error("a shared-world participant has no caller brain");
|
|
101
132
|
const base = planBase(config, {
|
|
102
133
|
dryRun: input.dryRun,
|
|
103
134
|
analysis,
|
|
135
|
+
participants: n,
|
|
104
136
|
});
|
|
105
137
|
return {
|
|
106
138
|
ok: true,
|
|
107
139
|
plan: {
|
|
108
140
|
...base,
|
|
141
|
+
...(atOnce.lowered === undefined ? {} : { warnings: [atOnce.lowered] }),
|
|
109
142
|
route: "shared-world",
|
|
110
143
|
actor,
|
|
111
144
|
plane,
|
|
112
|
-
concurrency:
|
|
113
|
-
sessionTimeoutMs
|
|
145
|
+
concurrency: atOnce.count,
|
|
146
|
+
sessionTimeoutMs,
|
|
114
147
|
brain,
|
|
115
148
|
caps: planCaps(config),
|
|
116
149
|
requirements: base.dryRun
|
|
@@ -144,8 +177,7 @@ const DEFAULT_APP_URL_SESSION_MS = 30 * 60_000;
|
|
|
144
177
|
function defaultSessionTimeoutMs(plane, ceilingMs) {
|
|
145
178
|
if (plane.kind !== "provisioned")
|
|
146
179
|
return DEFAULT_APP_URL_SESSION_MS;
|
|
147
|
-
const
|
|
148
|
-
const room = ceilingMs - SUBJECT_PROVISION_BUDGET_MS - stateBudgetMs - SANDBOX_TIMEOUT_BUFFER_MS;
|
|
180
|
+
const room = ceilingMs - sandboxHeadroomMs({ seed: plane.subject.state.seed ?? [] });
|
|
149
181
|
return Math.max(MIN_DERIVED_SESSION_MS, Math.min(MAX_DERIVED_SESSION_MS, room));
|
|
150
182
|
}
|
|
151
183
|
/** The provisioned plane's declared subject state. The external-public plane declares none. */
|