humanish 0.117.0 → 0.118.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -0
- package/dist/analysis/admission.d.ts +7 -2
- package/dist/analysis/admission.js +12 -3
- package/dist/analysis/automatic-config.d.ts +6 -1
- package/dist/analysis/automatic-config.js +12 -11
- package/dist/analysis/execute.d.ts +19 -7
- package/dist/analysis/execute.js +55 -17
- package/dist/cli/commands/analyze.js +2 -1
- package/dist/cli/commands/study-format.js +8 -11
- package/dist/cli/commands/study-route-computer-use.js +2 -2
- package/dist/cli/commands/study-route-scripted.js +2 -2
- package/dist/cli/commands/study-route-shared-world.js +5 -3
- package/dist/cli/commands/study-route-terminal.js +2 -2
- package/dist/cli/commands/study-run.js +3 -3
- package/dist/cli/commands/study.js +2 -2
- package/dist/cli/doctor.js +11 -1
- package/dist/cli/io.js +3 -2
- package/dist/comms/lease-store.d.ts +1 -1
- package/dist/comms/lease-store.js +3 -2
- package/dist/comms/receiving-types.d.ts +2 -0
- package/dist/comms/receiving-types.js +2 -0
- package/dist/observer-app.html +2 -2
- package/dist/routes/computer-use/fanout-bundle.js +2 -1
- package/dist/routes/computer-use/fanout-records.js +2 -1
- package/dist/routes/computer-use/participant-runs.js +3 -9
- package/dist/routes/computer-use/plan.d.ts +3 -1
- package/dist/routes/computer-use/plan.js +25 -7
- package/dist/routes/computer-use/single-bundle.js +7 -4
- package/dist/routes/computer-use/subject-projection.js +2 -0
- package/dist/routes/scripted/plan.d.ts +4 -1
- package/dist/routes/scripted/plan.js +25 -2
- package/dist/routes/scripted/setup.js +1 -4
- package/dist/routes/scripted/types.d.ts +1 -1
- package/dist/routes/shared-world/participant-specs.js +2 -2
- package/dist/routes/shared-world/plan.d.ts +3 -1
- package/dist/routes/shared-world/plan.js +37 -6
- package/dist/routes/shared-world/route.js +2 -2
- package/dist/run/bundle-shape.js +2 -1
- package/dist/run/bundle.d.ts +5 -3
- package/dist/run/projection.js +7 -2
- package/dist/run/route-shell.js +10 -5
- package/dist/run/subject-name.d.ts +11 -0
- package/dist/run/subject-name.js +11 -0
- package/dist/study/automatic-analysis-plan.d.ts +28 -0
- package/dist/study/automatic-analysis-plan.js +57 -0
- package/dist/study/composition-rules.js +1 -1
- package/dist/study/config.js +6 -20
- package/dist/study/doctor.js +17 -3
- package/dist/study/parse/actors.d.ts +1 -1
- package/dist/study/parse/actors.js +1 -1
- package/dist/study/parse/front.d.ts +1 -1
- package/dist/study/parse/front.js +7 -1
- package/dist/study/plan-base.d.ts +2 -0
- package/dist/study/plan-base.js +3 -0
- package/dist/study/plan-participants.js +2 -5
- package/dist/study/plan-types.d.ts +8 -3
- package/dist/study/plan.d.ts +4 -4
- package/dist/study/plan.js +21 -12
- package/dist/study/preflight-result.js +2 -2
- package/dist/study/preflight.d.ts +2 -1
- package/dist/study/preflight.js +12 -3
- package/dist/study/routing.d.ts +5 -4
- package/dist/study/routing.js +16 -7
- package/dist/study/summary.d.ts +3 -3
- package/dist/study/summary.js +3 -5
- package/dist/study/validation.d.ts +2 -2
- package/dist/study/validation.js +22 -14
- package/dist/substrates/e2b/lifetime.d.ts +75 -4
- package/dist/substrates/e2b/lifetime.js +134 -10
- package/dist/substrates/e2b/subject-sandbox.js +2 -6
- package/dist/tui-app.js +112 -112
- package/dist/verify/artifacts.d.ts +19 -4
- package/dist/verify/artifacts.js +39 -8
- package/dist/verify/verify.js +6 -15
- package/docs/contracts/run-bundle.md +7 -6
- package/docs/contracts/schemas.md +14 -8
- package/docs/contracts/study-analysis.md +2 -2
- package/docs/principles/invariants-and-defaults.md +1 -1
- package/docs/product/automatic-analysis.md +4 -3
- package/package.json +1 -1
|
@@ -5,6 +5,7 @@ import { redactText } from "../../evidence/redaction.js";
|
|
|
5
5
|
import { REVIEW_SCHEMA, bundleArtifacts, bundleHead, } from "../../run/bundle.js";
|
|
6
6
|
import { aggregateTaskFunnels, formatParticipantOutcomes, formatRunTaskFunnel, gapsListClause, withCuaReviewProvenance, } from "../../run/outcomes.js";
|
|
7
7
|
import { providerResourcesForOutcome, publicSafeAppUrlLabel } from "./bundle-parts.js";
|
|
8
|
+
import { subjectName } from "../../run/subject-name.js";
|
|
8
9
|
import { participantFeedbackCandidates } from "./participant-feedback.js";
|
|
9
10
|
import { participantFactsOf } from "./participant-facts.js";
|
|
10
11
|
import { judgeParticipantRecords } from "../../run/judge.js";
|
|
@@ -44,7 +45,7 @@ function fanoutSummary(args, facts) {
|
|
|
44
45
|
return `${people} ${count === 1 ? "is" : "are"} each using their own copy of the app. The run is still going, so nothing here is final.`;
|
|
45
46
|
}
|
|
46
47
|
if (args.dryRun) {
|
|
47
|
-
return `${rerun}Dry run: ${people} would each use their own copy of ${publicSafeAppUrlLabel(args.appUrl)}. No desktops were launched and $0 was spent.`;
|
|
48
|
+
return `${rerun}Dry run: ${people} would each use their own copy of ${subjectName(publicSafeAppUrlLabel(args.appUrl), args.aggregateSubject)}. No desktops were launched and $0 was spent.`;
|
|
48
49
|
}
|
|
49
50
|
const tasks = facts.tasks === undefined ? "" : ` Tasks: ${facts.tasks}.`;
|
|
50
51
|
return `${rerun}${people} took part, each in their own copy of the app: ${facts.outcomes}${gapsListClause(count - facts.passed, count)}.${tasks}`;
|
|
@@ -4,6 +4,7 @@ import { participantAssignment } from "../../study/participant-assignment.js";
|
|
|
4
4
|
import { participantEvent, participantRecord, participantStream, } from "../../run/participant-records.js";
|
|
5
5
|
import { participantCaption } from "../../run/participant-caption.js";
|
|
6
6
|
import { participantSummary } from "../../run/outcomes.js";
|
|
7
|
+
import { subjectName } from "../../run/subject-name.js";
|
|
7
8
|
import { declaredScreenForRender } from "../../substrates/e2b/desktop-geometry.js";
|
|
8
9
|
import { describeSubjectState, phaseEventIdSuffix, publicSafeAppUrlLabel } from "./bundle-parts.js";
|
|
9
10
|
import { participantSubjectEnv } from "./types.js";
|
|
@@ -60,7 +61,7 @@ function fanoutParticipantRecord(args, spec, index, view) {
|
|
|
60
61
|
mode: "browser-sim",
|
|
61
62
|
progress: args.inProgress === true && outcome === undefined ? 20 : 100,
|
|
62
63
|
currentStep: reason,
|
|
63
|
-
summary: participantSummary(name, { name: "the app", target: publicTargetUrl }, {
|
|
64
|
+
summary: participantSummary(name, { name: "the app", target: subjectName(publicTargetUrl, view.subject) }, {
|
|
64
65
|
trace: session?.trace,
|
|
65
66
|
skippedReason: outcome?.skippedReason,
|
|
66
67
|
sessionError: outcome?.sessionError,
|
|
@@ -3,7 +3,6 @@
|
|
|
3
3
|
// paths, and the public plan printed before any sandbox or provider call. loadCuaParticipants
|
|
4
4
|
// compiles the committed personas first and narrows a rerun to its selected participants.
|
|
5
5
|
import { isLocalBrowserStudy, LOCAL_BROWSER_LIFETIME_MS, } from "../../substrates/local/runtime-config.js";
|
|
6
|
-
import { DEFAULT_STATE_STEP_TIMEOUT_MS } from "../../subject/state.js";
|
|
7
6
|
import { resolveParticipant } from "../../run/participant.js";
|
|
8
7
|
import {} from "../../study/types.js";
|
|
9
8
|
import { scrubPersonaBrief } from "../../study/persona.js";
|
|
@@ -12,7 +11,7 @@ import {} from "../../run/bundle.js";
|
|
|
12
11
|
import { participantAssignment } from "../../study/participant-assignment.js";
|
|
13
12
|
import { resolveCommittedPersonas } from "../../study/persona-resolve.js";
|
|
14
13
|
import { CUA_FANOUT_STRATEGY, CUA_MAX_CONCURRENCY_ENV, DEFAULT_APP_URL_SESSION_TIMEOUT_MS, MIN_DERIVED_SESSION_TIMEOUT_MS, } from "./types.js";
|
|
15
|
-
import {
|
|
14
|
+
import { sandboxHeadroomMs } from "../../substrates/e2b/lifetime.js";
|
|
16
15
|
import { readPositiveInt } from "../../study/parse/values.js";
|
|
17
16
|
import { digestUrl } from "./bundle-parts.js";
|
|
18
17
|
import { composeParticipantInstructions, DEFAULT_MISSION } from "./participant-prompt.js";
|
|
@@ -26,8 +25,7 @@ export function defaultSessionTimeoutMs(config, ceilingMs) {
|
|
|
26
25
|
const provisionedRoute = config.subject.source === "clone" || config.subject.source === "local-tree";
|
|
27
26
|
if (!provisionedRoute)
|
|
28
27
|
return DEFAULT_APP_URL_SESSION_TIMEOUT_MS;
|
|
29
|
-
const
|
|
30
|
-
const room = ceilingMs - SUBJECT_PROVISION_BUDGET_MS - stateBudgetMs - SANDBOX_TIMEOUT_BUFFER_MS;
|
|
28
|
+
const room = ceilingMs - sandboxHeadroomMs({ seed: config.subject.state?.seed ?? [] });
|
|
31
29
|
return Math.max(MIN_DERIVED_SESSION_TIMEOUT_MS, Math.min(DEFAULT_APP_URL_SESSION_TIMEOUT_MS, room));
|
|
32
30
|
}
|
|
33
31
|
/** Per-participant sandbox deadline (each one owns its own desktop). Mirrors the
|
|
@@ -41,13 +39,9 @@ export function resolveParticipantSandboxMs(config, ceilingMs) {
|
|
|
41
39
|
return LOCAL_BROWSER_LIFETIME_MS;
|
|
42
40
|
const timeoutMs = config.execution?.timeoutMs ?? defaultSessionTimeoutMs(config, ceilingMs);
|
|
43
41
|
const provisionedRoute = config.subject.source === "clone" || config.subject.source === "local-tree";
|
|
44
|
-
const stateBudgetMs = provisionedRoute
|
|
45
|
-
? (config.subject.state?.seed ?? []).reduce((sum, step) => sum + (step.timeoutMs ?? DEFAULT_STATE_STEP_TIMEOUT_MS), 0)
|
|
46
|
-
: 0;
|
|
47
42
|
return (config.execution?.desktop?.sandboxTimeoutMs ??
|
|
48
43
|
timeoutMs +
|
|
49
|
-
(provisionedRoute ?
|
|
50
|
-
SANDBOX_TIMEOUT_BUFFER_MS);
|
|
44
|
+
sandboxHeadroomMs(provisionedRoute ? { seed: config.subject.state?.seed ?? [] } : undefined));
|
|
51
45
|
}
|
|
52
46
|
/**
|
|
53
47
|
* The in-flight participant bound a study declares. Defaults to the participant count: every
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { type CuaActorDescriptor } from "../../actors/registry.js";
|
|
2
2
|
import type { ComputerUsePlan, ComputerUseRunner, RoutePlanResult, RouteRefusal } from "../../study/plan-types.js";
|
|
3
3
|
import type { StudyConfig, StudySubjectState } from "../../study/types.js";
|
|
4
|
-
import { type SandboxCeiling } from "../../substrates/e2b/lifetime.js";
|
|
4
|
+
import { type ConcurrentSandboxes, type SandboxCeiling } from "../../substrates/e2b/lifetime.js";
|
|
5
5
|
import { type ComputerUseRunInput, type CuaActorStudyErrorCode } from "./types.js";
|
|
6
6
|
/** The error a computer-use study returns before a run starts. */
|
|
7
7
|
export interface ComputerUseRefusal extends RouteRefusal<"computer-use", CuaActorStudyErrorCode> {
|
|
@@ -48,4 +48,6 @@ export declare function planComputerUseStudy(config: StudyConfig, input: {
|
|
|
48
48
|
readonly rerun?: ComputerUseRunInput["rerun"];
|
|
49
49
|
/** The longest sandbox lifetime the operator's E2B plan allows (sandboxCeiling). */
|
|
50
50
|
readonly sandboxCeiling: SandboxCeiling;
|
|
51
|
+
/** How many sandboxes the operator's E2B plan runs at once (concurrentSandboxes). */
|
|
52
|
+
readonly concurrentSandboxes: ConcurrentSandboxes;
|
|
51
53
|
}): ComputerUsePlanResult;
|
|
@@ -9,12 +9,12 @@ import { studyUrlCredentialReason } from "../../study/url-credentials.js";
|
|
|
9
9
|
import { subjectStateInvalidReason } from "../../study/parse/subject-state.js";
|
|
10
10
|
import { brainOf, callerBrainOf, planCaps, desktopRequirements, planBase, provisionedSubject, } from "../../study/plan-base.js";
|
|
11
11
|
import { computerUseParticipants, declaredTargets } from "../../study/plan-participants.js";
|
|
12
|
-
import {
|
|
12
|
+
import { computerUseParticipantCount, participantCountReason } from "../../study/routing.js";
|
|
13
13
|
import { cloneTargetValidationReason, computerUseValidationReason, desktopMediaValidationReason, outputTokenLimitValidationReason, receivingEmailValidationReason, taskProtocolValidationReason, waitLimitValidationReason, } from "../../study/validation.js";
|
|
14
14
|
import { desktopCliProductReason } from "../../study/composition-rules.js";
|
|
15
15
|
import { isLocalBrowserStudy } from "../../substrates/local/runtime-config.js";
|
|
16
16
|
import { boundedConcurrency, defaultSessionTimeoutMs, resolveParticipantSandboxMs, } from "./participant-runs.js";
|
|
17
|
-
import { ceilingAdvice } from "../../substrates/e2b/lifetime.js";
|
|
17
|
+
import { ceilingAdvice, participantsAtOnce, } from "../../substrates/e2b/lifetime.js";
|
|
18
18
|
import {} from "./types.js";
|
|
19
19
|
/** The registered descriptor for a planned actor id; planComputerUseStudy checked the registry. */
|
|
20
20
|
export function cuaDescriptorOf(actor) {
|
|
@@ -192,6 +192,19 @@ function rosterShapeReason(config, ceilingMs) {
|
|
|
192
192
|
const headroomMs = derivedSandboxMs - sessionMs;
|
|
193
193
|
return invalid(`execution.timeoutMs ${Math.round(sessionMs / 60_000)}m derives a ${Math.round(derivedSandboxMs / 60_000)}m sandbox deadline, and a sandbox may not live longer than ${ceilingMs / 60_000}m. The deadline is the session budget plus ${Math.round(headroomMs / 60_000)}m of provisioning and teardown headroom${provisionedRoute ? " (this route clones, installs, builds and serves the subject before the actor starts)" : ""}. Lower execution.timeoutMs to at most ${Math.round((ceilingMs - headroomMs) / 60_000)}m, or set execution.desktop.sandboxTimeoutMs explicitly.${ceilingAdvice(derivedSandboxMs)}`);
|
|
194
194
|
}
|
|
195
|
+
/**
|
|
196
|
+
* How many of the runner's participants run at once. Hosted desktops share the E2B plan's
|
|
197
|
+
* concurrent sandboxes; the local VM has its own capacity rules and the in-process route runs one
|
|
198
|
+
* participant.
|
|
199
|
+
*/
|
|
200
|
+
function concurrencyOf(config, runner, limit) {
|
|
201
|
+
const declared = config.execution?.concurrency;
|
|
202
|
+
const n = runner.participants.length;
|
|
203
|
+
const concurrency = boundedConcurrency(declared, n);
|
|
204
|
+
return runner.desktop === "e2b-desktop"
|
|
205
|
+
? participantsAtOnce({ concurrency, declared: declared !== undefined, participants: n }, limit)
|
|
206
|
+
: { ok: true, count: concurrency };
|
|
207
|
+
}
|
|
195
208
|
/** The plan's rerun: the source run and, when given, the participants to rerun. */
|
|
196
209
|
function rerunPlan({ sourceRunId, participantIds, }) {
|
|
197
210
|
return { sourceRunId, ...(participantIds === undefined ? {} : { participantIds }) };
|
|
@@ -246,9 +259,11 @@ export function planComputerUseStudy(config, input) {
|
|
|
246
259
|
const productReason = desktopCliProductReason(config);
|
|
247
260
|
if (productReason)
|
|
248
261
|
return refuse("in-scope", "HUMANISH_COMPUTER_USE_SUBJECT_INVALID", productReason, actor);
|
|
262
|
+
// Counted before any participant is built, so a large --count is refused without building it.
|
|
263
|
+
const countReason = participantCountReason(computerUseParticipantCount(config, input.countOverride));
|
|
264
|
+
if (countReason)
|
|
265
|
+
return refuse("after-personas", "HUMANISH_COMPUTER_USE_FANOUT_INVALID", countReason, actor);
|
|
249
266
|
const participants = computerUseParticipants(config, input.countOverride);
|
|
250
|
-
if (participants.length > MAX_COMPUTER_USE_PARTICIPANTS)
|
|
251
|
-
return refuse("after-personas", "HUMANISH_COMPUTER_USE_FANOUT_INVALID", `Computer-use fan-out is capped at ${MAX_COMPUTER_USE_PARTICIPANTS} participants (resolved ${participants.length}); N concurrent paid desktops is real spend.`, actor);
|
|
252
267
|
const [first, ...rest] = participants;
|
|
253
268
|
if (driving.inProcess && rest.length > 0)
|
|
254
269
|
return refuse("after-personas", "HUMANISH_COMPUTER_USE_FANOUT_INVALID", "Fan-out to more than one participant is not supported on the in-process route (RunStudyOptions.inProcess): fan-out provisions one independent E2B desktop per participant, which the in-process route deliberately skips. Run a single in-process participant, or fan out on the E2B route.", actor);
|
|
@@ -278,21 +293,24 @@ export function planComputerUseStudy(config, input) {
|
|
|
278
293
|
: isLocalBrowserStudy(config)
|
|
279
294
|
? { desktop: "local-vm", brain, participants: [first, ...rest], subject: appUrlSubject }
|
|
280
295
|
: { desktop: "e2b-desktop", brain, participants: [first, ...rest], subject: hosted };
|
|
281
|
-
const
|
|
282
|
-
|
|
296
|
+
const atOnce = concurrencyOf(config, runner, input.concurrentSandboxes);
|
|
297
|
+
if (!atOnce.ok)
|
|
298
|
+
return refuse("after-personas", "HUMANISH_COMPUTER_USE_FANOUT_INVALID", atOnce.message, actor);
|
|
283
299
|
const provisioned = source === "clone" || source === "local-tree";
|
|
284
300
|
const base = planBase(config, {
|
|
285
301
|
dryRun: input.dryRun,
|
|
286
302
|
analysis,
|
|
303
|
+
participants: participants.length,
|
|
287
304
|
});
|
|
288
305
|
return {
|
|
289
306
|
ok: true,
|
|
290
307
|
plan: {
|
|
291
308
|
...base,
|
|
309
|
+
...(atOnce.lowered === undefined ? {} : { warnings: [atOnce.lowered] }),
|
|
292
310
|
route: "computer-use",
|
|
293
311
|
actor,
|
|
294
312
|
runner,
|
|
295
|
-
concurrency:
|
|
313
|
+
concurrency: atOnce.count,
|
|
296
314
|
sessionBudgetMs: config.execution?.timeoutMs ?? defaultSessionTimeoutMs(config, ceiling.ms),
|
|
297
315
|
sandboxMs: resolveParticipantSandboxMs(config, ceiling.ms),
|
|
298
316
|
caps: planCaps(config),
|
|
@@ -10,6 +10,7 @@ import { participantFeedbackCandidates } from "./participant-feedback.js";
|
|
|
10
10
|
import { judgeParticipantRecords } from "../../run/judge.js";
|
|
11
11
|
import { buildRunCostSummary } from "../../run/cost-summary.js";
|
|
12
12
|
import { participantEvent, participantIds, participantRecord, participantStream, } from "../../run/participant-records.js";
|
|
13
|
+
import { subjectName } from "../../run/subject-name.js";
|
|
13
14
|
import { plural } from "../../run/text.js";
|
|
14
15
|
/** The one participant's record and stream ids: sim-001 and stream-001. */
|
|
15
16
|
const SINGLE = participantIds(0);
|
|
@@ -79,6 +80,8 @@ function participantView(args, publicAppUrl) {
|
|
|
79
80
|
: (args.captureRedaction ?? "raw");
|
|
80
81
|
return {
|
|
81
82
|
publicAppUrl,
|
|
83
|
+
/** How the participant's sentence and the summary name the subject. */
|
|
84
|
+
subjectName: subjectName(publicAppUrl, args.subject),
|
|
82
85
|
participant,
|
|
83
86
|
tally: judgment.tally,
|
|
84
87
|
status,
|
|
@@ -90,7 +93,7 @@ function participantView(args, publicAppUrl) {
|
|
|
90
93
|
};
|
|
91
94
|
}
|
|
92
95
|
function singleSimulation(args, view) {
|
|
93
|
-
const {
|
|
96
|
+
const { status, reason } = view;
|
|
94
97
|
const name = participantCaption({
|
|
95
98
|
id: args.participantId ?? "lane-01",
|
|
96
99
|
personaId: args.persona.id,
|
|
@@ -103,7 +106,7 @@ function singleSimulation(args, view) {
|
|
|
103
106
|
mode: "browser-sim",
|
|
104
107
|
progress: args.inProgress === true ? 20 : 100,
|
|
105
108
|
currentStep: reason,
|
|
106
|
-
summary: participantSummary(name, { name: "the app", place: browserPlace(args), target:
|
|
109
|
+
summary: participantSummary(name, { name: "the app", place: browserPlace(args), target: view.subjectName }, {
|
|
107
110
|
trace: args.session?.trace,
|
|
108
111
|
sessionError: args.sessionError,
|
|
109
112
|
inProgress: args.inProgress === true,
|
|
@@ -285,11 +288,11 @@ function singleSummary(args, view, facts) {
|
|
|
285
288
|
return "1 participant is using the app. The run is still going, so nothing here is final.";
|
|
286
289
|
}
|
|
287
290
|
if (args.dryRun) {
|
|
288
|
-
return `Dry run: 1 participant would use ${view.
|
|
291
|
+
return `Dry run: 1 participant would use ${view.subjectName}. Nothing was launched and $0 was spent.`;
|
|
289
292
|
}
|
|
290
293
|
if (args.session === undefined) {
|
|
291
294
|
return args.sessionError === undefined
|
|
292
|
-
? `1 participant was planned against ${view.
|
|
295
|
+
? `1 participant was planned against ${view.subjectName}, but no session ran.`
|
|
293
296
|
: `1 participant was planned, but the run failed before their session finished: ${view.reason}`;
|
|
294
297
|
}
|
|
295
298
|
const ending = actorEnding(args.session.trace);
|
|
@@ -103,6 +103,8 @@ function participantSubjectProjection(args) {
|
|
|
103
103
|
if (args.subject.kind === "desktop-cli") {
|
|
104
104
|
return { source: "desktop-cli", product: args.subject.product.name, state: args.subjectState };
|
|
105
105
|
}
|
|
106
|
+
if (args.subject.kind === "local-app")
|
|
107
|
+
return { source: "local-app", state: args.subjectState };
|
|
106
108
|
return { source: "app-url", state: args.subjectState };
|
|
107
109
|
}
|
|
108
110
|
/**
|
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import type { StudyDeps } from "../../study/study-deps.js";
|
|
2
2
|
import type { RoutePlanResult, RouteRefusal, ScriptedPlan } from "../../study/plan-types.js";
|
|
3
3
|
import type { StudyConfig } from "../../study/types.js";
|
|
4
|
+
import { type SandboxCeiling } from "../../substrates/e2b/lifetime.js";
|
|
4
5
|
/** The error a scripted study returns before a run starts. */
|
|
5
|
-
export interface ScriptedRefusal extends RouteRefusal<"scripted", "HUMANISH_STUDY_ANALYSIS_INVALID" | "HUMANISH_STUDY_TASKS_UNSUPPORTED" | "HUMANISH_SCRIPTED_ACTOR_UNSUPPORTED" | "HUMANISH_SCRIPTED_SCENARIO_INVALID" | "HUMANISH_SCRIPTED_SUBJECT_UNSAFE"> {
|
|
6
|
+
export interface ScriptedRefusal extends RouteRefusal<"scripted", "HUMANISH_STUDY_ANALYSIS_INVALID" | "HUMANISH_STUDY_TASKS_UNSUPPORTED" | "HUMANISH_SCRIPTED_ACTOR_UNSUPPORTED" | "HUMANISH_SCRIPTED_SCENARIO_INVALID" | "HUMANISH_SCRIPTED_SUBJECT_UNSAFE" | "HUMANISH_SCRIPTED_SUBJECT_INVALID"> {
|
|
6
7
|
/**
|
|
7
8
|
* Set on the receiving, analysis and tasks refusals. The route returns those before it opens its
|
|
8
9
|
* run scope, so their result has its own field order and no automatic-analysis record.
|
|
@@ -25,4 +26,6 @@ export declare function planScriptedStudy(config: StudyConfig, input: {
|
|
|
25
26
|
readonly dryRun: boolean;
|
|
26
27
|
/** A test's injected browser means a live run needs no host browser. */
|
|
27
28
|
readonly injectedBrowser?: boolean;
|
|
29
|
+
/** The longest sandbox lifetime the operator's E2B plan allows (sandboxCeiling). */
|
|
30
|
+
readonly sandboxCeiling: SandboxCeiling;
|
|
28
31
|
}): ScriptedPlanResult;
|
|
@@ -7,9 +7,13 @@ import { browserSurfaces } from "../../actors/scripted-browser/types.js";
|
|
|
7
7
|
import { resolveAutomaticAnalysis } from "../../analysis/automatic-config.js";
|
|
8
8
|
import { isNonEmpty, planBase, provisionedSubject } from "../../study/plan-base.js";
|
|
9
9
|
import { cloneTargetValidationReason, desktopMediaValidationReason, taskProtocolValidationReason, } from "../../study/validation.js";
|
|
10
|
+
import { sandboxDeadlineRefusal } from "../../substrates/e2b/lifetime.js";
|
|
10
11
|
// Default surface roster is 1 (desktop only): the defaults-table single-participant row governs;
|
|
11
12
|
// `count: 2` is the declared override that adds the mobile surface.
|
|
12
13
|
const DEFAULT_SURFACE_COUNT = 1;
|
|
14
|
+
// Journey wall-clock budget per surface: 5 minutes. A scripted surface has zero model cost and
|
|
15
|
+
// sandbox-seconds are pennies; a short default only truncated slow-loading subjects.
|
|
16
|
+
const DEFAULT_SESSION_TIMEOUT_MS = 300_000;
|
|
13
17
|
/** The app URL a run's evidence names: a provisioned subject's getHost URL is never persisted. */
|
|
14
18
|
export function evidenceAppUrlOf(subject) {
|
|
15
19
|
return subject.kind === "clone" ? "[provisioned-subject]" : subject.appUrl;
|
|
@@ -66,6 +70,26 @@ export function planScriptedStudy(config, input) {
|
|
|
66
70
|
const scenarioRef = config.scenario;
|
|
67
71
|
if (!scenarioRef?.trim())
|
|
68
72
|
return refuse("HUMANISH_SCRIPTED_SCENARIO_INVALID", "A scripted-browser study needs `scenario`: the actor runs the browser steps in that scenario file.", { actor, appUrl: evidenceAppUrlOf(subject) });
|
|
73
|
+
const timeoutMs = config.execution?.timeoutMs;
|
|
74
|
+
const sessionTimeoutMs = timeoutMs ?? DEFAULT_SESSION_TIMEOUT_MS;
|
|
75
|
+
// A clone is served from an E2B sandbox that must outlive the journey; a loopback app runs on
|
|
76
|
+
// this machine and needs no sandbox.
|
|
77
|
+
if (subject.kind === "clone") {
|
|
78
|
+
const ceiling = input.sandboxCeiling;
|
|
79
|
+
const reason = ceiling.ok
|
|
80
|
+
? sandboxDeadlineRefusal({
|
|
81
|
+
name: "the subject sandbox",
|
|
82
|
+
sessionMs: sessionTimeoutMs,
|
|
83
|
+
sessionDeclared: timeoutMs !== undefined,
|
|
84
|
+
servedSubject: { seed: subject.state?.seed ?? [] },
|
|
85
|
+
}, ceiling.ms)
|
|
86
|
+
: ceiling.message;
|
|
87
|
+
if (reason)
|
|
88
|
+
return refuse("HUMANISH_SCRIPTED_SUBJECT_INVALID", reason, {
|
|
89
|
+
actor,
|
|
90
|
+
appUrl: evidenceAppUrlOf(subject),
|
|
91
|
+
});
|
|
92
|
+
}
|
|
69
93
|
const requirements = [];
|
|
70
94
|
if (!input.dryRun && subject.kind === "clone") {
|
|
71
95
|
requirements.push({ kind: "key", name: "E2B_API_KEY" });
|
|
@@ -76,7 +100,6 @@ export function planScriptedStudy(config, input) {
|
|
|
76
100
|
if (!input.dryRun && input.injectedBrowser !== true)
|
|
77
101
|
requirements.push({ kind: "host-browser" });
|
|
78
102
|
const persona = config.actor?.persona;
|
|
79
|
-
const timeoutMs = config.execution?.timeoutMs;
|
|
80
103
|
return {
|
|
81
104
|
ok: true,
|
|
82
105
|
plan: {
|
|
@@ -90,7 +113,7 @@ export function planScriptedStudy(config, input) {
|
|
|
90
113
|
scenarioRef,
|
|
91
114
|
surfaces: browserSurfaces.slice(0, config.surfaces?.length ?? DEFAULT_SURFACE_COUNT),
|
|
92
115
|
...(persona === undefined ? {} : { personaId: persona }),
|
|
93
|
-
|
|
116
|
+
sessionTimeoutMs,
|
|
94
117
|
requirements,
|
|
95
118
|
},
|
|
96
119
|
};
|
|
@@ -12,9 +12,6 @@ import { evidenceAppUrlOf } from "./plan.js";
|
|
|
12
12
|
import { resolveScriptedScenario } from "./scenario.js";
|
|
13
13
|
import {} from "./types.js";
|
|
14
14
|
import { refusedResult } from "../../run/study-result.js";
|
|
15
|
-
// Journey wall-clock budget per surface: 5 minutes. A scripted surface has zero model cost and
|
|
16
|
-
// sandbox-seconds are pennies; a short default only truncated slow-loading subjects.
|
|
17
|
-
const DEFAULT_SESSION_TIMEOUT_MS = 300_000;
|
|
18
15
|
/**
|
|
19
16
|
* Checks what the plan cannot: the scenario file, the E2B key and subject env of a live clone,
|
|
20
17
|
* and a browser to launch. Returns the refusal, or what the run needs.
|
|
@@ -83,7 +80,7 @@ export async function prepareScriptedRun(plan, input) {
|
|
|
83
80
|
if (refusal)
|
|
84
81
|
return { ok: false, result: failed(refusal.code, refusal.message) };
|
|
85
82
|
const surfaces = plan.surfaces;
|
|
86
|
-
const timeoutMs = plan.sessionTimeoutMs
|
|
83
|
+
const timeoutMs = plan.sessionTimeoutMs;
|
|
87
84
|
const persona = {
|
|
88
85
|
id: plan.personaId ?? "scripted-journey",
|
|
89
86
|
traitsApplied: [],
|
|
@@ -58,7 +58,7 @@ export interface ScriptedBrowserStudyResult extends AutomaticAnalysisResult, Stu
|
|
|
58
58
|
observer?: ObserverResult;
|
|
59
59
|
warnings: string[];
|
|
60
60
|
error?: {
|
|
61
|
-
code: StudyResultErrorCode | "HUMANISH_SCRIPTED_FAILED" | "HUMANISH_SCRIPTED_ACTOR_UNSUPPORTED" | "HUMANISH_SCRIPTED_SCENARIO_INVALID" | "HUMANISH_SCRIPTED_SUBJECT_UNSAFE" | "HUMANISH_SCRIPTED_BROWSER_MISSING" | "HUMANISH_SCRIPTED_KEYS_MISSING" | "HUMANISH_SCRIPTED_SUBJECT_ENV_MISSING" | "HUMANISH_SCRIPTED_GETHOST_UNAVAILABLE";
|
|
61
|
+
code: StudyResultErrorCode | "HUMANISH_SCRIPTED_FAILED" | "HUMANISH_SCRIPTED_ACTOR_UNSUPPORTED" | "HUMANISH_SCRIPTED_SCENARIO_INVALID" | "HUMANISH_SCRIPTED_SUBJECT_UNSAFE" | "HUMANISH_SCRIPTED_SUBJECT_INVALID" | "HUMANISH_SCRIPTED_BROWSER_MISSING" | "HUMANISH_SCRIPTED_KEYS_MISSING" | "HUMANISH_SCRIPTED_SUBJECT_ENV_MISSING" | "HUMANISH_SCRIPTED_GETHOST_UNAVAILABLE";
|
|
62
62
|
message: string;
|
|
63
63
|
};
|
|
64
64
|
}
|
|
@@ -11,7 +11,7 @@ import { desktopParticipantDeps, } from "../computer-use/participant-deps.js";
|
|
|
11
11
|
import { resolveCommittedPersonasForCwd } from "../../study/persona-resolve.js";
|
|
12
12
|
import { participantAssignment } from "../../study/participant-assignment.js";
|
|
13
13
|
import { redactText } from "../../evidence/redaction.js";
|
|
14
|
-
import {
|
|
14
|
+
import { sandboxHeadroomMs } from "../../substrates/e2b/lifetime.js";
|
|
15
15
|
// prose-check: model prompt (the participant model reads this, not a person)
|
|
16
16
|
const DEFAULT_MISSION = "You are one of MANY users hitting a shared web application at the same time. The browser is already open at the app. Accomplish your role's task, then stop.";
|
|
17
17
|
/** Resolve an actor's participant URL against the harness-minted getHost base (entry is a
|
|
@@ -134,7 +134,7 @@ export function participantRunDeps(ctx, live, scrubKnownValues) {
|
|
|
134
134
|
openaiApiKey: ctx.openaiApiKey,
|
|
135
135
|
e2bApiKey: ctx.e2bApiKey,
|
|
136
136
|
timeoutMs: ctx.timeoutMs,
|
|
137
|
-
sandboxMs: ctx.timeoutMs +
|
|
137
|
+
sandboxMs: ctx.timeoutMs + sandboxHeadroomMs(),
|
|
138
138
|
participantCount: plan.plane.participants.length,
|
|
139
139
|
artifactRoot: ctx.runPaths,
|
|
140
140
|
studyCwd: ctx.cwd,
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { type CuaActorDescriptor } from "../../actors/registry.js";
|
|
2
2
|
import type { RoutePlanResult, RouteRefusal, SharedWorldPlan } from "../../study/plan-types.js";
|
|
3
3
|
import type { StudyConfig, StudySubjectState } from "../../study/types.js";
|
|
4
|
-
import { type SandboxCeiling } from "../../substrates/e2b/lifetime.js";
|
|
4
|
+
import { type ConcurrentSandboxes, type SandboxCeiling } from "../../substrates/e2b/lifetime.js";
|
|
5
5
|
import type { ConcurrentSharedWorldStudyErrorCode } from "./types.js";
|
|
6
6
|
/** The error a shared-world study returns before a run starts. */
|
|
7
7
|
export interface SharedWorldRefusal extends RouteRefusal<"shared-world", ConcurrentSharedWorldStudyErrorCode> {
|
|
@@ -21,6 +21,8 @@ export declare function planSharedWorldStudy(config: StudyConfig, input: {
|
|
|
21
21
|
readonly hasRunSession?: boolean;
|
|
22
22
|
/** The longest sandbox lifetime the operator's E2B plan allows (sandboxCeiling). */
|
|
23
23
|
readonly sandboxCeiling: SandboxCeiling;
|
|
24
|
+
/** How many sandboxes the operator's E2B plan runs at once (concurrentSandboxes). */
|
|
25
|
+
readonly concurrentSandboxes: ConcurrentSandboxes;
|
|
24
26
|
}): SharedWorldPlanResult;
|
|
25
27
|
/** The provisioned plane's declared subject state. The external-public plane declares none. */
|
|
26
28
|
export declare function planeStateOf(plan: SharedWorldPlan): StudySubjectState | undefined;
|
|
@@ -12,8 +12,8 @@ import { PUBLIC_TARGET_OWNER_PATTERN } from "../../study/parse/subject.js";
|
|
|
12
12
|
import { REPO_SLUG_PATTERN } from "../../study/parse/values.js";
|
|
13
13
|
import { concurrentSharedWorldValidationReason, desktopMediaValidationReason, externalPublicSharedWorldValidationReason, outputTokenLimitValidationReason, receivingEmailValidationReason, taskProtocolValidationReason, waitLimitValidationReason, } from "../../study/validation.js";
|
|
14
14
|
import { unpricedCapCheck } from "../../study/requirements.js";
|
|
15
|
-
import {
|
|
16
|
-
import {
|
|
15
|
+
import { participantsAtOnce, sandboxDeadlineRefusal, sandboxHeadroomMs, } from "../../substrates/e2b/lifetime.js";
|
|
16
|
+
import { boundedConcurrency } from "../computer-use/participant-runs.js";
|
|
17
17
|
/** The registered descriptor for a planned actor; planSharedWorldStudy checked the registry. */
|
|
18
18
|
export function sharedWorldDescriptorOf(actorType) {
|
|
19
19
|
const descriptor = actorRegistry[actorType];
|
|
@@ -96,22 +96,54 @@ export function planSharedWorldStudy(config, input) {
|
|
|
96
96
|
const plane = planeOf(config);
|
|
97
97
|
if (plane === undefined)
|
|
98
98
|
throw new Error("shared-world validation admitted a plane it cannot plan");
|
|
99
|
+
// Every participant's desktop and, on the provisioned plane, the app's own sandbox share the E2B
|
|
100
|
+
// plan's concurrent sandboxes. A shared world needs two participants live together.
|
|
101
|
+
const declared = config.execution?.concurrency;
|
|
102
|
+
const n = plane.participants.length;
|
|
103
|
+
const atOnce = participantsAtOnce({
|
|
104
|
+
concurrency: boundedConcurrency(declared, n),
|
|
105
|
+
declared: declared !== undefined,
|
|
106
|
+
participants: n,
|
|
107
|
+
alongside: plane.kind === "provisioned" ? 1 : 0,
|
|
108
|
+
minimum: 2,
|
|
109
|
+
}, input.concurrentSandboxes);
|
|
110
|
+
if (!atOnce.ok)
|
|
111
|
+
return refuse(invalid, atOnce.message, actor);
|
|
112
|
+
const sessionTimeoutMs = config.execution?.timeoutMs ?? defaultSessionTimeoutMs(plane, ceiling.ms);
|
|
113
|
+
// The run's longest deadline: on the provisioned plane the subject sandbox, which serves the app
|
|
114
|
+
// until every participant ends; on the external-public plane each participant's own sandbox.
|
|
115
|
+
const deadlineReason = sandboxDeadlineRefusal(plane.kind === "provisioned"
|
|
116
|
+
? {
|
|
117
|
+
name: "the subject sandbox, which serves the app until every participant ends",
|
|
118
|
+
sessionMs: sessionTimeoutMs,
|
|
119
|
+
sessionDeclared: config.execution?.timeoutMs !== undefined,
|
|
120
|
+
servedSubject: { seed: plane.subject.state.seed ?? [] },
|
|
121
|
+
}
|
|
122
|
+
: {
|
|
123
|
+
name: "each participant's sandbox",
|
|
124
|
+
sessionMs: sessionTimeoutMs,
|
|
125
|
+
sessionDeclared: config.execution?.timeoutMs !== undefined,
|
|
126
|
+
}, ceiling.ms);
|
|
127
|
+
if (deadlineReason)
|
|
128
|
+
return refuse(invalid, deadlineReason, actor);
|
|
99
129
|
const brain = brainOf(config, false);
|
|
100
130
|
if (brain.kind === "caller")
|
|
101
131
|
throw new Error("a shared-world participant has no caller brain");
|
|
102
132
|
const base = planBase(config, {
|
|
103
133
|
dryRun: input.dryRun,
|
|
104
134
|
analysis,
|
|
135
|
+
participants: n,
|
|
105
136
|
});
|
|
106
137
|
return {
|
|
107
138
|
ok: true,
|
|
108
139
|
plan: {
|
|
109
140
|
...base,
|
|
141
|
+
...(atOnce.lowered === undefined ? {} : { warnings: [atOnce.lowered] }),
|
|
110
142
|
route: "shared-world",
|
|
111
143
|
actor,
|
|
112
144
|
plane,
|
|
113
|
-
concurrency:
|
|
114
|
-
sessionTimeoutMs
|
|
145
|
+
concurrency: atOnce.count,
|
|
146
|
+
sessionTimeoutMs,
|
|
115
147
|
brain,
|
|
116
148
|
caps: planCaps(config),
|
|
117
149
|
requirements: base.dryRun
|
|
@@ -145,8 +177,7 @@ const DEFAULT_APP_URL_SESSION_MS = 30 * 60_000;
|
|
|
145
177
|
function defaultSessionTimeoutMs(plane, ceilingMs) {
|
|
146
178
|
if (plane.kind !== "provisioned")
|
|
147
179
|
return DEFAULT_APP_URL_SESSION_MS;
|
|
148
|
-
const
|
|
149
|
-
const room = ceilingMs - SUBJECT_PROVISION_BUDGET_MS - stateBudgetMs - SANDBOX_TIMEOUT_BUFFER_MS;
|
|
180
|
+
const room = ceilingMs - sandboxHeadroomMs({ seed: plane.subject.state.seed ?? [] });
|
|
150
181
|
return Math.max(MIN_DERIVED_SESSION_MS, Math.min(MAX_DERIVED_SESSION_MS, room));
|
|
151
182
|
}
|
|
152
183
|
/** The provisioned plane's declared subject state. The external-public plane declares none. */
|
|
@@ -62,8 +62,8 @@ export function sharedWorldStudyRefusal(options, refusal) {
|
|
|
62
62
|
studyId: config.id,
|
|
63
63
|
actor: config.actor?.type ?? "",
|
|
64
64
|
participantCount: declared.length,
|
|
65
|
-
//
|
|
66
|
-
//
|
|
65
|
+
// A refused study never planned its concurrency, so its result reports the declared cap, else
|
|
66
|
+
// every declared participant.
|
|
67
67
|
concurrency: config.execution?.concurrency ?? Math.max(1, declared.length),
|
|
68
68
|
dryRun,
|
|
69
69
|
runId: options.runId,
|
package/dist/run/bundle-shape.js
CHANGED
|
@@ -216,7 +216,8 @@ function isRunSubjectProvenance(value) {
|
|
|
216
216
|
if (value.source !== "clone" &&
|
|
217
217
|
value.source !== "app-url" &&
|
|
218
218
|
value.source !== "local-tree" &&
|
|
219
|
-
value.source !== "desktop-cli"
|
|
219
|
+
value.source !== "desktop-cli" &&
|
|
220
|
+
value.source !== "local-app")
|
|
220
221
|
return false;
|
|
221
222
|
if (value.repo !== undefined && typeof value.repo !== "string")
|
|
222
223
|
return false;
|
package/dist/run/bundle.d.ts
CHANGED
|
@@ -158,7 +158,8 @@ export interface RunSubjectStateStepRecord {
|
|
|
158
158
|
* pre-existing bundles).
|
|
159
159
|
*/
|
|
160
160
|
export interface RunSubjectProvenance {
|
|
161
|
-
|
|
161
|
+
/** A run saved before `desktop-cli` or `local-app` was recorded says `app-url` for either. */
|
|
162
|
+
source: "clone" | "app-url" | "local-tree" | "desktop-cli" | "local-app";
|
|
162
163
|
/** Clone-route only. Honors policies.redactRepos exactly as the provenance event does. */
|
|
163
164
|
repo?: string;
|
|
164
165
|
/** Desktop-cli only: the product the study declares in `subject.product.name`. */
|
|
@@ -186,8 +187,9 @@ export interface RunSubjectProvenance {
|
|
|
186
187
|
* an external DB is still unpinned overall).
|
|
187
188
|
* declared-not-run: steps declared but not (all) executed ok: dry-run contract bundles
|
|
188
189
|
* and failed live provisioning.
|
|
189
|
-
* undeclared: no subject.state block (stateless apps, app-url and desktop-cli
|
|
190
|
-
* the explicit "absence declared" marker that provenance requires when there
|
|
190
|
+
* undeclared: no subject.state block (stateless apps, app-url, local-app and desktop-cli
|
|
191
|
+
* subjects), the explicit "absence declared" marker that provenance requires when there
|
|
192
|
+
* is none.
|
|
191
193
|
* external-public: an operator-declared, operator-owned public deployment used
|
|
192
194
|
* directly as the shared plane; humanish neither provisioned nor seeded it (no getHost, no
|
|
193
195
|
* clone, no in-sandbox filesystem). It is not "seeded" (nothing was seeded) and not "unpinned" (this is
|
package/dist/run/projection.js
CHANGED
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
// - an unknown cost is `null` (declared absent), never 0;
|
|
8
8
|
// - truncation is reported, never silent;
|
|
9
9
|
// - a participant's reported thinking is quoted, never paraphrased or summarized.
|
|
10
|
+
import { indexedRunCost } from "./run-cost.js";
|
|
10
11
|
/**
|
|
11
12
|
* Group runs by study, newest-first within each study and by recency between studies. Runs with no study
|
|
12
13
|
* attribution are collected under `unattributed` rather than invented into a study. That is the
|
|
@@ -74,9 +75,13 @@ export function expectationFor(entries, mode) {
|
|
|
74
75
|
.map((entry) => entry.durationMs)
|
|
75
76
|
.filter((value) => typeof value === "number" && Number.isFinite(value) && value >= 0)
|
|
76
77
|
.sort((a, b) => a - b);
|
|
78
|
+
// Each run's cost is the one its row and card show (src/run/run-cost.ts): participants and
|
|
79
|
+
// desktops plus its analyses. A run whose participants and desktops have no figure stays
|
|
80
|
+
// unpriced below, so an analysis alone never stands in for what a run cost.
|
|
77
81
|
const costs = finished
|
|
78
|
-
.
|
|
79
|
-
.
|
|
82
|
+
.filter((entry) => typeof entry.estimatedCostUsd === "number" && Number.isFinite(entry.estimatedCostUsd))
|
|
83
|
+
.map((entry) => indexedRunCost(entry).total?.usd)
|
|
84
|
+
.filter((value) => value !== undefined)
|
|
80
85
|
.sort((a, b) => a - b);
|
|
81
86
|
// A cost is unknown whether it was declared absent (`null`) or never recorded at all
|
|
82
87
|
// (`undefined`: every fail-closed exit finalized with no outcome). Counting only the first
|
package/dist/run/route-shell.js
CHANGED
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
// either is made here once. The preview route records no run and has no shell.
|
|
5
5
|
import { completeAutomaticAnalysis } from "../analysis/automatic-completion.js";
|
|
6
6
|
import { resolveAutomaticAnalysis } from "../analysis/automatic-config.js";
|
|
7
|
+
import { withParticipantLimitSkip } from "../study/automatic-analysis-plan.js";
|
|
7
8
|
import { runScope } from "./run.js";
|
|
8
9
|
import { withTransientCommsSecrets } from "./transient-comms-secrets.js";
|
|
9
10
|
/**
|
|
@@ -12,11 +13,15 @@ import { withTransientCommsSecrets } from "./transient-comms-secrets.js";
|
|
|
12
13
|
* it takes closes the run it started, and only a run that published its final bundle is analyzed.
|
|
13
14
|
*/
|
|
14
15
|
export async function admitRoute(shell) {
|
|
15
|
-
const complete = (input, result, finished) =>
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
16
|
+
const complete = (input, result, finished) =>
|
|
17
|
+
// A live run of more participants than one analysis reads records the skip and starts nothing.
|
|
18
|
+
shell.analysis?.skip !== undefined && !result.dryRun
|
|
19
|
+
? Promise.resolve(withParticipantLimitSkip(result, shell.analysis.trigger))
|
|
20
|
+
: completeAutomaticAnalysis(result, finished, shell.analysis?.config, input, {
|
|
21
|
+
...(shell.analysis === undefined ? {} : { trigger: shell.analysis.trigger }),
|
|
22
|
+
preferLargerOutput: shell.analysis?.preferLargerOutput === true,
|
|
23
|
+
...(shell.analysisRefusal === undefined ? {} : { refusal: shell.analysisRefusal }),
|
|
24
|
+
});
|
|
20
25
|
const admission = await shell.admit();
|
|
21
26
|
if (!admission.ok)
|
|
22
27
|
return {
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/** A subject as a run records it: its source and, for a product used at a desktop, the product. */
|
|
2
|
+
interface NamedSubject {
|
|
3
|
+
readonly source: string;
|
|
4
|
+
readonly product?: string | undefined;
|
|
5
|
+
}
|
|
6
|
+
/**
|
|
7
|
+
* The URL a participant opens names the subject. A product used at a desktop has no URL, so it is
|
|
8
|
+
* named with its source, as in `humanish (desktop-cli)`. Empty when there is neither.
|
|
9
|
+
*/
|
|
10
|
+
export declare function subjectName(url: string, subject: NamedSubject): string;
|
|
11
|
+
export {};
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
// The words a reader sees for a run's subject. The `subject:` line of `humanish run`, the run's
|
|
2
|
+
// summary and each participant's sentence name it the same way.
|
|
3
|
+
/**
|
|
4
|
+
* The URL a participant opens names the subject. A product used at a desktop has no URL, so it is
|
|
5
|
+
* named with its source, as in `humanish (desktop-cli)`. Empty when there is neither.
|
|
6
|
+
*/
|
|
7
|
+
export function subjectName(url, subject) {
|
|
8
|
+
if (url !== "" || subject.product === undefined)
|
|
9
|
+
return url;
|
|
10
|
+
return `${subject.product} (${subject.source})`;
|
|
11
|
+
}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import { type AutomaticAnalysisResult } from "../analysis/automatic-completion.js";
|
|
2
|
+
import { type AutomaticAnalysisBudget } from "../analysis/automatic-config.js";
|
|
3
|
+
import type { StudyRoute } from "./routing.js";
|
|
4
|
+
/** The reason a run records for an analysis its study has too many participants for. */
|
|
5
|
+
export declare const PARTICIPANT_LIMIT_SKIP = "AUTOMATIC_ANALYSIS_PARTICIPANT_LIMIT";
|
|
6
|
+
/** The CLI's words for that reason, after "analysis: ". */
|
|
7
|
+
export declare const PARTICIPANT_LIMIT_SKIP_TEXT: string;
|
|
8
|
+
/** The skip a study of `participants` gets, or undefined when its analysis runs. */
|
|
9
|
+
export declare function analysisSkipFor(participants: number): typeof PARTICIPANT_LIMIT_SKIP | undefined;
|
|
10
|
+
/** The analysis budget the plan line shows, and the skip when the analysis will not run. */
|
|
11
|
+
export type StudyAnalysisBudget = AutomaticAnalysisBudget & {
|
|
12
|
+
readonly skip?: typeof PARTICIPANT_LIMIT_SKIP;
|
|
13
|
+
};
|
|
14
|
+
/**
|
|
15
|
+
* The budget of a study's automatic analysis after a live run of `participants`, or undefined
|
|
16
|
+
* when none is planned. A skipped analysis carries no expected cost, since it spends nothing.
|
|
17
|
+
*/
|
|
18
|
+
export declare function studyAnalysisBudget(raw: unknown, route: StudyRoute, participants: number): StudyAnalysisBudget | undefined;
|
|
19
|
+
/** The line `run` and `study check` print about the analysis after a live run. */
|
|
20
|
+
export declare function formatStudyAnalysisBudget(budget: StudyAnalysisBudget): string;
|
|
21
|
+
/** A run's result with its analysis recorded as skipped for its participant count. */
|
|
22
|
+
export declare function withParticipantLimitSkip<T>(result: T, trigger: "default" | "explicit"): T & AutomaticAnalysisResult;
|
|
23
|
+
/**
|
|
24
|
+
* Whether a run's automatic analysis lets the command exit 0. A participant-count skip of the
|
|
25
|
+
* analysis a study did not ask for passes, like the default analysis's missing-key skip; a skip of
|
|
26
|
+
* an explicit request fails, like that request's missing key.
|
|
27
|
+
*/
|
|
28
|
+
export declare function studyAnalysisSucceeded(result: AutomaticAnalysisResult): boolean;
|