humanish 0.110.2 → 0.110.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +10 -15
- package/CHANGELOG.md +27 -0
- package/CONTEXT.md +1 -1
- package/dist/actors/computer-use/actor.js +2 -2
- package/dist/actors/computer-use/loop/ending.js +1 -1
- package/dist/actors/registry.js +1 -1
- package/dist/actors/scripted-browser/actor.js +17 -8
- package/dist/actors/scripted-browser/launch-env.d.ts +12 -0
- package/dist/actors/scripted-browser/launch-env.js +50 -0
- package/dist/analysis/automatic-completion.d.ts +2 -1
- package/dist/analysis/automatic-completion.js +15 -2
- package/dist/cli/commands/migrate.js +1 -1
- package/dist/cli/commands/observe.js +3 -2
- package/dist/cli/commands/runs.js +5 -2
- package/dist/cli/commands/study-route-computer-use.js +3 -2
- package/dist/cli/commands/study-route-preview.js +2 -1
- package/dist/cli/commands/study-route-shared-world.js +3 -3
- package/dist/cli/commands/study-run.js +9 -19
- package/dist/cli/commands/study.js +4 -1
- package/dist/cli/findings.d.ts +5 -0
- package/dist/cli/findings.js +21 -0
- package/dist/cli/io.d.ts +2 -1
- package/dist/cli/io.js +3 -2
- package/dist/cli/observer-follow.d.ts +6 -2
- package/dist/cli/observer-follow.js +15 -5
- package/dist/evidence/redaction.js +15 -4
- package/dist/routes/computer-use/live-phase.js +8 -1
- package/dist/routes/computer-use/local-vm.js +2 -1
- package/dist/routes/computer-use/participant-runs.d.ts +6 -9
- package/dist/routes/computer-use/participant-runs.js +16 -36
- package/dist/routes/computer-use/plan.d.ts +2 -2
- package/dist/routes/computer-use/plan.js +7 -6
- package/dist/routes/computer-use/route.d.ts +5 -18
- package/dist/routes/computer-use/route.js +19 -79
- package/dist/routes/computer-use/setup.d.ts +3 -2
- package/dist/routes/computer-use/setup.js +2 -1
- package/dist/routes/computer-use/types.d.ts +6 -10
- package/dist/routes/scripted/plan.js +5 -4
- package/dist/routes/scripted/route.d.ts +4 -17
- package/dist/routes/scripted/route.js +15 -43
- package/dist/routes/scripted/surface-sessions.d.ts +2 -0
- package/dist/routes/scripted/surface-sessions.js +2 -1
- package/dist/routes/scripted/types.d.ts +1 -6
- package/dist/routes/shared-world/participant-specs.js +5 -1
- package/dist/routes/shared-world/plan.js +7 -6
- package/dist/routes/shared-world/route.d.ts +5 -11
- package/dist/routes/shared-world/route.js +27 -66
- package/dist/routes/shared-world/types.d.ts +2 -6
- package/dist/routes/terminal/plan.js +4 -3
- package/dist/routes/terminal/route.d.ts +3 -13
- package/dist/routes/terminal/route.js +21 -60
- package/dist/routes/terminal/session.d.ts +2 -3
- package/dist/routes/terminal/session.js +3 -3
- package/dist/routes/terminal/types.d.ts +2 -7
- package/dist/run/display.d.ts +10 -0
- package/dist/run/display.js +22 -0
- package/dist/run/reclaim.d.ts +4 -2
- package/dist/run/reclaim.js +61 -18
- package/dist/run/route-shell.d.ts +60 -0
- package/dist/run/route-shell.js +51 -0
- package/dist/run/run.d.ts +16 -1
- package/dist/run/run.js +33 -5
- package/dist/run/status.d.ts +8 -0
- package/dist/run/status.js +6 -0
- package/dist/run/stored-runs.js +26 -11
- package/dist/study/composition-rules.js +33 -33
- package/dist/study/config.d.ts +9 -2
- package/dist/study/config.js +14 -5
- package/dist/study/doctor.js +5 -4
- package/dist/study/init-templates.d.ts +1 -1
- package/dist/study/init-templates.js +43 -34
- package/dist/study/init.js +1 -1
- package/dist/study/{convert.d.ts → migrate/convert.d.ts} +1 -1
- package/dist/study/{convert.js → migrate/convert.js} +29 -31
- package/dist/study/{migrate.d.ts → migrate/migrate.d.ts} +1 -1
- package/dist/study/{migrate.js → migrate/migrate.js} +2 -2
- package/dist/study/{owned-files.js → migrate/owned-files.js} +1 -1
- package/dist/study/migrate/v2.d.ts +68 -0
- package/dist/study/migrate/v2.js +480 -0
- package/dist/study/parse/actors.d.ts +46 -1
- package/dist/study/parse/actors.js +116 -69
- package/dist/study/parse/execution.js +2 -2
- package/dist/study/parse/study-v3.d.ts +26 -0
- package/dist/study/parse/study-v3.js +53 -35
- package/dist/study/parse/study.d.ts +38 -0
- package/dist/study/parse/study.js +225 -0
- package/dist/study/parse/subject.js +1 -1
- package/dist/study/persona-resolve.d.ts +2 -4
- package/dist/study/persona-resolve.js +2 -1
- package/dist/study/plan-base.d.ts +1 -1
- package/dist/study/plan-base.js +5 -4
- package/dist/study/plan-participants.js +12 -11
- package/dist/study/plan.js +4 -3
- package/dist/study/preflight-probes.js +2 -2
- package/dist/study/preflight.js +2 -2
- package/dist/study/routing.js +12 -15
- package/dist/study/run-study-options.js +3 -3
- package/dist/study/study-fields.d.ts +28 -0
- package/dist/study/study-fields.js +51 -0
- package/dist/study/summary.js +13 -17
- package/dist/study/types.d.ts +40 -0
- package/dist/study/url-credentials.js +2 -1
- package/dist/study/validation.d.ts +2 -0
- package/dist/study/validation.js +46 -41
- package/dist/study/warnings.js +5 -4
- package/dist/substrates/local/runtime-config.js +9 -9
- package/dist/tui-app.js +3 -3
- package/docs/architecture/actor-contract.md +1 -1
- package/docs/architecture/external-public-shared-world.md +1 -1
- package/docs/architecture/terminal-product-route.md +13 -13
- package/docs/contracts/schemas.md +13 -4
- package/docs/contracts/study-analysis.md +5 -2
- package/docs/product/automatic-analysis.md +1 -1
- package/package.json +1 -1
- /package/dist/study/{owned-files.d.ts → migrate/owned-files.d.ts} +0 -0
|
@@ -6,7 +6,6 @@ import type { RunScope } from "../../run/run.js";
|
|
|
6
6
|
import type { buildRuntimeAuth } from "./credentials.js";
|
|
7
7
|
import type { TerminalPlan } from "../../study/plan-types.js";
|
|
8
8
|
import type { StudyDeps } from "../../study/study-deps.js";
|
|
9
|
-
import type { StudyConfig } from "../../study/types.js";
|
|
10
9
|
import type { ObserverResult } from "../../observer/render.js";
|
|
11
10
|
import { type RunAdapterScore, type RunBundle, type RunFeedbackCandidate, type RunScorerProvenance } from "../../run/bundle.js";
|
|
12
11
|
import { type StudyResultIdentity } from "../../run/study-result.js";
|
|
@@ -97,15 +96,13 @@ export interface TerminalScorer {
|
|
|
97
96
|
*/
|
|
98
97
|
deriveFeedback?: (ctx: TerminalProductScoringContext) => RunFeedbackCandidate[] | Promise<RunFeedbackCandidate[]>;
|
|
99
98
|
}
|
|
100
|
-
|
|
99
|
+
/** What a terminal run takes besides its plan. The plan carries the config, dry run and study. */
|
|
100
|
+
export interface TerminalRunInput {
|
|
101
101
|
/** Cancels post-run analysis only. */
|
|
102
102
|
analysisSignal?: AbortSignal;
|
|
103
103
|
/** Reports the analysis window to onEvent; built by normalizeRunStudyOptions. */
|
|
104
104
|
emit?: (event: StudyEvent) => void;
|
|
105
105
|
cwd: string;
|
|
106
|
-
config: StudyConfig;
|
|
107
|
-
/** Resolved upstream (scenario.mode + CLI override); defaults safe (dry-run). */
|
|
108
|
-
dryRun: boolean;
|
|
109
106
|
open?: boolean;
|
|
110
107
|
runId?: string;
|
|
111
108
|
/**
|
|
@@ -312,8 +309,6 @@ export interface TerminalLedgers {
|
|
|
312
309
|
/** The no-spend proof derived from `cost`. Never an independent assertion. */
|
|
313
310
|
noSpendProof: NoSpendProof;
|
|
314
311
|
}
|
|
315
|
-
/** What a terminal run takes besides its plan. The plan carries the config, dry run and study. */
|
|
316
|
-
export type TerminalRunInput = Omit<RunTerminalProductStudyOptions, "config" | "dryRun">;
|
|
317
312
|
/** A live terminal plan: its caps are the fail-closed ones planTerminalStudy required. */
|
|
318
313
|
export type LiveTerminalPlan = Extract<TerminalPlan, {
|
|
319
314
|
readonly dryRun: false;
|
package/dist/run/display.d.ts
CHANGED
|
@@ -57,3 +57,13 @@ export declare function bundleDisplayFacts(bundle: DisplayedBundle, status?: unk
|
|
|
57
57
|
* execution failed never reads as a pass beside its participants' verdict.
|
|
58
58
|
*/
|
|
59
59
|
export declare function reviewOutcome(bundle: DisplayedBundle): string;
|
|
60
|
+
/**
|
|
61
|
+
* The summary and gaps a review shows beside its run's display. A run stopped while a participant
|
|
62
|
+
* was still running keeps the review its last live flush wrote, which describes that session as
|
|
63
|
+
* running. Once runDisplay reads the run as interrupted, the review says how many participants the
|
|
64
|
+
* stop cut off instead.
|
|
65
|
+
*/
|
|
66
|
+
export declare function displayedReview<T extends {
|
|
67
|
+
summary: string;
|
|
68
|
+
gaps: string[];
|
|
69
|
+
}>(review: T, bundle: DisplayedBundle, display: RunDisplay): T;
|
package/dist/run/display.js
CHANGED
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
// same facts, so a run whose own ok is false shows as a pass on none of them, and two surfaces
|
|
4
4
|
// cannot disagree about one run.
|
|
5
5
|
import { classifyRunStatus, isRunStatusRecord } from "./status.js";
|
|
6
|
+
import { plural } from "./text.js";
|
|
6
7
|
const LABELS = {
|
|
7
8
|
running: "running",
|
|
8
9
|
interrupted: "interrupted",
|
|
@@ -104,3 +105,24 @@ export function bundleDisplayFacts(bundle, status, nowMs = Date.now()) {
|
|
|
104
105
|
export function reviewOutcome(bundle) {
|
|
105
106
|
return runDisplayLine(runDisplay(bundleDisplayFacts(bundle)));
|
|
106
107
|
}
|
|
108
|
+
/**
|
|
109
|
+
* The summary and gaps a review shows beside its run's display. A run stopped while a participant
|
|
110
|
+
* was still running keeps the review its last live flush wrote, which describes that session as
|
|
111
|
+
* running. Once runDisplay reads the run as interrupted, the review says how many participants the
|
|
112
|
+
* stop cut off instead.
|
|
113
|
+
*/
|
|
114
|
+
export function displayedReview(review, bundle, display) {
|
|
115
|
+
const records = bundle.simulations ?? [];
|
|
116
|
+
const running = records.filter((record) => record?.status === "running").length;
|
|
117
|
+
if (display.state !== "interrupted" || running === 0)
|
|
118
|
+
return review;
|
|
119
|
+
const signal = bundle.outcome?.state === "interrupted" ? ` by ${bundle.outcome.signal}` : "";
|
|
120
|
+
const one = running === 1;
|
|
121
|
+
return {
|
|
122
|
+
...review,
|
|
123
|
+
summary: `Interrupted${signal} while ${running} of ${plural(records.length, "participant")} ${one ? "was" : "were"} still running.`,
|
|
124
|
+
gaps: [
|
|
125
|
+
`${plural(running, "participant")} ${one ? "has" : "have"} no verdict: the run stopped before ${one ? "its session" : "their sessions"} ended.`,
|
|
126
|
+
],
|
|
127
|
+
};
|
|
128
|
+
}
|
package/dist/run/reclaim.d.ts
CHANGED
|
@@ -20,9 +20,10 @@ export interface ReclaimResult {
|
|
|
20
20
|
createsInFlight: number;
|
|
21
21
|
/**
|
|
22
22
|
* Why there was nothing to look for: `dry-run` when run.json or status.json records the run as
|
|
23
|
-
* a dry run, which creates no sandboxes
|
|
23
|
+
* a dry run, which creates no sandboxes, and `no-sandbox` when status.json records that the
|
|
24
|
+
* route created none for this run. Reclaim then contacts E2B for nothing.
|
|
24
25
|
*/
|
|
25
|
-
reason?:
|
|
26
|
+
reason?: NoSandboxReason;
|
|
26
27
|
warnings: string[];
|
|
27
28
|
error?: {
|
|
28
29
|
code: "HUMANISH_RECLAIM_RUN_NOT_FOUND" | "HUMANISH_RECLAIM_E2B_DEBUG" | "HUMANISH_RECLAIM_MODULE_UNAVAILABLE" | "HUMANISH_RECLAIM_RECEIPTS_UNREADABLE";
|
|
@@ -47,6 +48,7 @@ export declare function reclaimRunSandboxes(cwd: string, runInput: string, hooks
|
|
|
47
48
|
* also an alias (a run named `latest`) cannot reach another run's receipts.
|
|
48
49
|
*/
|
|
49
50
|
export declare function reclaimPinnedRunSandboxes(cwd: string, runPaths: PreparedRunArtifactPaths, hooks?: ReclaimHooks): Promise<ReclaimResult>;
|
|
51
|
+
type NoSandboxReason = "dry-run" | "no-sandbox";
|
|
50
52
|
/**
|
|
51
53
|
* Kill the sandboxes left by interrupted `humanish study check` probes. A probe journals under
|
|
52
54
|
* .humanish/preflight/<probe-id>/ and removes the journal after a confirmed kill, so what is left
|
package/dist/run/reclaim.js
CHANGED
|
@@ -15,6 +15,7 @@ import { readRunJsonIfExists, resolveRunPath } from "./locate.js";
|
|
|
15
15
|
import { RUN_BUNDLE_FILE } from "./bundle.js";
|
|
16
16
|
import { RUN_STATUS_FILE } from "./status.js";
|
|
17
17
|
import { isRecord } from "./type-guards.js";
|
|
18
|
+
import { isRunOutcome } from "./bundle-shape.js";
|
|
18
19
|
import { parseSandboxOwners, parseSandboxReceipts, SANDBOX_RECEIPTS_ARTIFACT, } from "./sandbox-receipts.js";
|
|
19
20
|
import { sandboxOwnerTags, } from "./sandbox-creates.js";
|
|
20
21
|
import { earlierOutcomes, GONE, RECLAIM_RECEIPT_ARTIFACT, RECLAIM_RESULT_SCHEMA, receiptWriter, rootState, searchTagSets, settledWithin, targetSet, } from "./reclaim-outcomes.js";
|
|
@@ -64,8 +65,9 @@ async function reclaimRun(cwd, runInput, locate, hooks) {
|
|
|
64
65
|
}
|
|
65
66
|
const runId = runIdOf(runPaths);
|
|
66
67
|
// The signal handler passes the creates it stopped and decides from what they reported.
|
|
67
|
-
|
|
68
|
-
|
|
68
|
+
const reason = hooks.creates === undefined ? await recordedNoSandbox(runPaths) : undefined;
|
|
69
|
+
if (reason !== undefined)
|
|
70
|
+
return { ...base, ok: true, state: "clean", runId, reason };
|
|
69
71
|
const reclaimed = await reclaimRoot(runPaths, runId, hooks, warnings);
|
|
70
72
|
if (reclaimed.kind === "module-unavailable")
|
|
71
73
|
return {
|
|
@@ -93,29 +95,70 @@ async function reclaimRun(cwd, runInput, locate, hooks) {
|
|
|
93
95
|
};
|
|
94
96
|
}
|
|
95
97
|
/**
|
|
96
|
-
*
|
|
97
|
-
* `dry-run`.
|
|
98
|
-
*
|
|
99
|
-
*
|
|
98
|
+
* Why the run's own records say it created no sandbox. `dry-run`: run.json and status.json, each
|
|
99
|
+
* one present, say `dry-run`. `no-sandbox`: status.json, written at start, names this run and
|
|
100
|
+
* records `sandboxes: none`, which the route sets only when it has no way to create one (a
|
|
101
|
+
* scripted run against an app-url subject), and run.json agrees (scriptedAppUrlBundle). A status
|
|
102
|
+
* record alone can be stale or copied from another run, so it never decides on its own, and an
|
|
103
|
+
* empty journal alone never makes a run clean. A journal with any line in it, or an earlier
|
|
104
|
+
* reclaim receipt that names a sandbox, means a create ran, so that run is reclaimed as a live one
|
|
105
|
+
* whatever its records say.
|
|
100
106
|
*/
|
|
101
|
-
async function
|
|
102
|
-
const
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
107
|
+
async function recordedNoSandbox(runPaths) {
|
|
108
|
+
const bundle = await readRunJsonIfExists(runPaths, RUN_BUNDLE_FILE);
|
|
109
|
+
const status = await readRunJsonIfExists(runPaths, RUN_STATUS_FILE);
|
|
110
|
+
const records = [bundle, status].filter(isRecord);
|
|
111
|
+
const reason = records.length > 0 && records.every((record) => record.mode === "dry-run")
|
|
112
|
+
? "dry-run"
|
|
113
|
+
: isRecord(status) &&
|
|
114
|
+
status.runId === runIdOf(runPaths) &&
|
|
115
|
+
status.sandboxes === "none" &&
|
|
116
|
+
scriptedAppUrlBundle(bundle, runIdOf(runPaths))
|
|
117
|
+
? "no-sandbox"
|
|
118
|
+
: undefined;
|
|
119
|
+
if (reason === undefined)
|
|
120
|
+
return undefined;
|
|
110
121
|
if (!(await containedPathAbsent(runPaths, RECLAIM_RECEIPT_ARTIFACT))) {
|
|
111
122
|
const earlier = await readRunJsonIfExists(runPaths, RECLAIM_RECEIPT_ARTIFACT);
|
|
112
123
|
if (!isRecord(earlier) || !Array.isArray(earlier.outcomes) || earlier.outcomes.length > 0)
|
|
113
|
-
return
|
|
124
|
+
return undefined;
|
|
114
125
|
}
|
|
115
126
|
if (await containedPathAbsent(runPaths, SANDBOX_RECEIPTS_ARTIFACT))
|
|
116
|
-
return
|
|
127
|
+
return reason;
|
|
117
128
|
const journal = await readContainedRegularFile(runPaths, SANDBOX_RECEIPTS_ARTIFACT);
|
|
118
|
-
return journal !== null && journal.toString("utf8").trim() === "";
|
|
129
|
+
return journal !== null && journal.toString("utf8").trim() === "" ? reason : undefined;
|
|
130
|
+
}
|
|
131
|
+
/** The execution failure kinds that leave a sandbox or a provider's resources unconfirmed. */
|
|
132
|
+
const CLEANUP_KINDS = new Set(["sandbox-cleanup", "provider-cleanup"]);
|
|
133
|
+
/**
|
|
134
|
+
* run.json for `runId` agrees that its run could create no sandbox: the scripted route wrote it
|
|
135
|
+
* (each lifecycle event is a `scripted-lab.` one), its subject is not a provisioned clone (a clone
|
|
136
|
+
* records `subject`), it lists no provider resource and no desktop minutes, and its outcome marks
|
|
137
|
+
* no sandbox or provider cleanup unconfirmed. A run.json that is missing, unreadable or says
|
|
138
|
+
* otherwise does not agree.
|
|
139
|
+
*/
|
|
140
|
+
function scriptedAppUrlBundle(bundle, runId) {
|
|
141
|
+
if (!isRecord(bundle) || bundle.runId !== runId || bundle.subject !== undefined)
|
|
142
|
+
return false;
|
|
143
|
+
const { lifecycle, providerResources, cost, outcome } = bundle;
|
|
144
|
+
const scripted = Array.isArray(lifecycle) &&
|
|
145
|
+
lifecycle.length > 0 &&
|
|
146
|
+
lifecycle.every((entry) => isRecord(entry) &&
|
|
147
|
+
typeof entry.event === "string" &&
|
|
148
|
+
entry.event.startsWith("scripted-lab."));
|
|
149
|
+
if (!scripted)
|
|
150
|
+
return false;
|
|
151
|
+
if (providerResources !== undefined &&
|
|
152
|
+
!(Array.isArray(providerResources) && providerResources.length === 0))
|
|
153
|
+
return false;
|
|
154
|
+
if (cost !== undefined && !(isRecord(cost) && cost.desktopMinutes === null))
|
|
155
|
+
return false;
|
|
156
|
+
if (!isRunOutcome(outcome))
|
|
157
|
+
return false;
|
|
158
|
+
if (outcome.state !== "finished")
|
|
159
|
+
return true;
|
|
160
|
+
const { failures, warnings = [] } = outcome.execution;
|
|
161
|
+
return ![...failures, ...warnings].some((failure) => CLEANUP_KINDS.has(failure.kind));
|
|
119
162
|
}
|
|
120
163
|
/**
|
|
121
164
|
* Kill the sandboxes left by interrupted `humanish study check` probes. A probe journals under
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
import { type AnalysisInput } from "../analysis/automatic-completion.js";
|
|
2
|
+
import type { AutomaticAnalysisDeps } from "../analysis/automatic.js";
|
|
3
|
+
import type { AdmittedPlan, StudyResult } from "../run-study.js";
|
|
4
|
+
import type { PlannedAnalysis } from "../study/plan-types.js";
|
|
5
|
+
import type { LateScorer } from "../study/route-inputs.js";
|
|
6
|
+
import type { StudyRoute } from "../study/routing.js";
|
|
7
|
+
import type { StudyConfig } from "../study/types.js";
|
|
8
|
+
import { type RunScope } from "./run.js";
|
|
9
|
+
type ShellRoute = Exclude<StudyRoute, "preview">;
|
|
10
|
+
/** A route's local checks: the refused result, or what its run continues from. */
|
|
11
|
+
export type RouteAdmission<R extends ShellRoute, A> = {
|
|
12
|
+
readonly ok: false;
|
|
13
|
+
readonly result: StudyResult<R>;
|
|
14
|
+
} | {
|
|
15
|
+
readonly ok: true;
|
|
16
|
+
readonly admitted: A;
|
|
17
|
+
};
|
|
18
|
+
/** A route's own parts of its run. Everything around them is admitRoute's. */
|
|
19
|
+
export interface RouteShell<R extends ShellRoute, I extends AnalysisInput, A> {
|
|
20
|
+
readonly route: R;
|
|
21
|
+
/** The plan's automatic analysis, which both the refused result and the run complete. */
|
|
22
|
+
readonly analysis: PlannedAnalysis | undefined;
|
|
23
|
+
/** The route's input as the checks admitted it. */
|
|
24
|
+
readonly input: I;
|
|
25
|
+
/** The checks that need no scorer, made before any run scope opens. */
|
|
26
|
+
admit(): Promise<RouteAdmission<R, A>> | RouteAdmission<R, A>;
|
|
27
|
+
/** The run inside its run scope, with the input a late scorer joined. */
|
|
28
|
+
runInScope(admitted: A, input: I, scope: RunScope): Promise<StudyResult<R>>;
|
|
29
|
+
/** Layers a scorer the CLI loaded after the checks over the input. Scripted takes no scorer. */
|
|
30
|
+
readonly withScorer?: (input: I, scorer: LateScorer | undefined) => I;
|
|
31
|
+
/** Computer use's reason to skip analysis: a local VM study whose cleanup is unconfirmed. */
|
|
32
|
+
readonly analysisRefusal?: AutomaticAnalysisDeps["refusal"];
|
|
33
|
+
/**
|
|
34
|
+
* Computer use and shared world receive email. The run and its analysis share one scope for the
|
|
35
|
+
* secrets it registers. The other routes run outside any scope, where scrubbing changes nothing.
|
|
36
|
+
*/
|
|
37
|
+
readonly commsSecrets?: boolean;
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Runs a route's checks and returns the refusal they made or the route's run. A refusal carries the
|
|
41
|
+
* analysis record of a run that never started. The run opens its own run scope, so whichever exit
|
|
42
|
+
* it takes closes the run it started, and only a run that published its final bundle is analyzed.
|
|
43
|
+
*/
|
|
44
|
+
export declare function admitRoute<R extends ShellRoute, I extends AnalysisInput, A>(shell: RouteShell<R, I, A>): Promise<AdmittedPlan<R>>;
|
|
45
|
+
/**
|
|
46
|
+
* A refused study's result with the analysis record a refusal gets: it starts no run, so a declared
|
|
47
|
+
* or default analysis is recorded as skipped.
|
|
48
|
+
*/
|
|
49
|
+
export declare function completeRefusalAnalysis<T extends {
|
|
50
|
+
cwd: string;
|
|
51
|
+
runId: string;
|
|
52
|
+
dryRun: boolean;
|
|
53
|
+
}>(refused: T, config: StudyConfig, input: AnalysisInput): Promise<T>;
|
|
54
|
+
/** What a refused study's envelope reads besides the route's input. */
|
|
55
|
+
export interface RefusedStudy {
|
|
56
|
+
readonly config: StudyConfig;
|
|
57
|
+
/** The run's resolved dry run. */
|
|
58
|
+
readonly dryRun: boolean;
|
|
59
|
+
}
|
|
60
|
+
export {};
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
// The shell the computer-use, scripted, terminal and shared-world routes share. A route supplies
|
|
2
|
+
// its local checks and the body it runs inside the run scope. admitRoute completes the automatic
|
|
3
|
+
// analysis of a refusal and of a run, and wraps the result as the route's outcome, so a change to
|
|
4
|
+
// either is made here once. The preview route records no run and has no shell.
|
|
5
|
+
import { completeAutomaticAnalysis } from "../analysis/automatic-completion.js";
|
|
6
|
+
import { resolveAutomaticAnalysis } from "../analysis/automatic-config.js";
|
|
7
|
+
import { runScope } from "./run.js";
|
|
8
|
+
import { withTransientCommsSecrets } from "./transient-comms-secrets.js";
|
|
9
|
+
/**
|
|
10
|
+
* Runs a route's checks and returns the refusal they made or the route's run. A refusal carries the
|
|
11
|
+
* analysis record of a run that never started. The run opens its own run scope, so whichever exit
|
|
12
|
+
* it takes closes the run it started, and only a run that published its final bundle is analyzed.
|
|
13
|
+
*/
|
|
14
|
+
export async function admitRoute(shell) {
|
|
15
|
+
const complete = (input, result, finished) => completeAutomaticAnalysis(result, finished, shell.analysis?.config, input, {
|
|
16
|
+
...(shell.analysis === undefined ? {} : { trigger: shell.analysis.trigger }),
|
|
17
|
+
preferLargerOutput: shell.analysis?.preferLargerOutput === true,
|
|
18
|
+
...(shell.analysisRefusal === undefined ? {} : { refusal: shell.analysisRefusal }),
|
|
19
|
+
});
|
|
20
|
+
const admission = await shell.admit();
|
|
21
|
+
if (!admission.ok)
|
|
22
|
+
return {
|
|
23
|
+
ok: false,
|
|
24
|
+
outcome: routeOutcome(shell.route, await complete(shell.input, admission.result, undefined)),
|
|
25
|
+
};
|
|
26
|
+
const { admitted } = admission;
|
|
27
|
+
return {
|
|
28
|
+
ok: true,
|
|
29
|
+
run: async (scorer) => {
|
|
30
|
+
// The scorer joins the input only; whatever the checks admitted stays what the run uses.
|
|
31
|
+
const input = shell.withScorer?.(shell.input, scorer) ?? shell.input;
|
|
32
|
+
const run = async () => {
|
|
33
|
+
const { result, finished } = await runScope((scope) => shell.runInScope(admitted, input, scope));
|
|
34
|
+
return complete(input, result, finished);
|
|
35
|
+
};
|
|
36
|
+
return routeOutcome(shell.route, await (shell.commsSecrets ? withTransientCommsSecrets(run) : run()));
|
|
37
|
+
},
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* A refused study's result with the analysis record a refusal gets: it starts no run, so a declared
|
|
42
|
+
* or default analysis is recorded as skipped.
|
|
43
|
+
*/
|
|
44
|
+
export function completeRefusalAnalysis(refused, config, input) {
|
|
45
|
+
const analysis = resolveAutomaticAnalysis(config.review?.analysis);
|
|
46
|
+
return completeAutomaticAnalysis(refused, undefined, analysis.ok ? analysis.config : undefined, input, { trigger: config.review?.analysis === undefined ? "default" : "explicit" });
|
|
47
|
+
}
|
|
48
|
+
/** A result as its route's outcome. StudyOutcome pairs each route with its result type. */
|
|
49
|
+
function routeOutcome(route, result) {
|
|
50
|
+
return { route, result };
|
|
51
|
+
}
|
package/dist/run/run.d.ts
CHANGED
|
@@ -11,6 +11,8 @@ interface StartRunOptions {
|
|
|
11
11
|
mintRunId: () => string;
|
|
12
12
|
mode: "dry-run" | "live";
|
|
13
13
|
study?: RunStudyProvenance | undefined;
|
|
14
|
+
/** `none` when the route creates no sandbox for this run; status.json records it for reclaim. */
|
|
15
|
+
sandboxes?: "none" | undefined;
|
|
14
16
|
/** review.md for the published bundle. */
|
|
15
17
|
renderReview: (bundle: RunBundle) => string;
|
|
16
18
|
/** Used by `FinishedRun.renderObserver`; `render` is the `StudyDeps.renderObserver` seam. */
|
|
@@ -52,6 +54,12 @@ interface Run {
|
|
|
52
54
|
* can never overwrite the final bundle. Routes that throttle snapshots own their timers.
|
|
53
55
|
*/
|
|
54
56
|
writeSnapshot(bundle: RunBundle): Promise<void>;
|
|
57
|
+
/**
|
|
58
|
+
* A participant's session started: its actor began to act on the subject. FinishedRun's
|
|
59
|
+
* participantsRan reads this, so a session that throws before it returns a trace still counts,
|
|
60
|
+
* and a run that failed before any session (a refused E2B key) does not.
|
|
61
|
+
*/
|
|
62
|
+
participantStarted(): void;
|
|
55
63
|
/**
|
|
56
64
|
* The one final publication: run.json with `outcome` set from `outcome`, then the status
|
|
57
65
|
* outcome copied from it, then review.json, review.md, events.ndjson,
|
|
@@ -92,7 +100,14 @@ export declare class FinishedRun {
|
|
|
92
100
|
readonly runId: string;
|
|
93
101
|
/** The paths created by startRun and validated at the start of `finish`. */
|
|
94
102
|
readonly paths: PreparedRunArtifactPaths;
|
|
95
|
-
constructor(key: typeof issueKey, runId: string, paths: PreparedRunArtifactPaths, observer: ObserverTarget | undefined, outcome: RecordedOutcome, recordFailure: (failure: ExecutionFailure) => Promise<RecordedOutcome
|
|
103
|
+
constructor(key: typeof issueKey, runId: string, paths: PreparedRunArtifactPaths, observer: ObserverTarget | undefined, outcome: RecordedOutcome, recordFailure: (failure: ExecutionFailure) => Promise<RecordedOutcome>, interrupted: () => boolean, participantsRan: () => boolean);
|
|
104
|
+
/**
|
|
105
|
+
* A signal stopped this run's process: status.json and run.json record it `interrupted`. Read
|
|
106
|
+
* when asked, since the signal can arrive after the bundle was published.
|
|
107
|
+
*/
|
|
108
|
+
get interrupted(): boolean;
|
|
109
|
+
/** The route reported a participant's session started (Run.participantStarted). */
|
|
110
|
+
get participantsRan(): boolean;
|
|
96
111
|
static isIssued(value: unknown): value is FinishedRun;
|
|
97
112
|
/**
|
|
98
113
|
* The run's ok and execution outcome as run.json records them: what the route passed to
|
package/dist/run/run.js
CHANGED
|
@@ -27,18 +27,33 @@ const issueKey = Symbol("FinishedRun");
|
|
|
27
27
|
export class FinishedRun {
|
|
28
28
|
#observer;
|
|
29
29
|
#recordFailure;
|
|
30
|
+
#interrupted;
|
|
30
31
|
#outcome;
|
|
31
32
|
runId;
|
|
32
33
|
/** The paths created by startRun and validated at the start of `finish`. */
|
|
33
34
|
paths;
|
|
34
|
-
|
|
35
|
+
#participantsRan;
|
|
36
|
+
constructor(key, runId, paths, observer, outcome, recordFailure, interrupted, participantsRan) {
|
|
35
37
|
if (key !== issueKey)
|
|
36
38
|
throw new Error("Only Run.finish issues a FinishedRun.");
|
|
37
39
|
this.runId = runId;
|
|
38
40
|
this.paths = paths;
|
|
41
|
+
this.#participantsRan = participantsRan;
|
|
39
42
|
this.#observer = observer;
|
|
40
43
|
this.#outcome = outcome;
|
|
41
44
|
this.#recordFailure = recordFailure;
|
|
45
|
+
this.#interrupted = interrupted;
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* A signal stopped this run's process: status.json and run.json record it `interrupted`. Read
|
|
49
|
+
* when asked, since the signal can arrive after the bundle was published.
|
|
50
|
+
*/
|
|
51
|
+
get interrupted() {
|
|
52
|
+
return this.#interrupted();
|
|
53
|
+
}
|
|
54
|
+
/** The route reported a participant's session started (Run.participantStarted). */
|
|
55
|
+
get participantsRan() {
|
|
56
|
+
return this.#participantsRan();
|
|
42
57
|
}
|
|
43
58
|
static isIssued(value) {
|
|
44
59
|
return typeof value === "object" && value !== null && #observer in value;
|
|
@@ -97,7 +112,7 @@ function observerTarget(options) {
|
|
|
97
112
|
};
|
|
98
113
|
}
|
|
99
114
|
function runPublisher(args) {
|
|
100
|
-
const { options, runId, paths, runStatus, now, admit } = args;
|
|
115
|
+
const { options, runId, paths, runStatus, now, admit, participantsRan } = args;
|
|
101
116
|
const observer = observerTarget(options);
|
|
102
117
|
let pointerWritten = false;
|
|
103
118
|
// The bundle of the last write asked for, and the outcome every later write carries once a
|
|
@@ -167,7 +182,7 @@ function runPublisher(args) {
|
|
|
167
182
|
await writeBundleFiles(bundle, finishedOutcome(current), (publicBundle) => runStatus.restate(runStatusOutcome(publicBundle))).catch(() => undefined);
|
|
168
183
|
return current;
|
|
169
184
|
}));
|
|
170
|
-
return new FinishedRun(issueKey, runId, paths, observer, current, recordFailure);
|
|
185
|
+
return new FinishedRun(issueKey, runId, paths, observer, current, recordFailure, () => runStatus.interrupted, participantsRan);
|
|
171
186
|
};
|
|
172
187
|
const interrupt = async (signal) => {
|
|
173
188
|
interrupted = { state: "interrupted", ok: false, signal, at: new Date(now()).toISOString() };
|
|
@@ -222,7 +237,8 @@ export async function runScope(fn) {
|
|
|
222
237
|
const created = await createRunArtifactPaths(options.cwd, runId);
|
|
223
238
|
if (!created.ok)
|
|
224
239
|
return created;
|
|
225
|
-
// beginRunStatus reads only the mode
|
|
240
|
+
// beginRunStatus reads only the mode, the study provenance and the sandboxes field from
|
|
241
|
+
// the run's options.
|
|
226
242
|
const runStatus = beginRunStatus(created.paths, { ...options, runId });
|
|
227
243
|
status = runStatus;
|
|
228
244
|
let interruptBundle;
|
|
@@ -251,7 +267,16 @@ export async function runScope(fn) {
|
|
|
251
267
|
const openRun = (options, runId, paths, runStatus) => {
|
|
252
268
|
const now = options.now ?? Date.now;
|
|
253
269
|
const createdAt = new Date(now()).toISOString();
|
|
254
|
-
|
|
270
|
+
let participantsRan = false;
|
|
271
|
+
const publisher = runPublisher({
|
|
272
|
+
options,
|
|
273
|
+
runId,
|
|
274
|
+
paths,
|
|
275
|
+
runStatus,
|
|
276
|
+
now,
|
|
277
|
+
admit,
|
|
278
|
+
participantsRan: () => participantsRan,
|
|
279
|
+
});
|
|
255
280
|
let finishCalled = false;
|
|
256
281
|
const checkIdentity = (bundle) => bundle.runId !== runId || bundle.mode !== options.mode
|
|
257
282
|
? "The bundle names another run or mode than this run."
|
|
@@ -262,6 +287,9 @@ export async function runScope(fn) {
|
|
|
262
287
|
mode: options.mode,
|
|
263
288
|
...(options.study === undefined ? {} : { study: options.study }),
|
|
264
289
|
paths,
|
|
290
|
+
participantStarted() {
|
|
291
|
+
participantsRan = true;
|
|
292
|
+
},
|
|
265
293
|
writeSnapshot(bundle) {
|
|
266
294
|
if (closed)
|
|
267
295
|
return refuse("The run scope has closed.");
|
package/dist/run/status.d.ts
CHANGED
|
@@ -53,6 +53,11 @@ export interface RunStatusRecord {
|
|
|
53
53
|
* Read either through studyProvenanceOf.
|
|
54
54
|
*/
|
|
55
55
|
lab?: RunStudyProvenance;
|
|
56
|
+
/**
|
|
57
|
+
* `none` when the route recorded at start that this run creates no sandbox, as a scripted run
|
|
58
|
+
* against an app-url subject does. Absent when the run may create one. Reclaim reads it.
|
|
59
|
+
*/
|
|
60
|
+
sandboxes?: "none";
|
|
56
61
|
/** The pid that owns the run, for local liveness checks. */
|
|
57
62
|
pid: number;
|
|
58
63
|
startedAt: string;
|
|
@@ -80,11 +85,14 @@ export interface RunStatusHandle {
|
|
|
80
85
|
* `running` or `finished` over it. When the run had finished it writes nothing, waits for the
|
|
81
86
|
* writes already queued and resolves false. */
|
|
82
87
|
interrupt(signal: RunInterruptSignal): Promise<boolean>;
|
|
88
|
+
/** True from the moment `interrupt` accepts a signal, before its write lands. */
|
|
89
|
+
readonly interrupted: boolean;
|
|
83
90
|
}
|
|
84
91
|
export interface BeginRunStatusOptions {
|
|
85
92
|
runId: string;
|
|
86
93
|
mode: "dry-run" | "live";
|
|
87
94
|
study?: RunStudyProvenance | undefined;
|
|
95
|
+
sandboxes?: "none" | undefined;
|
|
88
96
|
}
|
|
89
97
|
/**
|
|
90
98
|
* Start a run's status record and keep it fresh. Fire-and-forget by design: a status write that
|
package/dist/run/status.js
CHANGED
|
@@ -64,11 +64,13 @@ export function beginRunStatus(runPaths, options) {
|
|
|
64
64
|
state: "running",
|
|
65
65
|
mode: options.mode,
|
|
66
66
|
...studyFields(options.study),
|
|
67
|
+
...(options.sandboxes === undefined ? {} : { sandboxes: options.sandboxes }),
|
|
67
68
|
pid: process.pid,
|
|
68
69
|
startedAt,
|
|
69
70
|
updatedAt: startedAt,
|
|
70
71
|
};
|
|
71
72
|
let finished = false;
|
|
73
|
+
let interrupted = false;
|
|
72
74
|
let finishedRecord;
|
|
73
75
|
let writing = Promise.resolve();
|
|
74
76
|
const write = (record) => {
|
|
@@ -119,11 +121,15 @@ export function beginRunStatus(runPaths, options) {
|
|
|
119
121
|
return false;
|
|
120
122
|
}
|
|
121
123
|
finished = true;
|
|
124
|
+
interrupted = true;
|
|
122
125
|
clearInterval(timer);
|
|
123
126
|
const completedAt = iso();
|
|
124
127
|
await write({ ...base, state: "interrupted", updatedAt: completedAt, completedAt, signal });
|
|
125
128
|
return true;
|
|
126
129
|
},
|
|
130
|
+
get interrupted() {
|
|
131
|
+
return interrupted;
|
|
132
|
+
},
|
|
127
133
|
};
|
|
128
134
|
return handle;
|
|
129
135
|
}
|
package/dist/run/stored-runs.js
CHANGED
|
@@ -5,7 +5,7 @@ import { bindExistingRunArtifactPaths, RUNS_RELATIVE_ROOT, isSafeRunIdSegment, r
|
|
|
5
5
|
import { assertPreparedSelectedOutputDirectory, bindExistingManagedHumanishOutputDirectory, } from "./contained-output.js";
|
|
6
6
|
import { RUN_BUNDLE_FILE } from "./bundle.js";
|
|
7
7
|
import { isReviewSummary, isRunOutcome } from "./bundle-shape.js";
|
|
8
|
-
import { bundleDisplayFacts, runDisplay } from "./display.js";
|
|
8
|
+
import { bundleDisplayFacts, displayedReview, runDisplay, } from "./display.js";
|
|
9
9
|
import { RUN_STATUS_FILE } from "./status.js";
|
|
10
10
|
import { readLatest, readRunJsonIfExists, resolveRunPath } from "./locate.js";
|
|
11
11
|
import { withCuaReviewProvenance } from "./outcomes.js";
|
|
@@ -101,22 +101,33 @@ export async function listRuns(cwdInput) {
|
|
|
101
101
|
* that is not an object naming a run gets none.
|
|
102
102
|
*/
|
|
103
103
|
async function listedRunDisplay(runPaths, bundle) {
|
|
104
|
-
|
|
104
|
+
const displayed = displayedBundle(bundle);
|
|
105
|
+
if (displayed === undefined)
|
|
105
106
|
return {};
|
|
106
|
-
const
|
|
107
|
-
|
|
108
|
-
|
|
107
|
+
const status = displayed.outcome === undefined
|
|
108
|
+
? await readRunJsonIfExists(runPaths, RUN_STATUS_FILE)
|
|
109
|
+
: undefined;
|
|
110
|
+
return { display: runDisplay(bundleDisplayFacts(displayed, status)) };
|
|
111
|
+
}
|
|
112
|
+
/** The fields runDisplay reads from an unchecked run.json; none when it does not name a run. */
|
|
113
|
+
function displayedBundle(bundle) {
|
|
114
|
+
if (!isRecord(bundle) || typeof bundle.runId !== "string")
|
|
115
|
+
return undefined;
|
|
116
|
+
return {
|
|
109
117
|
runId: bundle.runId,
|
|
110
118
|
...(typeof bundle.mode === "string" ? { mode: bundle.mode } : {}),
|
|
111
119
|
...(isRecord(bundle.review) && typeof bundle.review.verdict === "string"
|
|
112
120
|
? { review: { verdict: bundle.review.verdict } }
|
|
113
121
|
: {}),
|
|
114
122
|
...(Array.isArray(bundle.simulations)
|
|
115
|
-
? {
|
|
123
|
+
? {
|
|
124
|
+
simulations: bundle.simulations
|
|
125
|
+
.filter(isRecord)
|
|
126
|
+
.map((record) => (typeof record.status === "string" ? { status: record.status } : {})),
|
|
127
|
+
}
|
|
116
128
|
: {}),
|
|
117
|
-
...(outcome
|
|
118
|
-
}
|
|
119
|
-
return { display: runDisplay(facts) };
|
|
129
|
+
...(isRunOutcome(bundle.outcome) ? { outcome: bundle.outcome } : {}),
|
|
130
|
+
};
|
|
120
131
|
}
|
|
121
132
|
function runsUnavailableResult(cwd, error) {
|
|
122
133
|
return {
|
|
@@ -159,10 +170,14 @@ export async function readReview(cwdInput, runInput) {
|
|
|
159
170
|
const projected = isRecord(bundle) && Array.isArray(bundle.streams)
|
|
160
171
|
? withCuaReviewProvenance(review, bundle.streams.filter(isRecord))
|
|
161
172
|
: review;
|
|
173
|
+
const displayed = displayedBundle(bundle);
|
|
174
|
+
const { display } = runPaths ? await listedRunDisplay(runPaths, bundle) : {};
|
|
162
175
|
return {
|
|
163
|
-
...
|
|
176
|
+
...(displayed === undefined || display === undefined
|
|
177
|
+
? projected
|
|
178
|
+
: displayedReview(projected, displayed, display)),
|
|
164
179
|
path: path.relative(cwd, path.join(runPaths.absoluteRunRoot, "review.json")),
|
|
165
180
|
runId: path.basename(runPaths.absoluteRunRoot),
|
|
166
|
-
...(
|
|
181
|
+
...(display === undefined ? {} : { display }),
|
|
167
182
|
};
|
|
168
183
|
}
|