humanish 0.110.0 → 0.110.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +15 -12
- package/CHANGELOG.md +26 -0
- package/CONTEXT.md +8 -6
- package/README.md +103 -53
- package/dist/actors/computer-use/loop/ending.js +4 -4
- package/dist/actors/local-agent/claude-participant.d.ts +74 -0
- package/dist/actors/local-agent/claude-participant.js +243 -0
- package/dist/actors/local-agent/claude-session.d.ts +16 -3
- package/dist/actors/local-agent/claude-session.js +64 -23
- package/dist/actors/local-agent/cli.d.ts +17 -0
- package/dist/actors/local-agent/cli.js +111 -24
- package/dist/actors/local-agent/readiness.d.ts +2 -2
- package/dist/actors/local-agent/readiness.js +20 -12
- package/dist/cli/commands/analyze.js +21 -5
- package/dist/cli/commands/observe.js +15 -2
- package/dist/cli/commands/run-signals.d.ts +2 -0
- package/dist/cli/commands/run-signals.js +106 -45
- package/dist/cli/commands/runs.d.ts +5 -0
- package/dist/cli/commands/runs.js +113 -74
- package/dist/cli/commands/setup.js +23 -14
- package/dist/cli/commands/study-route-computer-use.js +9 -3
- package/dist/cli/commands/study-route-open.js +1 -0
- package/dist/cli/commands/study-route-preview.js +1 -0
- package/dist/cli/commands/study-route-scripted.js +2 -0
- package/dist/cli/commands/study-route-shared-world.js +6 -2
- package/dist/cli/commands/study-route-terminal.js +2 -0
- package/dist/cli/commands/watch.js +14 -4
- package/dist/cli/doctor.d.ts +4 -1
- package/dist/cli/doctor.js +51 -10
- package/dist/cli/findings.d.ts +103 -0
- package/dist/cli/findings.js +317 -0
- package/dist/cli/first-run-path.d.ts +32 -11
- package/dist/cli/first-run-path.js +113 -69
- package/dist/cli/invocation.d.ts +7 -0
- package/dist/cli/invocation.js +32 -0
- package/dist/cli/io.d.ts +3 -1
- package/dist/cli/io.js +8 -15
- package/dist/cli/observer-follow.d.ts +20 -0
- package/dist/cli/observer-follow.js +36 -1
- package/dist/cli/orientation.js +16 -14
- package/dist/cli/product-sentence.d.ts +3 -3
- package/dist/cli/product-sentence.js +3 -3
- package/dist/cli/version.d.ts +1 -0
- package/dist/cli/version.js +11 -0
- package/dist/evidence/encoded-text.d.ts +9 -6
- package/dist/evidence/encoded-text.js +84 -19
- package/dist/evidence/redaction.d.ts +2 -0
- package/dist/evidence/redaction.js +123 -21
- package/dist/feedback/export-bundle.js +2 -2
- package/dist/keys/env-file.d.ts +4 -0
- package/dist/keys/env-file.js +17 -0
- package/dist/keys/key-resolution.js +6 -5
- package/dist/observer/data.d.ts +14 -1
- package/dist/observer/data.js +19 -9
- package/dist/observer/exposure.js +1 -1
- package/dist/observer/library.d.ts +4 -0
- package/dist/observer/library.js +21 -17
- package/dist/observer/render.js +10 -3
- package/dist/observer/run-routes.d.ts +2 -0
- package/dist/observer/run-routes.js +51 -35
- package/dist/observer/serve.js +2 -1
- package/dist/observer-app.html +3 -3
- package/dist/routes/computer-use/bundle.js +3 -1
- package/dist/routes/computer-use/fanout-records.js +1 -1
- package/dist/routes/computer-use/participant-execution.js +3 -1
- package/dist/routes/computer-use/participant-model.js +5 -2
- package/dist/routes/computer-use/plan.js +9 -4
- package/dist/routes/computer-use/preflight.js +1 -1
- package/dist/routes/computer-use/result.js +17 -28
- package/dist/routes/computer-use/single-bundle.js +7 -1
- package/dist/routes/scripted/bundle.js +4 -2
- package/dist/routes/scripted/plan.js +2 -2
- package/dist/routes/scripted/result.d.ts +3 -5
- package/dist/routes/scripted/result.js +16 -22
- package/dist/routes/scripted/scenario.js +4 -4
- package/dist/routes/shared-world/bundle-records.js +6 -1
- package/dist/routes/shared-world/bundle.js +2 -0
- package/dist/routes/shared-world/plan.js +6 -2
- package/dist/routes/shared-world/result.js +22 -22
- package/dist/routes/terminal/bundle.js +6 -3
- package/dist/routes/terminal/credentials.d.ts +5 -5
- package/dist/routes/terminal/credentials.js +14 -6
- package/dist/routes/terminal/dry-run.js +8 -12
- package/dist/routes/terminal/ledger.d.ts +3 -3
- package/dist/routes/terminal/ledger.js +6 -6
- package/dist/routes/terminal/live-finish.js +15 -13
- package/dist/routes/terminal/plan.js +12 -4
- package/dist/routes/terminal/result.d.ts +6 -7
- package/dist/routes/terminal/result.js +6 -22
- package/dist/routes/terminal/route.js +5 -4
- package/dist/routes/terminal/runtime-auth.d.ts +6 -0
- package/dist/routes/terminal/runtime-auth.js +5 -0
- package/dist/routes/terminal/session.js +1 -1
- package/dist/run/bundle-shape.d.ts +3 -1
- package/dist/run/bundle-shape.js +25 -1
- package/dist/run/bundle.d.ts +24 -0
- package/dist/run/display.d.ts +59 -0
- package/dist/run/display.js +106 -0
- package/dist/run/dry-run.js +13 -8
- package/dist/run/judge.d.ts +3 -2
- package/dist/run/judge.js +12 -1
- package/dist/run/local-only-files.d.ts +13 -0
- package/dist/run/local-only-files.js +35 -0
- package/dist/run/pricing.js +1 -1
- package/dist/run/projection.d.ts +0 -2
- package/dist/run/projection.js +0 -11
- package/dist/run/reclaim-outcomes.d.ts +89 -0
- package/dist/run/reclaim-outcomes.js +213 -0
- package/dist/run/reclaim.d.ts +23 -17
- package/dist/run/reclaim.js +213 -114
- package/dist/run/results.d.ts +10 -20
- package/dist/run/results.js +2 -2
- package/dist/run/run-index.d.ts +11 -1
- package/dist/run/run-index.js +11 -0
- package/dist/run/run-not-found.js +3 -2
- package/dist/run/run.d.ts +34 -18
- package/dist/run/run.js +166 -67
- package/dist/run/sandbox-creates.d.ts +59 -0
- package/dist/run/sandbox-creates.js +140 -0
- package/dist/run/sandbox-receipts.d.ts +20 -5
- package/dist/run/sandbox-receipts.js +56 -7
- package/dist/run/stats.d.ts +5 -0
- package/dist/run/stats.js +8 -4
- package/dist/run/status.d.ts +6 -9
- package/dist/run/status.js +7 -23
- package/dist/run/stored-runs.d.ts +5 -14
- package/dist/run/stored-runs.js +31 -146
- package/dist/run/streams.d.ts +3 -2
- package/dist/study/agents-section.d.ts +21 -0
- package/dist/study/agents-section.js +86 -0
- package/dist/study/doctor.d.ts +1 -1
- package/dist/study/doctor.js +8 -4
- package/dist/study/init-templates.js +0 -95
- package/dist/study/init.js +45 -37
- package/dist/study/keys.d.ts +12 -0
- package/dist/study/keys.js +32 -0
- package/dist/study/local-browser-host.d.ts +11 -0
- package/dist/study/local-browser-host.js +42 -0
- package/dist/study/parse/actors.js +9 -1
- package/dist/study/parse/subject.js +12 -0
- package/dist/study/parse/url-credentials.d.ts +9 -0
- package/dist/study/parse/url-credentials.js +50 -0
- package/dist/study/persona.d.ts +4 -0
- package/dist/study/persona.js +4 -2
- package/dist/study/preflight-probes.js +2 -1
- package/dist/study/types.d.ts +8 -6
- package/dist/study/url-credentials.d.ts +6 -0
- package/dist/study/url-credentials.js +34 -0
- package/dist/substrates/e2b/peer-install.d.ts +51 -0
- package/dist/substrates/e2b/peer-install.js +161 -0
- package/dist/substrates/e2b/sandbox.d.ts +34 -6
- package/dist/substrates/e2b/sandbox.js +141 -30
- package/dist/substrates/e2b/sdk.d.ts +33 -0
- package/dist/substrates/e2b/sdk.js +21 -21
- package/dist/substrates/local/firecracker-desktop.js +3 -2
- package/dist/substrates/local/runtime-host.js +5 -4
- package/dist/substrates/local/runtime.js +5 -4
- package/dist/substrates/shell.d.ts +2 -0
- package/dist/substrates/shell.js +4 -0
- package/dist/tui/contract.js +5 -4
- package/dist/tui-app.js +133 -133
- package/dist/verify/liveness.d.ts +19 -3
- package/dist/verify/liveness.js +47 -25
- package/dist/verify/verify.d.ts +3 -0
- package/dist/verify/verify.js +8 -3
- package/docs/architecture/actor-contract.md +2 -1
- package/docs/architecture/observer.md +22 -7
- package/docs/architecture/task-protocol-support.md +13 -13
- package/docs/architecture/terminal-product-route.md +15 -13
- package/docs/contracts/run-bundle.md +95 -10
- package/docs/contracts/schemas.md +44 -26
- package/docs/contracts/study-analysis.md +45 -0
- package/docs/contracts/study-costs.md +8 -0
- package/docs/decisions/0001-run-bundle-is-the-source-of-truth.md +3 -0
- package/docs/decisions/0003-contained-paths-and-cleanup-authority.md +9 -3
- package/docs/principles/invariants-and-defaults.md +11 -8
- package/package.json +6 -3
- package/skills/humanish/SKILL.md +38 -20
|
@@ -2,9 +2,11 @@ import { redactText } from "../../evidence/redaction.js";
|
|
|
2
2
|
import { participantAssignment } from "../../study/participant-assignment.js";
|
|
3
3
|
import { REVIEW_SCHEMA, bundleArtifacts, bundleHead, } from "../../run/bundle.js";
|
|
4
4
|
import { verdictText } from "../../run/judge.js";
|
|
5
|
+
import { reviewOutcome } from "../../run/display.js";
|
|
5
6
|
import {} from "../../run/streams.js";
|
|
6
7
|
import { participantEvent, participantIds, participantRecord, participantStream, } from "../../run/participant-records.js";
|
|
7
8
|
import { TERMINAL_EVENTS_ARTIFACT, TERMINAL_LEDGERS_ARTIFACT, TERMINAL_TRANSCRIPT_ARTIFACT, } from "../../run/terminal-contract.js";
|
|
9
|
+
import { DEFAULT_RUNTIME_AUTH } from "./runtime-auth.js";
|
|
8
10
|
import { describeMeasuredSpend, noSpendLineMeasured, noSpendNotEstablished } from "./ledger.js";
|
|
9
11
|
/**
|
|
10
12
|
* Project the terminal-product run into a humanish.run-bundle.v1 (no schema change: a new
|
|
@@ -58,7 +60,7 @@ export function buildTerminalProductBundle(args) {
|
|
|
58
60
|
type: "terminal-lab.credentials.declared",
|
|
59
61
|
// Names-only evidence: the runtime-auth channel is declared; no value is ever
|
|
60
62
|
// recorded. The deny-by-default policies are recorded so the credential posture is auditable.
|
|
61
|
-
message: `Runtime auth channel: ${args.runtimeAuth ??
|
|
63
|
+
message: `Runtime auth channel: ${args.runtimeAuth ?? `${DEFAULT_RUNTIME_AUTH}, the default`} (names only; values never persist; the live engine applies the selected key placement, while this dry-run performs no injection). Credential policies (deny-by-default): allowPrivateRepoAccess=${args.policies.allowPrivateRepoAccess}, allowProviderCredentials=${args.policies.allowProviderCredentials}, allowPaymentCredentials=${args.policies.allowPaymentCredentials}, allowGitHubMutation=${args.policies.allowGitHubMutation}.`,
|
|
62
64
|
}),
|
|
63
65
|
participantEvent(TERMINAL_IDS, {
|
|
64
66
|
id: "event-003-caps",
|
|
@@ -72,7 +74,7 @@ export function buildTerminalProductBundle(args) {
|
|
|
72
74
|
at: args.run.createdAt,
|
|
73
75
|
level: "info",
|
|
74
76
|
type: "terminal-lab.contract.ready",
|
|
75
|
-
message: "Dry-run bundle ready. Switch
|
|
77
|
+
message: "Dry-run bundle ready. Switch mode to live with the required runtime auth and caps to exercise the in-sandbox agent route, captured exec stream, and declared runtime-auth placement.",
|
|
76
78
|
}),
|
|
77
79
|
];
|
|
78
80
|
const review = {
|
|
@@ -172,7 +174,7 @@ export function buildLiveTerminalProductBundle(args) {
|
|
|
172
174
|
// so a green run never silently over-claims a fully-proven $0.
|
|
173
175
|
...(noSpend.unmeasuredLines.length > 0
|
|
174
176
|
? [
|
|
175
|
-
`${noSpendLineMeasured(noSpend) ? noSpendNotEstablished(noSpend.maxUsd ?? 0) : "No-spend proof is partial."} ${describeMeasuredSpend(args.ledgers.cost, args.trace.tokenUsage)}
|
|
177
|
+
`${noSpendLineMeasured(noSpend) ? noSpendNotEstablished(noSpend.maxUsd ?? 0) : "No-spend proof is partial."} ${describeMeasuredSpend(args.ledgers.cost, args.trace.tokenUsage)} humanish has no source for the missing lines, and a study cannot supply one.`,
|
|
176
178
|
]
|
|
177
179
|
: []),
|
|
178
180
|
],
|
|
@@ -283,6 +285,7 @@ export function renderTerminalReviewMarkdown(bundle) {
|
|
|
283
285
|
`- run: ${bundle.runId}`,
|
|
284
286
|
`- mode: ${bundle.mode}`,
|
|
285
287
|
`- verdict: ${verdictText(bundle.review.verdict, bundle.mode)}`,
|
|
288
|
+
`- outcome: ${reviewOutcome(bundle)}`,
|
|
286
289
|
`- summary: ${bundle.review.summary}`,
|
|
287
290
|
`- mission: ${bundle.scenario.goal}`,
|
|
288
291
|
...(subject ? [`- subject: ${subject.message}`] : []),
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
import type { StudyRuntimeAuth } from "../../study/types.js";
|
|
2
2
|
/**
|
|
3
|
-
* Resolve the runtime key on the host.
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
* credential is forwarded. The real value must never be logged or persisted in either mode.
|
|
3
|
+
* Resolve the runtime key on the host. openai-egress, the default, returns an inert command env
|
|
4
|
+
* while retaining the actual value for the external transform and literal redaction; openai-env
|
|
5
|
+
* passes it command-scoped. Only CODEX_API_KEY/OPENAI_API_KEY are accepted as sources. No other
|
|
6
|
+
* operator credential is forwarded. The real value must never be logged or persisted in either mode.
|
|
7
7
|
*/
|
|
8
8
|
export declare function buildRuntimeAuth(args: {
|
|
9
|
-
/** Undefined
|
|
9
|
+
/** Undefined selects openai-egress, so the raw key stays out of the sandbox unless a study opts in. */
|
|
10
10
|
runtimeAuth: StudyRuntimeAuth | undefined;
|
|
11
11
|
/** The operator environment the key value is read from (process.env or a test fake). */
|
|
12
12
|
env: Record<string, string | undefined>;
|
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
import { E2B_SYSTEM_CA_BUNDLE, OPENAI_EGRESS_PLACEHOLDER } from "./runtime-auth.js";
|
|
1
|
+
import { DEFAULT_RUNTIME_AUTH, E2B_SYSTEM_CA_BUNDLE, OPENAI_EGRESS_PLACEHOLDER, } from "./runtime-auth.js";
|
|
2
2
|
import { TERMINAL_PRODUCT_STUDY_PROVIDER_METADATA } from "./types.js";
|
|
3
3
|
/**
|
|
4
|
-
* Resolve the runtime key on the host.
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
* credential is forwarded. The real value must never be logged or persisted in either mode.
|
|
4
|
+
* Resolve the runtime key on the host. openai-egress, the default, returns an inert command env
|
|
5
|
+
* while retaining the actual value for the external transform and literal redaction; openai-env
|
|
6
|
+
* passes it command-scoped. Only CODEX_API_KEY/OPENAI_API_KEY are accepted as sources. No other
|
|
7
|
+
* operator credential is forwarded. The real value must never be logged or persisted in either mode.
|
|
8
8
|
*/
|
|
9
9
|
export function buildRuntimeAuth(args) {
|
|
10
10
|
// The "openai-env" channel accepts CODEX_API_KEY or OPENAI_API_KEY as the runtime key source
|
|
@@ -46,7 +46,15 @@ export function buildRuntimeAuth(args) {
|
|
|
46
46
|
// GITHUB_TOKEN/GH_TOKEN, no payment/deploy/db/media key, excluded by construction. When the
|
|
47
47
|
// source was OPENAI_API_KEY, the same value is also injected as CODEX_API_KEY so codex exec's
|
|
48
48
|
// documented single-invocation auth channel is populated either way (see the comment above).
|
|
49
|
-
const mode = args.runtimeAuth ??
|
|
49
|
+
const mode = args.runtimeAuth ?? DEFAULT_RUNTIME_AUTH;
|
|
50
|
+
// Only the exact openai-env value places the raw key in the command, so a misspelled mode from a
|
|
51
|
+
// config that skipped the planner is refused, never read as openai-env.
|
|
52
|
+
if (mode !== "openai-egress" && mode !== "openai-env")
|
|
53
|
+
return {
|
|
54
|
+
ok: false,
|
|
55
|
+
code: "HUMANISH_TERMINAL_CREDENTIAL_DENIED",
|
|
56
|
+
message: "Runtime auth must be openai-egress or openai-env; no key was placed.",
|
|
57
|
+
};
|
|
50
58
|
const envs = mode === "openai-egress"
|
|
51
59
|
? // Codex documents this verified-TLS trust channel. The stock image's default OpenSSL CA
|
|
52
60
|
// file can be absent even though E2B has installed its proxy CA in the system bundle.
|
|
@@ -68,20 +68,16 @@ export async function runDryTerminalStudy(args) {
|
|
|
68
68
|
verdict: judgment.verdict,
|
|
69
69
|
});
|
|
70
70
|
bundle.events.push(runtimeDeclaredEvent(plan.runtime, createdAt));
|
|
71
|
-
const
|
|
71
|
+
const policy = OUTCOME_POLICIES.terminal;
|
|
72
|
+
const execution = judgeExecution([], policy);
|
|
73
|
+
const finished = await run.finish(bundle, {
|
|
74
|
+
ok: resultOk({ judgment, execution, scorerFailures: [], policy }),
|
|
75
|
+
execution,
|
|
76
|
+
policy,
|
|
77
|
+
});
|
|
72
78
|
const observer = await finished.renderObserver();
|
|
73
79
|
await validatePreparedRunArtifactPaths(finished.paths);
|
|
74
|
-
const
|
|
75
|
-
const execution = judgeExecution(observer.ok
|
|
76
|
-
? []
|
|
77
|
-
: [
|
|
78
|
-
{
|
|
79
|
-
kind: "evidence",
|
|
80
|
-
message: observer.error?.message ?? "Observer failed for the terminal-product run.",
|
|
81
|
-
},
|
|
82
|
-
], policy);
|
|
83
|
-
const ok = resultOk({ judgment, execution, scorerFailures: [], policy });
|
|
84
|
-
await finished.recordOutcome({ ok, execution });
|
|
80
|
+
const { ok } = finished.outcome;
|
|
85
81
|
return {
|
|
86
82
|
...studyResultIdentity("terminal", plan.studyId),
|
|
87
83
|
ok,
|
|
@@ -7,9 +7,9 @@ import type { CostLine, NoSpendProof, TerminalCostLedger } from "./types.js";
|
|
|
7
7
|
* - The `provider` line is populated from the actor trace's tokenUsage.costUsd when the trace
|
|
8
8
|
* carries it (a measured value, incl. a measured 0). When the trace carries no costUsd, the
|
|
9
9
|
* provider line is `null` = not measured (never guessed to 0 just because no-spend was intended).
|
|
10
|
-
* - product/media/payment are `null` by default: core has no signal for those categories
|
|
11
|
-
*
|
|
12
|
-
* `injectedLines` lets a test
|
|
10
|
+
* - product/media/payment are `null` by default: core has no signal for those categories, and
|
|
11
|
+
* only a test can supply one, through `StudyDeps.costProbe`.
|
|
12
|
+
* `injectedLines` lets a test supply known spend for a category,
|
|
13
13
|
* exercising the fail-closed cap enforcement deterministically without a real billable run.
|
|
14
14
|
*/
|
|
15
15
|
export declare function buildCostLedger(args: {
|
|
@@ -6,9 +6,9 @@ import { COST_CATEGORIES } from "../../run/terminal-contract.js";
|
|
|
6
6
|
* - The `provider` line is populated from the actor trace's tokenUsage.costUsd when the trace
|
|
7
7
|
* carries it (a measured value, incl. a measured 0). When the trace carries no costUsd, the
|
|
8
8
|
* provider line is `null` = not measured (never guessed to 0 just because no-spend was intended).
|
|
9
|
-
* - product/media/payment are `null` by default: core has no signal for those categories
|
|
10
|
-
*
|
|
11
|
-
* `injectedLines` lets a test
|
|
9
|
+
* - product/media/payment are `null` by default: core has no signal for those categories, and
|
|
10
|
+
* only a test can supply one, through `StudyDeps.costProbe`.
|
|
11
|
+
* `injectedLines` lets a test supply known spend for a category,
|
|
12
12
|
* exercising the fail-closed cap enforcement deterministically without a real billable run.
|
|
13
13
|
*/
|
|
14
14
|
export function buildCostLedger(args) {
|
|
@@ -39,7 +39,7 @@ export function buildCostLedger(args) {
|
|
|
39
39
|
usd: null,
|
|
40
40
|
count: null,
|
|
41
41
|
source: "unmeasured",
|
|
42
|
-
note: `${category} spend not measured: humanish has no ${category} spend signal
|
|
42
|
+
note: `${category} spend not measured: humanish has no ${category} spend signal, and a study cannot supply one. Recorded as null, never guessed as 0.`,
|
|
43
43
|
});
|
|
44
44
|
const lines = {
|
|
45
45
|
product: args.injectedLines?.product ?? unmeasured("product"),
|
|
@@ -149,7 +149,7 @@ export function evaluateCapsAgainstLedger(ledger, caps) {
|
|
|
149
149
|
const overLines = COST_CATEGORIES.filter((c) => ledger.lines[c].usd !== null && ledger.lines[c].usd > 0);
|
|
150
150
|
return {
|
|
151
151
|
ok: false,
|
|
152
|
-
message: `Stopped: measured spend ${ledger.knownTotalUsd} USD passed
|
|
152
|
+
message: `Stopped: measured spend ${ledger.knownTotalUsd} USD passed caps.maxUsd=${caps.maxUsd}${overLines.length > 0 ? ` (non-zero lines: ${overLines.join(", ")})` : ""}. The cap stops the run; it is not a warning.`,
|
|
153
153
|
};
|
|
154
154
|
}
|
|
155
155
|
if (caps.maxJobs !== undefined) {
|
|
@@ -162,7 +162,7 @@ export function evaluateCapsAgainstLedger(ledger, caps) {
|
|
|
162
162
|
if (knownJobs > caps.maxJobs) {
|
|
163
163
|
return {
|
|
164
164
|
ok: false,
|
|
165
|
-
message: `Stopped: ${knownJobs} measured billable jobs passed
|
|
165
|
+
message: `Stopped: ${knownJobs} measured billable jobs passed caps.maxJobs=${caps.maxJobs}.`,
|
|
166
166
|
};
|
|
167
167
|
}
|
|
168
168
|
}
|
|
@@ -9,7 +9,7 @@ import { normalizeLocalActorTranscript, TERMINAL_EVENTS_ARTIFACT, TERMINAL_LEDGE
|
|
|
9
9
|
import { applyAdapterExtensionSeam } from "./adapter.js";
|
|
10
10
|
import { buildLiveTerminalProductBundle } from "./bundle.js";
|
|
11
11
|
import { buildCostLedger, buildNoSpendProof, describeMeasuredSpend, evaluateCapsAgainstLedger, noSpendLineMeasured, noSpendNotEstablished, } from "./ledger.js";
|
|
12
|
-
import { foldScorerFailures, judgeExecution, judgeTerminal, OUTCOME_POLICIES, } from "../../run/judge.js";
|
|
12
|
+
import { foldScorerFailures, judgeExecution, judgeTerminal, OUTCOME_POLICIES, resultOk, } from "../../run/judge.js";
|
|
13
13
|
import { terminalExecutionFailures, terminalStudyResult, terminalParticipantFacts, } from "./result.js";
|
|
14
14
|
import { parseTerminalTokenUsage } from "./token-usage.js";
|
|
15
15
|
import { buildTerminalActorTrace, scrubSplitKnownValues, tailOf } from "./trace.js";
|
|
@@ -60,7 +60,7 @@ async function settleLiveLedgers(inputs, trace, normalizedTranscript) {
|
|
|
60
60
|
// --- Spend ledger + no-spend proof + full caps enforcement (fail-closed). ---
|
|
61
61
|
// The cost ledger is derived, with the null discipline: provider spend from the trace's
|
|
62
62
|
// tokenUsage.costUsd when present (else null = not measured), product/media/payment null by
|
|
63
|
-
// default (core has no signal). The costProbe hook lets tests
|
|
63
|
+
// default (core has no signal). The costProbe hook lets tests inject known
|
|
64
64
|
// spend to exercise the fail-closed cap without a real billable run.
|
|
65
65
|
const injectedLines = costProbe?.(trace.tokenUsage?.costUsd === undefined ? {} : { tokenCostUsd: trace.tokenUsage.costUsd });
|
|
66
66
|
if (costProbe)
|
|
@@ -171,19 +171,24 @@ export async function finishLiveTerminalSession(inputs) {
|
|
|
171
171
|
// The one final verdict fold: scoring can only make the judged verdict stricter.
|
|
172
172
|
bundle.review = foldScorerFailures(bundle.review, scorer.failures);
|
|
173
173
|
await validatePreparedRunArtifactPaths(runPaths);
|
|
174
|
-
const finished = await run.finish(bundle);
|
|
175
|
-
const observer = await finished.renderObserver();
|
|
176
|
-
await validatePreparedRunArtifactPaths(runPaths);
|
|
177
174
|
// A failing agent is captured evidence on this route: only the execution and a declared scorer
|
|
178
|
-
// fail ok.
|
|
175
|
+
// fail ok. A declared scorer that did not render a pass (status:"fail", malformed or thrown) is a
|
|
176
|
+
// gate, so it fails ok as well as the persisted verdict.
|
|
177
|
+
const policy = OUTCOME_POLICIES.terminal;
|
|
179
178
|
const execution = judgeExecution(terminalExecutionFailures({
|
|
180
179
|
participant,
|
|
181
180
|
capFailure,
|
|
182
181
|
sessionReason: sanitize(session.reason),
|
|
183
182
|
cleanup: session.cleanup,
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
183
|
+
}), policy);
|
|
184
|
+
const finished = await run.finish(bundle, {
|
|
185
|
+
ok: resultOk({ judgment, execution, scorerFailures: scorer.failures, policy }),
|
|
186
|
+
execution,
|
|
187
|
+
policy,
|
|
188
|
+
});
|
|
189
|
+
const observer = await finished.renderObserver();
|
|
190
|
+
await validatePreparedRunArtifactPaths(runPaths);
|
|
191
|
+
return terminalStudyResult({
|
|
187
192
|
cwd,
|
|
188
193
|
studyId: plan.studyId,
|
|
189
194
|
actorId: plan.actor,
|
|
@@ -199,13 +204,10 @@ export async function finishLiveTerminalSession(inputs) {
|
|
|
199
204
|
noSpendProof,
|
|
200
205
|
capFailure,
|
|
201
206
|
declaredScorerFailure: scorer.failures[0],
|
|
202
|
-
|
|
203
|
-
execution,
|
|
207
|
+
ok: finished.outcome.ok,
|
|
204
208
|
observer,
|
|
205
209
|
warnings,
|
|
206
210
|
});
|
|
207
|
-
await finished.recordOutcome({ ok: result.ok, execution });
|
|
208
|
-
return result;
|
|
209
211
|
}
|
|
210
212
|
/** Persist the terminal evidence: redacted events, normalized transcript, ledgers, actor trace. */
|
|
211
213
|
async function writeTerminalEvidence(runPaths, evidence) {
|
|
@@ -6,6 +6,7 @@ import { resolveAutomaticAnalysis } from "../../analysis/automatic-config.js";
|
|
|
6
6
|
import { isReasoningEffort } from "../../actors/reasoning-effort.js";
|
|
7
7
|
import { actorRegistry, isTerminalActorDescriptor } from "../../actors/registry.js";
|
|
8
8
|
import { isNonEmpty, planBase } from "../../study/plan-base.js";
|
|
9
|
+
import { studyUrlCredentialReason } from "../../study/url-credentials.js";
|
|
9
10
|
import { desktopMediaValidationReason, taskProtocolValidationReason, } from "../../study/validation.js";
|
|
10
11
|
import { MAX_SANDBOX_MS } from "../../substrates/e2b/lifetime.js";
|
|
11
12
|
import { NODE_BOOTSTRAP_TIMEOUT_MS } from "../../subject/node-bootstrap.js";
|
|
@@ -26,6 +27,13 @@ export function planTerminalStudy(config, input) {
|
|
|
26
27
|
const mediaReason = desktopMediaValidationReason(config);
|
|
27
28
|
if (mediaReason)
|
|
28
29
|
return refuse("HUMANISH_TERMINAL_SUBJECT_INVALID", mediaReason);
|
|
30
|
+
// A library caller's config skips the parser, which refuses these first.
|
|
31
|
+
const credential = studyUrlCredentialReason(config);
|
|
32
|
+
if (credential)
|
|
33
|
+
return refuse("HUMANISH_TERMINAL_SUBJECT_INVALID", credential);
|
|
34
|
+
const runtimeAuth = config.execution?.runtimeAuth;
|
|
35
|
+
if (runtimeAuth !== undefined && runtimeAuth !== "openai-egress" && runtimeAuth !== "openai-env")
|
|
36
|
+
return refuse("HUMANISH_TERMINAL_CREDENTIAL_DENIED", "`execution.runtimeAuth` must be openai-egress or openai-env. Leave it out for openai-egress, which keeps the OpenAI key outside the sandbox.");
|
|
29
37
|
const analysis = resolveAutomaticAnalysis(config.review?.analysis);
|
|
30
38
|
if (!analysis.ok)
|
|
31
39
|
return refuse("HUMANISH_STUDY_ANALYSIS_INVALID", analysis.message);
|
|
@@ -35,7 +43,7 @@ export function planTerminalStudy(config, input) {
|
|
|
35
43
|
const actorType = config.actors[0]?.type ?? "";
|
|
36
44
|
const descriptor = actorRegistry[actorType];
|
|
37
45
|
if (!descriptor || !isTerminalActorDescriptor(descriptor))
|
|
38
|
-
return refuse("HUMANISH_TERMINAL_ACTOR_UNSUPPORTED", `
|
|
46
|
+
return refuse("HUMANISH_TERMINAL_ACTOR_UNSUPPORTED", `actor.type "${actorType}" is not a registered terminal actor.`);
|
|
39
47
|
const runtimeVersion = config.execution?.runtime?.version;
|
|
40
48
|
const actor = config.actors[0];
|
|
41
49
|
if ((config.execution?.runtime !== undefined && !isExactRuntimeVersion(runtimeVersion)) ||
|
|
@@ -87,14 +95,14 @@ export function planTerminalStudy(config, input) {
|
|
|
87
95
|
const maxUsd = caps?.maxUsd;
|
|
88
96
|
const maxMinutes = caps?.maxMinutes;
|
|
89
97
|
if (caps === undefined || maxUsd === undefined || maxMinutes === undefined || maxMinutes <= 0)
|
|
90
|
-
return refuse("HUMANISH_TERMINAL_CAPS_MISSING", "A live terminal-product run gives the agent in the sandbox access to a model provider, so it needs a cap: set
|
|
98
|
+
return refuse("HUMANISH_TERMINAL_CAPS_MISSING", "A live terminal-product run gives the agent in the sandbox access to a model provider, so it needs a cap: set caps with maxUsd (0 means no spend) and a positive maxMinutes (the time limit for the codex command). The key is used only while the cap is in force.", descriptor.id);
|
|
91
99
|
// maxUsd is checked against the cost ledger after the session, and only known lines can trip it.
|
|
92
100
|
// The ledger's provider line takes only provider-reported cost, which Codex does not report (the
|
|
93
101
|
// token estimate in run.json is not a measurement), and core has no product, media or payment
|
|
94
102
|
// signal, so without a costProbe every line is null and a positive maxUsd could never trip. That
|
|
95
103
|
// cap would promise a bound nothing enforces, so it is refused; maxMinutes bounds a live run.
|
|
96
104
|
if (maxUsd > 0 && input.hasCostProbe !== true)
|
|
97
|
-
return refuse("HUMANISH_TERMINAL_UNPRICED_CAP", `
|
|
105
|
+
return refuse("HUMANISH_TERMINAL_UNPRICED_CAP", `caps.maxUsd=${maxUsd} cannot be enforced: Codex reports no provider cost for the participant (its token cost is only estimated after the run) and no product, media or payment spend is measured, so a positive dollar cap can never trip. Set caps.maxUsd to 0 and bound the run with caps.maxMinutes, the codex command's wall-clock kill. No sandbox was created and the runtime key was not used.`, descriptor.id);
|
|
98
106
|
// The sandbox's timeout covers the steps before the codex command, maxMinutes and the teardown
|
|
99
107
|
// buffer, and E2B refuses a sandbox over an hour. Refuse here, with the arithmetic, rather than
|
|
100
108
|
// after the plan prints, from a provider 400 that names neither knob.
|
|
@@ -102,7 +110,7 @@ export function planTerminalStudy(config, input) {
|
|
|
102
110
|
const sandboxTimeoutMs = terminalSandboxTimeoutMs({ maxMinutes, productInstall });
|
|
103
111
|
if (sandboxTimeoutMs > MAX_SANDBOX_MS) {
|
|
104
112
|
const headroomMinutes = (sandboxTimeoutMs - maxMinutes * 60_000) / 60_000;
|
|
105
|
-
return refuse("HUMANISH_TERMINAL_CAPS_INVALID", `
|
|
113
|
+
return refuse("HUMANISH_TERMINAL_CAPS_INVALID", `caps.maxMinutes ${maxMinutes} derives a ${sandboxTimeoutMs / 60_000}m sandbox deadline, and a sandbox may not live longer than ${MAX_SANDBOX_MS / 60_000}m. The deadline is maxMinutes plus ${headroomMinutes}m: the Node bootstrap (${NODE_BOOTSTRAP_TIMEOUT_MS / 60_000}m), the runtime version check (${TERMINAL_RUNTIME_VERSION_TIMEOUT_MS / 60_000}m)${productInstall ? `, the product setup (${PRODUCT_SETUP_TIMEOUT_MS / 60_000}m)` : ""} and the teardown buffer (${TERMINAL_SANDBOX_TIMEOUT_BUFFER_MS / 60_000}m). Lower caps.maxMinutes to at most ${MAX_SANDBOX_MS / 60_000 - headroomMinutes}. No sandbox was created and the runtime key was not used.`, descriptor.id);
|
|
106
114
|
}
|
|
107
115
|
return {
|
|
108
116
|
ok: true,
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { ObserverResult } from "../../observer/render.js";
|
|
2
2
|
import type { ActorCompletionReason, ActorStatus, ActorTrace } from "../../actors/contract.js";
|
|
3
|
-
import { type ExecutionFailure, type
|
|
3
|
+
import { type ExecutionFailure, type ParticipantFacts } from "../../run/judge.js";
|
|
4
4
|
import { type NoSpendProof, type TerminalCostLedger, type TerminalLedgers, type TerminalProductStudyResult } from "./types.js";
|
|
5
5
|
/**
|
|
6
6
|
* What judge.ts reads from a finished terminal session, after a blown cap has overridden it. It has
|
|
@@ -8,8 +8,9 @@ import { type NoSpendProof, type TerminalCostLedger, type TerminalLedgers, type
|
|
|
8
8
|
*/
|
|
9
9
|
export declare function terminalParticipantFacts(trace: ActorTrace, sessionError: string | undefined): ParticipantFacts;
|
|
10
10
|
/**
|
|
11
|
-
* The live run's execution failures: a harness error, a blown spend
|
|
12
|
-
*
|
|
11
|
+
* The live run's execution failures before the Observer renders: a harness error, a blown spend
|
|
12
|
+
* cap and an unproven sandbox teardown. FinishedRun.renderObserver adds an Observer that did not
|
|
13
|
+
* render.
|
|
13
14
|
*/
|
|
14
15
|
export declare function terminalExecutionFailures(args: {
|
|
15
16
|
participant: ParticipantFacts;
|
|
@@ -18,7 +19,6 @@ export declare function terminalExecutionFailures(args: {
|
|
|
18
19
|
/** Already scrubbed and redacted. */
|
|
19
20
|
sessionReason: string;
|
|
20
21
|
cleanup: TerminalLedgers["cleanup"];
|
|
21
|
-
observer: Pick<ObserverResult, "ok" | "error">;
|
|
22
22
|
}): ExecutionFailure[];
|
|
23
23
|
/** The terminal-product study result for a live run, from its session, cleanup and cost ledger. */
|
|
24
24
|
export declare function terminalStudyResult(args: {
|
|
@@ -39,9 +39,8 @@ export declare function terminalStudyResult(args: {
|
|
|
39
39
|
/** The cap check's message when known spend or jobs exceeded the caps. */
|
|
40
40
|
capFailure: string | undefined;
|
|
41
41
|
declaredScorerFailure: string | undefined;
|
|
42
|
-
/**
|
|
43
|
-
|
|
44
|
-
execution: ExecutionOutcome;
|
|
42
|
+
/** run.json's outcome.ok (FinishedRun.outcome). On this evidence route it does not read whether the agent passed. */
|
|
43
|
+
ok: boolean;
|
|
45
44
|
observer: ObserverResult;
|
|
46
45
|
warnings: string[];
|
|
47
46
|
}): TerminalProductStudyResult;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { participantHarnessFailed, } from "../../run/judge.js";
|
|
2
2
|
import {} from "./types.js";
|
|
3
3
|
import { studyResultIdentity } from "../../run/study-result.js";
|
|
4
4
|
/**
|
|
@@ -16,11 +16,12 @@ export function terminalParticipantFacts(trace, sessionError) {
|
|
|
16
16
|
};
|
|
17
17
|
}
|
|
18
18
|
/**
|
|
19
|
-
* The live run's execution failures: a harness error, a blown spend
|
|
20
|
-
*
|
|
19
|
+
* The live run's execution failures before the Observer renders: a harness error, a blown spend
|
|
20
|
+
* cap and an unproven sandbox teardown. FinishedRun.renderObserver adds an Observer that did not
|
|
21
|
+
* render.
|
|
21
22
|
*/
|
|
22
23
|
export function terminalExecutionFailures(args) {
|
|
23
|
-
const { participant, cleanup
|
|
24
|
+
const { participant, cleanup } = args;
|
|
24
25
|
return [
|
|
25
26
|
...(participantHarnessFailed(participant)
|
|
26
27
|
? [{ kind: "harness", message: participant.sessionError ?? args.sessionReason }]
|
|
@@ -34,34 +35,17 @@ export function terminalExecutionFailures(args) {
|
|
|
34
35
|
message: `Sandbox teardown unproven (killed=${cleanup.killed}, remaining=${cleanup.remaining}): ${cleanup.reason}`,
|
|
35
36
|
},
|
|
36
37
|
]),
|
|
37
|
-
...(observer.ok
|
|
38
|
-
? []
|
|
39
|
-
: [
|
|
40
|
-
{
|
|
41
|
-
kind: "evidence",
|
|
42
|
-
message: observer.error?.message ?? "Observer failed for the terminal-product run.",
|
|
43
|
-
},
|
|
44
|
-
]),
|
|
45
38
|
];
|
|
46
39
|
}
|
|
47
40
|
/** The terminal-product study result for a live run, from its session, cleanup and cost ledger. */
|
|
48
41
|
export function terminalStudyResult(args) {
|
|
49
|
-
const { cwd, studyId, actorId, productName, runId, sessionStatus, completionReason, sessionReason, sessionError, sandboxId, cleanup, cost, noSpendProof, capFailure, declaredScorerFailure,
|
|
42
|
+
const { cwd, studyId, actorId, productName, runId, sessionStatus, completionReason, sessionReason, sessionError, sandboxId, cleanup, cost, noSpendProof, capFailure, declaredScorerFailure, ok, observer, warnings, } = args;
|
|
50
43
|
// The run's exit code: verified evidence, no harness error and proven cleanup. A blocked/
|
|
51
44
|
// timed-out agent run is still ok-as-evidence at the bundle level (the failure is the evidence),
|
|
52
45
|
// but the study result surfaces ok:false on a harness error or unproven teardown (fail-closed).
|
|
53
46
|
// remaining===0 is the by-id-confirmed-reclaimed state; remaining===1 (still present) and
|
|
54
47
|
// remaining===-1 (kill(id) itself failed) are both unproven by design.
|
|
55
48
|
const cleanupProven = cleanup.killed && cleanup.remaining === 0;
|
|
56
|
-
// A config-declared scorer that failed to render a pass (status:"fail" / malformed / throw) fails the
|
|
57
|
-
// run result as well as the persisted verdict: the keystone route's declared rubric is a gate, so
|
|
58
|
-
// its fail must drive exit code. Library callers never set this (additive, back-compat).
|
|
59
|
-
const ok = resultOk({
|
|
60
|
-
judgment,
|
|
61
|
-
execution,
|
|
62
|
-
scorerFailures: declaredScorerFailure === undefined ? [] : [declaredScorerFailure],
|
|
63
|
-
policy: OUTCOME_POLICIES.terminal,
|
|
64
|
-
});
|
|
65
49
|
return {
|
|
66
50
|
...studyResultIdentity("terminal", studyId),
|
|
67
51
|
ok,
|
|
@@ -11,10 +11,11 @@
|
|
|
11
11
|
// The safety contract (`docs/history/goals/terminal-product-lane/goal.md`) is enforced by construction in
|
|
12
12
|
// the files each item names, and checked by the verifier (verify/actor.ts
|
|
13
13
|
// validateTerminalProductEvidence):
|
|
14
|
-
// 1. Explicit key placement. openai-
|
|
15
|
-
//
|
|
16
|
-
//
|
|
17
|
-
//
|
|
14
|
+
// 1. Explicit key placement. openai-egress (default) sends the raw runtime key only in the
|
|
15
|
+
// host-side E2B header transform and passes an inert command placeholder. The proxy is
|
|
16
|
+
// spendable by every sandbox process from creation; this protects the raw key, not provider
|
|
17
|
+
// spending. Opt-in openai-env injects the raw key command-scoped, never through
|
|
18
|
+
// Sandbox.create({envs}).
|
|
18
19
|
// Enforced in credentials.ts and runtime-auth.ts.
|
|
19
20
|
// 2. Fail-closed cap. The live key is never exercised without scenario.caps in force: maxUsd
|
|
20
21
|
// (default/require 0 = no-spend) + maxMinutes (wall-clock kill of the codex command).
|
|
@@ -1,4 +1,10 @@
|
|
|
1
|
+
import type { StudyRuntimeAuth } from "../../study/types.js";
|
|
1
2
|
import type { E2BNetworkOptions } from "../../substrates/e2b/sdk.js";
|
|
3
|
+
/**
|
|
4
|
+
* The runtime auth a study gets when it declares none. A shell agent can read its own command env,
|
|
5
|
+
* so the raw key stays in the host-side E2B request unless a study declares openai-env.
|
|
6
|
+
*/
|
|
7
|
+
export declare const DEFAULT_RUNTIME_AUTH: StudyRuntimeAuth;
|
|
2
8
|
/** E2B envd installs the sandbox-specific proxy CA into this system bundle before routing. */
|
|
3
9
|
export declare const E2B_SYSTEM_CA_BUNDLE = "/etc/ssl/certs/ca-certificates.crt";
|
|
4
10
|
/** An inert nonsecret value satisfies Codex exec's auth prerequisite. E2B replaces the header. */
|
|
@@ -1,3 +1,8 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The runtime auth a study gets when it declares none. A shell agent can read its own command env,
|
|
3
|
+
* so the raw key stays in the host-side E2B request unless a study declares openai-env.
|
|
4
|
+
*/
|
|
5
|
+
export const DEFAULT_RUNTIME_AUTH = "openai-egress";
|
|
1
6
|
const OPENAI_EGRESS_HOST = "api.openai.com";
|
|
2
7
|
/** E2B envd installs the sandbox-specific proxy CA into this system bundle before routing. */
|
|
3
8
|
export const E2B_SYSTEM_CA_BUNDLE = "/etc/ssl/certs/ca-certificates.crt";
|
|
@@ -20,7 +20,7 @@ export function checkLiveTerminalMachine(plan, input, warnings) {
|
|
|
20
20
|
const { maxUsd } = plan.caps;
|
|
21
21
|
// planTerminalStudy refuses a positive maxUsd without a costProbe, so one is present here.
|
|
22
22
|
if (maxUsd > 0) {
|
|
23
|
-
warnings.push(`
|
|
23
|
+
warnings.push(`caps.maxUsd=${maxUsd} is checked after the session against the measured cost lines; a line left null (unmeasured) never trips it. caps.maxMinutes bounds the run while it runs.`);
|
|
24
24
|
}
|
|
25
25
|
// --- Deny-by-default credentials: build the command-scoped allowlist. ---
|
|
26
26
|
const runtimeEnv = buildRuntimeAuth({ runtimeAuth: plan.runtime.auth, env });
|
|
@@ -1,6 +1,8 @@
|
|
|
1
|
-
import { type ReviewSummary, type RunAdapterScore, type RunBundle } from "./bundle.js";
|
|
1
|
+
import { type ReviewSummary, type RunAdapterScore, type RunBundle, type RunOutcome } from "./bundle.js";
|
|
2
2
|
import { type StoredCleanupResult, type RunPointer } from "./results.js";
|
|
3
3
|
export declare function isRunBundle(value: unknown): value is RunBundle;
|
|
4
|
+
/** run.json's `outcome`: a finished run's ok and execution outcome, or the signal that stopped it. */
|
|
5
|
+
export declare function isRunOutcome(value: unknown): value is RunOutcome;
|
|
4
6
|
export declare function isCleanupResult(value: unknown): value is StoredCleanupResult;
|
|
5
7
|
export declare function isRunAdapterScore(value: unknown): value is RunAdapterScore;
|
|
6
8
|
export declare const ARCHIVE_SHA256_PATTERN: RegExp;
|
package/dist/run/bundle-shape.js
CHANGED
|
@@ -5,6 +5,7 @@ import { CLEANUP_SCHEMA, } from "./results.js";
|
|
|
5
5
|
import { isRunSimulationStatus, isRunStream, isRunStreamKind } from "./stream-shape.js";
|
|
6
6
|
import { isLocalEvidenceArtifactPath } from "./paths.js";
|
|
7
7
|
import { isRunFeedbackCandidate } from "./feedback-shape.js";
|
|
8
|
+
import { EXECUTION_FAILURE_KINDS } from "./judge.js";
|
|
8
9
|
import { isSharedWorldEvidence } from "./shared-world-shape.js";
|
|
9
10
|
import { isNonNegativeSafeInteger, isPositiveSafeInteger, isRecord } from "./type-guards.js";
|
|
10
11
|
export function isRunBundle(value) {
|
|
@@ -64,7 +65,30 @@ export function isRunBundle(value) {
|
|
|
64
65
|
value.adapterArtifacts.every(isRunAdapterArtifact))) &&
|
|
65
66
|
(value.providerResources === undefined ||
|
|
66
67
|
(Array.isArray(value.providerResources) &&
|
|
67
|
-
value.providerResources.every(isRunProviderResource)))
|
|
68
|
+
value.providerResources.every(isRunProviderResource))) &&
|
|
69
|
+
(value.outcome === undefined || isRunOutcome(value.outcome)));
|
|
70
|
+
}
|
|
71
|
+
function isExecutionFailureList(value) {
|
|
72
|
+
return (Array.isArray(value) &&
|
|
73
|
+
value.every((failure) => isRecord(failure) &&
|
|
74
|
+
typeof failure.kind === "string" &&
|
|
75
|
+
EXECUTION_FAILURE_KINDS.includes(failure.kind) &&
|
|
76
|
+
typeof failure.message === "string"));
|
|
77
|
+
}
|
|
78
|
+
/** run.json's `outcome`: a finished run's ok and execution outcome, or the signal that stopped it. */
|
|
79
|
+
export function isRunOutcome(value) {
|
|
80
|
+
if (!isRecord(value))
|
|
81
|
+
return false;
|
|
82
|
+
if (value.state === "interrupted")
|
|
83
|
+
return (value.ok === false &&
|
|
84
|
+
(value.signal === "SIGINT" || value.signal === "SIGTERM" || value.signal === "SIGHUP") &&
|
|
85
|
+
typeof value.at === "string");
|
|
86
|
+
return (value.state === "finished" &&
|
|
87
|
+
typeof value.ok === "boolean" &&
|
|
88
|
+
isRecord(value.execution) &&
|
|
89
|
+
typeof value.execution.succeeded === "boolean" &&
|
|
90
|
+
isExecutionFailureList(value.execution.failures) &&
|
|
91
|
+
(value.execution.warnings === undefined || isExecutionFailureList(value.execution.warnings)));
|
|
68
92
|
}
|
|
69
93
|
function isRunProviderResource(value) {
|
|
70
94
|
return (isRecord(value) &&
|
package/dist/run/bundle.d.ts
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
import type { CommsReceivingEvidence } from "../comms/receiving-types.js";
|
|
2
2
|
import { type CapturedGitState } from "./git-state.js";
|
|
3
|
+
import type { ExecutionOutcome } from "./judge.js";
|
|
3
4
|
import type { SharedWorldEvidence } from "./shared-world-evidence.js";
|
|
4
5
|
import { type RunStudyProvenance } from "./study-provenance.js";
|
|
6
|
+
import type { RunInterruptSignal } from "./status.js";
|
|
5
7
|
import type { RunSimulationStatus, RunStream, RunStreamKind } from "./streams.js";
|
|
6
8
|
export declare const RUN_BUNDLE_SCHEMA = "humanish.run-bundle.v1";
|
|
7
9
|
/** The run bundle file, relative to the run directory. */
|
|
@@ -252,6 +254,13 @@ export interface RunBundle {
|
|
|
252
254
|
events: string;
|
|
253
255
|
};
|
|
254
256
|
review: ReviewSummary;
|
|
257
|
+
/**
|
|
258
|
+
* How the run ended as an execution. `review.verdict` is what the participants experienced;
|
|
259
|
+
* this is whether the run worked, which no surface may show as a pass when `ok` is false. Absent
|
|
260
|
+
* while the run is in progress, when its process died without writing it, and on runs recorded
|
|
261
|
+
* before this field, whose status.json holds the outcome instead.
|
|
262
|
+
*/
|
|
263
|
+
outcome?: RunOutcome;
|
|
255
264
|
feedbackCandidates: RunFeedbackCandidate[];
|
|
256
265
|
/** Structured subject provenance. Optional and additive: emitted by the
|
|
257
266
|
* computer-use and shared-world backends and the clone scripted-browser route; tolerated absent
|
|
@@ -347,6 +356,21 @@ export interface RunBundle {
|
|
|
347
356
|
*/
|
|
348
357
|
cost?: RunCostSummary;
|
|
349
358
|
}
|
|
359
|
+
/**
|
|
360
|
+
* How a run ended. `finished`: Run.finish published the final bundle with the route result's `ok`
|
|
361
|
+
* and its execution outcome (`judgeExecution` in src/run/judge.ts). `interrupted`: the CLI's signal
|
|
362
|
+
* handler stopped the run before it finished; run.json is then the last live flush.
|
|
363
|
+
*/
|
|
364
|
+
export type RunOutcome = {
|
|
365
|
+
state: "finished";
|
|
366
|
+
ok: boolean;
|
|
367
|
+
execution: ExecutionOutcome;
|
|
368
|
+
} | {
|
|
369
|
+
state: "interrupted";
|
|
370
|
+
ok: false;
|
|
371
|
+
signal: RunInterruptSignal;
|
|
372
|
+
at: string;
|
|
373
|
+
};
|
|
350
374
|
/**
|
|
351
375
|
* One contributing cost line of a RunCostSummary. A line is present even when it cannot be priced
|
|
352
376
|
* (it records that we tried and could not): an unpriceable line carries estimatedCostUsd: null + a
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import type { RunOutcome } from "./bundle.js";
|
|
2
|
+
import type { ExecutionFailure } from "./judge.js";
|
|
3
|
+
import { type RunLiveness } from "./status.js";
|
|
4
|
+
/** The states a run is shown in. */
|
|
5
|
+
export type RunDisplayState = "running" | "interrupted" | "passed" | "failed" | "blocked" | "timed_out" | "no_verdict" | "dry_run" | "unknown";
|
|
6
|
+
/** A run as a surface shows it. */
|
|
7
|
+
export interface RunDisplay {
|
|
8
|
+
state: RunDisplayState;
|
|
9
|
+
/** The state in words: "passed", "timed out", "no verdict". */
|
|
10
|
+
label: string;
|
|
11
|
+
/** How a surface colors it: `pass` only for `passed`, `fail` only for `failed`. */
|
|
12
|
+
tone: "pass" | "fail" | "warn" | "live" | "neutral";
|
|
13
|
+
/** The first execution failure that failed the run, as `kind: message`. */
|
|
14
|
+
reason?: string;
|
|
15
|
+
}
|
|
16
|
+
/** What runDisplay reads: whether the run is alive, what its participants experienced, and its ok. */
|
|
17
|
+
export interface RunDisplayFacts {
|
|
18
|
+
liveness: RunLiveness;
|
|
19
|
+
mode?: string | null | undefined;
|
|
20
|
+
/** `review.verdict`. */
|
|
21
|
+
verdict?: string | undefined;
|
|
22
|
+
/** The run's own ok from run.json's outcome; absent when the run recorded none. */
|
|
23
|
+
ok?: boolean | undefined;
|
|
24
|
+
/** The first execution failure that failed the run. */
|
|
25
|
+
failure?: ExecutionFailure | undefined;
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* A run's state. A live or interrupted run is that, whatever its last flush said. A finished run
|
|
29
|
+
* shows its participants' blocked, timed-out or failed verdict; otherwise a run whose ok is false
|
|
30
|
+
* is failed, even with a pass verdict, because its execution failed. Only then does a pass verdict
|
|
31
|
+
* show as passed.
|
|
32
|
+
*/
|
|
33
|
+
export declare function runDisplay(facts: RunDisplayFacts): RunDisplay;
|
|
34
|
+
/** A state in words: "passed", "timed out", "no verdict". */
|
|
35
|
+
export declare function runDisplayLabel(state: RunDisplayState): string;
|
|
36
|
+
/** What bundleDisplayFacts reads from run.json; a listing reads it from an unchecked file. */
|
|
37
|
+
export interface DisplayedBundle {
|
|
38
|
+
runId: string;
|
|
39
|
+
mode?: string | undefined;
|
|
40
|
+
review?: {
|
|
41
|
+
verdict?: string | undefined;
|
|
42
|
+
} | undefined;
|
|
43
|
+
simulations?: ReadonlyArray<{
|
|
44
|
+
status?: string | undefined;
|
|
45
|
+
} | undefined> | undefined;
|
|
46
|
+
outcome?: RunOutcome | undefined;
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* The facts for a run from its bundle. run.json's `outcome` decides when it has one. A bundle
|
|
50
|
+
* without one is in progress, or was recorded before the field existed: its status record, when
|
|
51
|
+
* given and naming the run, then says whether the run is alive and, for the older run, holds its ok.
|
|
52
|
+
* With no record, a bundle whose simulations are still running is live.
|
|
53
|
+
*/
|
|
54
|
+
export declare function bundleDisplayFacts(bundle: DisplayedBundle, status?: unknown, nowMs?: number): RunDisplayFacts;
|
|
55
|
+
/**
|
|
56
|
+
* How review.md says the run ended: the display of the bundle it is rendered from, so a run whose
|
|
57
|
+
* execution failed never reads as a pass beside its participants' verdict.
|
|
58
|
+
*/
|
|
59
|
+
export declare function reviewOutcome(bundle: DisplayedBundle): string;
|