humanish 0.110.0 → 0.110.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (132) hide show
  1. package/ARCHITECTURE.md +14 -11
  2. package/CHANGELOG.md +13 -0
  3. package/CONTEXT.md +8 -6
  4. package/README.md +96 -47
  5. package/dist/actors/computer-use/loop/ending.js +4 -4
  6. package/dist/actors/local-agent/claude-participant.d.ts +74 -0
  7. package/dist/actors/local-agent/claude-participant.js +243 -0
  8. package/dist/actors/local-agent/claude-session.d.ts +16 -3
  9. package/dist/actors/local-agent/claude-session.js +64 -23
  10. package/dist/actors/local-agent/cli.d.ts +11 -0
  11. package/dist/actors/local-agent/cli.js +103 -24
  12. package/dist/actors/local-agent/readiness.d.ts +2 -2
  13. package/dist/actors/local-agent/readiness.js +18 -11
  14. package/dist/cli/commands/analyze.js +21 -5
  15. package/dist/cli/commands/run-signals.d.ts +2 -0
  16. package/dist/cli/commands/run-signals.js +96 -44
  17. package/dist/cli/commands/runs.d.ts +5 -0
  18. package/dist/cli/commands/runs.js +86 -70
  19. package/dist/cli/commands/study-route-computer-use.js +2 -0
  20. package/dist/cli/commands/study-route-scripted.js +2 -0
  21. package/dist/cli/commands/study-route-shared-world.js +2 -0
  22. package/dist/cli/commands/study-route-terminal.js +2 -0
  23. package/dist/cli/doctor.d.ts +4 -1
  24. package/dist/cli/doctor.js +42 -4
  25. package/dist/cli/findings.d.ts +103 -0
  26. package/dist/cli/findings.js +316 -0
  27. package/dist/cli/first-run-path.d.ts +5 -4
  28. package/dist/cli/first-run-path.js +14 -7
  29. package/dist/cli/io.js +2 -1
  30. package/dist/cli/product-sentence.d.ts +3 -3
  31. package/dist/cli/product-sentence.js +3 -3
  32. package/dist/keys/env-file.d.ts +4 -0
  33. package/dist/keys/env-file.js +17 -0
  34. package/dist/observer/data.d.ts +14 -1
  35. package/dist/observer/data.js +19 -9
  36. package/dist/observer/library.d.ts +4 -0
  37. package/dist/observer/library.js +18 -15
  38. package/dist/observer/render.js +8 -2
  39. package/dist/observer/run-routes.d.ts +2 -0
  40. package/dist/observer/run-routes.js +43 -34
  41. package/dist/observer-app.html +3 -3
  42. package/dist/routes/computer-use/bundle.js +3 -1
  43. package/dist/routes/computer-use/fanout-records.js +1 -1
  44. package/dist/routes/computer-use/participant-execution.js +3 -1
  45. package/dist/routes/computer-use/participant-model.js +5 -2
  46. package/dist/routes/computer-use/plan.js +4 -4
  47. package/dist/routes/computer-use/preflight.js +1 -1
  48. package/dist/routes/computer-use/result.js +17 -28
  49. package/dist/routes/computer-use/single-bundle.js +7 -1
  50. package/dist/routes/scripted/bundle.js +4 -2
  51. package/dist/routes/scripted/plan.js +2 -2
  52. package/dist/routes/scripted/result.d.ts +3 -5
  53. package/dist/routes/scripted/result.js +16 -22
  54. package/dist/routes/scripted/scenario.js +4 -4
  55. package/dist/routes/shared-world/bundle-records.js +6 -1
  56. package/dist/routes/shared-world/bundle.js +2 -0
  57. package/dist/routes/shared-world/plan.js +1 -1
  58. package/dist/routes/shared-world/result.js +22 -22
  59. package/dist/routes/terminal/bundle.js +4 -2
  60. package/dist/routes/terminal/dry-run.js +8 -12
  61. package/dist/routes/terminal/ledger.d.ts +3 -3
  62. package/dist/routes/terminal/ledger.js +6 -6
  63. package/dist/routes/terminal/live-finish.js +15 -13
  64. package/dist/routes/terminal/plan.js +4 -4
  65. package/dist/routes/terminal/result.d.ts +6 -7
  66. package/dist/routes/terminal/result.js +6 -22
  67. package/dist/routes/terminal/session.js +1 -1
  68. package/dist/run/bundle-shape.d.ts +3 -1
  69. package/dist/run/bundle-shape.js +25 -1
  70. package/dist/run/bundle.d.ts +24 -0
  71. package/dist/run/display.d.ts +59 -0
  72. package/dist/run/display.js +106 -0
  73. package/dist/run/dry-run.js +11 -7
  74. package/dist/run/judge.d.ts +3 -2
  75. package/dist/run/judge.js +10 -0
  76. package/dist/run/pricing.js +1 -1
  77. package/dist/run/projection.d.ts +0 -2
  78. package/dist/run/projection.js +0 -11
  79. package/dist/run/reclaim-outcomes.d.ts +89 -0
  80. package/dist/run/reclaim-outcomes.js +213 -0
  81. package/dist/run/reclaim.d.ts +18 -17
  82. package/dist/run/reclaim.js +180 -113
  83. package/dist/run/results.d.ts +10 -20
  84. package/dist/run/results.js +2 -2
  85. package/dist/run/run-index.d.ts +11 -1
  86. package/dist/run/run-index.js +11 -0
  87. package/dist/run/run.d.ts +34 -18
  88. package/dist/run/run.js +166 -67
  89. package/dist/run/sandbox-creates.d.ts +59 -0
  90. package/dist/run/sandbox-creates.js +138 -0
  91. package/dist/run/sandbox-receipts.d.ts +14 -0
  92. package/dist/run/sandbox-receipts.js +44 -0
  93. package/dist/run/stats.d.ts +5 -0
  94. package/dist/run/stats.js +8 -4
  95. package/dist/run/status.d.ts +6 -9
  96. package/dist/run/status.js +7 -23
  97. package/dist/run/stored-runs.d.ts +5 -14
  98. package/dist/run/stored-runs.js +31 -146
  99. package/dist/run/streams.d.ts +3 -2
  100. package/dist/study/doctor.d.ts +1 -1
  101. package/dist/study/doctor.js +5 -2
  102. package/dist/study/init.js +14 -3
  103. package/dist/study/keys.d.ts +12 -0
  104. package/dist/study/keys.js +32 -0
  105. package/dist/study/persona.d.ts +4 -0
  106. package/dist/study/persona.js +4 -2
  107. package/dist/substrates/e2b/peer-install.d.ts +51 -0
  108. package/dist/substrates/e2b/peer-install.js +161 -0
  109. package/dist/substrates/e2b/sandbox.d.ts +34 -6
  110. package/dist/substrates/e2b/sandbox.js +128 -35
  111. package/dist/substrates/e2b/sdk.d.ts +33 -0
  112. package/dist/substrates/e2b/sdk.js +21 -21
  113. package/dist/substrates/shell.d.ts +2 -0
  114. package/dist/substrates/shell.js +4 -0
  115. package/dist/tui-app.js +133 -133
  116. package/dist/verify/liveness.d.ts +19 -3
  117. package/dist/verify/liveness.js +46 -25
  118. package/dist/verify/verify.d.ts +3 -0
  119. package/dist/verify/verify.js +6 -2
  120. package/docs/architecture/actor-contract.md +2 -1
  121. package/docs/architecture/observer.md +22 -7
  122. package/docs/architecture/task-protocol-support.md +13 -13
  123. package/docs/architecture/terminal-product-route.md +1 -1
  124. package/docs/contracts/run-bundle.md +75 -6
  125. package/docs/contracts/schemas.md +39 -22
  126. package/docs/contracts/study-analysis.md +45 -0
  127. package/docs/contracts/study-costs.md +8 -0
  128. package/docs/decisions/0001-run-bundle-is-the-source-of-truth.md +3 -0
  129. package/docs/decisions/0003-contained-paths-and-cleanup-authority.md +9 -3
  130. package/docs/principles/invariants-and-defaults.md +11 -8
  131. package/package.json +4 -3
  132. package/skills/humanish/SKILL.md +16 -2
@@ -7,9 +7,9 @@ import type { CostLine, NoSpendProof, TerminalCostLedger } from "./types.js";
7
7
  * - The `provider` line is populated from the actor trace's tokenUsage.costUsd when the trace
8
8
  * carries it (a measured value, incl. a measured 0). When the trace carries no costUsd, the
9
9
  * provider line is `null` = not measured (never guessed to 0 just because no-spend was intended).
10
- * - product/media/payment are `null` by default: core has no signal for those categories; an
11
- * adapter may provide one through the shipped costProbe seam.
12
- * `injectedLines` lets a test or adapter supply known spend for a category,
10
+ * - product/media/payment are `null` by default: core has no signal for those categories, and
11
+ * only a test can supply one, through `StudyDeps.costProbe`.
12
+ * `injectedLines` lets a test supply known spend for a category,
13
13
  * exercising the fail-closed cap enforcement deterministically without a real billable run.
14
14
  */
15
15
  export declare function buildCostLedger(args: {
@@ -6,9 +6,9 @@ import { COST_CATEGORIES } from "../../run/terminal-contract.js";
6
6
  * - The `provider` line is populated from the actor trace's tokenUsage.costUsd when the trace
7
7
  * carries it (a measured value, incl. a measured 0). When the trace carries no costUsd, the
8
8
  * provider line is `null` = not measured (never guessed to 0 just because no-spend was intended).
9
- * - product/media/payment are `null` by default: core has no signal for those categories; an
10
- * adapter may provide one through the shipped costProbe seam.
11
- * `injectedLines` lets a test or adapter supply known spend for a category,
9
+ * - product/media/payment are `null` by default: core has no signal for those categories, and
10
+ * only a test can supply one, through `StudyDeps.costProbe`.
11
+ * `injectedLines` lets a test supply known spend for a category,
12
12
  * exercising the fail-closed cap enforcement deterministically without a real billable run.
13
13
  */
14
14
  export function buildCostLedger(args) {
@@ -39,7 +39,7 @@ export function buildCostLedger(args) {
39
39
  usd: null,
40
40
  count: null,
41
41
  source: "unmeasured",
42
- note: `${category} spend not measured: humanish has no ${category} spend signal for this run, and an adapter may supply one through costProbe. Recorded as null, never guessed as 0.`,
42
+ note: `${category} spend not measured: humanish has no ${category} spend signal, and a study cannot supply one. Recorded as null, never guessed as 0.`,
43
43
  });
44
44
  const lines = {
45
45
  product: args.injectedLines?.product ?? unmeasured("product"),
@@ -149,7 +149,7 @@ export function evaluateCapsAgainstLedger(ledger, caps) {
149
149
  const overLines = COST_CATEGORIES.filter((c) => ledger.lines[c].usd !== null && ledger.lines[c].usd > 0);
150
150
  return {
151
151
  ok: false,
152
- message: `Stopped: measured spend ${ledger.knownTotalUsd} USD passed scenario.caps.maxUsd=${caps.maxUsd}${overLines.length > 0 ? ` (non-zero lines: ${overLines.join(", ")})` : ""}. The cap stops the run; it is not a warning.`,
152
+ message: `Stopped: measured spend ${ledger.knownTotalUsd} USD passed caps.maxUsd=${caps.maxUsd}${overLines.length > 0 ? ` (non-zero lines: ${overLines.join(", ")})` : ""}. The cap stops the run; it is not a warning.`,
153
153
  };
154
154
  }
155
155
  if (caps.maxJobs !== undefined) {
@@ -162,7 +162,7 @@ export function evaluateCapsAgainstLedger(ledger, caps) {
162
162
  if (knownJobs > caps.maxJobs) {
163
163
  return {
164
164
  ok: false,
165
- message: `Stopped: ${knownJobs} measured billable jobs passed scenario.caps.maxJobs=${caps.maxJobs}.`,
165
+ message: `Stopped: ${knownJobs} measured billable jobs passed caps.maxJobs=${caps.maxJobs}.`,
166
166
  };
167
167
  }
168
168
  }
@@ -9,7 +9,7 @@ import { normalizeLocalActorTranscript, TERMINAL_EVENTS_ARTIFACT, TERMINAL_LEDGE
9
9
  import { applyAdapterExtensionSeam } from "./adapter.js";
10
10
  import { buildLiveTerminalProductBundle } from "./bundle.js";
11
11
  import { buildCostLedger, buildNoSpendProof, describeMeasuredSpend, evaluateCapsAgainstLedger, noSpendLineMeasured, noSpendNotEstablished, } from "./ledger.js";
12
- import { foldScorerFailures, judgeExecution, judgeTerminal, OUTCOME_POLICIES, } from "../../run/judge.js";
12
+ import { foldScorerFailures, judgeExecution, judgeTerminal, OUTCOME_POLICIES, resultOk, } from "../../run/judge.js";
13
13
  import { terminalExecutionFailures, terminalStudyResult, terminalParticipantFacts, } from "./result.js";
14
14
  import { parseTerminalTokenUsage } from "./token-usage.js";
15
15
  import { buildTerminalActorTrace, scrubSplitKnownValues, tailOf } from "./trace.js";
@@ -60,7 +60,7 @@ async function settleLiveLedgers(inputs, trace, normalizedTranscript) {
60
60
  // --- Spend ledger + no-spend proof + full caps enforcement (fail-closed). ---
61
61
  // The cost ledger is derived, with the null discipline: provider spend from the trace's
62
62
  // tokenUsage.costUsd when present (else null = not measured), product/media/payment null by
63
- // default (core has no signal). The costProbe hook lets tests or adapters inject known
63
+ // default (core has no signal). The costProbe hook lets tests inject known
64
64
  // spend to exercise the fail-closed cap without a real billable run.
65
65
  const injectedLines = costProbe?.(trace.tokenUsage?.costUsd === undefined ? {} : { tokenCostUsd: trace.tokenUsage.costUsd });
66
66
  if (costProbe)
@@ -171,19 +171,24 @@ export async function finishLiveTerminalSession(inputs) {
171
171
  // The one final verdict fold: scoring can only make the judged verdict stricter.
172
172
  bundle.review = foldScorerFailures(bundle.review, scorer.failures);
173
173
  await validatePreparedRunArtifactPaths(runPaths);
174
- const finished = await run.finish(bundle);
175
- const observer = await finished.renderObserver();
176
- await validatePreparedRunArtifactPaths(runPaths);
177
174
  // A failing agent is captured evidence on this route: only the execution and a declared scorer
178
- // fail ok.
175
+ // fail ok. A declared scorer that did not render a pass (status:"fail", malformed or thrown) is a
176
+ // gate, so it fails ok as well as the persisted verdict.
177
+ const policy = OUTCOME_POLICIES.terminal;
179
178
  const execution = judgeExecution(terminalExecutionFailures({
180
179
  participant,
181
180
  capFailure,
182
181
  sessionReason: sanitize(session.reason),
183
182
  cleanup: session.cleanup,
184
- observer,
185
- }), OUTCOME_POLICIES.terminal);
186
- const result = terminalStudyResult({
183
+ }), policy);
184
+ const finished = await run.finish(bundle, {
185
+ ok: resultOk({ judgment, execution, scorerFailures: scorer.failures, policy }),
186
+ execution,
187
+ policy,
188
+ });
189
+ const observer = await finished.renderObserver();
190
+ await validatePreparedRunArtifactPaths(runPaths);
191
+ return terminalStudyResult({
187
192
  cwd,
188
193
  studyId: plan.studyId,
189
194
  actorId: plan.actor,
@@ -199,13 +204,10 @@ export async function finishLiveTerminalSession(inputs) {
199
204
  noSpendProof,
200
205
  capFailure,
201
206
  declaredScorerFailure: scorer.failures[0],
202
- judgment,
203
- execution,
207
+ ok: finished.outcome.ok,
204
208
  observer,
205
209
  warnings,
206
210
  });
207
- await finished.recordOutcome({ ok: result.ok, execution });
208
- return result;
209
211
  }
210
212
  /** Persist the terminal evidence: redacted events, normalized transcript, ledgers, actor trace. */
211
213
  async function writeTerminalEvidence(runPaths, evidence) {
@@ -35,7 +35,7 @@ export function planTerminalStudy(config, input) {
35
35
  const actorType = config.actors[0]?.type ?? "";
36
36
  const descriptor = actorRegistry[actorType];
37
37
  if (!descriptor || !isTerminalActorDescriptor(descriptor))
38
- return refuse("HUMANISH_TERMINAL_ACTOR_UNSUPPORTED", `actors[0].type "${actorType}" is not a registered terminal actor.`);
38
+ return refuse("HUMANISH_TERMINAL_ACTOR_UNSUPPORTED", `actor.type "${actorType}" is not a registered terminal actor.`);
39
39
  const runtimeVersion = config.execution?.runtime?.version;
40
40
  const actor = config.actors[0];
41
41
  if ((config.execution?.runtime !== undefined && !isExactRuntimeVersion(runtimeVersion)) ||
@@ -87,14 +87,14 @@ export function planTerminalStudy(config, input) {
87
87
  const maxUsd = caps?.maxUsd;
88
88
  const maxMinutes = caps?.maxMinutes;
89
89
  if (caps === undefined || maxUsd === undefined || maxMinutes === undefined || maxMinutes <= 0)
90
- return refuse("HUMANISH_TERMINAL_CAPS_MISSING", "A live terminal-product run gives the agent in the sandbox access to a model provider, so it needs a cap: set scenario.caps with maxUsd (0 means no spend) and a positive maxMinutes (the time limit for the codex command). The key is used only while the cap is in force.", descriptor.id);
90
+ return refuse("HUMANISH_TERMINAL_CAPS_MISSING", "A live terminal-product run gives the agent in the sandbox access to a model provider, so it needs a cap: set caps with maxUsd (0 means no spend) and a positive maxMinutes (the time limit for the codex command). The key is used only while the cap is in force.", descriptor.id);
91
91
  // maxUsd is checked against the cost ledger after the session, and only known lines can trip it.
92
92
  // The ledger's provider line takes only provider-reported cost, which Codex does not report (the
93
93
  // token estimate in run.json is not a measurement), and core has no product, media or payment
94
94
  // signal, so without a costProbe every line is null and a positive maxUsd could never trip. That
95
95
  // cap would promise a bound nothing enforces, so it is refused; maxMinutes bounds a live run.
96
96
  if (maxUsd > 0 && input.hasCostProbe !== true)
97
- return refuse("HUMANISH_TERMINAL_UNPRICED_CAP", `scenario.caps.maxUsd=${maxUsd} cannot be enforced: Codex reports no provider cost for the participant (its token cost is only estimated after the run) and no product, media or payment spend is measured, so a positive dollar cap can never trip. Set scenario.caps.maxUsd to 0 and bound the run with scenario.caps.maxMinutes, the codex command's wall-clock kill. No sandbox was created and the runtime key was not used.`, descriptor.id);
97
+ return refuse("HUMANISH_TERMINAL_UNPRICED_CAP", `caps.maxUsd=${maxUsd} cannot be enforced: Codex reports no provider cost for the participant (its token cost is only estimated after the run) and no product, media or payment spend is measured, so a positive dollar cap can never trip. Set caps.maxUsd to 0 and bound the run with caps.maxMinutes, the codex command's wall-clock kill. No sandbox was created and the runtime key was not used.`, descriptor.id);
98
98
  // The sandbox's timeout covers the steps before the codex command, maxMinutes and the teardown
99
99
  // buffer, and E2B refuses a sandbox over an hour. Refuse here, with the arithmetic, rather than
100
100
  // after the plan prints, from a provider 400 that names neither knob.
@@ -102,7 +102,7 @@ export function planTerminalStudy(config, input) {
102
102
  const sandboxTimeoutMs = terminalSandboxTimeoutMs({ maxMinutes, productInstall });
103
103
  if (sandboxTimeoutMs > MAX_SANDBOX_MS) {
104
104
  const headroomMinutes = (sandboxTimeoutMs - maxMinutes * 60_000) / 60_000;
105
- return refuse("HUMANISH_TERMINAL_CAPS_INVALID", `scenario.caps.maxMinutes ${maxMinutes} derives a ${sandboxTimeoutMs / 60_000}m sandbox deadline, and a sandbox may not live longer than ${MAX_SANDBOX_MS / 60_000}m. The deadline is maxMinutes plus ${headroomMinutes}m: the Node bootstrap (${NODE_BOOTSTRAP_TIMEOUT_MS / 60_000}m), the runtime version check (${TERMINAL_RUNTIME_VERSION_TIMEOUT_MS / 60_000}m)${productInstall ? `, the product setup (${PRODUCT_SETUP_TIMEOUT_MS / 60_000}m)` : ""} and the teardown buffer (${TERMINAL_SANDBOX_TIMEOUT_BUFFER_MS / 60_000}m). Lower scenario.caps.maxMinutes to at most ${MAX_SANDBOX_MS / 60_000 - headroomMinutes}. No sandbox was created and the runtime key was not used.`, descriptor.id);
105
+ return refuse("HUMANISH_TERMINAL_CAPS_INVALID", `caps.maxMinutes ${maxMinutes} derives a ${sandboxTimeoutMs / 60_000}m sandbox deadline, and a sandbox may not live longer than ${MAX_SANDBOX_MS / 60_000}m. The deadline is maxMinutes plus ${headroomMinutes}m: the Node bootstrap (${NODE_BOOTSTRAP_TIMEOUT_MS / 60_000}m), the runtime version check (${TERMINAL_RUNTIME_VERSION_TIMEOUT_MS / 60_000}m)${productInstall ? `, the product setup (${PRODUCT_SETUP_TIMEOUT_MS / 60_000}m)` : ""} and the teardown buffer (${TERMINAL_SANDBOX_TIMEOUT_BUFFER_MS / 60_000}m). Lower caps.maxMinutes to at most ${MAX_SANDBOX_MS / 60_000 - headroomMinutes}. No sandbox was created and the runtime key was not used.`, descriptor.id);
106
106
  }
107
107
  return {
108
108
  ok: true,
@@ -1,6 +1,6 @@
1
1
  import type { ObserverResult } from "../../observer/render.js";
2
2
  import type { ActorCompletionReason, ActorStatus, ActorTrace } from "../../actors/contract.js";
3
- import { type ExecutionFailure, type ExecutionOutcome, type Judgment, type ParticipantFacts } from "../../run/judge.js";
3
+ import { type ExecutionFailure, type ParticipantFacts } from "../../run/judge.js";
4
4
  import { type NoSpendProof, type TerminalCostLedger, type TerminalLedgers, type TerminalProductStudyResult } from "./types.js";
5
5
  /**
6
6
  * What judge.ts reads from a finished terminal session, after a blown cap has overridden it. It has
@@ -8,8 +8,9 @@ import { type NoSpendProof, type TerminalCostLedger, type TerminalLedgers, type
8
8
  */
9
9
  export declare function terminalParticipantFacts(trace: ActorTrace, sessionError: string | undefined): ParticipantFacts;
10
10
  /**
11
- * The live run's execution failures: a harness error, a blown spend cap, an unproven sandbox
12
- * teardown, and an Observer that failed.
11
+ * The live run's execution failures before the Observer renders: a harness error, a blown spend
12
+ * cap and an unproven sandbox teardown. FinishedRun.renderObserver adds an Observer that did not
13
+ * render.
13
14
  */
14
15
  export declare function terminalExecutionFailures(args: {
15
16
  participant: ParticipantFacts;
@@ -18,7 +19,6 @@ export declare function terminalExecutionFailures(args: {
18
19
  /** Already scrubbed and redacted. */
19
20
  sessionReason: string;
20
21
  cleanup: TerminalLedgers["cleanup"];
21
- observer: Pick<ObserverResult, "ok" | "error">;
22
22
  }): ExecutionFailure[];
23
23
  /** The terminal-product study result for a live run, from its session, cleanup and cost ledger. */
24
24
  export declare function terminalStudyResult(args: {
@@ -39,9 +39,8 @@ export declare function terminalStudyResult(args: {
39
39
  /** The cap check's message when known spend or jobs exceeded the caps. */
40
40
  capFailure: string | undefined;
41
41
  declaredScorerFailure: string | undefined;
42
- /** The run's judgment. On this evidence route ok does not read whether the agent passed. */
43
- judgment: Judgment;
44
- execution: ExecutionOutcome;
42
+ /** run.json's outcome.ok (FinishedRun.outcome). On this evidence route it does not read whether the agent passed. */
43
+ ok: boolean;
45
44
  observer: ObserverResult;
46
45
  warnings: string[];
47
46
  }): TerminalProductStudyResult;
@@ -1,4 +1,4 @@
1
- import { OUTCOME_POLICIES, participantHarnessFailed, resultOk, } from "../../run/judge.js";
1
+ import { participantHarnessFailed, } from "../../run/judge.js";
2
2
  import {} from "./types.js";
3
3
  import { studyResultIdentity } from "../../run/study-result.js";
4
4
  /**
@@ -16,11 +16,12 @@ export function terminalParticipantFacts(trace, sessionError) {
16
16
  };
17
17
  }
18
18
  /**
19
- * The live run's execution failures: a harness error, a blown spend cap, an unproven sandbox
20
- * teardown, and an Observer that failed.
19
+ * The live run's execution failures before the Observer renders: a harness error, a blown spend
20
+ * cap and an unproven sandbox teardown. FinishedRun.renderObserver adds an Observer that did not
21
+ * render.
21
22
  */
22
23
  export function terminalExecutionFailures(args) {
23
- const { participant, cleanup, observer } = args;
24
+ const { participant, cleanup } = args;
24
25
  return [
25
26
  ...(participantHarnessFailed(participant)
26
27
  ? [{ kind: "harness", message: participant.sessionError ?? args.sessionReason }]
@@ -34,34 +35,17 @@ export function terminalExecutionFailures(args) {
34
35
  message: `Sandbox teardown unproven (killed=${cleanup.killed}, remaining=${cleanup.remaining}): ${cleanup.reason}`,
35
36
  },
36
37
  ]),
37
- ...(observer.ok
38
- ? []
39
- : [
40
- {
41
- kind: "evidence",
42
- message: observer.error?.message ?? "Observer failed for the terminal-product run.",
43
- },
44
- ]),
45
38
  ];
46
39
  }
47
40
  /** The terminal-product study result for a live run, from its session, cleanup and cost ledger. */
48
41
  export function terminalStudyResult(args) {
49
- const { cwd, studyId, actorId, productName, runId, sessionStatus, completionReason, sessionReason, sessionError, sandboxId, cleanup, cost, noSpendProof, capFailure, declaredScorerFailure, judgment, execution, observer, warnings, } = args;
42
+ const { cwd, studyId, actorId, productName, runId, sessionStatus, completionReason, sessionReason, sessionError, sandboxId, cleanup, cost, noSpendProof, capFailure, declaredScorerFailure, ok, observer, warnings, } = args;
50
43
  // The run's exit code: verified evidence, no harness error and proven cleanup. A blocked/
51
44
  // timed-out agent run is still ok-as-evidence at the bundle level (the failure is the evidence),
52
45
  // but the study result surfaces ok:false on a harness error or unproven teardown (fail-closed).
53
46
  // remaining===0 is the by-id-confirmed-reclaimed state; remaining===1 (still present) and
54
47
  // remaining===-1 (kill(id) itself failed) are both unproven by design.
55
48
  const cleanupProven = cleanup.killed && cleanup.remaining === 0;
56
- // A config-declared scorer that failed to render a pass (status:"fail" / malformed / throw) fails the
57
- // run result as well as the persisted verdict: the keystone route's declared rubric is a gate, so
58
- // its fail must drive exit code. Library callers never set this (additive, back-compat).
59
- const ok = resultOk({
60
- judgment,
61
- execution,
62
- scorerFailures: declaredScorerFailure === undefined ? [] : [declaredScorerFailure],
63
- policy: OUTCOME_POLICIES.terminal,
64
- });
65
49
  return {
66
50
  ...studyResultIdentity("terminal", studyId),
67
51
  ok,
@@ -20,7 +20,7 @@ export function checkLiveTerminalMachine(plan, input, warnings) {
20
20
  const { maxUsd } = plan.caps;
21
21
  // planTerminalStudy refuses a positive maxUsd without a costProbe, so one is present here.
22
22
  if (maxUsd > 0) {
23
- warnings.push(`scenario.caps.maxUsd=${maxUsd} is checked after the session against the lines the costProbe measures; lines it leaves null (unmeasured) never trip it. scenario.caps.maxMinutes bounds the run while it runs.`);
23
+ warnings.push(`caps.maxUsd=${maxUsd} is checked after the session against the measured cost lines; a line left null (unmeasured) never trips it. caps.maxMinutes bounds the run while it runs.`);
24
24
  }
25
25
  // --- Deny-by-default credentials: build the command-scoped allowlist. ---
26
26
  const runtimeEnv = buildRuntimeAuth({ runtimeAuth: plan.runtime.auth, env });
@@ -1,6 +1,8 @@
1
- import { type ReviewSummary, type RunAdapterScore, type RunBundle } from "./bundle.js";
1
+ import { type ReviewSummary, type RunAdapterScore, type RunBundle, type RunOutcome } from "./bundle.js";
2
2
  import { type StoredCleanupResult, type RunPointer } from "./results.js";
3
3
  export declare function isRunBundle(value: unknown): value is RunBundle;
4
+ /** run.json's `outcome`: a finished run's ok and execution outcome, or the signal that stopped it. */
5
+ export declare function isRunOutcome(value: unknown): value is RunOutcome;
4
6
  export declare function isCleanupResult(value: unknown): value is StoredCleanupResult;
5
7
  export declare function isRunAdapterScore(value: unknown): value is RunAdapterScore;
6
8
  export declare const ARCHIVE_SHA256_PATTERN: RegExp;
@@ -5,6 +5,7 @@ import { CLEANUP_SCHEMA, } from "./results.js";
5
5
  import { isRunSimulationStatus, isRunStream, isRunStreamKind } from "./stream-shape.js";
6
6
  import { isLocalEvidenceArtifactPath } from "./paths.js";
7
7
  import { isRunFeedbackCandidate } from "./feedback-shape.js";
8
+ import { EXECUTION_FAILURE_KINDS } from "./judge.js";
8
9
  import { isSharedWorldEvidence } from "./shared-world-shape.js";
9
10
  import { isNonNegativeSafeInteger, isPositiveSafeInteger, isRecord } from "./type-guards.js";
10
11
  export function isRunBundle(value) {
@@ -64,7 +65,30 @@ export function isRunBundle(value) {
64
65
  value.adapterArtifacts.every(isRunAdapterArtifact))) &&
65
66
  (value.providerResources === undefined ||
66
67
  (Array.isArray(value.providerResources) &&
67
- value.providerResources.every(isRunProviderResource))));
68
+ value.providerResources.every(isRunProviderResource))) &&
69
+ (value.outcome === undefined || isRunOutcome(value.outcome)));
70
+ }
71
+ function isExecutionFailureList(value) {
72
+ return (Array.isArray(value) &&
73
+ value.every((failure) => isRecord(failure) &&
74
+ typeof failure.kind === "string" &&
75
+ EXECUTION_FAILURE_KINDS.includes(failure.kind) &&
76
+ typeof failure.message === "string"));
77
+ }
78
+ /** run.json's `outcome`: a finished run's ok and execution outcome, or the signal that stopped it. */
79
+ export function isRunOutcome(value) {
80
+ if (!isRecord(value))
81
+ return false;
82
+ if (value.state === "interrupted")
83
+ return (value.ok === false &&
84
+ (value.signal === "SIGINT" || value.signal === "SIGTERM" || value.signal === "SIGHUP") &&
85
+ typeof value.at === "string");
86
+ return (value.state === "finished" &&
87
+ typeof value.ok === "boolean" &&
88
+ isRecord(value.execution) &&
89
+ typeof value.execution.succeeded === "boolean" &&
90
+ isExecutionFailureList(value.execution.failures) &&
91
+ (value.execution.warnings === undefined || isExecutionFailureList(value.execution.warnings)));
68
92
  }
69
93
  function isRunProviderResource(value) {
70
94
  return (isRecord(value) &&
@@ -1,7 +1,9 @@
1
1
  import type { CommsReceivingEvidence } from "../comms/receiving-types.js";
2
2
  import { type CapturedGitState } from "./git-state.js";
3
+ import type { ExecutionOutcome } from "./judge.js";
3
4
  import type { SharedWorldEvidence } from "./shared-world-evidence.js";
4
5
  import { type RunStudyProvenance } from "./study-provenance.js";
6
+ import type { RunInterruptSignal } from "./status.js";
5
7
  import type { RunSimulationStatus, RunStream, RunStreamKind } from "./streams.js";
6
8
  export declare const RUN_BUNDLE_SCHEMA = "humanish.run-bundle.v1";
7
9
  /** The run bundle file, relative to the run directory. */
@@ -252,6 +254,13 @@ export interface RunBundle {
252
254
  events: string;
253
255
  };
254
256
  review: ReviewSummary;
257
+ /**
258
+ * How the run ended as an execution. `review.verdict` is what the participants experienced;
259
+ * this is whether the run worked, which no surface may show as a pass when `ok` is false. Absent
260
+ * while the run is in progress, when its process died without writing it, and on runs recorded
261
+ * before this field, whose status.json holds the outcome instead.
262
+ */
263
+ outcome?: RunOutcome;
255
264
  feedbackCandidates: RunFeedbackCandidate[];
256
265
  /** Structured subject provenance. Optional and additive: emitted by the
257
266
  * computer-use and shared-world backends and the clone scripted-browser route; tolerated absent
@@ -347,6 +356,21 @@ export interface RunBundle {
347
356
  */
348
357
  cost?: RunCostSummary;
349
358
  }
359
+ /**
360
+ * How a run ended. `finished`: Run.finish published the final bundle with the route result's `ok`
361
+ * and its execution outcome (`judgeExecution` in src/run/judge.ts). `interrupted`: the CLI's signal
362
+ * handler stopped the run before it finished; run.json is then the last live flush.
363
+ */
364
+ export type RunOutcome = {
365
+ state: "finished";
366
+ ok: boolean;
367
+ execution: ExecutionOutcome;
368
+ } | {
369
+ state: "interrupted";
370
+ ok: false;
371
+ signal: RunInterruptSignal;
372
+ at: string;
373
+ };
350
374
  /**
351
375
  * One contributing cost line of a RunCostSummary. A line is present even when it cannot be priced
352
376
  * (it records that we tried and could not): an unpriceable line carries estimatedCostUsd: null + a
@@ -0,0 +1,59 @@
1
+ import type { RunOutcome } from "./bundle.js";
2
+ import type { ExecutionFailure } from "./judge.js";
3
+ import { type RunLiveness } from "./status.js";
4
+ /** The states a run is shown in. */
5
+ export type RunDisplayState = "running" | "interrupted" | "passed" | "failed" | "blocked" | "timed_out" | "no_verdict" | "dry_run" | "unknown";
6
+ /** A run as a surface shows it. */
7
+ export interface RunDisplay {
8
+ state: RunDisplayState;
9
+ /** The state in words: "passed", "timed out", "no verdict". */
10
+ label: string;
11
+ /** How a surface colors it: `pass` only for `passed`, `fail` only for `failed`. */
12
+ tone: "pass" | "fail" | "warn" | "live" | "neutral";
13
+ /** The first execution failure that failed the run, as `kind: message`. */
14
+ reason?: string;
15
+ }
16
+ /** What runDisplay reads: whether the run is alive, what its participants experienced, and its ok. */
17
+ export interface RunDisplayFacts {
18
+ liveness: RunLiveness;
19
+ mode?: string | null | undefined;
20
+ /** `review.verdict`. */
21
+ verdict?: string | undefined;
22
+ /** The run's own ok from run.json's outcome; absent when the run recorded none. */
23
+ ok?: boolean | undefined;
24
+ /** The first execution failure that failed the run. */
25
+ failure?: ExecutionFailure | undefined;
26
+ }
27
+ /**
28
+ * A run's state. A live or interrupted run is that, whatever its last flush said. A finished run
29
+ * shows its participants' blocked, timed-out or failed verdict; otherwise a run whose ok is false
30
+ * is failed, even with a pass verdict, because its execution failed. Only then does a pass verdict
31
+ * show as passed.
32
+ */
33
+ export declare function runDisplay(facts: RunDisplayFacts): RunDisplay;
34
+ /** A state in words: "passed", "timed out", "no verdict". */
35
+ export declare function runDisplayLabel(state: RunDisplayState): string;
36
+ /** What bundleDisplayFacts reads from run.json; a listing reads it from an unchecked file. */
37
+ export interface DisplayedBundle {
38
+ runId: string;
39
+ mode?: string | undefined;
40
+ review?: {
41
+ verdict?: string | undefined;
42
+ } | undefined;
43
+ simulations?: ReadonlyArray<{
44
+ status?: string | undefined;
45
+ } | undefined> | undefined;
46
+ outcome?: RunOutcome | undefined;
47
+ }
48
+ /**
49
+ * The facts for a run from its bundle. run.json's `outcome` decides when it has one. A bundle
50
+ * without one is in progress, or was recorded before the field existed: its status record, when
51
+ * given and naming the run, then says whether the run is alive and, for the older run, holds its ok.
52
+ * With no record, a bundle whose simulations are still running is live.
53
+ */
54
+ export declare function bundleDisplayFacts(bundle: DisplayedBundle, status?: unknown, nowMs?: number): RunDisplayFacts;
55
+ /**
56
+ * How review.md says the run ended: the display of the bundle it is rendered from, so a run whose
57
+ * execution failed never reads as a pass beside its participants' verdict.
58
+ */
59
+ export declare function reviewOutcome(bundle: DisplayedBundle): string;
@@ -0,0 +1,106 @@
1
+ // How a run reads wherever a surface says whether it passed: the TUI, the Observer and its run
2
+ // library, `humanish runs`, `humanish stats` and review.md. Each of them calls runDisplay with the
3
+ // same facts, so a run whose own ok is false shows as a pass on none of them, and two surfaces
4
+ // cannot disagree about one run.
5
+ import { classifyRunStatus, isRunStatusRecord } from "./status.js";
6
+ const LABELS = {
7
+ running: "running",
8
+ interrupted: "interrupted",
9
+ passed: "passed",
10
+ failed: "failed",
11
+ blocked: "blocked",
12
+ timed_out: "timed out",
13
+ no_verdict: "no verdict",
14
+ dry_run: "dry run",
15
+ unknown: "unknown",
16
+ };
17
+ const TONES = {
18
+ running: "live",
19
+ interrupted: "warn",
20
+ passed: "pass",
21
+ failed: "fail",
22
+ blocked: "warn",
23
+ timed_out: "warn",
24
+ no_verdict: "warn",
25
+ dry_run: "neutral",
26
+ unknown: "neutral",
27
+ };
28
+ function shown(state, reason) {
29
+ return {
30
+ state,
31
+ label: LABELS[state],
32
+ tone: TONES[state],
33
+ ...(reason === undefined ? {} : { reason }),
34
+ };
35
+ }
36
+ /**
37
+ * A run's state. A live or interrupted run is that, whatever its last flush said. A finished run
38
+ * shows its participants' blocked, timed-out or failed verdict; otherwise a run whose ok is false
39
+ * is failed, even with a pass verdict, because its execution failed. Only then does a pass verdict
40
+ * show as passed.
41
+ */
42
+ export function runDisplay(facts) {
43
+ if (facts.liveness === "running")
44
+ return shown("running");
45
+ if (facts.liveness === "interrupted")
46
+ return shown("interrupted");
47
+ const reason = facts.ok === false && facts.failure !== undefined
48
+ ? `${facts.failure.kind}: ${facts.failure.message}`
49
+ : undefined;
50
+ if (facts.verdict === "blocked")
51
+ return shown("blocked", reason);
52
+ if (facts.verdict === "timed_out")
53
+ return shown("timed_out", reason);
54
+ if (facts.verdict === "fail" || facts.ok === false)
55
+ return shown("failed", reason);
56
+ if (facts.verdict === "pass")
57
+ return shown("passed");
58
+ if (facts.verdict === "contract_proof_only")
59
+ return shown(facts.mode === "dry-run" ? "dry_run" : "no_verdict");
60
+ return shown("unknown");
61
+ }
62
+ /** A state in words: "passed", "timed out", "no verdict". */
63
+ export function runDisplayLabel(state) {
64
+ return LABELS[state];
65
+ }
66
+ /** A display as one line of text: the label, then why the run failed when it says. */
67
+ function runDisplayLine(display) {
68
+ return display.reason === undefined ? display.label : `${display.label}: ${display.reason}`;
69
+ }
70
+ /**
71
+ * The facts for a run from its bundle. run.json's `outcome` decides when it has one. A bundle
72
+ * without one is in progress, or was recorded before the field existed: its status record, when
73
+ * given and naming the run, then says whether the run is alive and, for the older run, holds its ok.
74
+ * With no record, a bundle whose simulations are still running is live.
75
+ */
76
+ export function bundleDisplayFacts(bundle, status, nowMs = Date.now()) {
77
+ const base = { mode: bundle.mode, verdict: bundle.review?.verdict };
78
+ const outcome = bundle.outcome;
79
+ if (outcome?.state === "interrupted")
80
+ return { ...base, liveness: "interrupted" };
81
+ if (outcome?.state === "finished")
82
+ return {
83
+ ...base,
84
+ liveness: "finished",
85
+ ok: outcome.ok,
86
+ failure: outcome.execution.failures[0],
87
+ };
88
+ if (isRunStatusRecord(status) && status.runId === bundle.runId)
89
+ return {
90
+ ...base,
91
+ liveness: classifyRunStatus(status, nowMs),
92
+ ...(typeof status.outcome?.ok === "boolean" ? { ok: status.outcome.ok } : {}),
93
+ ...(status.outcome?.execution?.failures[0] === undefined
94
+ ? {}
95
+ : { failure: status.outcome.execution.failures[0] }),
96
+ };
97
+ const inProgress = (bundle.simulations ?? []).some((simulation) => simulation?.status === "running");
98
+ return { ...base, liveness: inProgress ? "running" : "finished" };
99
+ }
100
+ /**
101
+ * How review.md says the run ended: the display of the bundle it is rendered from, so a run whose
102
+ * execution failed never reads as a pass beside its participants' verdict.
103
+ */
104
+ export function reviewOutcome(bundle) {
105
+ return runDisplayLine(runDisplay(bundleDisplayFacts(bundle)));
106
+ }
@@ -10,6 +10,7 @@ import { participantEvent, participantIdsOf, participantRecord, participantStrea
10
10
  import { implicitProjectDirectoryExists, readPackageName, validateCwd } from "./project.js";
11
11
  import { loadDryRunInputs } from "./dry-run-inputs.js";
12
12
  import { judgeExecution, judgePreview, OUTCOME_POLICIES, resultOk, verdictText, } from "./judge.js";
13
+ import { reviewOutcome } from "./display.js";
13
14
  /**
14
15
  * The preview route's run. The run scope closes the run it started on every exit, including
15
16
  * the fail-closed ones. It renders an Observer only when `options.observer` asks for one.
@@ -123,16 +124,17 @@ async function runDryRunInScope(options, scope) {
123
124
  review: createReviewSummary(judgment.verdict),
124
125
  feedbackCandidates: [],
125
126
  };
126
- const finished = await run.finish(bundle);
127
- const observer = options.observer === undefined ? undefined : await renderPreviewObserver(finished, warnings);
128
127
  // The preview's policy keeps an Observer that did not render a warning: the bundle is still
129
128
  // evidence, and `export` can render it later.
130
129
  const policy = OUTCOME_POLICIES.preview;
131
- const execution = judgeExecution(observer === undefined || observer.ok
132
- ? []
133
- : [{ kind: "evidence", message: observer.error?.message ?? "render failed" }], policy);
134
- const ok = resultOk({ judgment, execution, scorerFailures: [], policy });
135
- await finished.recordOutcome({ ok, execution });
130
+ const execution = judgeExecution([], policy);
131
+ const finished = await run.finish(bundle, {
132
+ ok: resultOk({ judgment, execution, scorerFailures: [], policy }),
133
+ execution,
134
+ policy,
135
+ });
136
+ const observer = options.observer === undefined ? undefined : await renderPreviewObserver(finished, warnings);
137
+ const { ok } = finished.outcome;
136
138
  return {
137
139
  schema: "humanish.run-result.v1",
138
140
  ok,
@@ -175,6 +177,8 @@ Mode: ${bundle.mode}
175
177
 
176
178
  Verdict: ${verdictText(bundle.review.verdict, bundle.mode)}
177
179
 
180
+ Outcome: ${reviewOutcome(bundle)}
181
+
178
182
  ${bundle.review.summary}
179
183
 
180
184
  ## Public-Safety
@@ -91,8 +91,9 @@ export interface Judgment {
91
91
  }
92
92
  /** The judgment for a verdict: a run passes when its verdict is pass, and a dry run passes. */
93
93
  export declare function judgmentOf(verdict: Verdict, dryRun: boolean): Judgment;
94
- /** What kind of execution failure a run recorded. */
95
- type ExecutionFailureKind = "harness" | "provider-cleanup" | "provider-policy" | "sandbox-cleanup" | "evidence" | "cap" | "run";
94
+ /** The kinds of execution failure a run records. */
95
+ export declare const EXECUTION_FAILURE_KINDS: readonly ["harness", "provider-cleanup", "provider-policy", "sandbox-cleanup", "evidence", "cap", "run"];
96
+ type ExecutionFailureKind = (typeof EXECUTION_FAILURE_KINDS)[number];
96
97
  /** One way the run failed as an execution. The message is scrubbed; status.json shows it. */
97
98
  export interface ExecutionFailure {
98
99
  kind: ExecutionFailureKind;
package/dist/run/judge.js CHANGED
@@ -108,6 +108,16 @@ export function verdictForStatus(status) {
108
108
  export function judgmentOf(verdict, dryRun) {
109
109
  return { verdict, passed: dryRun || verdict === "pass" };
110
110
  }
111
+ /** The kinds of execution failure a run records. */
112
+ export const EXECUTION_FAILURE_KINDS = [
113
+ "harness",
114
+ "provider-cleanup",
115
+ "provider-policy",
116
+ "sandbox-cleanup",
117
+ "evidence",
118
+ "cap",
119
+ "run",
120
+ ];
111
121
  /**
112
122
  * The execution failure for one sandbox whose release is unconfirmed. `warning` is the route's
113
123
  * release warning, already scrubbed. `recovery` says how to release it by hand; without one,
@@ -101,7 +101,7 @@ const DESKTOP_RESOURCE_RATE = {
101
101
  * since an npm install cannot edit this file.
102
102
  */
103
103
  export function unpricedCapMessage(model) {
104
- return `execution.caps sets a spend cap (maxUsd or maxTotalUsd), but humanish has no rate for model "${model}", so it cannot enforce the cap and refuses to start. Priced models: ${Object.keys(MODEL_RATES).sort().join(", ")}. Use one of them, or remove maxUsd and maxTotalUsd to run without a cap.`;
104
+ return `caps sets a spend cap (maxUsd or maxTotalUsd), but humanish has no rate for model "${model}", so it cannot enforce the cap and refuses to start. Priced models: ${Object.keys(MODEL_RATES).sort().join(", ")}. Use one of them, or remove maxUsd and maxTotalUsd to run without a cap.`;
105
105
  }
106
106
  export const DESKTOP_RATE = {
107
107
  usdPerMinute: 0.00888,
@@ -99,8 +99,6 @@ export declare function listWindow(args: {
99
99
  start: number;
100
100
  end: number;
101
101
  };
102
- /** The label for a run's state, in the register the surfaces share. */
103
- export declare function livenessLabel(entry: Pick<RunIndexEntry, "liveness" | "verdict">): string;
104
102
  /** A study as the study list shows it: what is declared, joined to what actually happened. */
105
103
  export interface StudyRow {
106
104
  /**