humanish 0.111.4 → 0.112.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. package/CHANGELOG.md +18 -0
  2. package/dist/actors/computer-use/openai-conversation.d.ts +91 -0
  3. package/dist/actors/computer-use/openai-conversation.js +209 -0
  4. package/dist/actors/computer-use/openai-provider.js +25 -41
  5. package/dist/actors/contract.d.ts +2 -0
  6. package/dist/analysis/diagnostics.js +7 -12
  7. package/dist/analysis/execute.d.ts +1 -1
  8. package/dist/analysis/execute.js +35 -5
  9. package/dist/analysis/service.js +7 -1
  10. package/dist/analysis/types.d.ts +23 -0
  11. package/dist/analysis/validation.d.ts +80 -35
  12. package/dist/analysis/validation.js +58 -11
  13. package/dist/cli/commands/runs.js +4 -10
  14. package/dist/cli/findings.d.ts +34 -3
  15. package/dist/cli/findings.js +136 -18
  16. package/dist/cli/io.d.ts +2 -1
  17. package/dist/cli/io.js +3 -2
  18. package/dist/feedback/draft.js +14 -4
  19. package/dist/observer/run-routes.js +8 -4
  20. package/dist/observer-app.html +4 -4
  21. package/dist/routes/computer-use/bundle.d.ts +1 -1
  22. package/dist/routes/computer-use/bundle.js +6 -16
  23. package/dist/routes/computer-use/fanout-bundle.js +40 -37
  24. package/dist/routes/computer-use/fanout-records.d.ts +2 -0
  25. package/dist/routes/computer-use/fanout-records.js +18 -26
  26. package/dist/routes/computer-use/participant-execution.js +4 -11
  27. package/dist/routes/computer-use/participant-facts.d.ts +2 -2
  28. package/dist/routes/computer-use/participant-facts.js +10 -1
  29. package/dist/routes/computer-use/result.js +7 -15
  30. package/dist/routes/computer-use/single-bundle.d.ts +2 -0
  31. package/dist/routes/computer-use/single-bundle.js +81 -37
  32. package/dist/routes/scripted/bundle.d.ts +1 -1
  33. package/dist/routes/scripted/bundle.js +9 -19
  34. package/dist/routes/scripted/surface-records.d.ts +0 -1
  35. package/dist/routes/scripted/surface-records.js +27 -13
  36. package/dist/routes/shared-world/bundle-records.d.ts +2 -2
  37. package/dist/routes/shared-world/bundle-records.js +21 -28
  38. package/dist/routes/shared-world/bundle.d.ts +1 -1
  39. package/dist/routes/shared-world/bundle.js +70 -78
  40. package/dist/routes/shared-world/participant-specs.d.ts +1 -2
  41. package/dist/routes/shared-world/participant-specs.js +0 -8
  42. package/dist/routes/shared-world/result.js +8 -12
  43. package/dist/routes/terminal/bundle.d.ts +1 -1
  44. package/dist/routes/terminal/bundle.js +10 -19
  45. package/dist/routes/terminal/dry-run.js +1 -0
  46. package/dist/routes/terminal/live-sandbox.js +9 -2
  47. package/dist/routes/terminal/plan.js +1 -0
  48. package/dist/routes/terminal/runtime.d.ts +1 -0
  49. package/dist/routes/terminal/runtime.js +1 -0
  50. package/dist/routes/terminal/session.js +1 -0
  51. package/dist/run/costs.js +9 -20
  52. package/dist/run/detail.d.ts +1 -1
  53. package/dist/run/detail.js +10 -1
  54. package/dist/run/display.d.ts +2 -2
  55. package/dist/run/display.js +15 -18
  56. package/dist/run/dry-run.js +10 -25
  57. package/dist/run/judge.d.ts +24 -0
  58. package/dist/run/judge.js +54 -0
  59. package/dist/run/liveness.d.ts +18 -0
  60. package/dist/run/liveness.js +24 -0
  61. package/dist/run/outcomes.d.ts +8 -1
  62. package/dist/run/outcomes.js +58 -12
  63. package/dist/run/participant-caption.d.ts +30 -0
  64. package/dist/run/participant-caption.js +50 -0
  65. package/dist/run/participant-records.d.ts +1 -1
  66. package/dist/run/review-markdown.d.ts +11 -0
  67. package/dist/run/review-markdown.js +22 -0
  68. package/dist/run/run-cost.d.ts +7 -0
  69. package/dist/run/run-cost.js +25 -5
  70. package/dist/run/run-index.d.ts +4 -50
  71. package/dist/run/run-index.js +57 -87
  72. package/dist/run/run.d.ts +2 -2
  73. package/dist/run/run.js +5 -3
  74. package/dist/run/stats.js +2 -12
  75. package/dist/run/transient-comms-secrets.d.ts +8 -3
  76. package/dist/run/transient-comms-secrets.js +21 -1
  77. package/dist/study/keys.d.ts +1 -0
  78. package/dist/study/keys.js +1 -1
  79. package/dist/study/parse/execution.js +7 -1
  80. package/dist/study/plan-types.d.ts +1 -0
  81. package/dist/study/types.d.ts +4 -1
  82. package/dist/study/warnings.d.ts +2 -1
  83. package/dist/study/warnings.js +3 -2
  84. package/dist/tui-app.js +73 -73
  85. package/dist/verify/liveness.d.ts +2 -2
  86. package/dist/verify/liveness.js +7 -4
  87. package/docs/architecture/terminal-product-route.md +14 -0
  88. package/docs/contracts/run-bundle.md +10 -0
  89. package/docs/contracts/schemas.md +9 -1
  90. package/docs/contracts/study-analysis.md +71 -22
  91. package/package.json +2 -1
  92. package/dist/actors/computer-use/openai-requests.d.ts +0 -62
  93. package/dist/actors/computer-use/openai-requests.js +0 -144
@@ -1,12 +1,14 @@
1
1
  import { redactText } from "../../evidence/redaction.js";
2
2
  import { participantAssignment } from "../../study/participant-assignment.js";
3
3
  import { REVIEW_SCHEMA, bundleArtifacts, bundleHead, } from "../../run/bundle.js";
4
- import { verdictText } from "../../run/judge.js";
5
- import { reviewOutcome } from "../../run/display.js";
4
+ import {} from "../../run/judge.js";
5
+ import { renderReviewMarkdown } from "../../run/review-markdown.js";
6
6
  import {} from "../../run/streams.js";
7
7
  import { participantEvent, participantIds, participantRecord, participantStream, } from "../../run/participant-records.js";
8
8
  import { TERMINAL_EVENTS_ARTIFACT, TERMINAL_LEDGERS_ARTIFACT, TERMINAL_TRANSCRIPT_ARTIFACT, } from "../../run/terminal-contract.js";
9
9
  import { DEFAULT_RUNTIME_AUTH } from "./runtime-auth.js";
10
+ import { participantCaption } from "../../run/participant-caption.js";
11
+ import { formatParticipantOutcomes, tallyParticipantOutcomes } from "../../run/outcomes.js";
10
12
  import { describeMeasuredSpend, noSpendLineMeasured, noSpendNotEstablished } from "./ledger.js";
11
13
  /**
12
14
  * Project the terminal-product run into a humanish.run-bundle.v1 (no schema change: a new
@@ -80,7 +82,7 @@ export function buildTerminalProductBundle(args) {
80
82
  const review = {
81
83
  schema: REVIEW_SCHEMA,
82
84
  verdict: args.verdict,
83
- summary: reason,
85
+ summary: `Dry run: 1 participant would try ${args.productName} from a terminal. No sandbox was created, no key was used and $0 was spent.`,
84
86
  gaps: [
85
87
  "This dry run did not execute the live in-sandbox agent route; it checks the evidence shape only, not live behavior, scale, or adoption.",
86
88
  "No exec-stream, transcript, substrate, cost, or cleanup artifacts were produced because no live session ran; live verification requires those artifacts.",
@@ -164,7 +166,7 @@ export function buildLiveTerminalProductBundle(args) {
164
166
  const review = {
165
167
  schema: REVIEW_SCHEMA,
166
168
  verdict: args.verdict,
167
- summary: args.sessionReason,
169
+ summary: `1 participant took part: ${formatParticipantOutcomes(tallyParticipantOutcomes([args.trace.status]))}. How it ended: ${args.sessionReason}`,
168
170
  gaps: [
169
171
  ...(args.trace.status === "passed"
170
172
  ? []
@@ -212,7 +214,7 @@ function terminalParticipant(args, session) {
212
214
  const stream = participantStream(TERMINAL_IDS, {
213
215
  assignment: participantAssignment({ mission: args.mission }),
214
216
  kind: "terminal",
215
- label: `Terminal agent · ${args.studyId}`,
217
+ label: participantCaption({ personaId: args.persona.id }),
216
218
  status: session.status,
217
219
  transport: "snapshot",
218
220
  updatedAt: session.updatedAt,
@@ -275,25 +277,14 @@ function describeCaps(caps) {
275
277
  parts.push(`maxMinutes=${caps.maxMinutes}`);
276
278
  return parts.length > 0 ? parts.join(", ") : "empty";
277
279
  }
278
- export function renderTerminalReviewMarkdown(bundle) {
280
+ export function renderTerminalReviewMarkdown(bundle, status) {
279
281
  const subject = bundle.events.find((event) => event.type === "terminal-lab.subject.declared");
280
282
  const credentials = bundle.events.find((event) => event.type === "terminal-lab.credentials.declared");
281
283
  const caps = bundle.events.find((event) => event.type === "terminal-lab.caps.declared");
282
- return [
283
- `# ${bundle.scenario.title}`,
284
- "",
285
- `- run: ${bundle.runId}`,
286
- `- mode: ${bundle.mode}`,
287
- `- verdict: ${verdictText(bundle.review.verdict, bundle.mode)}`,
288
- `- outcome: ${reviewOutcome(bundle)}`,
289
- `- summary: ${bundle.review.summary}`,
284
+ return renderReviewMarkdown(bundle, [
290
285
  `- mission: ${bundle.scenario.goal}`,
291
286
  ...(subject ? [`- subject: ${subject.message}`] : []),
292
287
  ...(credentials ? [`- credentials: ${credentials.message}`] : []),
293
288
  ...(caps ? [`- caps: ${caps.message}`] : []),
294
- ...(bundle.review.gaps.length > 0
295
- ? ["", "## Gaps", ...bundle.review.gaps.map((gap) => `- ${gap}`)]
296
- : []),
297
- "",
298
- ].join("\n");
289
+ ], { status });
299
290
  }
@@ -124,6 +124,7 @@ function runtimeDeclaredEvent(runtime, createdAt) {
124
124
  level: "info",
125
125
  type: "terminal-lab.runtime.declared",
126
126
  message: redactText(JSON.stringify(declaredRuntimeProvenance({
127
+ doNotTrack: runtime.doNotTrack,
127
128
  ...(version === undefined ? {} : { version }),
128
129
  model,
129
130
  modelSource,
@@ -261,7 +261,10 @@ export class LiveTerminalSandbox {
261
261
  // The install step may run the product itself (release:dogfood's does: `humanish init
262
262
  // --yes`); on the 0.67.0 dogfood that one command arrived unmarked while the participant's
263
263
  // nine others carried the marker.
264
- envs: { HUMANISH_STUDY_PARTICIPANT: "1" },
264
+ envs: {
265
+ HUMANISH_STUDY_PARTICIPANT: "1",
266
+ ...(plan.runtime.doNotTrack ? { DO_NOT_TRACK: "1" } : {}),
267
+ },
265
268
  requestTimeoutMs,
266
269
  timeoutMs: PRODUCT_SETUP_TIMEOUT_MS,
267
270
  });
@@ -314,7 +317,11 @@ export class LiveTerminalSandbox {
314
317
  // The selected command env (raw key or inert placeholder). The participant
315
318
  // marker rides the same command: humanish telemetry from inside a study reads as a new
316
319
  // adopter otherwise.
317
- envs: { ...runtimeEnv.envs, HUMANISH_STUDY_PARTICIPANT: "1" },
320
+ envs: {
321
+ ...runtimeEnv.envs,
322
+ HUMANISH_STUDY_PARTICIPANT: "1",
323
+ ...(plan.runtime.doNotTrack ? { DO_NOT_TRACK: "1" } : {}),
324
+ },
318
325
  requestTimeoutMs,
319
326
  timeoutMs: wallClockMs,
320
327
  onStdout: (data) => recordStreamedTerminalChunk("stdout", data),
@@ -69,6 +69,7 @@ export function planTerminalStudy(config, input) {
69
69
  ...(actor?.persona === undefined ? {} : { personaId: actor.persona }),
70
70
  ...(actor?.mission === undefined ? {} : { mission: actor.mission }),
71
71
  runtime: {
72
+ doNotTrack: config.execution?.terminal?.doNotTrack !== false,
72
73
  ...(runtimeVersion === undefined ? {} : { version: runtimeVersion }),
73
74
  // Codex's own default can change with any release and its JSON stream does not name it, so
74
75
  // the route always passes a model: the declared one, else humanish's participant default.
@@ -8,6 +8,7 @@ export declare function parseTerminalRuntimeVersion(stdout: string): string | un
8
8
  export declare function buildRuntimeVersionCommand(requestedVersion?: string): string;
9
9
  export declare function buildRuntimeExecPrefix(version: string, model: string, reasoningEffort?: ReasoningEffort): string;
10
10
  export declare function declaredRuntimeProvenance(args: {
11
+ doNotTrack: boolean;
11
12
  version?: string;
12
13
  model: string;
13
14
  modelSource: "declared" | "humanish_default";
@@ -35,6 +35,7 @@ export function declaredRuntimeProvenance(args) {
35
35
  return {
36
36
  schema: "humanish.actor-runtime.v1",
37
37
  package: TERMINAL_RUNTIME_PACKAGE,
38
+ doNotTrack: args.doNotTrack,
38
39
  requestedVersion: args.version ?? "latest",
39
40
  versionStatus: "unobserved",
40
41
  requestedModel: args.model,
@@ -75,6 +75,7 @@ export async function runLiveTerminalSession(args) {
75
75
  });
76
76
  const { version, model, modelSource, reasoningEffort } = plan.runtime;
77
77
  const runtime = declaredRuntimeProvenance({
78
+ doNotTrack: plan.runtime.doNotTrack,
78
79
  ...(version === undefined ? {} : { version }),
79
80
  model: sanitize(model),
80
81
  modelSource,
package/dist/run/costs.js CHANGED
@@ -1,4 +1,4 @@
1
- import { round6 } from "./pricing.js";
1
+ import { runEstimateUsd, sumEstimatedUsd } from "./run-cost.js";
2
2
  import { bindExistingRunArtifactPaths } from "./paths.js";
3
3
  import { contradictsAccountBilling } from "../verify/costs.js";
4
4
  import { EVIDENCE_LIMITS } from "../analysis/evidence.js";
@@ -20,13 +20,9 @@ export function emptyCostTotals() {
20
20
  analysisHistoryUncertainRuns: 0,
21
21
  };
22
22
  }
23
- const isKnownUsd = (value) => typeof value === "number" && Number.isFinite(value) && value >= 0;
24
- function sumKnown(a, b) {
25
- return a === null && b === null ? null : round6((a ?? 0) + (b ?? 0));
26
- }
27
23
  export function addCostTotals(into, next) {
28
24
  for (const key of ["estimatedTotalUsd", "runEstimatedUsd", "analysisEstimatedUsd"]) {
29
- into[key] = sumKnown(into[key], next[key]);
25
+ into[key] = sumEstimatedUsd(into[key], next[key]);
30
26
  }
31
27
  for (const key of [
32
28
  "incompleteRunEstimates",
@@ -49,11 +45,7 @@ export async function readCostTotals(cwd, entry) {
49
45
  const analysisWarnings = [];
50
46
  // A dry run makes no model request and creates no desktop, so with no figure it costs $0. A
51
47
  // recorded unknown (null) figure stays unknown.
52
- costs.runEstimatedUsd = isKnownUsd(entry.estimatedCostUsd)
53
- ? entry.estimatedCostUsd
54
- : entry.estimatedCostUsd === undefined && entry.mode === "dry-run"
55
- ? 0
56
- : null;
48
+ costs.runEstimatedUsd = runEstimateUsd(entry);
57
49
  try {
58
50
  const prepared = await bindExistingRunArtifactPaths(cwd, entry.runId);
59
51
  const bytes = await readBoundedFile(prepared, RUN_BUNDLE_FILE, EVIDENCE_LIMITS.sourceBytes);
@@ -70,9 +62,7 @@ export async function readCostTotals(cwd, entry) {
70
62
  runWarnings.push("RUN_COST_ID_MISMATCH");
71
63
  }
72
64
  if (bundle?.cost !== undefined) {
73
- costs.runEstimatedUsd = isKnownUsd(bundle.cost?.estimatedTotalUsd)
74
- ? bundle.cost.estimatedTotalUsd
75
- : null;
65
+ costs.runEstimatedUsd = runEstimateUsd({ estimatedCostUsd: bundle.cost?.estimatedTotalUsd });
76
66
  if (bundle.cost?.fullyEstimated !== true || costs.runEstimatedUsd === null) {
77
67
  costs.incompleteRunEstimates = 1;
78
68
  runWarnings.push("RUN_COST_PARTIAL_OR_UNKNOWN");
@@ -102,7 +92,7 @@ export async function readCostTotals(cwd, entry) {
102
92
  costs.analysisHistoryUncertainRuns = 1;
103
93
  analysisWarnings.push("STUDY_COST_ACCOUNTING_UNAVAILABLE");
104
94
  }
105
- costs.estimatedTotalUsd = sumKnown(costs.runEstimatedUsd, costs.analysisEstimatedUsd);
95
+ costs.estimatedTotalUsd = sumEstimatedUsd(costs.runEstimatedUsd, costs.analysisEstimatedUsd);
106
96
  return {
107
97
  runId: entry.runId,
108
98
  costs,
@@ -153,16 +143,15 @@ export async function readAnalysisAccounting(prepared) {
153
143
  }
154
144
  if (!usage.dispatched) {
155
145
  accounting.notDispatched += 1;
156
- accounting.estimatedUsd = sumKnown(accounting.estimatedUsd, 0);
146
+ accounting.estimatedUsd = sumEstimatedUsd(accounting.estimatedUsd, 0);
157
147
  continue;
158
148
  }
159
149
  accounting.dispatched += 1;
160
150
  if (record.receipt)
161
151
  providers.add(record.receipt.provider);
162
- if (isKnownUsd(usage.estimatedCostUsd)) {
163
- accounting.estimatedUsd = sumKnown(accounting.estimatedUsd, usage.estimatedCostUsd);
164
- }
165
- if (!isKnownUsd(usage.estimatedCostUsd) || !usage.usageComplete)
152
+ const estimatedUsd = runEstimateUsd({ estimatedCostUsd: usage.estimatedCostUsd });
153
+ accounting.estimatedUsd = sumEstimatedUsd(accounting.estimatedUsd, estimatedUsd);
154
+ if (estimatedUsd === null || !usage.usageComplete)
166
155
  accounting.unpriced += 1;
167
156
  }
168
157
  // A skipped or queued automatic job says nothing about earlier manual requests. Only final
@@ -11,7 +11,7 @@ interface RunThought {
11
11
  export interface RunParticipant {
12
12
  /** Stream id: stable within a run, and what distinguishes participants of one study. */
13
13
  id: string;
14
- /** What to call them on screen: the participant's own label, else the persona id. */
14
+ /** What to call them on screen: the caption the Observer shows, else the persona id. */
15
15
  label: string;
16
16
  /** The persona id, when the trace recorded one. */
17
17
  personaId?: string;
@@ -12,6 +12,8 @@ import path from "node:path";
12
12
  import { estimateActorCostForExecution } from "./pricing.js";
13
13
  import { resolveRunPath } from "./locate.js";
14
14
  import { RUN_BUNDLE_FILE } from "./bundle.js";
15
+ import { savedCaption } from "./participant-caption.js";
16
+ import { streamParticipantIdOf } from "./participant-records.js";
15
17
  import { readContainedRegularFile } from "./contained-output.js";
16
18
  import { isPathInside, resolvePhysicalCwd } from "./paths.js";
17
19
  const RUN_DETAIL_SCHEMA = "humanish.run-detail.v1";
@@ -47,13 +49,20 @@ function participantFrom(stream, index) {
47
49
  const trace = stream.liveActor ?? stream.actor ?? {};
48
50
  const id = stream.id ?? `stream-${index + 1}`;
49
51
  const personaId = trace.persona?.id;
52
+ const participantId = streamParticipantIdOf(stream);
50
53
  // Computed once: calling it twice to test-then-use reads as though the two could differ.
51
54
  const thought = latestThought(trace);
52
55
  const status = trace.status ?? stream.status;
53
56
  const thoughts = (trace.items ?? []).filter((item) => item?.kind === "reasoning").length;
54
57
  return {
55
58
  id,
56
- label: stream.label ?? personaId ?? id,
59
+ label: stream.label === undefined
60
+ ? (personaId ?? id)
61
+ : savedCaption({
62
+ label: stream.label,
63
+ ...(participantId === undefined ? {} : { participantId }),
64
+ ...(personaId === undefined ? {} : { personaId }),
65
+ }),
57
66
  ...(personaId === undefined ? {} : { personaId }),
58
67
  traits: Array.isArray(trace.persona?.traitsApplied) ? trace.persona.traitsApplied : [],
59
68
  ...(status === undefined ? {} : { status }),
@@ -49,14 +49,14 @@ export interface DisplayedBundle {
49
49
  * The facts for a run from its bundle. run.json's `outcome` decides when it has one. A bundle
50
50
  * without one is in progress, or was recorded before the field existed: its status record, when
51
51
  * given and naming the run, then says whether the run is alive and, for the older run, holds its ok.
52
- * With no record, a bundle whose simulations are still running is live.
52
+ * With no record, a bundle whose simulations are still running is interrupted.
53
53
  */
54
54
  export declare function bundleDisplayFacts(bundle: DisplayedBundle, status?: unknown, nowMs?: number): RunDisplayFacts;
55
55
  /**
56
56
  * How review.md says the run ended: the display of the bundle it is rendered from, so a run whose
57
57
  * execution failed never reads as a pass beside its participants' verdict.
58
58
  */
59
- export declare function reviewOutcome(bundle: DisplayedBundle): string;
59
+ export declare function reviewOutcome(bundle: DisplayedBundle, status?: unknown): string;
60
60
  /**
61
61
  * The summary and gaps a review shows beside its run's display. A run stopped while a participant
62
62
  * was still running keeps the review its last live flush wrote, which describes that session as
@@ -2,7 +2,8 @@
2
2
  // library, `humanish runs`, `humanish stats` and review.md. Each of them calls runDisplay with the
3
3
  // same facts, so a run whose own ok is false shows as a pass on none of them, and two surfaces
4
4
  // cannot disagree about one run.
5
- import { classifyRunStatus, isRunStatusRecord } from "./status.js";
5
+ import {} from "./status.js";
6
+ import { runLiveness } from "./liveness.js";
6
7
  import { plural } from "./text.js";
7
8
  const LABELS = {
8
9
  running: "running",
@@ -72,38 +73,34 @@ function runDisplayLine(display) {
72
73
  * The facts for a run from its bundle. run.json's `outcome` decides when it has one. A bundle
73
74
  * without one is in progress, or was recorded before the field existed: its status record, when
74
75
  * given and naming the run, then says whether the run is alive and, for the older run, holds its ok.
75
- * With no record, a bundle whose simulations are still running is live.
76
+ * With no record, a bundle whose simulations are still running is interrupted.
76
77
  */
77
78
  export function bundleDisplayFacts(bundle, status, nowMs = Date.now()) {
78
- const base = { mode: bundle.mode, verdict: bundle.review?.verdict };
79
+ const { liveness, record } = runLiveness(bundle.runId, status, bundle, nowMs);
80
+ const base = { liveness, mode: bundle.mode, verdict: bundle.review?.verdict };
79
81
  const outcome = bundle.outcome;
80
82
  if (outcome?.state === "interrupted")
81
- return { ...base, liveness: "interrupted" };
83
+ return base;
82
84
  if (outcome?.state === "finished")
83
85
  return {
84
86
  ...base,
85
- liveness: "finished",
86
87
  ok: outcome.ok,
87
88
  failure: outcome.execution.failures[0],
88
89
  };
89
- if (isRunStatusRecord(status) && status.runId === bundle.runId)
90
- return {
91
- ...base,
92
- liveness: classifyRunStatus(status, nowMs),
93
- ...(typeof status.outcome?.ok === "boolean" ? { ok: status.outcome.ok } : {}),
94
- ...(status.outcome?.execution?.failures[0] === undefined
95
- ? {}
96
- : { failure: status.outcome.execution.failures[0] }),
97
- };
98
- const inProgress = (bundle.simulations ?? []).some((simulation) => simulation?.status === "running");
99
- return { ...base, liveness: inProgress ? "running" : "finished" };
90
+ return {
91
+ ...base,
92
+ ...(typeof record?.outcome?.ok === "boolean" ? { ok: record.outcome.ok } : {}),
93
+ ...(record?.outcome?.execution?.failures[0] === undefined
94
+ ? {}
95
+ : { failure: record.outcome.execution.failures[0] }),
96
+ };
100
97
  }
101
98
  /**
102
99
  * How review.md says the run ended: the display of the bundle it is rendered from, so a run whose
103
100
  * execution failed never reads as a pass beside its participants' verdict.
104
101
  */
105
- export function reviewOutcome(bundle) {
106
- return runDisplayLine(runDisplay(bundleDisplayFacts(bundle)));
102
+ export function reviewOutcome(bundle, status) {
103
+ return runDisplayLine(runDisplay(bundleDisplayFacts(bundle, status)));
107
104
  }
108
105
  /**
109
106
  * The summary and gaps a review shows beside its run's display. A run stopped while a participant
@@ -9,8 +9,8 @@ import {} from "./streams.js";
9
9
  import { participantEvent, participantIdsOf, participantRecord, participantStream, } from "./participant-records.js";
10
10
  import { implicitProjectDirectoryExists, readPackageName, validateCwd } from "./project.js";
11
11
  import { loadDryRunInputs } from "./dry-run-inputs.js";
12
- import { judgeExecution, judgePreview, OUTCOME_POLICIES, resultOk, verdictText, } from "./judge.js";
13
- import { reviewOutcome } from "./display.js";
12
+ import { judgeExecution, judgePreview, OUTCOME_POLICIES, resultOk } from "./judge.js";
13
+ import { renderReviewMarkdown } from "./review-markdown.js";
14
14
  import { cli } from "../cli/invocation.js";
15
15
  /**
16
16
  * The preview route's run. The run scope closes the run it started on every exit, including
@@ -68,7 +68,7 @@ async function runDryRunInScope(options, scope) {
68
68
  mode: options.dryRun ? "dry-run" : "live",
69
69
  study: options.study,
70
70
  warnings: options.warnings,
71
- renderReview: renderReviewMarkdown,
71
+ renderReview: renderPreviewReview,
72
72
  // The run starts at the time its id and source were stamped with.
73
73
  now: () => Date.parse(createdAt),
74
74
  ...(options.observer === undefined ? {} : { observer: { open: options.observer.open } }),
@@ -170,28 +170,13 @@ function createReviewSummary(verdict) {
170
170
  };
171
171
  }
172
172
  /** The preview's review.md. */
173
- function renderReviewMarkdown(bundle) {
174
- return `# humanish Run Review
175
-
176
- Run: ${bundle.runId}
177
-
178
- Mode: ${bundle.mode}
179
-
180
- Verdict: ${verdictText(bundle.review.verdict, bundle.mode)}
181
-
182
- Outcome: ${reviewOutcome(bundle)}
183
-
184
- ${bundle.review.summary}
185
-
186
- ## Public-Safety
187
-
188
- - Redaction: ${bundle.redaction.status}
189
- - Notes: ${bundle.redaction.notes}
190
-
191
- ## Gaps
192
-
193
- ${bundle.review.gaps.map((gap) => `- ${gap}`).join("\n")}
194
- `;
173
+ function renderPreviewReview(bundle, status) {
174
+ return renderReviewMarkdown(bundle, [
175
+ "## Public-Safety",
176
+ "",
177
+ `- Redaction: ${bundle.redaction.status}`,
178
+ `- Notes: ${bundle.redaction.notes}`,
179
+ ], { style: "preview", status });
195
180
  }
196
181
  /**
197
182
  * Render through the finished run, so a directory swapped in under the same id is never rendered.
@@ -41,6 +41,30 @@ export interface ParticipantFacts {
41
41
  /** selfReportedBlocker held for its session. */
42
42
  selfReportedBlocker: boolean;
43
43
  }
44
+ /** The participant facts its bundle and result can name. */
45
+ export interface ParticipantRecordFacts extends ParticipantFacts {
46
+ id?: string;
47
+ reason?: string;
48
+ skippedReason?: string;
49
+ reportedFriction?: boolean;
50
+ inProgress?: boolean;
51
+ }
52
+ /** Judge participant evidence for the bundle and the result in the same order. */
53
+ export declare function judgeParticipantRecords(participants: readonly ParticipantRecordFacts[], wording?: {
54
+ sessionLabel?: string;
55
+ missingSessionMessage?: string;
56
+ runningReason?: string;
57
+ contractReason?: string;
58
+ }): {
59
+ participants: {
60
+ status: "abandoned" | "blocked" | "contract_proof_only" | "failed" | "incomplete" | "passed" | "running" | "timed_out";
61
+ judgedStatus: "abandoned" | "blocked" | "failed" | "incomplete" | "passed" | "timed_out" | undefined;
62
+ reason: string;
63
+ notPassedMessage: string;
64
+ gapLine: string | undefined;
65
+ }[];
66
+ tally: import("./bundle.js").ParticipantOutcomes;
67
+ };
44
68
  /**
45
69
  * The status a participant is judged under. A harness failure is failed, and a skipped participant
46
70
  * is blocked, as the fan-out bundle and the result write it. Otherwise it is the tallied
package/dist/run/judge.js CHANGED
@@ -12,6 +12,7 @@
12
12
  // 4. foldScorerFailures, which every route with a scorer applies before the run finishes;
13
13
  // 5. the outcome: judgeExecution applies the route's policy to its execution failures, and
14
14
  // resultOk gives the result's ok.
15
+ import { tallyParticipantOutcomes } from "./outcomes.js";
15
16
  import { cli } from "../cli/invocation.js";
16
17
  /**
17
18
  * The participant said it reached the goal having taken no action and said nothing, and no stop
@@ -35,6 +36,59 @@ export function selfReportedBlocker(ending) {
35
36
  ending.closingReportReadsBlocked &&
36
37
  !ending.stopConditionMatched);
37
38
  }
39
+ /** Judge participant evidence for the bundle and the result in the same order. */
40
+ export function judgeParticipantRecords(participants, wording = {}) {
41
+ const records = participants.map((participant) => {
42
+ const status = participant.inProgress
43
+ ? "running"
44
+ : participant.skipped
45
+ ? "blocked"
46
+ : (participant.status ??
47
+ (participant.sessionError === undefined ? "contract_proof_only" : "failed"));
48
+ const judged = participant.inProgress ? undefined : judgedStatus(participant);
49
+ return {
50
+ status,
51
+ judgedStatus: judged,
52
+ reason: participant.inProgress
53
+ ? (wording.runningReason ??
54
+ "Live computer-use session is running; stream auth URL is available only through the attached Observer server.")
55
+ : (participant.skippedReason ??
56
+ participant.reason ??
57
+ participant.sessionError ??
58
+ wording.contractReason ??
59
+ "Dry run: the evidence shape was written without launching a desktop or spending provider tokens."),
60
+ notPassedMessage: participantNotPassedMessage(participant, wording),
61
+ gapLine: participant.inProgress || judged === "passed"
62
+ ? undefined
63
+ : `${participant.id ?? "Participant"}: ${participant.skippedReason ?? participant.sessionError ?? participant.reason ?? "did not pass"}`,
64
+ };
65
+ });
66
+ const terminal = records.filter((_, index) => participants[index].status !== undefined && !participants[index].inProgress);
67
+ return {
68
+ participants: records,
69
+ tally: tallyParticipantOutcomes(terminal.map((participant) => participant.judgedStatus), participants
70
+ .filter((participant) => participant.status !== undefined && !participant.inProgress)
71
+ .map((participant) => participant.reportedFriction === true)),
72
+ };
73
+ }
74
+ function participantNotPassedMessage(participant, wording) {
75
+ const label = wording.sessionLabel ?? "Computer-use session";
76
+ if (participant.sessionError !== undefined)
77
+ return participant.sessionError;
78
+ if (participant.skippedReason !== undefined)
79
+ return participant.skippedReason;
80
+ if (participant.noEngagement)
81
+ return "Actor took no actions and produced no message (likely a blank/still-loading screen); not a credible goal_satisfied.";
82
+ if (participant.selfReportedBlocker)
83
+ return "Actor reported goal_satisfied while its final message described a blocker or asked for missing instructions; not a credible pass.";
84
+ if (participant.completionReason === "harness_error")
85
+ return `${label} ended with a harness error: ${participant.reason}`;
86
+ if (participant.status === undefined && wording.missingSessionMessage !== undefined)
87
+ return wording.missingSessionMessage;
88
+ if (participant.status !== "passed")
89
+ return `${label} ended with ${participant.status ?? "unknown"}: ${participant.reason ?? "no terminal reason"}`;
90
+ return (wording.missingSessionMessage ?? "The computer-use run did not produce a terminal session.");
91
+ }
38
92
  /**
39
93
  * The status a participant is judged under. A harness failure is failed, and a skipped participant
40
94
  * is blocked, as the fan-out bundle and the result write it. Otherwise it is the tallied
@@ -0,0 +1,18 @@
1
+ import type { RunOutcome } from "./bundle.js";
2
+ import { type RunLiveness, type RunStatusRecord } from "./status.js";
3
+ interface LivenessBundle {
4
+ simulations?: ReadonlyArray<{
5
+ status?: string | undefined;
6
+ } | undefined> | undefined;
7
+ outcome?: RunOutcome | undefined;
8
+ }
9
+ /**
10
+ * One run's liveness: the bundle outcome, then usable status, then bundle-only evidence.
11
+ * `readRunIndex` reads in the same order; `humanish verify` calls this with both in hand. `record`
12
+ * is the status record the liveness came from, when there was one.
13
+ */
14
+ export declare function runLiveness(runId: string, status: unknown, bundle: LivenessBundle, nowMs: number): {
15
+ liveness: RunLiveness;
16
+ record?: RunStatusRecord;
17
+ };
18
+ export {};
@@ -0,0 +1,24 @@
1
+ import { classifyRunStatus, isRunStatusRecord, } from "./status.js";
2
+ /**
3
+ * The bundle-only reading, for a run with no usable status record. A bundle on disk usually means
4
+ * the run reached its final write. But a live run now flushes an in-progress bundle as it goes (so
5
+ * anything asking what a participant is doing has something to read), and that bundle marks its
6
+ * simulations `running`. With no status record there is no freshness to judge, and the
7
+ * reading of "it started, and nothing here says it finished" is interrupted, not finished.
8
+ */
9
+ function bundleLiveness(bundle) {
10
+ const inProgress = (bundle.simulations ?? []).some((simulation) => simulation?.status === "running");
11
+ return inProgress ? "interrupted" : "finished";
12
+ }
13
+ /**
14
+ * One run's liveness: the bundle outcome, then usable status, then bundle-only evidence.
15
+ * `readRunIndex` reads in the same order; `humanish verify` calls this with both in hand. `record`
16
+ * is the status record the liveness came from, when there was one.
17
+ */
18
+ export function runLiveness(runId, status, bundle, nowMs) {
19
+ if (bundle.outcome !== undefined)
20
+ return { liveness: bundle.outcome.state };
21
+ return isRunStatusRecord(status) && status.runId === runId
22
+ ? { liveness: classifyRunStatus(status, nowMs), record: status }
23
+ : { liveness: bundleLiveness(bundle) };
24
+ }
@@ -1,4 +1,4 @@
1
- import { type ActorStatus } from "../actors/contract.js";
1
+ import { type ActorStatus, type ActorTrace } from "../actors/contract.js";
2
2
  import { type CuaGoalSource } from "../actors/goal-source.js";
3
3
  import type { TaskFunnel } from "../study/tasks.js";
4
4
  import type { ParticipantOutcomes, ReviewSummary, RunTaskFunnel } from "./bundle.js";
@@ -16,6 +16,13 @@ type ParticipantOutcomeDetail = {
16
16
  label?: string;
17
17
  goalSource?: CuaGoalSource;
18
18
  };
19
+ /**
20
+ * How one participant's session ended, in words a participant summary can end on. A computer-use
21
+ * goal says where the claim came from, as the run's participant line does.
22
+ */
23
+ export declare function sessionEndingInWords(trace: ActorTrace): string;
24
+ /** "; the gaps list the other 2", or "all 3" when nobody passed. Empty when the gaps list nobody. */
25
+ export declare function gapsListClause(listed: number, total: number): string;
19
26
  /** One line a stakeholder can read, with the denominator attached to every number. */
20
27
  export declare function formatParticipantOutcomes(outcomes: ParticipantOutcomes, terminalCauses?: readonly ParticipantOutcomeDetail[]): string;
21
28
  /** Presentation details tolerate optional legacy actor payloads without changing their tallies. */