humanish 0.111.4 → 0.113.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (158) hide show
  1. package/CHANGELOG.md +37 -0
  2. package/README.md +11 -0
  3. package/TELEMETRY.md +21 -0
  4. package/dist/actors/codex/restricted-participant-policy.d.ts +21 -1
  5. package/dist/actors/codex/restricted-participant-policy.js +20 -2
  6. package/dist/actors/codex/restricted-participant.d.ts +1 -1
  7. package/dist/actors/codex/restricted-participant.js +2 -1
  8. package/dist/actors/computer-use/loop/debrief.d.ts +12 -4
  9. package/dist/actors/computer-use/loop/debrief.js +57 -20
  10. package/dist/actors/computer-use/loop/ending.d.ts +5 -2
  11. package/dist/actors/computer-use/loop/impressions.d.ts +14 -0
  12. package/dist/actors/computer-use/loop/impressions.js +40 -0
  13. package/dist/actors/computer-use/loop/session.d.ts +3 -1
  14. package/dist/actors/computer-use/loop/session.js +2 -0
  15. package/dist/actors/computer-use/loop/trace.js +2 -0
  16. package/dist/actors/computer-use/loop/types.d.ts +8 -1
  17. package/dist/actors/computer-use/loop.d.ts +2 -1
  18. package/dist/actors/computer-use/loop.js +15 -3
  19. package/dist/actors/computer-use/openai-conversation.d.ts +96 -0
  20. package/dist/actors/computer-use/openai-conversation.js +238 -0
  21. package/dist/actors/computer-use/openai-provider.js +38 -48
  22. package/dist/actors/contract.d.ts +45 -3
  23. package/dist/actors/contract.js +13 -0
  24. package/dist/analysis/diagnostics.js +7 -12
  25. package/dist/analysis/evidence-sources.d.ts +3 -1
  26. package/dist/analysis/evidence-sources.js +10 -1
  27. package/dist/analysis/evidence.js +2 -2
  28. package/dist/analysis/execute.d.ts +1 -1
  29. package/dist/analysis/execute.js +37 -5
  30. package/dist/analysis/service.js +7 -1
  31. package/dist/analysis/types.d.ts +23 -0
  32. package/dist/analysis/validation.d.ts +80 -35
  33. package/dist/analysis/validation.js +58 -11
  34. package/dist/cli/commands/runs.js +37 -20
  35. package/dist/cli/commands/setup.js +95 -6
  36. package/dist/cli/commands/study-format.js +7 -4
  37. package/dist/cli/commands/study.js +1 -1
  38. package/dist/cli/commands/tui.js +1 -1
  39. package/dist/cli/doctor.d.ts +2 -0
  40. package/dist/cli/doctor.js +18 -0
  41. package/dist/cli/findings.d.ts +34 -3
  42. package/dist/cli/findings.js +136 -18
  43. package/dist/cli/io.d.ts +4 -1
  44. package/dist/cli/io.js +3 -2
  45. package/dist/cli/program.js +35 -1
  46. package/dist/cli/secret-prompt.d.ts +5 -1
  47. package/dist/cli/secret-prompt.js +6 -2
  48. package/dist/cli/update-check-worker.d.ts +1 -0
  49. package/dist/cli/update-check-worker.js +5 -0
  50. package/dist/cli/update-check.d.ts +46 -0
  51. package/dist/cli/update-check.js +203 -0
  52. package/dist/feedback/draft.js +32 -4
  53. package/dist/keys/key-resolution.js +2 -0
  54. package/dist/keys/key-status.d.ts +37 -0
  55. package/dist/keys/key-status.js +68 -0
  56. package/dist/observer/run-routes.js +8 -4
  57. package/dist/observer-app.html +4 -4
  58. package/dist/routes/computer-use/bundle.d.ts +1 -1
  59. package/dist/routes/computer-use/bundle.js +6 -16
  60. package/dist/routes/computer-use/fanout-bundle.js +40 -37
  61. package/dist/routes/computer-use/fanout-records.d.ts +2 -0
  62. package/dist/routes/computer-use/fanout-records.js +18 -26
  63. package/dist/routes/computer-use/local-vm.js +9 -2
  64. package/dist/routes/computer-use/participant-desktop.d.ts +6 -0
  65. package/dist/routes/computer-use/participant-execution.js +13 -12
  66. package/dist/routes/computer-use/participant-facts.d.ts +2 -2
  67. package/dist/routes/computer-use/participant-facts.js +10 -1
  68. package/dist/routes/computer-use/rerun-selection.js +23 -21
  69. package/dist/routes/computer-use/result.js +10 -17
  70. package/dist/routes/computer-use/self-report.js +7 -1
  71. package/dist/routes/computer-use/setup.js +14 -0
  72. package/dist/routes/computer-use/single-bundle.d.ts +2 -0
  73. package/dist/routes/computer-use/single-bundle.js +81 -37
  74. package/dist/routes/computer-use/types.d.ts +4 -1
  75. package/dist/routes/scripted/bundle.d.ts +1 -1
  76. package/dist/routes/scripted/bundle.js +9 -19
  77. package/dist/routes/scripted/surface-records.d.ts +0 -1
  78. package/dist/routes/scripted/surface-records.js +27 -13
  79. package/dist/routes/shared-world/bundle-records.d.ts +2 -2
  80. package/dist/routes/shared-world/bundle-records.js +24 -30
  81. package/dist/routes/shared-world/bundle.d.ts +1 -1
  82. package/dist/routes/shared-world/bundle.js +71 -79
  83. package/dist/routes/shared-world/participant-specs.d.ts +1 -2
  84. package/dist/routes/shared-world/participant-specs.js +0 -8
  85. package/dist/routes/shared-world/result.js +8 -12
  86. package/dist/routes/terminal/bundle.d.ts +1 -1
  87. package/dist/routes/terminal/bundle.js +10 -19
  88. package/dist/routes/terminal/dry-run.js +1 -0
  89. package/dist/routes/terminal/live-sandbox.js +9 -2
  90. package/dist/routes/terminal/plan.js +1 -0
  91. package/dist/routes/terminal/runtime.d.ts +1 -0
  92. package/dist/routes/terminal/runtime.js +1 -0
  93. package/dist/routes/terminal/session.js +1 -0
  94. package/dist/run/costs.js +9 -20
  95. package/dist/run/detail.d.ts +1 -1
  96. package/dist/run/detail.js +13 -4
  97. package/dist/run/display.d.ts +2 -2
  98. package/dist/run/display.js +15 -18
  99. package/dist/run/dry-run.js +10 -25
  100. package/dist/run/judge.d.ts +24 -0
  101. package/dist/run/judge.js +54 -0
  102. package/dist/run/liveness.d.ts +18 -0
  103. package/dist/run/liveness.js +24 -0
  104. package/dist/run/outcomes.d.ts +8 -1
  105. package/dist/run/outcomes.js +58 -12
  106. package/dist/run/participant-caption.d.ts +31 -0
  107. package/dist/run/participant-caption.js +60 -0
  108. package/dist/run/participant-records.d.ts +4 -2
  109. package/dist/run/participant-records.js +6 -0
  110. package/dist/run/review-markdown.d.ts +11 -0
  111. package/dist/run/review-markdown.js +22 -0
  112. package/dist/run/run-cost.d.ts +7 -0
  113. package/dist/run/run-cost.js +25 -5
  114. package/dist/run/run-index.d.ts +4 -50
  115. package/dist/run/run-index.js +57 -87
  116. package/dist/run/run.d.ts +2 -2
  117. package/dist/run/run.js +5 -3
  118. package/dist/run/stats.js +2 -12
  119. package/dist/run/transient-comms-secrets.d.ts +8 -3
  120. package/dist/run/transient-comms-secrets.js +21 -1
  121. package/dist/run-study.js +17 -1
  122. package/dist/study/discover.js +6 -1
  123. package/dist/study/doctor.d.ts +2 -0
  124. package/dist/study/doctor.js +45 -7
  125. package/dist/study/init-templates.js +10 -5
  126. package/dist/study/keys.d.ts +1 -0
  127. package/dist/study/keys.js +1 -1
  128. package/dist/study/parse/execution.js +7 -1
  129. package/dist/study/plan-types.d.ts +1 -0
  130. package/dist/study/plan.js +3 -2
  131. package/dist/study/preflight-result.d.ts +1 -0
  132. package/dist/study/preflight-result.js +1 -1
  133. package/dist/study/preflight.d.ts +2 -0
  134. package/dist/study/preflight.js +54 -14
  135. package/dist/study/types.d.ts +4 -1
  136. package/dist/study/warnings.d.ts +7 -1
  137. package/dist/study/warnings.js +82 -4
  138. package/dist/substrates/local/capacity.d.ts +75 -0
  139. package/dist/substrates/local/capacity.js +143 -0
  140. package/dist/substrates/local/firecracker-desktop.d.ts +8 -0
  141. package/dist/substrates/local/firecracker-desktop.js +34 -3
  142. package/dist/substrates/local/runtime-host.d.ts +16 -3
  143. package/dist/substrates/local/runtime-host.js +49 -4
  144. package/dist/substrates/local/runtime.d.ts +10 -0
  145. package/dist/substrates/local/runtime.js +44 -5
  146. package/dist/tui-app.js +75 -75
  147. package/dist/verify/liveness.d.ts +2 -2
  148. package/dist/verify/liveness.js +7 -4
  149. package/docs/architecture/actor-contract.md +20 -6
  150. package/docs/architecture/terminal-product-route.md +14 -0
  151. package/docs/contracts/feedback.md +8 -1
  152. package/docs/contracts/run-bundle.md +10 -0
  153. package/docs/contracts/schemas.md +65 -34
  154. package/docs/contracts/study-analysis.md +86 -22
  155. package/package.json +2 -1
  156. package/skills/humanish/SKILL.md +21 -10
  157. package/dist/actors/computer-use/openai-requests.d.ts +0 -62
  158. package/dist/actors/computer-use/openai-requests.js +0 -144
@@ -2,7 +2,8 @@
2
2
  // library, `humanish runs`, `humanish stats` and review.md. Each of them calls runDisplay with the
3
3
  // same facts, so a run whose own ok is false shows as a pass on none of them, and two surfaces
4
4
  // cannot disagree about one run.
5
- import { classifyRunStatus, isRunStatusRecord } from "./status.js";
5
+ import {} from "./status.js";
6
+ import { runLiveness } from "./liveness.js";
6
7
  import { plural } from "./text.js";
7
8
  const LABELS = {
8
9
  running: "running",
@@ -72,38 +73,34 @@ function runDisplayLine(display) {
72
73
  * The facts for a run from its bundle. run.json's `outcome` decides when it has one. A bundle
73
74
  * without one is in progress, or was recorded before the field existed: its status record, when
74
75
  * given and naming the run, then says whether the run is alive and, for the older run, holds its ok.
75
- * With no record, a bundle whose simulations are still running is live.
76
+ * With no record, a bundle whose simulations are still running is interrupted.
76
77
  */
77
78
  export function bundleDisplayFacts(bundle, status, nowMs = Date.now()) {
78
- const base = { mode: bundle.mode, verdict: bundle.review?.verdict };
79
+ const { liveness, record } = runLiveness(bundle.runId, status, bundle, nowMs);
80
+ const base = { liveness, mode: bundle.mode, verdict: bundle.review?.verdict };
79
81
  const outcome = bundle.outcome;
80
82
  if (outcome?.state === "interrupted")
81
- return { ...base, liveness: "interrupted" };
83
+ return base;
82
84
  if (outcome?.state === "finished")
83
85
  return {
84
86
  ...base,
85
- liveness: "finished",
86
87
  ok: outcome.ok,
87
88
  failure: outcome.execution.failures[0],
88
89
  };
89
- if (isRunStatusRecord(status) && status.runId === bundle.runId)
90
- return {
91
- ...base,
92
- liveness: classifyRunStatus(status, nowMs),
93
- ...(typeof status.outcome?.ok === "boolean" ? { ok: status.outcome.ok } : {}),
94
- ...(status.outcome?.execution?.failures[0] === undefined
95
- ? {}
96
- : { failure: status.outcome.execution.failures[0] }),
97
- };
98
- const inProgress = (bundle.simulations ?? []).some((simulation) => simulation?.status === "running");
99
- return { ...base, liveness: inProgress ? "running" : "finished" };
90
+ return {
91
+ ...base,
92
+ ...(typeof record?.outcome?.ok === "boolean" ? { ok: record.outcome.ok } : {}),
93
+ ...(record?.outcome?.execution?.failures[0] === undefined
94
+ ? {}
95
+ : { failure: record.outcome.execution.failures[0] }),
96
+ };
100
97
  }
101
98
  /**
102
99
  * How review.md says the run ended: the display of the bundle it is rendered from, so a run whose
103
100
  * execution failed never reads as a pass beside its participants' verdict.
104
101
  */
105
- export function reviewOutcome(bundle) {
106
- return runDisplayLine(runDisplay(bundleDisplayFacts(bundle)));
102
+ export function reviewOutcome(bundle, status) {
103
+ return runDisplayLine(runDisplay(bundleDisplayFacts(bundle, status)));
107
104
  }
108
105
  /**
109
106
  * The summary and gaps a review shows beside its run's display. A run stopped while a participant
@@ -9,8 +9,8 @@ import {} from "./streams.js";
9
9
  import { participantEvent, participantIdsOf, participantRecord, participantStream, } from "./participant-records.js";
10
10
  import { implicitProjectDirectoryExists, readPackageName, validateCwd } from "./project.js";
11
11
  import { loadDryRunInputs } from "./dry-run-inputs.js";
12
- import { judgeExecution, judgePreview, OUTCOME_POLICIES, resultOk, verdictText, } from "./judge.js";
13
- import { reviewOutcome } from "./display.js";
12
+ import { judgeExecution, judgePreview, OUTCOME_POLICIES, resultOk } from "./judge.js";
13
+ import { renderReviewMarkdown } from "./review-markdown.js";
14
14
  import { cli } from "../cli/invocation.js";
15
15
  /**
16
16
  * The preview route's run. The run scope closes the run it started on every exit, including
@@ -68,7 +68,7 @@ async function runDryRunInScope(options, scope) {
68
68
  mode: options.dryRun ? "dry-run" : "live",
69
69
  study: options.study,
70
70
  warnings: options.warnings,
71
- renderReview: renderReviewMarkdown,
71
+ renderReview: renderPreviewReview,
72
72
  // The run starts at the time its id and source were stamped with.
73
73
  now: () => Date.parse(createdAt),
74
74
  ...(options.observer === undefined ? {} : { observer: { open: options.observer.open } }),
@@ -170,28 +170,13 @@ function createReviewSummary(verdict) {
170
170
  };
171
171
  }
172
172
  /** The preview's review.md. */
173
- function renderReviewMarkdown(bundle) {
174
- return `# humanish Run Review
175
-
176
- Run: ${bundle.runId}
177
-
178
- Mode: ${bundle.mode}
179
-
180
- Verdict: ${verdictText(bundle.review.verdict, bundle.mode)}
181
-
182
- Outcome: ${reviewOutcome(bundle)}
183
-
184
- ${bundle.review.summary}
185
-
186
- ## Public-Safety
187
-
188
- - Redaction: ${bundle.redaction.status}
189
- - Notes: ${bundle.redaction.notes}
190
-
191
- ## Gaps
192
-
193
- ${bundle.review.gaps.map((gap) => `- ${gap}`).join("\n")}
194
- `;
173
+ function renderPreviewReview(bundle, status) {
174
+ return renderReviewMarkdown(bundle, [
175
+ "## Public-Safety",
176
+ "",
177
+ `- Redaction: ${bundle.redaction.status}`,
178
+ `- Notes: ${bundle.redaction.notes}`,
179
+ ], { style: "preview", status });
195
180
  }
196
181
  /**
197
182
  * Render through the finished run, so a directory swapped in under the same id is never rendered.
@@ -41,6 +41,30 @@ export interface ParticipantFacts {
41
41
  /** selfReportedBlocker held for its session. */
42
42
  selfReportedBlocker: boolean;
43
43
  }
44
+ /** The participant facts its bundle and result can name. */
45
+ export interface ParticipantRecordFacts extends ParticipantFacts {
46
+ id?: string;
47
+ reason?: string;
48
+ skippedReason?: string;
49
+ reportedFriction?: boolean;
50
+ inProgress?: boolean;
51
+ }
52
+ /** Judge participant evidence for the bundle and the result in the same order. */
53
+ export declare function judgeParticipantRecords(participants: readonly ParticipantRecordFacts[], wording?: {
54
+ sessionLabel?: string;
55
+ missingSessionMessage?: string;
56
+ runningReason?: string;
57
+ contractReason?: string;
58
+ }): {
59
+ participants: {
60
+ status: "abandoned" | "blocked" | "contract_proof_only" | "failed" | "incomplete" | "passed" | "running" | "timed_out";
61
+ judgedStatus: "abandoned" | "blocked" | "failed" | "incomplete" | "passed" | "timed_out" | undefined;
62
+ reason: string;
63
+ notPassedMessage: string;
64
+ gapLine: string | undefined;
65
+ }[];
66
+ tally: import("./bundle.js").ParticipantOutcomes;
67
+ };
44
68
  /**
45
69
  * The status a participant is judged under. A harness failure is failed, and a skipped participant
46
70
  * is blocked, as the fan-out bundle and the result write it. Otherwise it is the tallied
package/dist/run/judge.js CHANGED
@@ -12,6 +12,7 @@
12
12
  // 4. foldScorerFailures, which every route with a scorer applies before the run finishes;
13
13
  // 5. the outcome: judgeExecution applies the route's policy to its execution failures, and
14
14
  // resultOk gives the result's ok.
15
+ import { tallyParticipantOutcomes } from "./outcomes.js";
15
16
  import { cli } from "../cli/invocation.js";
16
17
  /**
17
18
  * The participant said it reached the goal having taken no action and said nothing, and no stop
@@ -35,6 +36,59 @@ export function selfReportedBlocker(ending) {
35
36
  ending.closingReportReadsBlocked &&
36
37
  !ending.stopConditionMatched);
37
38
  }
39
+ /** Judge participant evidence for the bundle and the result in the same order. */
40
+ export function judgeParticipantRecords(participants, wording = {}) {
41
+ const records = participants.map((participant) => {
42
+ const status = participant.inProgress
43
+ ? "running"
44
+ : participant.skipped
45
+ ? "blocked"
46
+ : (participant.status ??
47
+ (participant.sessionError === undefined ? "contract_proof_only" : "failed"));
48
+ const judged = participant.inProgress ? undefined : judgedStatus(participant);
49
+ return {
50
+ status,
51
+ judgedStatus: judged,
52
+ reason: participant.inProgress
53
+ ? (wording.runningReason ??
54
+ "Live computer-use session is running; stream auth URL is available only through the attached Observer server.")
55
+ : (participant.skippedReason ??
56
+ participant.reason ??
57
+ participant.sessionError ??
58
+ wording.contractReason ??
59
+ "Dry run: the evidence shape was written without launching a desktop or spending provider tokens."),
60
+ notPassedMessage: participantNotPassedMessage(participant, wording),
61
+ gapLine: participant.inProgress || judged === "passed"
62
+ ? undefined
63
+ : `${participant.id ?? "Participant"}: ${participant.skippedReason ?? participant.sessionError ?? participant.reason ?? "did not pass"}`,
64
+ };
65
+ });
66
+ const terminal = records.filter((_, index) => participants[index].status !== undefined && !participants[index].inProgress);
67
+ return {
68
+ participants: records,
69
+ tally: tallyParticipantOutcomes(terminal.map((participant) => participant.judgedStatus), participants
70
+ .filter((participant) => participant.status !== undefined && !participant.inProgress)
71
+ .map((participant) => participant.reportedFriction === true)),
72
+ };
73
+ }
74
+ function participantNotPassedMessage(participant, wording) {
75
+ const label = wording.sessionLabel ?? "Computer-use session";
76
+ if (participant.sessionError !== undefined)
77
+ return participant.sessionError;
78
+ if (participant.skippedReason !== undefined)
79
+ return participant.skippedReason;
80
+ if (participant.noEngagement)
81
+ return "Actor took no actions and produced no message (likely a blank/still-loading screen); not a credible goal_satisfied.";
82
+ if (participant.selfReportedBlocker)
83
+ return "Actor reported goal_satisfied while its final message described a blocker or asked for missing instructions; not a credible pass.";
84
+ if (participant.completionReason === "harness_error")
85
+ return `${label} ended with a harness error: ${participant.reason}`;
86
+ if (participant.status === undefined && wording.missingSessionMessage !== undefined)
87
+ return wording.missingSessionMessage;
88
+ if (participant.status !== "passed")
89
+ return `${label} ended with ${participant.status ?? "unknown"}: ${participant.reason ?? "no terminal reason"}`;
90
+ return (wording.missingSessionMessage ?? "The computer-use run did not produce a terminal session.");
91
+ }
38
92
  /**
39
93
  * The status a participant is judged under. A harness failure is failed, and a skipped participant
40
94
  * is blocked, as the fan-out bundle and the result write it. Otherwise it is the tallied
@@ -0,0 +1,18 @@
1
+ import type { RunOutcome } from "./bundle.js";
2
+ import { type RunLiveness, type RunStatusRecord } from "./status.js";
3
+ interface LivenessBundle {
4
+ simulations?: ReadonlyArray<{
5
+ status?: string | undefined;
6
+ } | undefined> | undefined;
7
+ outcome?: RunOutcome | undefined;
8
+ }
9
+ /**
10
+ * One run's liveness: the bundle outcome, then usable status, then bundle-only evidence.
11
+ * `readRunIndex` reads in the same order; `humanish verify` calls this with both in hand. `record`
12
+ * is the status record the liveness came from, when there was one.
13
+ */
14
+ export declare function runLiveness(runId: string, status: unknown, bundle: LivenessBundle, nowMs: number): {
15
+ liveness: RunLiveness;
16
+ record?: RunStatusRecord;
17
+ };
18
+ export {};
@@ -0,0 +1,24 @@
1
+ import { classifyRunStatus, isRunStatusRecord, } from "./status.js";
2
+ /**
3
+ * The bundle-only reading, for a run with no usable status record. A bundle on disk usually means
4
+ * the run reached its final write. But a live run now flushes an in-progress bundle as it goes (so
5
+ * anything asking what a participant is doing has something to read), and that bundle marks its
6
+ * simulations `running`. With no status record there is no freshness to judge, and the
7
+ * reading of "it started, and nothing here says it finished" is interrupted, not finished.
8
+ */
9
+ function bundleLiveness(bundle) {
10
+ const inProgress = (bundle.simulations ?? []).some((simulation) => simulation?.status === "running");
11
+ return inProgress ? "interrupted" : "finished";
12
+ }
13
+ /**
14
+ * One run's liveness: the bundle outcome, then usable status, then bundle-only evidence.
15
+ * `readRunIndex` reads in the same order; `humanish verify` calls this with both in hand. `record`
16
+ * is the status record the liveness came from, when there was one.
17
+ */
18
+ export function runLiveness(runId, status, bundle, nowMs) {
19
+ if (bundle.outcome !== undefined)
20
+ return { liveness: bundle.outcome.state };
21
+ return isRunStatusRecord(status) && status.runId === runId
22
+ ? { liveness: classifyRunStatus(status, nowMs), record: status }
23
+ : { liveness: bundleLiveness(bundle) };
24
+ }
@@ -1,4 +1,4 @@
1
- import { type ActorStatus } from "../actors/contract.js";
1
+ import { type ActorStatus, type ActorTrace } from "../actors/contract.js";
2
2
  import { type CuaGoalSource } from "../actors/goal-source.js";
3
3
  import type { TaskFunnel } from "../study/tasks.js";
4
4
  import type { ParticipantOutcomes, ReviewSummary, RunTaskFunnel } from "./bundle.js";
@@ -16,6 +16,13 @@ type ParticipantOutcomeDetail = {
16
16
  label?: string;
17
17
  goalSource?: CuaGoalSource;
18
18
  };
19
+ /**
20
+ * How one participant's session ended, in words a participant summary can end on. A computer-use
21
+ * goal says where the claim came from, as the run's participant line does.
22
+ */
23
+ export declare function sessionEndingInWords(trace: ActorTrace): string;
24
+ /** "; the gaps list the other 2", or "all 3" when nobody passed. Empty when the gaps list nobody. */
25
+ export declare function gapsListClause(listed: number, total: number): string;
19
26
  /** One line a stakeholder can read, with the denominator attached to every number. */
20
27
  export declare function formatParticipantOutcomes(outcomes: ParticipantOutcomes, terminalCauses?: readonly ParticipantOutcomeDetail[]): string;
21
28
  /** Presentation details tolerate optional legacy actor payloads without changing their tallies. */
@@ -1,6 +1,7 @@
1
1
  import { ACTOR_STATUSES } from "../actors/contract.js";
2
2
  import { cuaGoalSource, isCuaTrace, CUA_COMPLETION_NOTE, } from "../actors/goal-source.js";
3
3
  import { actorEnding } from "../actors/stop-cause.js";
4
+ import { plural } from "./text.js";
4
5
  import { isNonNegativeSafeInteger, isRecord } from "./type-guards.js";
5
6
  /** Tally participant outcomes from actor statuses. Statuses this does not recognise are counted in
6
7
  * `total` but nowhere else, so the parts can never exceed the whole. */
@@ -113,6 +114,46 @@ function participantCompletionLine(outcomes, terminalCauses) {
113
114
  }
114
115
  return goalLine;
115
116
  }
117
+ /**
118
+ * How one participant's session ended, in words a participant summary can end on. A computer-use
119
+ * goal says where the claim came from, as the run's participant line does.
120
+ */
121
+ export function sessionEndingInWords(trace) {
122
+ switch (trace.completionReason) {
123
+ case "goal_satisfied": {
124
+ const source = cuaGoalSource(trace);
125
+ if (source === "participant_report")
126
+ return "reported reaching the goal";
127
+ return source === "condition_matched" ? "met its completion condition" : "reached the goal";
128
+ }
129
+ case "turn_completed":
130
+ return "said it was done";
131
+ case "gave_up":
132
+ return "gave up";
133
+ case "blocked_approval":
134
+ return "was blocked waiting for an approval";
135
+ case "timed_out":
136
+ return "ran out of time";
137
+ case "budget_reached":
138
+ return "was stopped by a time or spend limit";
139
+ case "actor_error":
140
+ return "stopped on an error";
141
+ case "step_failed":
142
+ return "failed a scripted step";
143
+ case "harness_error":
144
+ return "was lost to a harness failure";
145
+ default:
146
+ return "ended";
147
+ }
148
+ }
149
+ /** "; the gaps list the other 2", or "all 3" when nobody passed. Empty when the gaps list nobody. */
150
+ export function gapsListClause(listed, total) {
151
+ if (listed <= 0)
152
+ return "";
153
+ if (listed < total)
154
+ return `; the gaps list the other ${listed}`;
155
+ return total === 1 ? "; the gaps say why" : `; the gaps list all ${total}`;
156
+ }
116
157
  /** One line a stakeholder can read, with the denominator attached to every number. */
117
158
  export function formatParticipantOutcomes(outcomes, terminalCauses = []) {
118
159
  if (outcomes.total === 0)
@@ -189,20 +230,25 @@ export function withCuaReviewProvenance(review, streams) {
189
230
  : !details.some((entry) => entry.goalSource !== undefined)))
190
231
  return review;
191
232
  const outcomes = formatParticipantOutcomes(review.participants, details);
192
- // Preserve rerun context, participant narration and adapter-specific findings. Refreshing a
193
- // historical summary qualifies its old tally instead of silently discarding that context.
194
- const header = `Run gate: ${review.verdict}. Participants: ${outcomes}.${review.tasks ? ` Tasks: ${formatRunTaskFunnel(review.tasks)}.` : ""}`;
195
- const prefix = `${header} Recorded summary: `;
196
- const recorded = review.summary.startsWith(prefix)
197
- ? review.summary.slice(prefix.length)
198
- : review.summary;
199
- const oldGoal = `${review.participants.reachedGoal}/${review.participants.total} reached the goal`;
200
- const qualified = recorded
201
- .split(oldGoal)
202
- .join(participantCompletionLine(review.participants, details));
233
+ const tasks = review.tasks ? ` Tasks: ${formatRunTaskFunnel(review.tasks)}.` : "";
234
+ // A summary that states the bare count gets it qualified where it stands, keeping rerun context,
235
+ // the participant's own words and adapter findings around it.
236
+ const { reachedGoal, total } = review.participants;
237
+ const bareLine = `${reachedGoal}/${total} reached the goal`;
238
+ const qualifiedLine = participantCompletionLine(review.participants, details);
239
+ // A summary with no count gets it in a header. Earlier releases wrote a header in other words.
240
+ const header = `${plural(total, "participant")} took part: ${outcomes}.${tasks} `;
241
+ const olderHeader = `Run gate: ${review.verdict}. Participants: ${outcomes}.${tasks} Recorded summary: `;
242
+ const summary = review.summary.startsWith(olderHeader)
243
+ ? `${header}${review.summary.slice(olderHeader.length)}`
244
+ : review.summary.includes(bareLine)
245
+ ? review.summary.split(bareLine).join(qualifiedLine)
246
+ : review.summary.includes(qualifiedLine)
247
+ ? review.summary
248
+ : `${header}${review.summary}`;
203
249
  return {
204
250
  ...review,
205
- summary: `${prefix}${qualified}`,
251
+ summary,
206
252
  gaps: review.participants.reachedGoal === 0
207
253
  ? review.gaps
208
254
  : [...review.gaps.filter((gap) => gap !== CUA_COMPLETION_NOTE), CUA_COMPLETION_NOTE],
@@ -0,0 +1,31 @@
1
+ /** A participant, as much of it as a caption reads. */
2
+ export interface CaptionFacts {
3
+ /** The roster id, on a route that has a roster. */
4
+ readonly id?: string;
5
+ /** The persona, which names the participant when there is no roster id or it is only a number. */
6
+ readonly personaId: string;
7
+ /** The participant's device, when it has one. */
8
+ readonly device?: {
9
+ readonly name: string;
10
+ readonly preset: {
11
+ readonly isMobile: boolean;
12
+ };
13
+ };
14
+ }
15
+ /** "Lobby host", or "Player two, phone" for a participant on a phone. */
16
+ export declare function participantCaption(participant: CaptionFacts): string;
17
+ /** The fields of a saved stream that name its participant. */
18
+ export interface SavedCaptionFacts {
19
+ readonly label: string;
20
+ /** The participant id a computer-use stream records. */
21
+ readonly participantId?: string;
22
+ readonly personaId?: string;
23
+ }
24
+ /**
25
+ * The caption a saved stream shows: its label, which a route writes with `participantCaption`.
26
+ * Earlier releases wrote ids and taxonomy into the label (`lane-01 · browser`, `CUA participant
27
+ * <id>: <study>`, `Concurrent persona <id> (type:<t> / ...) · <study>`), so a stream saved that way
28
+ * is captioned again from the id it names. Older terminal and scripted labels use the
29
+ * recorded persona; a scripted mobile surface also identifies a phone.
30
+ */
31
+ export declare function savedCaption(stream: SavedCaptionFacts): string;
@@ -0,0 +1,60 @@
1
+ // The words a reader sees for one participant. Routes write the caption as the stream label, and
2
+ // the Observer, the TUI and findings show that label, so every surface names the participant the
3
+ // same way. The participant's taxonomy and ids stay in the bundle data, out of the caption.
4
+ /** Ids `participantIdAt` makes for an undeclared roster entry. They number a position. */
5
+ const NUMBERED_ID = /^(lane|role)-\d+$/;
6
+ /** An id in words: `lobby-host` reads "Lobby host". */
7
+ function inWords(id) {
8
+ const words = id
9
+ .split(/[-_\s]+/)
10
+ .filter(Boolean)
11
+ .join(" ");
12
+ return `${words.charAt(0).toUpperCase()}${words.slice(1)}`;
13
+ }
14
+ /** A phone or a tablet. A desktop of any size goes unnamed, since most participants use one. */
15
+ function deviceInWords(device) {
16
+ if (device === undefined)
17
+ return undefined;
18
+ if (device.preset.isMobile)
19
+ return "phone";
20
+ return device.name === "tablet" ? "tablet" : undefined;
21
+ }
22
+ /** "Lobby host", or "Player two, phone" for a participant on a phone. */
23
+ export function participantCaption(participant) {
24
+ const { id, personaId } = participant;
25
+ const name = inWords(id === undefined || NUMBERED_ID.test(id) ? personaId : id);
26
+ const device = deviceInWords(participant.device);
27
+ return device === undefined ? name : `${name}, ${device}`;
28
+ }
29
+ const OLDER_SHARED_WORLD_LABEL = /^Concurrent persona (\S+)/;
30
+ const OLDER_COMPUTER_USE_LABEL = /^CUA |^\S+ · browser$/;
31
+ const OLDER_TERMINAL_OR_SCRIPTED_LABEL = /^(Terminal agent|Desktop browser surface|Mobile browser surface) · /;
32
+ /**
33
+ * The caption a saved stream shows: its label, which a route writes with `participantCaption`.
34
+ * Earlier releases wrote ids and taxonomy into the label (`lane-01 · browser`, `CUA participant
35
+ * <id>: <study>`, `Concurrent persona <id> (type:<t> / ...) · <study>`), so a stream saved that way
36
+ * is captioned again from the id it names. Older terminal and scripted labels use the
37
+ * recorded persona; a scripted mobile surface also identifies a phone.
38
+ */
39
+ export function savedCaption(stream) {
40
+ const { label, participantId, personaId } = stream;
41
+ const sharedWorldId = OLDER_SHARED_WORLD_LABEL.exec(label)?.[1];
42
+ if (sharedWorldId !== undefined) {
43
+ return participantCaption({ id: sharedWorldId, personaId: personaId ?? sharedWorldId });
44
+ }
45
+ if (OLDER_TERMINAL_OR_SCRIPTED_LABEL.test(label) && personaId !== undefined) {
46
+ return participantCaption({
47
+ personaId,
48
+ ...(label.startsWith("Mobile browser surface · ")
49
+ ? { device: { name: "mobile", preset: { isMobile: true } } }
50
+ : {}),
51
+ });
52
+ }
53
+ if (OLDER_COMPUTER_USE_LABEL.test(label) && personaId !== undefined) {
54
+ return participantCaption({
55
+ ...(participantId === undefined ? {} : { id: participantId }),
56
+ personaId,
57
+ });
58
+ }
59
+ return label;
60
+ }
@@ -1,4 +1,4 @@
1
- import type { RunEvent, RunProviderResource, RunSimulation } from "./bundle.js";
1
+ import type { RunBundle, RunEvent, RunProviderResource, RunSimulation } from "./bundle.js";
2
2
  import type { RunStream } from "./streams.js";
3
3
  /** The ids a participant's saved records carry: its record id and its stream id. */
4
4
  export interface ParticipantIds {
@@ -25,4 +25,6 @@ export declare function recordIdOf(stream: RunStream): string;
25
25
  /** The record id a saved event names, when it names one. */
26
26
  export declare function eventRecordIdOf(event: RunEvent): string | undefined;
27
27
  /** The participant id an adapter recorded on a saved stream (a fan-out `lane-NN` id), when it has one. */
28
- export declare function streamParticipantIdOf(stream: RunStream): string | undefined;
28
+ export declare function streamParticipantIdOf(stream: Pick<RunStream, "laneId">): string | undefined;
29
+ /** The persona recorded for a stream, including dry runs without an actor trace. */
30
+ export declare function recordedPersonaId(bundle: Partial<Pick<RunBundle, "simulations">>, stream: Partial<Pick<RunStream, "simId">>): string | undefined;
@@ -61,3 +61,9 @@ export function eventRecordIdOf(event) {
61
61
  export function streamParticipantIdOf(stream) {
62
62
  return stream.laneId;
63
63
  }
64
+ /** The persona recorded for a stream, including dry runs without an actor trace. */
65
+ export function recordedPersonaId(bundle, stream) {
66
+ if (stream.simId === undefined || !Array.isArray(bundle.simulations))
67
+ return undefined;
68
+ return bundle.simulations.find((record) => record?.id === stream.simId)?.personaId;
69
+ }
@@ -0,0 +1,11 @@
1
+ import type { RunBundle } from "./bundle.js";
2
+ type ReviewBundle = Pick<RunBundle, "runId" | "mode" | "review"> & Partial<Pick<RunBundle, "outcome" | "simulations">> & {
3
+ scenario: Pick<RunBundle["scenario"], "title">;
4
+ };
5
+ /** The review's common head and gaps, around the route's evidence lines. */
6
+ export declare function renderReviewMarkdown(bundle: ReviewBundle, middle: readonly string[], options?: {
7
+ beforeVerdict?: readonly string[];
8
+ style?: "preview";
9
+ status?: unknown;
10
+ }): string;
11
+ export {};
@@ -0,0 +1,22 @@
1
+ import { reviewOutcome } from "./display.js";
2
+ import { verdictText } from "./judge.js";
3
+ /** The review's common head and gaps, around the route's evidence lines. */
4
+ export function renderReviewMarkdown(bundle, middle, options = {}) {
5
+ const preview = options.style === "preview";
6
+ const field = (name, value) => preview ? [`${name[0].toUpperCase()}${name.slice(1)}: ${value}`, ""] : [`- ${name}: ${value}`];
7
+ return [
8
+ `# ${preview ? "humanish Run Review" : bundle.scenario.title}`,
9
+ "",
10
+ ...field("run", bundle.runId),
11
+ ...field("mode", bundle.mode),
12
+ ...(options.beforeVerdict ?? []),
13
+ ...field("verdict", verdictText(bundle.review.verdict, bundle.mode)),
14
+ ...field("outcome", reviewOutcome(bundle, options.status)),
15
+ ...(preview ? [bundle.review.summary, ""] : field("summary", bundle.review.summary)),
16
+ ...middle,
17
+ ...(preview || bundle.review.gaps.length > 0
18
+ ? ["", "## Gaps", ...(preview ? [""] : []), ...bundle.review.gaps.map((gap) => `- ${gap}`)]
19
+ : []),
20
+ "",
21
+ ].join("\n");
22
+ }
@@ -34,6 +34,13 @@ export interface RunCost {
34
34
  }
35
35
  /** The accounting's analysis spend, or null when the run sent no analysis request. */
36
36
  export declare function analysisCostOf(accounting: Pick<AnalysisAccounting, "dispatched" | "unresolved" | "unpriced" | "estimatedUsd" | "providers" | "warnings">): RunAnalysisCost | null;
37
+ /** The recorded estimate, or a supported zero for a dry run with no cost record. */
38
+ export declare function runEstimateUsd(entry: {
39
+ estimatedCostUsd?: number | null | undefined;
40
+ mode?: string | null | undefined;
41
+ }): number | null;
42
+ /** Add known estimates, retaining unknown when neither component has a price. */
43
+ export declare function sumEstimatedUsd(a: number | null, b: number | null): number | null;
37
44
  export declare function runCost(subtotal: RunCostSubtotal | null | undefined, analysis?: RunAnalysisCost | null): RunCost;
38
45
  /**
39
46
  * A run index entry's cost: its participants-and-desktops figure, complete or a lower bound when
@@ -11,25 +11,45 @@ export function analysisCostOf(accounting) {
11
11
  providers: [...accounting.providers],
12
12
  };
13
13
  }
14
+ /** The recorded estimate, or a supported zero for a dry run with no cost record. */
15
+ export function runEstimateUsd(entry) {
16
+ const usd = entry.estimatedCostUsd;
17
+ if (typeof usd === "number" && Number.isFinite(usd) && usd >= 0)
18
+ return usd;
19
+ return usd === undefined && entry.mode === "dry-run" ? 0 : null;
20
+ }
21
+ /** Add known estimates, retaining unknown when neither component has a price. */
22
+ export function sumEstimatedUsd(a, b) {
23
+ const left = runEstimateUsd({ estimatedCostUsd: a });
24
+ const right = runEstimateUsd({ estimatedCostUsd: b });
25
+ return left === null && right === null ? null : round6((left ?? 0) + (right ?? 0));
26
+ }
14
27
  export function runCost(subtotal, analysis) {
15
28
  if (subtotal === null || subtotal === undefined)
16
29
  return { run: null, analysis: null, total: null };
17
30
  const run = {
18
- usd: typeof subtotal.estimatedTotalUsd === "number" ? subtotal.estimatedTotalUsd : null,
31
+ usd: runEstimateUsd({ estimatedCostUsd: subtotal.estimatedTotalUsd }),
19
32
  complete: subtotal.fullyEstimated === undefined ? null : subtotal.fullyEstimated,
20
33
  ratesAsOf: subtotal.ratesAsOf ?? null,
21
34
  placeholder: subtotal.placeholder === true,
22
35
  };
23
- const spent = analysis ?? null;
24
- const known = [run.usd, spent?.estimatedUsd].filter((usd) => typeof usd === "number");
25
- if (known.length === 0)
36
+ const spent = analysis == null
37
+ ? null
38
+ : {
39
+ ...analysis,
40
+ estimatedUsd: runEstimateUsd({ estimatedCostUsd: analysis.estimatedUsd }),
41
+ complete: analysis.complete &&
42
+ runEstimateUsd({ estimatedCostUsd: analysis.estimatedUsd }) !== null,
43
+ };
44
+ const totalUsd = sumEstimatedUsd(run.usd, spent?.estimatedUsd ?? null);
45
+ if (totalUsd === null)
26
46
  return { run, analysis: spent, total: null };
27
47
  const lowerBound = run.usd === null || run.complete === false || spent?.complete === false;
28
48
  return {
29
49
  run,
30
50
  analysis: spent,
31
51
  total: {
32
- usd: round6(known.reduce((sum, usd) => sum + usd, 0)),
52
+ usd: totalUsd,
33
53
  complete: lowerBound ? false : run.complete,
34
54
  },
35
55
  };