humanish 0.111.3 → 0.112.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (112) hide show
  1. package/CHANGELOG.md +33 -0
  2. package/dist/actors/computer-use/openai-conversation.d.ts +91 -0
  3. package/dist/actors/computer-use/openai-conversation.js +209 -0
  4. package/dist/actors/computer-use/openai-provider.js +25 -41
  5. package/dist/actors/contract.d.ts +2 -0
  6. package/dist/actors/scripted-browser/actor.d.ts +5 -0
  7. package/dist/actors/scripted-browser/actor.js +40 -19
  8. package/dist/actors/scripted-browser/journey.d.ts +2 -0
  9. package/dist/actors/scripted-browser/journey.js +2 -1
  10. package/dist/actors/scripted-browser/steps.d.ts +2 -0
  11. package/dist/actors/scripted-browser/steps.js +17 -0
  12. package/dist/analysis/diagnostics.js +7 -12
  13. package/dist/analysis/execute.d.ts +1 -1
  14. package/dist/analysis/execute.js +35 -5
  15. package/dist/analysis/service.js +7 -1
  16. package/dist/analysis/types.d.ts +23 -0
  17. package/dist/analysis/validation.d.ts +80 -35
  18. package/dist/analysis/validation.js +58 -11
  19. package/dist/cli/commands/runs.js +4 -10
  20. package/dist/cli/findings.d.ts +49 -4
  21. package/dist/cli/findings.js +179 -23
  22. package/dist/cli/io.d.ts +2 -1
  23. package/dist/cli/io.js +3 -2
  24. package/dist/evidence/secret-scrub.d.ts +7 -0
  25. package/dist/evidence/secret-scrub.js +21 -12
  26. package/dist/feedback/draft.js +14 -4
  27. package/dist/observer/run-routes.js +8 -4
  28. package/dist/observer-app.html +4 -4
  29. package/dist/routes/computer-use/bundle.d.ts +1 -1
  30. package/dist/routes/computer-use/bundle.js +6 -16
  31. package/dist/routes/computer-use/fanout-bundle.js +40 -37
  32. package/dist/routes/computer-use/fanout-records.d.ts +2 -0
  33. package/dist/routes/computer-use/fanout-records.js +18 -26
  34. package/dist/routes/computer-use/participant-execution.js +4 -11
  35. package/dist/routes/computer-use/participant-facts.d.ts +2 -2
  36. package/dist/routes/computer-use/participant-facts.js +10 -1
  37. package/dist/routes/computer-use/result.js +7 -15
  38. package/dist/routes/computer-use/single-bundle.d.ts +2 -0
  39. package/dist/routes/computer-use/single-bundle.js +81 -37
  40. package/dist/routes/scripted/bundle.d.ts +1 -1
  41. package/dist/routes/scripted/bundle.js +9 -19
  42. package/dist/routes/scripted/route.js +1 -0
  43. package/dist/routes/scripted/setup.js +53 -10
  44. package/dist/routes/scripted/surface-records.d.ts +0 -1
  45. package/dist/routes/scripted/surface-records.js +27 -13
  46. package/dist/routes/scripted/surface-sessions.d.ts +1 -1
  47. package/dist/routes/scripted/surface-sessions.js +2 -0
  48. package/dist/routes/shared-world/bundle-records.d.ts +2 -2
  49. package/dist/routes/shared-world/bundle-records.js +21 -28
  50. package/dist/routes/shared-world/bundle.d.ts +1 -1
  51. package/dist/routes/shared-world/bundle.js +70 -78
  52. package/dist/routes/shared-world/participant-specs.d.ts +1 -2
  53. package/dist/routes/shared-world/participant-specs.js +0 -8
  54. package/dist/routes/shared-world/result.js +8 -12
  55. package/dist/routes/terminal/bundle.d.ts +1 -1
  56. package/dist/routes/terminal/bundle.js +10 -19
  57. package/dist/routes/terminal/credentials.js +14 -1
  58. package/dist/routes/terminal/dry-run.js +1 -0
  59. package/dist/routes/terminal/live-finish.js +7 -4
  60. package/dist/routes/terminal/live-sandbox.js +9 -2
  61. package/dist/routes/terminal/plan.js +1 -0
  62. package/dist/routes/terminal/runtime.d.ts +1 -0
  63. package/dist/routes/terminal/runtime.js +1 -0
  64. package/dist/routes/terminal/session.js +2 -1
  65. package/dist/routes/terminal/trace.d.ts +4 -2
  66. package/dist/routes/terminal/trace.js +30 -35
  67. package/dist/run/costs.js +9 -20
  68. package/dist/run/detail.d.ts +1 -1
  69. package/dist/run/detail.js +10 -1
  70. package/dist/run/display.d.ts +2 -2
  71. package/dist/run/display.js +15 -18
  72. package/dist/run/dry-run.js +10 -25
  73. package/dist/run/judge.d.ts +24 -0
  74. package/dist/run/judge.js +54 -0
  75. package/dist/run/liveness.d.ts +18 -0
  76. package/dist/run/liveness.js +24 -0
  77. package/dist/run/outcomes.d.ts +8 -1
  78. package/dist/run/outcomes.js +58 -12
  79. package/dist/run/participant-caption.d.ts +30 -0
  80. package/dist/run/participant-caption.js +50 -0
  81. package/dist/run/participant-records.d.ts +1 -1
  82. package/dist/run/review-markdown.d.ts +11 -0
  83. package/dist/run/review-markdown.js +22 -0
  84. package/dist/run/run-cost.d.ts +7 -0
  85. package/dist/run/run-cost.js +25 -5
  86. package/dist/run/run-index.d.ts +4 -50
  87. package/dist/run/run-index.js +57 -87
  88. package/dist/run/run.d.ts +2 -2
  89. package/dist/run/run.js +5 -3
  90. package/dist/run/secrets.d.ts +15 -2
  91. package/dist/run/secrets.js +134 -9
  92. package/dist/run/stats.js +2 -12
  93. package/dist/run/transient-comms-secrets.d.ts +8 -3
  94. package/dist/run/transient-comms-secrets.js +21 -1
  95. package/dist/study/keys.d.ts +1 -0
  96. package/dist/study/keys.js +1 -1
  97. package/dist/study/parse/execution.js +7 -1
  98. package/dist/study/plan-types.d.ts +1 -0
  99. package/dist/study/requirements.js +2 -1
  100. package/dist/study/types.d.ts +4 -1
  101. package/dist/study/warnings.d.ts +2 -1
  102. package/dist/study/warnings.js +3 -2
  103. package/dist/tui-app.js +73 -73
  104. package/dist/verify/liveness.d.ts +2 -2
  105. package/dist/verify/liveness.js +7 -4
  106. package/docs/architecture/terminal-product-route.md +14 -0
  107. package/docs/contracts/run-bundle.md +10 -0
  108. package/docs/contracts/schemas.md +9 -1
  109. package/docs/contracts/study-analysis.md +76 -23
  110. package/package.json +2 -1
  111. package/dist/actors/computer-use/openai-requests.d.ts +0 -62
  112. package/dist/actors/computer-use/openai-requests.js +0 -144
@@ -7,3 +7,5 @@ export declare function parseBrowserPersonaJourneyFromScenario(args: {
7
7
  failure?: string;
8
8
  journey?: BrowserPersonaJourney;
9
9
  };
10
+ /** `value` lower-cased, with each run of other characters a `-`, cut to 80; else `fallback`. */
11
+ export declare function publicSafeToken(value: string | undefined, fallback: string): string;
@@ -172,7 +172,8 @@ function stringValue(value) {
172
172
  function booleanValue(value) {
173
173
  return typeof value === "boolean" ? value : undefined;
174
174
  }
175
- function publicSafeToken(value, fallback) {
175
+ /** `value` lower-cased, with each run of other characters a `-`, cut to 80; else `fallback`. */
176
+ export function publicSafeToken(value, fallback) {
176
177
  const candidate = (value ?? fallback)
177
178
  .toLowerCase()
178
179
  .replace(/[^a-z0-9_.-]+/g, "-")
@@ -18,6 +18,8 @@ export declare function buildBlockedBrowserPersonaSteps(args: {
18
18
  timestamp: string;
19
19
  urlPolicy?: ScriptedBrowserEvidenceUrlPolicy;
20
20
  }): BrowserPersonaStepCapture[];
21
+ /** A step as the trace records it: its label, reason, URL and assertion reasons scrubbed. */
22
+ export declare function scrubRecordedStep(step: BrowserPersonaStepCapture, scrub: (text: string) => string): BrowserPersonaStepCapture;
21
23
  export declare function screenshotPathForBrowserStep(surface: BrowserSurface, step: BrowserPersonaStepManifest | undefined): string;
22
24
  export declare function tracePathForBrowserSurface(surface: BrowserSurface): string;
23
25
  /**
@@ -191,6 +191,23 @@ export function buildBlockedBrowserPersonaSteps(args) {
191
191
  url: sanitizeBrowserEvidenceUrl(args.currentUrl, args.urlPolicy),
192
192
  }));
193
193
  }
194
+ /** A step as the trace records it: its label, reason, URL and assertion reasons scrubbed. */
195
+ export function scrubRecordedStep(step, scrub) {
196
+ return {
197
+ ...step,
198
+ ...(step.assertions === undefined
199
+ ? {}
200
+ : {
201
+ assertions: step.assertions.map((assertion) => ({
202
+ ...assertion,
203
+ reason: scrub(assertion.reason),
204
+ })),
205
+ }),
206
+ label: scrub(step.label),
207
+ reason: scrub(step.reason),
208
+ url: scrub(step.url),
209
+ };
210
+ }
194
211
  export function screenshotPathForBrowserStep(surface, step) {
195
212
  assertSafeOutputPathSegment(surface.id, "Browser surface id");
196
213
  if (step) {
@@ -1,9 +1,8 @@
1
1
  import { lstat, readdir, rmdir, unlink } from "node:fs/promises";
2
2
  import path from "node:path";
3
- import { decodeEscapes } from "../evidence/encoded-text.js";
4
3
  import { redactText } from "../evidence/redaction.js";
5
4
  import { assertPreparedSelectedOutputDirectory, assertSafeOutputPathSegment, bindExistingManagedHumanishOutputDirectory, prepareManagedHumanishOutputDirectory, writeContainedOutputFile, } from "../run/contained-output.js";
6
- import { scrubTransientCommsText, transientCommsEncodedScrub, } from "../run/transient-comms-secrets.js";
5
+ import { transientCommsKnownValueScrub } from "../run/transient-comms-secrets.js";
7
6
  import { ANALYSIS_ID_PATTERN } from "./types.js";
8
7
  // A rejected analyst response is kept for diagnosis under .humanish/analysis-diagnostics/<run>/,
9
8
  // outside every run directory. Export, verify and the Observer read run directories only, so the
@@ -14,19 +13,15 @@ const ANALYSIS_DIAGNOSTIC_SCHEMA = "humanish.analysis-rejected-output.v1";
14
13
  /** How many rejected outputs are kept across all runs; older ones are removed on each write. */
15
14
  export const MAX_ANALYSIS_DIAGNOSTICS = 20;
16
15
  const REDACTED = "[REDACTED_SECRET]";
17
- const markers = (text) => text.split(REDACTED).length - 1;
18
16
  /**
19
- * Scrub every string, key and scalar in the output. The encoded scrub finds a known value written
20
- * percent-encoded, escaped or base64-encoded and returns decoded text; a string where it finds
21
- * nothing keeps its original spelling, so an escape in a rejected quote stays visible, and still
22
- * gets the literal scrub. A number or boolean equal to a known value becomes the marker.
17
+ * Scrub every string, key and scalar in the output. The known-value scrub finds a value written
18
+ * percent-encoded, escaped or base64-encoded; a string where it finds nothing keeps its original
19
+ * spelling, so an escape in a rejected quote stays visible. A number or boolean equal to a known
20
+ * value becomes the marker.
23
21
  */
24
22
  function scrubber() {
25
- const encoded = transientCommsEncodedScrub();
26
- const text = (value) => {
27
- const found = encoded(value);
28
- return redactText(markers(found) > markers(decodeEscapes(value)) ? found : scrubTransientCommsText(value));
29
- };
23
+ const known = transientCommsKnownValueScrub();
24
+ const text = (value) => redactText(known(value));
30
25
  const scrub = (value) => {
31
26
  if (typeof value === "string")
32
27
  return text(value);
@@ -1,7 +1,7 @@
1
1
  import { type AnalysisArtifact, type AnalysisConfig, type AnalysisInput } from "./types.js";
2
2
  import { type AnalysisFetch, type AnalysisProvider } from "./provider.js";
3
3
  import type { RejectedAnalysisOutput } from "./diagnostics.js";
4
- export declare const ANALYSIS_PROMPT_VERSION = "study-evidence-6";
4
+ export declare const ANALYSIS_PROMPT_VERSION = "study-evidence-7";
5
5
  export declare const isSupportedAnalysisModel: (model: string) => boolean;
6
6
  /** The OpenAI output limit when a study omits maxOutputTokens, and the most it may ask for. */
7
7
  export declare const DEFAULT_ANALYSIS_MAX_OUTPUT_TOKENS = 16384;
@@ -3,14 +3,14 @@ import { highDetailImageTokens } from "./image-tokens.js";
3
3
  import { createHash, randomUUID } from "node:crypto";
4
4
  import { estimateActorCost, MODEL_RATES } from "../run/pricing.js";
5
5
  import { containsSensitive } from "../evidence/redaction.js";
6
- import { scrubTransientCommsText } from "../run/transient-comms-secrets.js";
6
+ import { transientCommsKnownValueScrub } from "../run/transient-comms-secrets.js";
7
7
  import { ANALYSIS_ID_PATTERN, ANALYSIS_SCHEMA, } from "./types.js";
8
8
  import { INPUT_IMAGE_DATA_URL, createAnalysisProvider, } from "./provider.js";
9
9
  import { checkAnalysisResult, hashAnalysisValue, analysisResponseSchema, analysisResultJsonSchema, validateAnalysisInputMetadata, } from "./validation.js";
10
10
  import { EVIDENCE_LIMITS } from "./evidence.js";
11
11
  import { truncatedFrameWarning, unknownNotificationsWarning, } from "../actors/codex/restricted-notifications.js";
12
12
  import { protocolAdditionsWarning, protocolIncompatibilityMessage, } from "../actors/codex/protocol-compat.js";
13
- export const ANALYSIS_PROMPT_VERSION = "study-evidence-6";
13
+ export const ANALYSIS_PROMPT_VERSION = "study-evidence-7";
14
14
  const SUPPORTED_ANALYSIS_MODELS = Object.freeze([
15
15
  "gpt-6-astra",
16
16
  "gpt-5.5",
@@ -55,7 +55,11 @@ Harness records describe the machinery running the study. Provisioning, runtime
55
55
 
56
56
  For each finding, consolidate repeated observations into one bounded problem or recovery. Each observation states one claim, labels its basis (visual, action, participant_statement, inference), cites supporting evidence, and declares its limitation. High confidence does not turn an inference into an observation. Show which distinct participants were affected and which were demonstrably exposed to the relevant interaction. Affected IDs must be a subset of exposed IDs, and both sets must be within included participants. Explain exposure; never automatically use the full panel as a denominator. Each affected participant must have supporting observation evidence.
57
57
 
58
- Keep every field as qualified as its evidence. This includes headlines, summaries, outcomes, impact, exposure, recovery, confidence and the premises of proposed next checks. A cautious observation cannot support an unqualified headline. When a useful concern is established only as participant feedback, say it was reported wherever the concern is summarized. A dispatched action establishes an attempt; its success needs resulting evidence. Use recovered only when the improvement is supported for the participants being grouped. If recovery differs across participants, describe those differences and use unknown for the combined recovery instead of erasing an unresolved case. Keep distinct problems separate, especially an unresolved result and a different problem that was corrected. Reversible exploration and unmeasured pauses are not automatically product defects.
58
+ Keep every evidence field as qualified as its evidence. This includes titles, summaries, outcomes, impact, exposure, recovery, confidence and the premises of proposed next checks. A cautious observation cannot support an unqualified title. When a useful concern is established only as participant feedback, say it was reported wherever the concern is summarized. A dispatched action establishes an attempt; its success needs resulting evidence. Use recovered only when the improvement is supported for the participants being grouped. If recovery differs across participants, describe those differences and use unknown for the combined recovery instead of erasing an unresolved case. Keep distinct problems separate, especially an unresolved result and a different problem that was corrected. Reversible exploration and unmeasured pauses are not automatically product defects.
59
+
60
+ Write for the person who will act on the report: a designer, product manager or developer who did not watch the session and does not know this tool's vocabulary. Give every finding a headline and an experience. The headline is one plain sentence about what happened to the people involved, as a colleague would say it out loud. The experience is one to three plain sentences: what the person was trying to do, what got in their way, and how it seemed to feel, using the participant's own words where a quote exists. Do not use participant stream IDs, evidence IDs, or this report's evidence vocabulary (exposure, exposed, stream, capture, corroborated, established, basis, provenance) in the headline or experience. Plain is not unqualified: a concern known only from what participants said reads as something they said ("Two players said they lost track of which control had focus"), and a cause the evidence does not show is not stated as fact. The title, summary and observations keep their current precision underneath.
61
+
62
+ Separately, review the supplied captures as an experienced product designer would and report designFindings: problems a designer would notice in what the screens show, whether or not any participant mentioned them. Look at the size of text and controls (too small to read or to hit comfortably at this window size), layout and density (spread too wide or packed too tight for the task), visual hierarchy, legibility and contrast, wording and labels, consistency between screens, feedback after actions, and anything that makes the product look unfinished or untrustworthy. Judge the screens as the participants' personas would meet them: who they are and what they came to do. Each design finding names the screen in plain words, says what a designer notices, why it matters to a person using the product, and one concrete suggestion. Each must cite at least one supplied capture that shows the problem, and seenByStreamIds lists only participants whose cited captures show it. Rate severity by the effect on a person using the product: major when it misleads or blocks, moderate when it slows or confuses, minor when it is polish. Report only what the captures show; no capture, no design finding. Use D1, D2, and so on as IDs. An empty designFindings array is correct when the captures show no design problem worth a designer's time. Write design findings in the same plain register as headlines.
59
63
 
60
64
  Order findings by observed task impact, replication among exposed participants, and recovery. Preserve severity and confidence as separate fields. Do not compute a numeric frustration score or universal priority score. Use F1, F2, and so on as local finding IDs. Explain ordering with concrete evidence. Provide a short, testable next check for each finding. Say when recovery was not observed rather than claiming it was impossible. Keep titles and summaries concise and specific. The overall summary must be grounded in participant reviews and observations, including successful outcomes and evidence limits.`;
61
65
  function instructions(config) {
@@ -213,9 +217,10 @@ export function preferLargerAnalysisOutput(input, config) {
213
217
  const expanded = { ...config, maxOutputTokens: MAX_ANALYSIS_OUTPUT_TOKENS };
214
218
  return estimateAnalysisAdmission(input, expanded).allowed ? expanded : config;
215
219
  }
216
- /** Scrub only generated prose. Source evidence, provenance and integrity hashes remain exact. */
220
+ /** Scrub only generated prose. Source evidence, provenance and integrity hashes remain exact. A
221
+ * known value is found as written and in its encoded forms. */
217
222
  function scrubGeneratedNarrative(result) {
218
- const scrub = scrubTransientCommsText;
223
+ const scrub = transientCommsKnownValueScrub();
219
224
  const observation = (value) => ({
220
225
  ...value,
221
226
  claim: scrub(value.claim),
@@ -239,6 +244,13 @@ function scrubGeneratedNarrative(result) {
239
244
  ...value.exposedStreamIds,
240
245
  ...value.observations.flatMap((item) => [item.basis, ...item.evidenceIds]),
241
246
  ]),
247
+ ...(result.designFindings ?? []).flatMap((value) => [
248
+ value.id,
249
+ value.severity,
250
+ value.confidence,
251
+ ...value.seenByStreamIds,
252
+ ...value.evidenceIds,
253
+ ]),
242
254
  ...(result.concernReviews ?? []).flatMap((value) => [
243
255
  value.basis,
244
256
  value.disposition,
@@ -263,12 +275,26 @@ function scrubGeneratedNarrative(result) {
263
275
  findings: result.findings.map((value) => ({
264
276
  ...value,
265
277
  title: scrub(value.title),
278
+ ...(value.headline === undefined ? {} : { headline: scrub(value.headline) }),
279
+ ...(value.experience === undefined ? {} : { experience: scrub(value.experience) }),
266
280
  summary: scrub(value.summary),
267
281
  exposureReason: scrub(value.exposureReason),
268
282
  nextStep: scrub(value.nextStep),
269
283
  priorityReason: scrub(value.priorityReason),
270
284
  observations: value.observations.map(observation),
271
285
  })),
286
+ ...(result.designFindings === undefined
287
+ ? {}
288
+ : {
289
+ designFindings: result.designFindings.map((value) => ({
290
+ ...value,
291
+ headline: scrub(value.headline),
292
+ screen: scrub(value.screen),
293
+ notice: scrub(value.notice),
294
+ whyItMatters: scrub(value.whyItMatters),
295
+ suggestion: scrub(value.suggestion),
296
+ })),
297
+ }),
272
298
  ...(result.concernReviews === undefined
273
299
  ? {}
274
300
  : {
@@ -281,6 +307,10 @@ function scrubGeneratedNarrative(result) {
281
307
  }
282
308
  const VALIDATION_FAILURES = Object.freeze({
283
309
  ANALYSIS_RESULT_SCHEMA_INVALID: "analysis_validation_failed_schema_invalid",
310
+ ANALYSIS_DESIGN_FINDING_ID_DUPLICATE: "analysis_validation_failed_design_finding_id_duplicate",
311
+ ANALYSIS_DESIGN_REFERENCE_INVALID: "analysis_validation_failed_design_reference_invalid",
312
+ ANALYSIS_DESIGN_WITHOUT_CAPTURE: "analysis_validation_failed_design_without_capture",
313
+ ANALYSIS_DESIGN_MEMBERSHIP_INVALID: "analysis_validation_failed_design_membership_invalid",
284
314
  ANALYSIS_INPUT_DUPLICATES: "analysis_validation_failed_input_duplicates",
285
315
  ANALYSIS_PARTICIPANT_COVERAGE_INVALID: "analysis_validation_failed_participant_coverage_invalid",
286
316
  ANALYSIS_PARTICIPANT_REFERENCE_INVALID: "analysis_validation_failed_participant_reference_invalid",
@@ -5,6 +5,7 @@ import { lstat, mkdir, realpath, rmdir } from "node:fs/promises";
5
5
  import path from "node:path";
6
6
  import { renderObserver } from "../observer/render.js";
7
7
  import { containsSensitive } from "../evidence/redaction.js";
8
+ import { transientCommsKnownValueScrub } from "../run/transient-comms-secrets.js";
8
9
  import { verifyRunPrepared } from "../verify/verify.js";
9
10
  import { loadRunBundlePrepared, resolveRunPath } from "../run/locate.js";
10
11
  import { physicalCwdOf, runIdOf, resolvePhysicalCwd, validatePreparedRunRootIdentity, } from "../run/paths.js";
@@ -397,7 +398,12 @@ export async function correctAnalysis(cwd, run, options) {
397
398
  const finding = analysis?.result?.findings.find((item) => item.id === options.findingId);
398
399
  if (loaded.state !== "ready" || !analysis || !finding)
399
400
  throw new Error("ANALYSIS_CORRECTION_SOURCE_UNAVAILABLE");
400
- if (containsSensitive(options.reason) || containsSensitive(options.replacementClaim ?? ""))
401
+ // A reviewer's claim replaces the finding's headline in every view, so it gets the same known
402
+ // value check as generated text: as written and encoded. Only a caller inside the run's secret
403
+ // scope holds those values; a separate `analyze correct` has only the pattern check.
404
+ const known = transientCommsKnownValueScrub();
405
+ const unsafe = (text) => containsSensitive(text) || known(text) !== text;
406
+ if (unsafe(options.reason) || unsafe(options.replacementClaim ?? ""))
401
407
  throw new Error("ANALYSIS_CORRECTION_TEXT_UNSAFE");
402
408
  const correction = {
403
409
  schema: ANALYSIS_CORRECTION_SCHEMA,
@@ -104,6 +104,11 @@ export interface AnalysisObservation {
104
104
  interface AnalysisFinding {
105
105
  id: string;
106
106
  title: string;
107
+ /** One plain sentence for a reader who did not watch the session. Absent in older reports. */
108
+ headline?: string;
109
+ /** One to three plain sentences: what the person tried, what got in the way, how it seemed to
110
+ * feel. Absent in older reports. */
111
+ experience?: string;
107
112
  summary: string;
108
113
  impact: "blocked_task" | "friction" | "recovery" | "uncertain";
109
114
  affectedStreamIds: string[];
@@ -115,6 +120,22 @@ interface AnalysisFinding {
115
120
  nextStep: string;
116
121
  priorityReason: string;
117
122
  }
123
+ /** A problem a product designer would see in the cited captures, whether or not a participant
124
+ * mentioned it. It cites at least one capture. */
125
+ interface AnalysisDesignFinding {
126
+ id: string;
127
+ headline: string;
128
+ /** The screen, in plain words. */
129
+ screen: string;
130
+ notice: string;
131
+ whyItMatters: string;
132
+ suggestion: string;
133
+ severity: "minor" | "moderate" | "major";
134
+ confidence: "low" | "medium" | "high";
135
+ /** Participants whose cited evidence shows the problem. */
136
+ seenByStreamIds: string[];
137
+ evidenceIds: string[];
138
+ }
118
139
  /** Evidence-grounded disposition of a material concern, not model reasoning. */
119
140
  interface AnalysisConcernReview extends AnalysisObservation {
120
141
  disposition: "finding" | "context" | "unsupported";
@@ -128,6 +149,8 @@ export interface AnalysisResult {
128
149
  concernReviews?: AnalysisConcernReview[];
129
150
  /** Highest priority first; counts are derived from distinct stream sets. */
130
151
  findings: AnalysisFinding[];
152
+ /** Absent in older reports. New analyses supply it, empty when the captures show no problem. */
153
+ designFindings?: AnalysisDesignFinding[];
131
154
  limitations: string[];
132
155
  }
133
156
  interface OpenAIAnalysisConfig {
@@ -1,6 +1,7 @@
1
1
  import { z } from "zod";
2
2
  import { type AnalysisArtifact, type AnalysisCorrection, type AnalysisInput, type AnalysisResult } from "./types.js";
3
- /** Historical artifacts may omit concernReviews; every new provider response must supply it. */
3
+ /** Historical artifacts may omit concernReviews, headlines, experiences and designFindings; every
4
+ * new provider response must supply them. */
4
5
  export declare const analysisResponseSchema: z.ZodObject<{
5
6
  summary: z.ZodString;
6
7
  participants: z.ZodArray<z.ZodObject<{
@@ -22,9 +23,30 @@ export declare const analysisResponseSchema: z.ZodObject<{
22
23
  }, z.core.$strict>>;
23
24
  limitations: z.ZodArray<z.ZodString>;
24
25
  }, z.core.$strict>>;
26
+ concernReviews: z.ZodNonOptional<z.ZodOptional<z.ZodArray<z.ZodObject<{
27
+ claim: z.ZodString;
28
+ basis: z.ZodEnum<{
29
+ action: "action";
30
+ inference: "inference";
31
+ participant_statement: "participant_statement";
32
+ visual: "visual";
33
+ }>;
34
+ evidenceIds: z.ZodArray<z.ZodString>;
35
+ limitation: z.ZodString;
36
+ disposition: z.ZodEnum<{
37
+ context: "context";
38
+ finding: "finding";
39
+ unsupported: "unsupported";
40
+ }>;
41
+ findingId: z.ZodNullable<z.ZodString>;
42
+ reason: z.ZodString;
43
+ }, z.core.$strict>>>>;
44
+ limitations: z.ZodArray<z.ZodString>;
25
45
  findings: z.ZodArray<z.ZodObject<{
26
46
  id: z.ZodString;
27
47
  title: z.ZodString;
48
+ headline: z.ZodNonOptional<z.ZodOptional<z.ZodString>>;
49
+ experience: z.ZodNonOptional<z.ZodOptional<z.ZodString>>;
28
50
  summary: z.ZodString;
29
51
  impact: z.ZodEnum<{
30
52
  blocked_task: "blocked_task";
@@ -59,25 +81,26 @@ export declare const analysisResponseSchema: z.ZodObject<{
59
81
  nextStep: z.ZodString;
60
82
  priorityReason: z.ZodString;
61
83
  }, z.core.$strict>>;
62
- concernReviews: z.ZodNonOptional<z.ZodOptional<z.ZodArray<z.ZodObject<{
63
- claim: z.ZodString;
64
- basis: z.ZodEnum<{
65
- action: "action";
66
- inference: "inference";
67
- participant_statement: "participant_statement";
68
- visual: "visual";
84
+ designFindings: z.ZodArray<z.ZodObject<{
85
+ id: z.ZodString;
86
+ headline: z.ZodString;
87
+ screen: z.ZodString;
88
+ notice: z.ZodString;
89
+ whyItMatters: z.ZodString;
90
+ suggestion: z.ZodString;
91
+ severity: z.ZodEnum<{
92
+ major: "major";
93
+ minor: "minor";
94
+ moderate: "moderate";
69
95
  }>;
70
- evidenceIds: z.ZodArray<z.ZodString>;
71
- limitation: z.ZodString;
72
- disposition: z.ZodEnum<{
73
- context: "context";
74
- finding: "finding";
75
- unsupported: "unsupported";
96
+ confidence: z.ZodEnum<{
97
+ high: "high";
98
+ low: "low";
99
+ medium: "medium";
76
100
  }>;
77
- findingId: z.ZodNullable<z.ZodString>;
78
- reason: z.ZodString;
79
- }, z.core.$strict>>>>;
80
- limitations: z.ZodArray<z.ZodString>;
101
+ seenByStreamIds: z.ZodArray<z.ZodString>;
102
+ evidenceIds: z.ZodArray<z.ZodString>;
103
+ }, z.core.$strict>>;
81
104
  }, z.core.$strict>;
82
105
  export declare const analysisResultJsonSchema: z.core.ZodStandardJSONSchemaPayload<z.ZodObject<{
83
106
  summary: z.ZodString;
@@ -100,9 +123,30 @@ export declare const analysisResultJsonSchema: z.core.ZodStandardJSONSchemaPaylo
100
123
  }, z.core.$strict>>;
101
124
  limitations: z.ZodArray<z.ZodString>;
102
125
  }, z.core.$strict>>;
126
+ concernReviews: z.ZodNonOptional<z.ZodOptional<z.ZodArray<z.ZodObject<{
127
+ claim: z.ZodString;
128
+ basis: z.ZodEnum<{
129
+ action: "action";
130
+ inference: "inference";
131
+ participant_statement: "participant_statement";
132
+ visual: "visual";
133
+ }>;
134
+ evidenceIds: z.ZodArray<z.ZodString>;
135
+ limitation: z.ZodString;
136
+ disposition: z.ZodEnum<{
137
+ context: "context";
138
+ finding: "finding";
139
+ unsupported: "unsupported";
140
+ }>;
141
+ findingId: z.ZodNullable<z.ZodString>;
142
+ reason: z.ZodString;
143
+ }, z.core.$strict>>>>;
144
+ limitations: z.ZodArray<z.ZodString>;
103
145
  findings: z.ZodArray<z.ZodObject<{
104
146
  id: z.ZodString;
105
147
  title: z.ZodString;
148
+ headline: z.ZodNonOptional<z.ZodOptional<z.ZodString>>;
149
+ experience: z.ZodNonOptional<z.ZodOptional<z.ZodString>>;
106
150
  summary: z.ZodString;
107
151
  impact: z.ZodEnum<{
108
152
  blocked_task: "blocked_task";
@@ -137,25 +181,26 @@ export declare const analysisResultJsonSchema: z.core.ZodStandardJSONSchemaPaylo
137
181
  nextStep: z.ZodString;
138
182
  priorityReason: z.ZodString;
139
183
  }, z.core.$strict>>;
140
- concernReviews: z.ZodNonOptional<z.ZodOptional<z.ZodArray<z.ZodObject<{
141
- claim: z.ZodString;
142
- basis: z.ZodEnum<{
143
- action: "action";
144
- inference: "inference";
145
- participant_statement: "participant_statement";
146
- visual: "visual";
184
+ designFindings: z.ZodArray<z.ZodObject<{
185
+ id: z.ZodString;
186
+ headline: z.ZodString;
187
+ screen: z.ZodString;
188
+ notice: z.ZodString;
189
+ whyItMatters: z.ZodString;
190
+ suggestion: z.ZodString;
191
+ severity: z.ZodEnum<{
192
+ major: "major";
193
+ minor: "minor";
194
+ moderate: "moderate";
147
195
  }>;
148
- evidenceIds: z.ZodArray<z.ZodString>;
149
- limitation: z.ZodString;
150
- disposition: z.ZodEnum<{
151
- context: "context";
152
- finding: "finding";
153
- unsupported: "unsupported";
196
+ confidence: z.ZodEnum<{
197
+ high: "high";
198
+ low: "low";
199
+ medium: "medium";
154
200
  }>;
155
- findingId: z.ZodNullable<z.ZodString>;
156
- reason: z.ZodString;
157
- }, z.core.$strict>>>>;
158
- limitations: z.ZodArray<z.ZodString>;
201
+ seenByStreamIds: z.ZodArray<z.ZodString>;
202
+ evidenceIds: z.ZodArray<z.ZodString>;
203
+ }, z.core.$strict>>;
159
204
  }, z.core.$strict>>;
160
205
  /** Stable hashing is independent of object insertion order and excludes no fields implicitly. */
161
206
  export declare function hashAnalysisValue(value: unknown): string;
@@ -27,6 +27,20 @@ const observationSchema = z
27
27
  limitation: text(2000),
28
28
  })
29
29
  .strict();
30
+ const designFindingSchema = z
31
+ .object({
32
+ id,
33
+ headline: label,
34
+ screen: label,
35
+ notice: text(1500).min(1),
36
+ whyItMatters: text(1000).min(1),
37
+ suggestion: text(1000).min(1),
38
+ severity: z.enum(["minor", "moderate", "major"]),
39
+ confidence: z.enum(["low", "medium", "high"]),
40
+ seenByStreamIds: sourceIds.min(1),
41
+ evidenceIds: refs,
42
+ })
43
+ .strict();
30
44
  const analysisResultSchema = z
31
45
  .object({
32
46
  summary: text(4000).min(1),
@@ -51,6 +65,8 @@ const analysisResultSchema = z
51
65
  .object({
52
66
  id,
53
67
  title: label,
68
+ headline: label.optional(),
69
+ experience: text(1200).min(1).optional(),
54
70
  summary: text(4000).min(1),
55
71
  impact: z.enum(["blocked_task", "friction", "recovery", "uncertain"]),
56
72
  affectedStreamIds: sourceIds.min(1),
@@ -64,6 +80,7 @@ const analysisResultSchema = z
64
80
  })
65
81
  .strict())
66
82
  .max(100),
83
+ designFindings: z.array(designFindingSchema).max(40).optional(),
67
84
  concernReviews: z
68
85
  .array(observationSchema
69
86
  .extend({
@@ -77,13 +94,25 @@ const analysisResultSchema = z
77
94
  limitations,
78
95
  })
79
96
  .strict();
80
- /** Historical artifacts may omit concernReviews; every new provider response must supply it. */
81
- export const analysisResponseSchema = analysisResultSchema.required({
82
- concernReviews: true,
83
- });
97
+ /** Historical artifacts may omit concernReviews, headlines, experiences and designFindings; every
98
+ * new provider response must supply them. */
99
+ export const analysisResponseSchema = analysisResultSchema
100
+ .extend({
101
+ findings: z
102
+ .array(analysisResultSchema.shape.findings.element.required({ headline: true, experience: true }))
103
+ .max(100),
104
+ designFindings: z.array(designFindingSchema).max(40),
105
+ })
106
+ .required({ concernReviews: true });
84
107
  export const analysisResultJsonSchema = z.toJSONSchema(analysisResponseSchema);
85
- const normalizedResult = ({ concernReviews, ...result }) => ({
108
+ const normalizedResult = ({ concernReviews, designFindings, findings, ...result }) => ({
86
109
  ...result,
110
+ findings: findings.map(({ headline, experience, ...finding }) => ({
111
+ ...finding,
112
+ ...(headline === undefined ? {} : { headline }),
113
+ ...(experience === undefined ? {} : { experience }),
114
+ })),
115
+ ...(designFindings === undefined ? {} : { designFindings }),
87
116
  ...(concernReviews === undefined ? {} : { concernReviews }),
88
117
  });
89
118
  const analysisCoverageSchema = z
@@ -349,6 +378,21 @@ export function checkAnalysisResult(input, value) {
349
378
  if (finding.affectedStreamIds.some((stream) => !citedStreams.has(stream)))
350
379
  errors.add("ANALYSIS_AFFECTED_WITHOUT_EVIDENCE");
351
380
  }
381
+ const designFindings = result.designFindings ?? [];
382
+ if (!distinct(designFindings.map((finding) => finding.id)))
383
+ errors.add("ANALYSIS_DESIGN_FINDING_ID_DUPLICATE");
384
+ for (const finding of designFindings) {
385
+ const refs = finding.evidenceIds.map((ref) => evidence.get(ref));
386
+ if (!distinct(finding.evidenceIds) || refs.some((ref) => !ref || !included.has(ref.streamId)))
387
+ errors.add("ANALYSIS_DESIGN_REFERENCE_INVALID");
388
+ if (!refs.some((ref) => ref?.capture !== null && ref?.capture !== undefined))
389
+ errors.add("ANALYSIS_DESIGN_WITHOUT_CAPTURE");
390
+ // Seen by means a cited capture of that participant shows the problem.
391
+ const captured = new Set(refs.flatMap((ref) => (ref?.capture ? [ref.streamId] : [])));
392
+ if (!distinct(finding.seenByStreamIds) ||
393
+ finding.seenByStreamIds.some((stream) => !included.has(stream) || !captured.has(stream)))
394
+ errors.add("ANALYSIS_DESIGN_MEMBERSHIP_INVALID");
395
+ }
352
396
  for (const review of result.concernReviews ?? []) {
353
397
  const citedStreams = checkObservation(review);
354
398
  const finding = result.findings.find((item) => item.id === review.findingId);
@@ -425,12 +469,15 @@ export function validateAnalysisArtifact(value) {
425
469
  (artifact.status === "complete" && !coverage.complete))
426
470
  throw new Error("ANALYSIS_STATUS_INVALID");
427
471
  if (artifact.result !== null) {
428
- // Concern accounting became required with revision 5. Keep the boundary stable
429
- // when the prompt version advances; a later prompt must not regain legacy omissions.
430
- const revision = /^study-evidence-(\d+)$/.exec(artifact.promptVersion)?.[1];
431
- if (revision !== undefined &&
432
- Number(revision) >= 5 &&
433
- artifact.result.concernReviews === undefined)
472
+ // Concern accounting became required with revision 5, and headlines, experiences and design
473
+ // findings with revision 7. Keep each boundary stable when the prompt version advances; a later
474
+ // prompt must not regain legacy omissions.
475
+ const revision = Number(/^study-evidence-(\d+)$/.exec(artifact.promptVersion)?.[1] ?? 0);
476
+ const result = artifact.result;
477
+ if ((revision >= 5 && result.concernReviews === undefined) ||
478
+ (revision >= 7 &&
479
+ (result.designFindings === undefined ||
480
+ result.findings.some((finding) => finding.headline === undefined || finding.experience === undefined))))
434
481
  throw new Error("ANALYSIS_RESULT_SCHEMA_INVALID");
435
482
  validateAnalysisResult({ ...artifact, images: [] }, artifact.result);
436
483
  }
@@ -6,6 +6,7 @@ import { DEFAULT_EXPORT_MAX_BYTES, exportRun, formatExportHuman } from "../../fe
6
6
  import { listRuns, readReview } from "../../run/stored-runs.js";
7
7
  import { verifyRun } from "../../verify/verify.js";
8
8
  import { reclaimPreflightSandboxes, reclaimRunSandboxes, } from "../../run/reclaim.js";
9
+ import { runDisplay } from "../../run/display.js";
9
10
  import { addRunOptions, handleRun } from "./run-command.js";
10
11
  import { applyEnvFileOption, CWD_OPTION_DESCRIPTION, DOTENV_OPTION_DESCRIPTION, dotenvPathOf, envFileAliasOption, JSON_OPTION_DESCRIPTION, RUN_OPTION_DESCRIPTION, writeResult, humanError, } from "../io.js";
11
12
  import { plural } from "../../run/text.js";
@@ -33,7 +34,7 @@ export function registerVerifyCommand(parent, io) {
33
34
  io.setExitCode(result.ok ? 0 : 2);
34
35
  });
35
36
  }
36
- const CLEANUP_RENAMED = "`cleanup` is now `reclaim --check`; `cleanup` is removed in 0.112.0.";
37
+ const CLEANUP_RENAMED = "`cleanup` is now `reclaim --check`; `cleanup` is removed in the first minor release on or after 2026-11-03.";
37
38
  /**
38
39
  * The deprecated `cleanup`, hidden from help: one warning, then exactly `reclaim --check`, with
39
40
  * its output and exit code. The old command read only what computer-use runs recorded and passed
@@ -244,13 +245,6 @@ const REVIEW_VERDICTS = {
244
245
  blocked: "blocked",
245
246
  timed_out: "timed out",
246
247
  };
247
- /** The state runDisplay gives a finished run whose own ok agrees with its verdict. */
248
- const VERDICT_STATES = {
249
- pass: "passed",
250
- fail: "failed",
251
- blocked: "blocked",
252
- timed_out: "timed_out",
253
- };
254
248
  /**
255
249
  * Review's headline: the run's display label, as `humanish runs`, review.md and the Observer show
256
250
  * it. A verdict that names another state follows in parentheses, so an interrupted run whose last
@@ -262,8 +256,8 @@ function reviewHeadline(verdict, display) {
262
256
  return REVIEW_VERDICTS[verdict];
263
257
  if (display.state === "no_verdict" || display.state === "dry_run")
264
258
  return `${display.label}; ${NOTHING_TESTED}`;
265
- const judged = VERDICT_STATES[verdict];
266
- return judged === undefined || judged === display.state
259
+ const judged = runDisplay({ liveness: "finished", verdict }).state;
260
+ return verdict === "contract_proof_only" || judged === display.state
267
261
  ? display.label
268
262
  : `${display.label} (verdict ${REVIEW_VERDICTS[verdict]})`;
269
263
  }