humanish 0.111.4 → 0.112.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. package/CHANGELOG.md +18 -0
  2. package/dist/actors/computer-use/openai-conversation.d.ts +91 -0
  3. package/dist/actors/computer-use/openai-conversation.js +209 -0
  4. package/dist/actors/computer-use/openai-provider.js +25 -41
  5. package/dist/actors/contract.d.ts +2 -0
  6. package/dist/analysis/diagnostics.js +7 -12
  7. package/dist/analysis/execute.d.ts +1 -1
  8. package/dist/analysis/execute.js +35 -5
  9. package/dist/analysis/service.js +7 -1
  10. package/dist/analysis/types.d.ts +23 -0
  11. package/dist/analysis/validation.d.ts +80 -35
  12. package/dist/analysis/validation.js +58 -11
  13. package/dist/cli/commands/runs.js +4 -10
  14. package/dist/cli/findings.d.ts +34 -3
  15. package/dist/cli/findings.js +136 -18
  16. package/dist/cli/io.d.ts +2 -1
  17. package/dist/cli/io.js +3 -2
  18. package/dist/feedback/draft.js +14 -4
  19. package/dist/observer/run-routes.js +8 -4
  20. package/dist/observer-app.html +4 -4
  21. package/dist/routes/computer-use/bundle.d.ts +1 -1
  22. package/dist/routes/computer-use/bundle.js +6 -16
  23. package/dist/routes/computer-use/fanout-bundle.js +40 -37
  24. package/dist/routes/computer-use/fanout-records.d.ts +2 -0
  25. package/dist/routes/computer-use/fanout-records.js +18 -26
  26. package/dist/routes/computer-use/participant-execution.js +4 -11
  27. package/dist/routes/computer-use/participant-facts.d.ts +2 -2
  28. package/dist/routes/computer-use/participant-facts.js +10 -1
  29. package/dist/routes/computer-use/result.js +7 -15
  30. package/dist/routes/computer-use/single-bundle.d.ts +2 -0
  31. package/dist/routes/computer-use/single-bundle.js +81 -37
  32. package/dist/routes/scripted/bundle.d.ts +1 -1
  33. package/dist/routes/scripted/bundle.js +9 -19
  34. package/dist/routes/scripted/surface-records.d.ts +0 -1
  35. package/dist/routes/scripted/surface-records.js +27 -13
  36. package/dist/routes/shared-world/bundle-records.d.ts +2 -2
  37. package/dist/routes/shared-world/bundle-records.js +21 -28
  38. package/dist/routes/shared-world/bundle.d.ts +1 -1
  39. package/dist/routes/shared-world/bundle.js +70 -78
  40. package/dist/routes/shared-world/participant-specs.d.ts +1 -2
  41. package/dist/routes/shared-world/participant-specs.js +0 -8
  42. package/dist/routes/shared-world/result.js +8 -12
  43. package/dist/routes/terminal/bundle.d.ts +1 -1
  44. package/dist/routes/terminal/bundle.js +10 -19
  45. package/dist/routes/terminal/dry-run.js +1 -0
  46. package/dist/routes/terminal/live-sandbox.js +9 -2
  47. package/dist/routes/terminal/plan.js +1 -0
  48. package/dist/routes/terminal/runtime.d.ts +1 -0
  49. package/dist/routes/terminal/runtime.js +1 -0
  50. package/dist/routes/terminal/session.js +1 -0
  51. package/dist/run/costs.js +9 -20
  52. package/dist/run/detail.d.ts +1 -1
  53. package/dist/run/detail.js +10 -1
  54. package/dist/run/display.d.ts +2 -2
  55. package/dist/run/display.js +15 -18
  56. package/dist/run/dry-run.js +10 -25
  57. package/dist/run/judge.d.ts +24 -0
  58. package/dist/run/judge.js +54 -0
  59. package/dist/run/liveness.d.ts +18 -0
  60. package/dist/run/liveness.js +24 -0
  61. package/dist/run/outcomes.d.ts +8 -1
  62. package/dist/run/outcomes.js +58 -12
  63. package/dist/run/participant-caption.d.ts +30 -0
  64. package/dist/run/participant-caption.js +50 -0
  65. package/dist/run/participant-records.d.ts +1 -1
  66. package/dist/run/review-markdown.d.ts +11 -0
  67. package/dist/run/review-markdown.js +22 -0
  68. package/dist/run/run-cost.d.ts +7 -0
  69. package/dist/run/run-cost.js +25 -5
  70. package/dist/run/run-index.d.ts +4 -50
  71. package/dist/run/run-index.js +57 -87
  72. package/dist/run/run.d.ts +2 -2
  73. package/dist/run/run.js +5 -3
  74. package/dist/run/stats.js +2 -12
  75. package/dist/run/transient-comms-secrets.d.ts +8 -3
  76. package/dist/run/transient-comms-secrets.js +21 -1
  77. package/dist/study/keys.d.ts +1 -0
  78. package/dist/study/keys.js +1 -1
  79. package/dist/study/parse/execution.js +7 -1
  80. package/dist/study/plan-types.d.ts +1 -0
  81. package/dist/study/types.d.ts +4 -1
  82. package/dist/study/warnings.d.ts +2 -1
  83. package/dist/study/warnings.js +3 -2
  84. package/dist/tui-app.js +73 -73
  85. package/dist/verify/liveness.d.ts +2 -2
  86. package/dist/verify/liveness.js +7 -4
  87. package/docs/architecture/terminal-product-route.md +14 -0
  88. package/docs/contracts/run-bundle.md +10 -0
  89. package/docs/contracts/schemas.md +9 -1
  90. package/docs/contracts/study-analysis.md +71 -22
  91. package/package.json +2 -1
  92. package/dist/actors/computer-use/openai-requests.d.ts +0 -62
  93. package/dist/actors/computer-use/openai-requests.js +0 -144
@@ -14,8 +14,8 @@ export interface UnfinishedRun {
14
14
  }
15
15
  /**
16
16
  * One warning, and the unfinished-run facts, when the run is not finished, by the run index's
17
- * rule: its status record when it has a usable one, else a participant still `running` in its
18
- * bundle's `simulations[]`. Nothing for a finished run.
17
+ * rule: its bundle outcome first, then usable status, then the bundle's simulations. Nothing
18
+ * for a finished run.
19
19
  */
20
20
  export declare function runNotFinished(runPaths: PreparedRunArtifactPaths, bundle: RunBundle, nowMs?: number): Promise<{
21
21
  warnings: string[];
@@ -7,7 +7,7 @@ import path from "node:path";
7
7
  import { readRunJsonIfExists, readRunTextIfExists } from "../run/locate.js";
8
8
  import { isRecord } from "../run/type-guards.js";
9
9
  import { RECLAIM_RECEIPT_ARTIFACT } from "../run/reclaim.js";
10
- import { runLiveness } from "../run/run-index.js";
10
+ import { runLiveness } from "../run/liveness.js";
11
11
  import { parseSandboxReceipts, SANDBOX_RECEIPTS_ARTIFACT } from "../run/sandbox-receipts.js";
12
12
  import { RUN_STATUS_FILE } from "../run/status.js";
13
13
  import { plural } from "../run/text.js";
@@ -17,8 +17,8 @@ const RUN_NOT_FINISHED = "RUN_NOT_FINISHED";
17
17
  const GONE_STATES = new Set(["killed", "already-gone"]);
18
18
  /**
19
19
  * One warning, and the unfinished-run facts, when the run is not finished, by the run index's
20
- * rule: its status record when it has a usable one, else a participant still `running` in its
21
- * bundle's `simulations[]`. Nothing for a finished run.
20
+ * rule: its bundle outcome first, then usable status, then the bundle's simulations. Nothing
21
+ * for a finished run.
22
22
  */
23
23
  export async function runNotFinished(runPaths, bundle, nowMs = Date.now()) {
24
24
  const runId = path.basename(runPaths.absoluteRunRoot);
@@ -27,7 +27,10 @@ export async function runNotFinished(runPaths, bundle, nowMs = Date.now()) {
27
27
  if (liveness === "finished")
28
28
  return { warnings: [] };
29
29
  const seen = [];
30
- if (record === undefined) {
30
+ if (bundle.outcome?.state === "interrupted") {
31
+ seen.push(`run.json records interruption by ${bundle.outcome.signal} at ${bundle.outcome.at}`);
32
+ }
33
+ else if (record === undefined) {
31
34
  const running = bundle.simulations.filter((simulation) => simulation.status === "running");
32
35
  seen.push(`the run has no usable ${RUN_STATUS_FILE} and ${running.length} of ${bundle.simulations.length} simulations are still running`);
33
36
  }
@@ -84,6 +84,20 @@ at the base tier, with cached input at the cached rate, and says so with
84
84
  checked against `caps.maxUsd` keeps the provider line unmeasured, and the studied
85
85
  product's no-spend boundary still needs separate interpretation.
86
86
 
87
+ ## Participant telemetry
88
+
89
+ Terminal participant commands and product setup receive `DO_NOT_TRACK=1` by default.
90
+ Their child processes inherit it, so studied CLIs that honor the convention can exclude
91
+ synthetic usage. To study the product's telemetry behavior, set
92
+ `execution.terminal.doNotTrack: false`; humanish then leaves the variable unset.
93
+ Only booleans are accepted, and `true` or an omitted field enables the default.
94
+
95
+ The resolved `doNotTrack` boolean is recorded with `humanish.actor-runtime.v1` in
96
+ `actor.json`, `streams[].actor.runtime` in `run.json`, and `runtime` in
97
+ `terminal-ledgers.json`. A dry run records it in the JSON message of its
98
+ `terminal-lab.runtime.declared` event. Older bundles omit it. This records the
99
+ command environment policy; it does not prove that the studied CLI honors the variable.
100
+
87
101
  ## Runtime auth: raw-key placement and remaining provider access
88
102
 
89
103
  `execution.runtimeAuth: openai-egress` is the default: a study that declares no
@@ -6,6 +6,13 @@
6
6
  source of truth. This page explains the stable public fields and the extension rules; it does not
7
7
  version the runtime separately.
8
8
 
9
+ Terminal runs record the resolved telemetry setting as `doNotTrack` in
10
+ `streams[].actor.runtime` (`humanish.actor-runtime.v1`), also retained in `actor.json`
11
+ and `terminal-ledgers.json`. `true` means participant and product setup commands receive
12
+ `DO_NOT_TRACK=1`; `false` means humanish leaves it unset. Dry runs record the same boolean
13
+ in the JSON message of the `terminal-lab.runtime.declared` event. Older bundles omit it.
14
+ The setting does not attest that the studied CLI honors the variable.
15
+
9
16
  ## Purpose
10
17
 
11
18
  A run bundle is the durable evidence packet for one harness run. It should be
@@ -154,6 +161,9 @@ writing it, and on runs recorded before the field existed. For those, readers
154
161
  take liveness from `status.json` and, for an older finished run, its `ok` and
155
162
  `execution` from `status.json`'s `outcome`. That record is the only copy of an
156
163
  older run's `ok`; for a run with `outcome` in `run.json`, `status.json` copies it.
164
+ The bundle's outcome wins even if the process stopped before updating `status.json`.
165
+ Without an outcome or a usable status record naming the run, a bundle with a simulation still
166
+ `running` reads as interrupted. A fresh status record keeps an in-progress bundle running.
157
167
 
158
168
  Every surface that shows whether a run passed calls `runDisplay`
159
169
  (`src/run/display.ts`) with the run's liveness, verdict and `ok`: the Observer
@@ -35,7 +35,7 @@ workflow without leaking private upstream truth into core.
35
35
  | Substrate | reserved (never shipped) | none |
36
36
  | Evidence stream | reserved (streams live inside the run bundle) | see [`run-bundle.md`](run-bundle.md) |
37
37
  | Review | `humanish.review.v1` | `contract-proof-review` |
38
- | Study analysis | `humanish.study-analysis.v1` | see [study analysis](study-analysis.md) and synthetic analysis fixtures in `tests/` |
38
+ | Study analysis | `humanish.study-analysis.v1` (prompt `study-evidence-7` adds `findings[].headline`, `findings[].experience` and `designFindings[]`; older artifacts omit them) | see [study analysis](study-analysis.md) and synthetic analysis fixtures in `tests/` |
39
39
  | Study analysis correction | `humanish.study-analysis-correction.v1` | see [study analysis](study-analysis.md#human-review-and-sharing) |
40
40
  | Analysis findings view | `humanish.analysis-findings.v1` (`review --json` field `analysis`; read-only projection of the selected analysis) | see [study analysis](study-analysis.md#reading-findings) |
41
41
  | Analysis execution receipt | `humanish.analysis-execution.v1` | see [study analysis](study-analysis.md#durable-records) |
@@ -296,6 +296,14 @@ instruction?, target?, entry?, host?, reasoningEffort?, stopWhen?, dwell? }` bec
296
296
  metadata; with `openai-egress` it stays in the host-side E2B header transform
297
297
  set at sandbox creation, and the agent gets a placeholder. A dry-run neither
298
298
  reads nor injects it;
299
+ - `execution.terminal.doNotTrack`: boolean, default `true`. Terminal participant and product
300
+ setup commands receive `DO_NOT_TRACK=1`. Set `false` to leave that variable unset when
301
+ studying the product's telemetry behavior. Non-boolean values are refused at parse.
302
+ The resolved boolean is `doNotTrack` in `humanish.actor-runtime.v1`: `actor.json`,
303
+ `streams[].actor.runtime` in `run.json`, and `runtime` in `terminal-ledgers.json` for live
304
+ runs; the JSON message of the `terminal-lab.runtime.declared` event for dry runs.
305
+ Older bundles omit this field. It records the command environment policy, not whether
306
+ the studied CLI honored it;
299
307
  - `mode`: `dry-run` (a synthetic bundle, no spend; the default) or `live`. On the
300
308
  scripted route `live` gates real browser actuation against the declared app.
301
309
  Provider spend stays $0 there by mechanism (no model runs);
@@ -161,6 +161,42 @@ thought. Older reports may omit this field and remain readable. The disclosure
161
161
  is model-generated assessment, not an independent completeness audit or a human
162
162
  reviewer annotation.
163
163
 
164
+ ## Headlines and design findings
165
+
166
+ The `study-evidence-7` prompt writes each finding twice. `headline` is one plain sentence about what
167
+ happened, for a designer, product manager or developer who did not watch the session.
168
+ `experience` is one to three plain sentences: what the person tried, what got in their way, and
169
+ how it seemed to feel. Neither names participant or evidence IDs. A concern known only from what
170
+ participants said still reads as something they said. `title`, `summary` and `observations` keep
171
+ their evidence-level wording, and validation applies to them as before.
172
+
173
+ The same prompt adds `designFindings`: problems a product designer would notice in the supplied
174
+ captures, whether or not a participant mentioned them. Each has an `id` (`D1`, `D2`, ...), a
175
+ `headline`, the `screen` in plain words, what a designer would `notice`, `whyItMatters` to a
176
+ person using the product, one `suggestion`, a `severity` (`major` when it misleads or blocks,
177
+ `moderate` when it slows or confuses, `minor` when it is polish), a `confidence`,
178
+ `seenByStreamIds` and `evidenceIds`. Validation rejects a design finding that cites no retained
179
+ capture, cites evidence outside the packet, repeats an ID, or lists a participant in
180
+ `seenByStreamIds` without a cited capture of that participant. An empty list means the review found no design
181
+ problem in the captures. The model judges legibility and size from images alone; it does not
182
+ measure text or control sizes.
183
+
184
+ New provider responses must include `headline` and `experience` on every finding and a
185
+ `designFindings` list. A stored artifact with prompt `study-evidence-7` or later must carry them
186
+ too. Artifacts from earlier prompts load without them and render by title and summary. The
187
+ Observer applies the same prompt-version rules and the same field limits when it reads a saved
188
+ analysis, and treats an analysis that breaks them as unavailable.
189
+
190
+ The narrative scrub removes a run's known transient values, such as a received one-time code or
191
+ link, from every generated text field. It finds each value as written and in its encoded forms:
192
+ percent-encoded, JSON-escaped, base64, base64url and hex, and where escapes split it. A field
193
+ with a value is stored decoded with the value replaced; a field without one keeps its spelling. A
194
+ known value in any of these forms in an ID, enum or reference refuses the response.
195
+
196
+ When a reviewer amends a finding, its headline and experience describe the replaced claim.
197
+ Text output, the Observer and feedback drafts show the reviewer's claim in their place and say the
198
+ finding was corrected; the stored analysis keeps the original.
199
+
164
200
  Every observation cites packet-local evidence IDs. The model cannot choose a
165
201
  filesystem path or fetch another resource. Validation checks participant
166
202
  membership, unique counts, quote fidelity, evidence type and reference
@@ -203,7 +239,7 @@ feedback quotes cite evidence IDs that the run's own `analysis.json` resolves.
203
239
  Participant reviews cite distinct packet-local evidence IDs belonging only to
204
240
  that participant's stream; source event IDs are not citation IDs. Shared
205
241
  interactions can cite multiple included participants in findings and concern
206
- reviews under the existing exposure rules. The `study-evidence-6` prompt makes
242
+ reviews under the existing exposure rules. The `study-evidence-6` prompt made
207
243
  this distinction explicit without changing the validator or historical reports.
208
244
 
209
245
  Elapsed replay time starts at the first retained capture. It is not a video
@@ -219,6 +255,12 @@ guarantee that the selected capture is the most relevant. Context-only and
219
255
  nonvisual evidence retain their basis and original event. All cited moments stay
220
256
  available, with exact recording links.
221
257
 
258
+ A finding with a headline leads its Observer row with the headline and its open panel with the
259
+ experience. The title, summary, assessment and observations sit in a closed **Evidence**
260
+ disclosure beneath. A **Design findings** section follows the ranked findings, grouped by
261
+ severity, with each cited capture as a thumbnail that opens that frame. Findings without a
262
+ headline render as described below.
263
+
222
264
  Confidence, recovery and the first full evidence limitation remain visible when
223
265
  a finding opens. Exposure and remaining unique limits are one disclosure away;
224
266
  observation details retain each original claim and its specific limitation.
@@ -291,31 +333,35 @@ is disabled. Existing adapter captures and logs retain their contained routes.
291
333
  ## Reading findings
292
334
 
293
335
  `humanish review` prints a run's analysis findings after its participant review, and
294
- `humanish analyze show` prints them alone. Each finding shows its title, impact, confidence and
295
- recovery, the participants it affected out of those exposed, the frames its evidence cites with
296
- their time since the first retained capture, the capture files, its next step and the latest
297
- human review note. A live run's human output ends with up to three findings, one line each, and
298
- the `review` command that prints all of them. `analyze show --json` prints the validated
336
+ `humanish analyze show` prints them alone. Each finding leads with its headline and experience,
337
+ then shows its title as `evidence:`, its impact, confidence and recovery, the participants it
338
+ affected out of those exposed, the frames its evidence cites with their time since the first
339
+ retained capture, the capture files, its next step and the latest human review note. A finding
340
+ without a headline starts at its title. Design findings follow, most severe first, each with its
341
+ screen, notice, why it matters, suggestion, who saw it and its capture files. A live run's human
342
+ output ends with up to three findings and up to three design findings, one line each, and the
343
+ `review` command that prints all of them. `analyze show --json` prints the validated
299
344
  analysis record and its corrections, as before. Reading findings never starts an analysis.
300
345
 
301
346
  `review --json` carries the same view as its `analysis` field, schema
302
347
  `humanish.analysis-findings.v1`, built in `src/cli/findings.ts`:
303
348
 
304
- | Field | Meaning |
305
- | ------------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
306
- | `schema` | `humanish.analysis-findings.v1`. |
307
- | `runId` | The run the view reads. |
308
- | `state` | `ready`, `stale`, `running`, `none`, `skipped`, `failed`, `dry_run` or `unavailable`. Only `ready` and `stale` carry findings. |
309
- | `reason` | The stable code behind a state other than `ready`, or null: an automatic analysis reason such as `AUTOMATIC_ANALYSIS_KEY_MISSING`, a failed attempt's error code, or a load warning such as `ANALYSIS_SOURCE_CHANGED`. |
310
- | `message` | What happened, in words. |
311
- | `next` | The command that gets findings for this run, or for a dry run the command that starts a live one. Null when no command can, as for a run whose participants left no evidence. |
312
- | `analysisId`, `status`, `provider`, `model`, `completedAt`, `estimatedCostUsd` | The analysis version `analyze show` selects: the newest ready one, else the newest stale one. Null without one. |
313
- | `runPath`, `path` | The run directory and the version's `analysis.json`, relative to the project directory. |
314
- | `summary`, `limitations` | The analysis's own summary and limitations. |
315
- | `findings[]` | Highest priority first: `id`, `title`, `summary`, `impact` (`blocked_task`, `friction`, `recovery`, `uncertain`), `confidence` (`low`, `medium`, `high`), `recovery` (`recovered`, `not_observed`, `unknown`), `affected[]` (`streamId`, `label`), `exposedCount`, `evidence[]`, `nextStep` and `correction`. |
316
- | `findings[].evidence[]` | Each cited item once: `id`, `streamId`, `kind`, `bases` (how the observations used it), `frame` (the latest retained capture's index from 0, null before any capture), `elapsedMs` (time since the first retained capture, not a video offset), `at` and `capture` (the file relative to the project directory, null for nonvisual evidence). |
317
- | `findings[].correction` | The latest `analyze correct` note on that finding version (`status`, `reason`, `replacementClaim`), or null. |
318
- | `warnings` | Load warnings for the selected version. |
349
+ | Field | Meaning |
350
+ | ------------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
351
+ | `schema` | `humanish.analysis-findings.v1`. |
352
+ | `runId` | The run the view reads. |
353
+ | `state` | `ready`, `stale`, `running`, `none`, `skipped`, `failed`, `dry_run` or `unavailable`. Only `ready` and `stale` carry findings. |
354
+ | `reason` | The stable code behind a state other than `ready`, or null: an automatic analysis reason such as `AUTOMATIC_ANALYSIS_KEY_MISSING`, a failed attempt's error code, or a load warning such as `ANALYSIS_SOURCE_CHANGED`. |
355
+ | `message` | What happened, in words. |
356
+ | `next` | The command that gets findings for this run, or for a dry run the command that starts a live one. Null when no command can, as for a run whose participants left no evidence. |
357
+ | `analysisId`, `status`, `provider`, `model`, `completedAt`, `estimatedCostUsd` | The analysis version `analyze show` selects: the newest ready one, else the newest stale one. Null without one. |
358
+ | `runPath`, `path` | The run directory and the version's `analysis.json`, relative to the project directory. |
359
+ | `summary`, `limitations` | The analysis's own summary and limitations. |
360
+ | `findings[]` | Highest priority first: `id`, `headline` and `experience` (null in analyses before `study-evidence-7`), `title`, `summary`, `impact` (`blocked_task`, `friction`, `recovery`, `uncertain`), `confidence` (`low`, `medium`, `high`), `recovery` (`recovered`, `not_observed`, `unknown`), `affected[]` (`streamId`, `label`), `exposedCount`, `evidence[]`, `nextStep` and `correction`. |
361
+ | `findings[].evidence[]` | Each cited item once: `id`, `streamId`, `kind`, `bases` (how the observations used it), `frame` (the latest retained capture's index from 0, null before any capture), `elapsedMs` (time since the first retained capture, not a video offset), `at` and `capture` (the file relative to the project directory, null for nonvisual evidence). |
362
+ | `findings[].correction` | The latest `analyze correct` note on that finding version (`status`, `reason`, `replacementClaim`), or null. When `status` is `amended`, `replacementClaim` replaces the finding's headline and experience. |
363
+ | `designFindings` | Most severe first: `id`, `headline`, `screen`, `notice`, `whyItMatters`, `suggestion`, `severity` (`major`, `moderate`, `minor`), `confidence`, `seenBy[]` (`streamId`, `label`) and `evidence[]` (as `findings[].evidence[]`, without `bases`). Null without an analysis result, or for an analysis written before design findings. |
364
+ | `warnings` | Load warnings for the selected version. |
319
365
 
320
366
  The states without findings:
321
367
 
@@ -352,7 +398,10 @@ humanish feedback issue --run latest --analysis <id> --finding F1 --repo owner/r
352
398
 
353
399
  Corrections append against exact analysis and finding hashes. Confirming one
354
400
  version does not approve a later claim. Dismissed findings cannot become feedback
355
- drafts; amendments preserve the original and record the replacement. Feedback
401
+ drafts; amendments preserve the original and record the replacement. A reason or replacement
402
+ claim that matches a sensitive pattern is refused. When the correction is made inside the run's
403
+ own invocation, one that holds a known transient value, as written or encoded, is refused too; a
404
+ separate `analyze correct` command has no transient values to check against. Feedback
356
405
  drafts include source/version/evidence references and remain explicitly
357
406
  independent of participant-authored candidates. No command above posts to GitHub.
358
407
  Each analysis has room for 256 correction inventory entries, including interrupted
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "humanish",
3
- "version": "0.111.4",
3
+ "version": "0.112.0",
4
4
  "description": "Synthetic user research for apps, CLIs, and agent-facing product flows.",
5
5
  "keywords": [
6
6
  "cli",
@@ -76,6 +76,7 @@
76
76
  "tui:test": "pnpm --filter humanish-tui test",
77
77
  "release:dogfood": "node scripts/release-dogfood.mjs",
78
78
  "bench": "tsx bench/run.ts",
79
+ "arch:bench": "tsx scripts/arch-bench.ts",
79
80
  "browser-control:proof": "node scripts/browser-control-proof.mjs",
80
81
  "docs:generate": "tsx scripts/generate-cli-docs.ts",
81
82
  "docs:check": "tsx scripts/generate-cli-docs.ts --check && tsx scripts/check-doc-paths.ts && tsx scripts/check-doc-symbols.ts && tsx scripts/check-doc-comments.ts && tsx scripts/check-doc-study-fields.ts",
@@ -1,62 +0,0 @@
1
- import type { CuaTurnRequest } from "./loop.js";
2
- import type { ReasoningEffort } from "../reasoning-effort.js";
3
- import type { ActorConversation } from "../contract.js";
4
- import { CarriedConversation } from "./openai-context.js";
5
- import { type OpenAiReasoningSummary, type ParsedOpenAiResponse } from "./openai-wire.js";
6
- /** Request settings fixed for a provider's whole session. */
7
- export interface OpenAiRequestSettings {
8
- readonly model: string;
9
- readonly reasoningEffort: ReasoningEffort;
10
- readonly maxOutputTokens?: number;
11
- readonly safetyIdentifier?: string;
12
- }
13
- /** The conversation as the provider has it after its last accepted reply. */
14
- export interface OpenAiConversationState {
15
- lastResponseId: string | undefined;
16
- /** computer_call ids the next request owes an output for. */
17
- pendingCallIds: string[];
18
- /** Accepted replies so far; the next request is number `replies + 1`. */
19
- replies: number;
20
- /** The whole conversation, which explicit-context requests carry within its token budget. */
21
- conversation: CarriedConversation;
22
- /** explicit_context after a zero-data-retention rejection or when configured. */
23
- mode: "previous_response_id" | "explicit_context";
24
- /** Undefined once the account or model rejected reasoning summaries. */
25
- reasoningSummary: OpenAiReasoningSummary | undefined;
26
- }
27
- /**
28
- * The items the next request adds to the conversation: the opening message with the first
29
- * screen, or an output for each pending computer_call with the latest screenshot and any hint.
30
- * Built once per turn, so every attempt at the request and the accepted reply see the same items.
31
- */
32
- export declare function turnInputItems(state: OpenAiConversationState, req: CuaTurnRequest): readonly unknown[];
33
- /**
34
- * The body of the next participant request: the opening request, or a continuation carrying
35
- * `sent` (turnInputItems). A threaded continuation names the previous response; an
36
- * explicit-context one carries the conversation ahead of `sent`.
37
- */
38
- export declare function turnRequestBody(settings: OpenAiRequestSettings, state: OpenAiConversationState, req: CuaTurnRequest, sent: readonly unknown[]): Record<string, unknown>;
39
- /** The body of the read-only closing report: no tools, a small output limit, a strict schema. */
40
- export declare function debriefRequestBody(settings: OpenAiRequestSettings, state: OpenAiConversationState, req: CuaTurnRequest, sent: readonly unknown[]): Record<string, unknown>;
41
- /**
42
- * Move the conversation on to an accepted reply to a request that added `sent`. A participant
43
- * reply cut off by the output limit is set aside, so the next request is the same one again
44
- * (outputLimitRetry); its actions never run, so no call output is owed for them.
45
- */
46
- export declare function acceptReply(state: OpenAiConversationState, parsed: ParsedOpenAiResponse, closing: boolean, sent: readonly unknown[]): void;
47
- /** The trace's account of how a session carried its conversation (ActorConversation). */
48
- export declare class ConversationRecord {
49
- private readonly now;
50
- private explicitReason;
51
- private rejection;
52
- private switchedAt;
53
- private switchedAtRequest;
54
- private summarizedTurns;
55
- private readonly requests;
56
- constructor(configured: boolean, now: () => number);
57
- /** The organization rejected server-side state, so the request being sent is explicit_context. */
58
- switched(rejection: NonNullable<ActorConversation["rejection"]>): void;
59
- /** A participant request just went out with `sent` after the carried conversation. */
60
- requested(state: OpenAiConversationState, sent: readonly unknown[]): void;
61
- snapshot(state: OpenAiConversationState): ActorConversation;
62
- }
@@ -1,144 +0,0 @@
1
- import { CarriedConversation, estimateTokens } from "./openai-context.js";
2
- import { buildCallOutput, buildContinuationRequest, buildInitialRequest, hintItems, openingMessage, } from "./openai-wire.js";
3
- /** The closing report's output limit: its own cap, or the declared limit when that is lower. */
4
- const CLOSING_OUTPUT_LIMIT = 1024;
5
- function requestContext(settings, state, instructions) {
6
- return {
7
- stateless: state.mode === "explicit_context",
8
- model: settings.model,
9
- instructions,
10
- reasoningEffort: settings.reasoningEffort,
11
- ...(settings.maxOutputTokens === undefined
12
- ? {}
13
- : { maxOutputTokens: settings.maxOutputTokens }),
14
- ...(state.reasoningSummary === undefined ? {} : { reasoningSummary: state.reasoningSummary }),
15
- ...(settings.safetyIdentifier === undefined
16
- ? {}
17
- : { safetyIdentifier: settings.safetyIdentifier }),
18
- };
19
- }
20
- /** No reply has been accepted yet, so the next request opens the conversation. */
21
- function isFirstTurn(state) {
22
- return state.replies === 0;
23
- }
24
- /**
25
- * The items the next request adds to the conversation: the opening message with the first
26
- * screen, or an output for each pending computer_call with the latest screenshot and any hint.
27
- * Built once per turn, so every attempt at the request and the accepted reply see the same items.
28
- */
29
- export function turnInputItems(state, req) {
30
- if (isFirstTurn(state))
31
- return [openingMessage(req.instructions, req.observation.screenshot)];
32
- return [
33
- ...state.pendingCallIds.map((id) => buildCallOutput(id, req.observation.screenshot, req.acknowledgedSafetyChecks)),
34
- ...hintItems(req.contextHint),
35
- ];
36
- }
37
- /**
38
- * The body of the next participant request: the opening request, or a continuation carrying
39
- * `sent` (turnInputItems). A threaded continuation names the previous response; an
40
- * explicit-context one carries the conversation ahead of `sent`.
41
- */
42
- export function turnRequestBody(settings, state, req, sent) {
43
- const ctx = requestContext(settings, state, req.instructions);
44
- if (isFirstTurn(state))
45
- return buildInitialRequest(ctx, req.observation.screenshot);
46
- return buildContinuationRequest({
47
- ctx,
48
- previousResponseId: state.lastResponseId,
49
- callOutputs: [...sent],
50
- ...(state.mode === "explicit_context"
51
- ? { explicitContextItems: state.conversation.carried() }
52
- : {}),
53
- });
54
- }
55
- /** The body of the read-only closing report: no tools, a small output limit, a strict schema. */
56
- export function debriefRequestBody(settings, state, req, sent) {
57
- return {
58
- ...turnRequestBody(settings, state, req, sent),
59
- tool_choice: "none",
60
- max_output_tokens: Math.min(settings.maxOutputTokens ?? CLOSING_OUTPUT_LIMIT, CLOSING_OUTPUT_LIMIT),
61
- text: {
62
- format: {
63
- type: "json_schema",
64
- name: "participant_closing_report",
65
- strict: true,
66
- schema: {
67
- type: "object",
68
- additionalProperties: false,
69
- required: ["summary", "frictionReports"],
70
- properties: {
71
- summary: { type: "string" },
72
- frictionReports: { type: "array", items: { type: "string" } },
73
- },
74
- },
75
- },
76
- },
77
- };
78
- }
79
- /**
80
- * Move the conversation on to an accepted reply to a request that added `sent`. A participant
81
- * reply cut off by the output limit is set aside, so the next request is the same one again
82
- * (outputLimitRetry); its actions never run, so no call output is owed for them.
83
- */
84
- export function acceptReply(state, parsed, closing, sent) {
85
- if (!closing && parsed.turn.interruption === "output_limit")
86
- return;
87
- if (parsed.turn.responseId !== undefined)
88
- state.lastResponseId = parsed.turn.responseId;
89
- state.pendingCallIds = parsed.callIds;
90
- state.replies += 1;
91
- state.conversation.accept(state.replies, sent, parsed.outputItems);
92
- }
93
- /** The trace's account of how a session carried its conversation (ActorConversation). */
94
- export class ConversationRecord {
95
- now;
96
- explicitReason;
97
- rejection;
98
- switchedAt;
99
- switchedAtRequest;
100
- summarizedTurns = 0;
101
- requests = [];
102
- constructor(configured, now) {
103
- this.now = now;
104
- if (configured)
105
- this.explicitReason = "configured";
106
- }
107
- /** The organization rejected server-side state, so the request being sent is explicit_context. */
108
- switched(rejection) {
109
- this.explicitReason = "zdr_rejection";
110
- this.rejection = rejection;
111
- this.switchedAt = new Date(this.now()).toISOString();
112
- // Counted like requests[], which includes replies set aside at their output limit.
113
- this.switchedAtRequest = this.requests.length + 1;
114
- }
115
- /** A participant request just went out with `sent` after the carried conversation. */
116
- requested(state, sent) {
117
- if (state.mode !== "explicit_context") {
118
- this.requests.push({ mode: "threaded" });
119
- return;
120
- }
121
- const carried = state.conversation.size();
122
- // Only summaries a request carried count; the conversation also trims after the last reply.
123
- this.summarizedTurns = carried.notedTurns;
124
- this.requests.push({
125
- mode: "explicit_context",
126
- carriedExchanges: carried.exchanges,
127
- carriedScreenshots: carried.screenshots,
128
- estimatedInputTokens: carried.estimatedTokens + estimateTokens(sent),
129
- });
130
- }
131
- snapshot(state) {
132
- return {
133
- mode: state.mode === "explicit_context" ? "explicit_context" : "threaded",
134
- ...(this.explicitReason === undefined ? {} : { explicitReason: this.explicitReason }),
135
- ...(this.rejection === undefined ? {} : { rejection: this.rejection }),
136
- ...(this.switchedAt === undefined ? {} : { switchedAt: this.switchedAt }),
137
- ...(this.switchedAtRequest === undefined
138
- ? {}
139
- : { switchedAtRequest: this.switchedAtRequest }),
140
- summarizedTurns: this.summarizedTurns,
141
- requests: this.requests.map((request) => ({ ...request })),
142
- };
143
- }
144
- }