humanish 0.122.0 → 0.124.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. package/CHANGELOG.md +29 -0
  2. package/CONTEXT.md +2 -2
  3. package/README.md +33 -6
  4. package/dist/actors/computer-use/loop/trace.js +6 -1
  5. package/dist/actors/computer-use/openai-conversation.d.ts +0 -6
  6. package/dist/actors/computer-use/openai-conversation.js +0 -8
  7. package/dist/actors/computer-use/openai-provider.js +5 -8
  8. package/dist/analysis/automatic.js +2 -1
  9. package/dist/analysis/evidence.js +41 -21
  10. package/dist/cli/commands/study-run.js +9 -3
  11. package/dist/evidence/image.js +82 -3
  12. package/dist/feedback/derivative-text.d.ts +12 -0
  13. package/dist/feedback/derivative-text.js +130 -0
  14. package/dist/feedback/export-bundle.js +39 -136
  15. package/dist/feedback/export.js +36 -9
  16. package/dist/observer/artifact.js +2 -9
  17. package/dist/observer/data.d.ts +9 -2
  18. package/dist/observer/data.js +17 -2
  19. package/dist/observer/page-data.d.ts +5 -0
  20. package/dist/observer/page-data.js +21 -0
  21. package/dist/observer/participant-items.d.ts +45 -0
  22. package/dist/observer/participant-items.js +85 -0
  23. package/dist/observer/render.js +2 -1
  24. package/dist/observer/run-routes.js +75 -22
  25. package/dist/observer-app.html +4 -4
  26. package/dist/run/bundle-shape.js +2 -2
  27. package/dist/run/bundle.d.ts +5 -2
  28. package/dist/run/bundle.js +4 -1
  29. package/dist/run/detail.d.ts +6 -0
  30. package/dist/run/detail.js +14 -1
  31. package/dist/run/locate.js +3 -1
  32. package/dist/run/projection.d.ts +7 -0
  33. package/dist/run/projection.js +10 -0
  34. package/dist/run/run.js +9 -2
  35. package/dist/run/stored-runs.js +10 -4
  36. package/dist/run/stream-shape.d.ts +11 -1
  37. package/dist/run/stream-shape.js +26 -2
  38. package/dist/run/stream-trace.d.ts +68 -0
  39. package/dist/run/stream-trace.js +260 -0
  40. package/dist/run/streams.d.ts +20 -0
  41. package/dist/study/bundle-estimate.d.ts +48 -0
  42. package/dist/study/bundle-estimate.js +159 -0
  43. package/dist/study/doctor.js +4 -0
  44. package/dist/study/plan-types.d.ts +3 -0
  45. package/dist/study/plan.js +42 -2
  46. package/dist/study/preflight.js +8 -0
  47. package/dist/tui-app.js +87 -87
  48. package/dist/verify/artifacts.d.ts +17 -10
  49. package/dist/verify/artifacts.js +22 -13
  50. package/dist/verify/scan-limit.d.ts +6 -0
  51. package/dist/verify/scan-limit.js +25 -0
  52. package/dist/verify/verify.d.ts +1 -1
  53. package/dist/verify/verify.js +68 -18
  54. package/docs/architecture/github-feedback-loop.md +1 -1
  55. package/docs/architecture/observer.md +24 -5
  56. package/docs/contracts/README.md +1 -1
  57. package/docs/contracts/run-bundle.md +89 -6
  58. package/docs/contracts/schemas.md +14 -7
  59. package/docs/contracts/study-analysis.md +1 -1
  60. package/docs/decisions/0002-observer-is-one-self-contained-file.md +2 -1
  61. package/package.json +7 -4
package/CHANGELOG.md CHANGED
@@ -8,6 +8,35 @@ The Unreleased section holds the full notes for the next version until it is tag
8
8
 
9
9
  ## Unreleased
10
10
 
11
+ ## 0.124.0: Small Observer files for long studies, faster verify, closing reports under zero data retention (2026-10-12)
12
+
13
+ humanish 0.124.0 keeps a long study's Observer files small, checks screenshots about eight times
14
+ faster, and asks an `openai-computer-use` participant for its closing report at a scheduled end
15
+ under zero data retention. `observer/observer-data.json` is now `humanish.observer-data.v2`: the
16
+ run and each participant without their trace items and screenshot entries, which the Observer
17
+ page carries compressed. On a synthetic run of 24 participants working 8 hours, observer-data.json
18
+ fell from 155 MiB to 0.33 MiB and index.html from 94 MiB to 16 MiB, so verify reads both. A
19
+ reader of observer-data.json that needs a participant's items reads the run's traces. verify of a
20
+ synthetic 8-hour run with 8,895 real-size screenshots took 51 s, down from 282 s.
21
+
22
+ [Release notes](https://github.com/danielgwilson/humanish/releases/tag/v0.124.0)
23
+
24
+ ## 0.123.0: Run bundle v2 writes each trace once, bundle size estimate (2026-10-12)
25
+
26
+ humanish 0.123.0 writes each participant's trace once, outside run.json, so long studies stay
27
+ within the bundle's size limits, and estimates a study's bundle before it runs. Run bundles are now
28
+ `humanish.run-bundle.v2`: a participant's trace items live only in its trace file
29
+ (`actor.json`, `actors/<stream>.json` or `actor-<surface>.json`), and run.json indexes each one by
30
+ path, size and sha256. On a synthetic 8-participant, 1,000-turn run, run.json fell from 17 MB to
31
+ 74 KB. Every reader loads traces through one loader that checks the index and refuses a missing,
32
+ altered or shared trace; a refused trace shows as evidence unavailable, and an unfinished run
33
+ grades `local_only`. Bundles from earlier versions still load unchanged; npm 0.122.0 and earlier
34
+ refuse a v2 bundle by name. `study check` and `run` estimate the bundle's size and file count and
35
+ warn before any sandbox is created when it may pass the analysis or verify limit, and the
36
+ public-safety scan's file limit grows with the number of participants.
37
+
38
+ [Release notes](https://github.com/danielgwilson/humanish/releases/tag/v0.123.0)
39
+
11
40
  ## 0.122.0: Participants end on the clock, a harness error ends only its participant (2026-10-12)
12
41
 
13
42
  humanish 0.122.0 lets a participant end at a scheduled time and keeps one participant's harness
package/CONTEXT.md CHANGED
@@ -12,7 +12,7 @@ bundle and output fields keep older spellings; the second table maps them.
12
12
  | Term | Meaning | Where it lives in the code |
13
13
  | ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
14
14
  | study | What a user designs and runs: one YAML file that declares the subject, the actor, the participants and the caps. The file lives at `humanish/studies/<id>.yaml` with schema `humanish.study.v3`, read by `humanish study` and `--study`. humanish refuses a 0.107 `humanish.lab.v2` file, and `humanish migrate` converts it (second table). | `src/study/`; `StudyConfig` in `src/study/types.ts`, parsed by `parseStudy` in `src/study/config.ts` |
15
- | run | One execution of a study, on disk: `.humanish/runs/<runId>/`, with `run.json` (`humanish.run-bundle.v1`) as the source of truth. | `src/run/`; `RunBundle` in `src/run/bundle.ts` |
15
+ | run | One execution of a study, on disk: `.humanish/runs/<runId>/`, with `run.json` (`humanish.run-bundle.v2`) as the source of truth and each participant's trace in its own file. | `src/run/`; `RunBundle` in `src/run/bundle.ts` |
16
16
  | dry run | A run with no live participant, from `--dry-run` or `mode: dry-run`. humanish writes a synthetic run bundle, review and Observer with no browser, model, keys or provider spend, and tests no product behavior. The `preview` route only dry-runs; "preview" names that route and nothing else. | `runDryRun` in `src/run/dry-run.ts`; `resolveStudyDryRun` in `src/study/plan.ts` |
17
17
  | participant | One simulated person, or agent, taking part in a study. The planner builds each one's record (id, persona id, assignment, labels and the route's own fields). Shared-world participants resolve the plan's records; computer-use participants rebuild the same records from the study config with `computerUseParticipants` (`src/study/plan-participants.ts`) and resolve those. `ResolvedParticipant` adds the bundle ids, persona and prompt. `run.json` records each one as a `simulations[]` entry with one or more `streams[]`. | `ResolvedParticipant` in `src/run/participant.ts`; the planned records in `src/study/plan-participants.ts`. Computer-use and shared-world participants run a `DesktopParticipantRun` (`src/routes/computer-use/types.ts`). The terminal session (`src/routes/terminal/session.ts`), scripted surfaces (`src/routes/scripted/plan.ts`) and fixture simulations (`src/run/dry-run.ts`) still build their own. |
18
18
  | persona | Who a participant is: traits, background, device. `humanish/personas/<id>.yaml`. | `src/study/persona.ts`, `src/study/persona-resolve.ts` |
@@ -36,7 +36,7 @@ bundle and output fields keep older spellings; the second table maps them.
36
36
  | topology | Whether participants share one instance of the subject. `route: shared-world` gives one for all; every other route gives each participant its own (`per-lane-worlds`). `run.json` records it as `sharedWorld.topology`. | `route` in `StudyConfig` (`src/study/types.ts`); `isSharedWorldComposition` in `src/study/routing.ts` |
37
37
  | plane | The one instance of the subject that a shared-world study's participants use together. A `provisioned` plane is a `clone` or `local-tree` subject that humanish serves in a sandbox, with a declared subject state. An `external-public` plane is a deployment already running at `subject.appUrl`, whose owner the study names in `subject.publicTarget.owner`. Each plane has at least two participants. | `SharedWorldPlane` in `src/study/plan-types.ts`; `src/routes/shared-world/` |
38
38
  | stream | One live or recorded view of a participant, such as a browser or a terminal, in `run.json` and the Observer. | `RunStream` in `src/run/streams.ts` |
39
- | Observer | The single-file review page built from `observer/observer-data.json` (`humanish.observer-data.v1`). | `observer/` (the page), `src/observer/` (data, render, serve) |
39
+ | Observer | The single-file review page built from `observer/observer-data.json` (`humanish.observer-data.v2`). | `observer/` (the page), `src/observer/` (data, render, serve) |
40
40
  | evidence | Files a run keeps: screenshots, traces, transcripts, recordings. `verify` checks them and grades share-safety. | `src/evidence/`; `verifyRun` in `src/verify/verify.ts` |
41
41
  | analysis | An independent model reading of a finished run's retained evidence, kept apart from participant feedback. Supported live runs request it automatically, and `humanish analyze` runs it on demand. It records findings and observations as `humanish.study-analysis.v1`. | `src/analysis/`; `runAutomaticAnalysis` in `src/analysis/automatic.ts`; `analyzeRun` in `src/analysis/service.ts`; `ANALYSIS_SCHEMA` in `src/analysis/types.ts` |
42
42
  | finding | A result of analysis, with citations to evidence in the run. Each finding groups the observations that support it. | `src/analysis/` |
package/README.md CHANGED
@@ -1,12 +1,23 @@
1
1
  # humanish
2
2
 
3
- Synthetic user research for apps, CLIs, and agent-facing product flows. AI participants with a persona and a task use your app on real desktops, and each run leaves evidence you can verify: what they did, where they got stuck, and what it cost.
3
+ [![npm version](https://img.shields.io/npm/v/humanish)](https://www.npmjs.com/package/humanish) [![npm downloads per month](https://img.shields.io/npm/dm/humanish)](https://www.npmjs.com/package/humanish) [![CI status on main](https://img.shields.io/github/actions/workflow/status/danielgwilson/humanish/ci.yml?branch=main&label=CI)](https://github.com/danielgwilson/humanish/actions/workflows/ci.yml?query=branch%3Amain) [![License: MIT](https://img.shields.io/npm/l/humanish)](LICENSE) [![Node.js version](https://img.shields.io/node/v/humanish)](https://nodejs.org/)
4
4
 
5
- [![The Observer grid of a saved eight-participant study: eight desktops in one multiplayer lobby, each tile a participant's live screen](https://humanish.dev/runs/lobby-0927/poster.jpg)](https://humanish.dev/demo)
5
+ Synthetic user research for apps, CLIs, and agent-facing product flows. AI participants with a persona and a task use your app on real desktops, and each run leaves evidence you can verify: what they did, where they got stuck, and what it cost.
6
6
 
7
- **[Watch a saved run](https://humanish.dev/demo).** Eight synthetic participants joined one
8
- lobby of a multiplayer game on its live deployment, each on its own hosted desktop. The page
9
- replays the real Observer; nothing runs from it.
7
+ <a href="https://humanish.dev/demo">
8
+ <picture>
9
+ <source media="(prefers-color-scheme: dark)" srcset="https://humanish.dev/runs/lobby-0927/observer-grid-dark.webp">
10
+ <img alt="The humanish Observer replaying a saved study five minutes into its 13-minute recording: a grid of eight participants' desktops in a multiplayer movie-guessing game, seven in the same round and one held at the join page, under tabs for 8 participants and 7 findings" src="https://humanish.dev/runs/lobby-0927/observer-grid-light.webp">
11
+ </picture>
12
+ </a>
13
+
14
+ **[Watch the replay](https://humanish.dev/demo).** The image is the Observer partway through a
15
+ saved study. Eight synthetic participants joined one lobby of a multiplayer movie-guessing game
16
+ on its live deployment, each on its own hosted desktop, and each tile shows one participant's
17
+ screen at the same moment of the recording. Six reached the final standings. Two were blocked,
18
+ among them Player six, who arrived after the game started and is still on the join page. The
19
+ Findings tab holds the seven findings the analysis wrote. The demo page replays this run in the
20
+ Observer, and nothing runs from it.
10
21
 
11
22
  ## Learn how a study works
12
23
 
@@ -19,6 +30,14 @@ disposable browser VM on your machine. On the computer-use route a model drives
19
30
  participant: every turn it reads a screenshot and answers with mouse and keyboard actions, until
20
31
  the participant finishes, gets stuck or reaches a limit.
21
32
 
33
+ <picture>
34
+ <source media="(prefers-color-scheme: dark)" srcset="https://humanish.dev/runs/lobby-0927/observer-player-dark.webp">
35
+ <img alt="The Observer with one participant open: its desktop during a round of a movie-guessing game, replay controls and a filmstrip of captures below, and on the right its trace, with the participant's reasoning and each TAB and ENTER keypress" src="https://humanish.dev/runs/lobby-0927/observer-player-light.webp">
36
+ </picture>
37
+
38
+ Player four from the run above, a keyboard-only participant: its screen, and beside it each
39
+ keypress and the reasoning it wrote before acting.
40
+
22
41
  The run writes its evidence to gitignored `.humanish/runs/<run id>/`: screenshots, every
23
42
  action, each participant's report and the estimated cost. The Observer replays it,
24
43
  `humanish review` prints the findings of the analysis that follows a live run, and
@@ -26,6 +45,14 @@ action, each participant's report and the estimated cost. The Observer replays i
26
45
  [How a study works](https://humanish.dev/docs/concepts) walks through one annotated study file,
27
46
  and the [study file reference](https://humanish.dev/docs/study-files) lists every field.
28
47
 
48
+ <picture>
49
+ <source media="(prefers-color-scheme: dark)" srcset="https://humanish.dev/runs/lobby-0927/observer-findings-dark.webp">
50
+ <img alt="The Observer's Findings tab: the first of seven findings, that three participants found keyboard guessing hard to track and one stopped mid-game, with the analysis's summary, what to check next, a cited capture and the timeline of evidence moments behind it" src="https://humanish.dev/runs/lobby-0927/observer-findings-light.webp">
51
+ </picture>
52
+
53
+ The first of the seven findings the analysis wrote for the same run, with the evidence moments
54
+ it cites.
55
+
29
56
  ## Try it without keys
30
57
 
31
58
  In a project directory, with Node.js 22.19 or newer:
@@ -308,7 +335,7 @@ and the library exports.
308
335
  - Breaking changes ship together in one minor release. Its release notes carry a migration
309
336
  table: each old form, its new form, and the change to make.
310
337
  - The study file and the run bundle carry versioned schemas, `humanish.study.v3` and
311
- `humanish.run-bundle.v1`. A breaking change to either one bumps its version.
338
+ `humanish.run-bundle.v2`. A breaking change to either one bumps its version.
312
339
 
313
340
  Pin the exact version in `package.json` and read [CHANGELOG.md](CHANGELOG.md) before upgrading.
314
341
 
@@ -65,7 +65,12 @@ export function statusForCompletionReason(reason) {
65
65
  }
66
66
  export function loopResult(session, stop, debrief) {
67
67
  const { provider, settings, trace: recorder, usage } = session;
68
- const { completionReason, reason, stopCause } = stop;
68
+ const { completionReason, stopCause } = stop;
69
+ // A participant whose session ended on the clock has its closing account as its reason, as one
70
+ // that ended the session itself has its last message; the stop cause keeps the scheduled end.
71
+ const reason = stopCause === "scheduled_end" && debrief?.report !== undefined
72
+ ? debrief.report.summary
73
+ : stop.reason;
69
74
  const completedAtMs = session.now();
70
75
  const status = statusForCompletionReason(completionReason);
71
76
  const ids = {};
@@ -65,12 +65,6 @@ export declare class OpenAiConversation {
65
65
  * from the request being sent. False when none are asked for, so the rejection stands.
66
66
  */
67
67
  dropReasoningSummaries(): boolean;
68
- /**
69
- * Whether the server holds the whole session, so a closing report threaded on it reads every
70
- * turn. An explicit-context conversation summarizes its oldest turns past its budget, configured
71
- * or switched to at runtime.
72
- */
73
- get serverHoldsSession(): boolean;
74
68
  /** The trace's account of how the conversation was carried (ActorConversation). */
75
69
  record(): ActorConversation;
76
70
  /**
@@ -72,14 +72,6 @@ export class OpenAiConversation {
72
72
  this.reasoningSummary = undefined;
73
73
  return true;
74
74
  }
75
- /**
76
- * Whether the server holds the whole session, so a closing report threaded on it reads every
77
- * turn. An explicit-context conversation summarizes its oldest turns past its budget, configured
78
- * or switched to at runtime.
79
- */
80
- get serverHoldsSession() {
81
- return this.mode === "threaded" && this.lastResponseId !== undefined;
82
- }
83
75
  /** The trace's account of how the conversation was carried (ActorConversation). */
84
76
  record() {
85
77
  return this.evidence.snapshot(this.mode);
@@ -509,14 +509,11 @@ export function createOpenAiResponsesProvider(options) {
509
509
  get conversation() {
510
510
  return conversation.record();
511
511
  },
512
- // A retrospective report reads the session the server holds; see serverHoldsSession.
513
- get debrief() {
514
- return conversation.serverHoldsSession
515
- ? (req, signal) => requestTurn(req, signal, "report")
516
- : undefined;
517
- },
518
- // Offered in both modes. An explicit_context conversation carries its newest screens whole and
519
- // earlier turns as text, which serves impressions; a closing report needs the whole session.
512
+ // Both are offered in both modes. A threaded request reads the session the server holds; an
513
+ // explicit_context one carries its newest screens whole and its oldest turns summarized past
514
+ // the budget, which the trace's conversation record counts. A participant that worked a long
515
+ // shift writes its closing account from what it carries, as a person writes one from memory.
516
+ debrief: (req, signal) => requestTurn(req, signal, "report"),
520
517
  requestImpressions: (req, signal) => requestTurn(req, signal, "impressions"),
521
518
  };
522
519
  }
@@ -4,6 +4,7 @@ import {} from "../run/bundle.js";
4
4
  import { refreshObserver, analyzeRun, readCompletedAnalysisSource, resolveAnalysisRun, } from "./service.js";
5
5
  import { preferLargerAnalysisOutput, ANALYSIS_PROMPT_VERSION } from "./execute.js";
6
6
  import { captureEvidence } from "./evidence.js";
7
+ import { loadRunTraces } from "../run/stream-trace.js";
7
8
  import { hashAnalysisValue } from "./validation.js";
8
9
  import { refusedCost } from "./admission.js";
9
10
  import { claimAutomaticAnalysis, readAutomaticAnalysisPrepared, requestAutomaticAnalysisCancellationPrepared, } from "./job.js";
@@ -88,7 +89,7 @@ async function readAutomaticSource(cwd, prepared, configInput, deps, hasKey) {
88
89
  let config = configInput;
89
90
  try {
90
91
  const bytes = await readCompletedAnalysisSource(cwd, prepared);
91
- const bundle = JSON.parse(bytes.toString("utf8"));
92
+ const bundle = await loadRunTraces(prepared, JSON.parse(bytes.toString("utf8")));
92
93
  if (bundle.streams.some((stream) => stream.actor?.stopCause === "harness_aborted"))
93
94
  return skipped("AUTOMATIC_ANALYSIS_ACTOR_CANCELLED");
94
95
  const participantEvidence = hasParticipantEvidence(bundle);
@@ -3,7 +3,8 @@ import { createHash } from "node:crypto";
3
3
  import { screenshotEvidenceError } from "../evidence/image.js";
4
4
  import { runIdOf } from "../run/paths.js";
5
5
  import { isRecord } from "../run/type-guards.js";
6
- import { RUN_BUNDLE_FILE } from "../run/bundle.js";
6
+ import { RUN_BUNDLE_FILE, RUN_BUNDLE_SCHEMA, RUN_BUNDLE_V1_SCHEMA, } from "../run/bundle.js";
7
+ import { loadStreamTraces } from "../run/stream-trace.js";
7
8
  import { ACTION_CAPTURE_VERSION, } from "./types.js";
8
9
  import { digestAnalysisInput, hashAnalysisValue, validateAnalysisInputMetadata, } from "./validation.js";
9
10
  import { boundedText, fairShares, hasUnmappedCaptures, participantAssignment, participantSource, sourceEntries, sourceOrder, impressionEventIds, } from "./evidence-sources.js";
@@ -29,7 +30,7 @@ function parseSource(prepared, bytes) {
29
30
  throw new Error("ANALYSIS_SOURCE_TOO_LARGE");
30
31
  const value = JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(bytes));
31
32
  if (!isRecord(value) ||
32
- value.schema !== "humanish.run-bundle.v1" ||
33
+ (value.schema !== RUN_BUNDLE_SCHEMA && value.schema !== RUN_BUNDLE_V1_SCHEMA) ||
33
34
  value.runId !== runIdOf(prepared) ||
34
35
  !Array.isArray(value.streams) ||
35
36
  value.streams.length > EVIDENCE_LIMITS.participants ||
@@ -49,24 +50,8 @@ function parseSource(prepared, bytes) {
49
50
  throw new Error("ANALYSIS_SOURCE_INVALID");
50
51
  }
51
52
  ids.add(stream.id);
52
- const actor = stream.actor;
53
- if (actor !== undefined &&
54
- (!isRecord(actor) || !Array.isArray(actor.items) || actor.items.length > 100000)) {
53
+ if (stream.actor !== undefined && !isRecord(stream.actor))
55
54
  throw new Error("ANALYSIS_SOURCE_INVALID");
56
- }
57
- const eventIds = new Set();
58
- for (const item of isRecord(actor) ? actor.items : []) {
59
- if (!isRecord(item) ||
60
- typeof item.id !== "string" ||
61
- item.id.length === 0 ||
62
- item.id.length > 256 ||
63
- eventIds.has(item.id) ||
64
- typeof item.kind !== "string" ||
65
- typeof item.title !== "string" ||
66
- (item.text !== undefined && typeof item.text !== "string"))
67
- throw new Error("ANALYSIS_SOURCE_INVALID");
68
- eventIds.add(item.id);
69
- }
70
55
  }
71
56
  for (const event of value.events) {
72
57
  if (!isRecord(event) ||
@@ -77,6 +62,41 @@ function parseSource(prepared, bytes) {
77
62
  }
78
63
  return value;
79
64
  }
65
+ /**
66
+ * run.json's bundle with each stream's trace loaded. The source limit applies to each trace file
67
+ * as it does to run.json; run.json's digest, which the analysis record binds, binds each trace's
68
+ * sha256.
69
+ */
70
+ async function readSource(prepared, bytes, traceBytes) {
71
+ const stored = parseSource(prepared, bytes);
72
+ if (stored.streams.some((stream) => (stream.trace?.bytes ?? 0) > traceBytes))
73
+ throw new Error("ANALYSIS_SOURCE_TOO_LARGE");
74
+ const loaded = await loadStreamTraces(prepared, stored, { maxBytes: traceBytes });
75
+ if (loaded.refused.length > 0)
76
+ throw new Error("ANALYSIS_SOURCE_INVALID");
77
+ for (const stream of loaded.bundle.streams)
78
+ validateTraceItems(stream.actor);
79
+ return loaded.bundle;
80
+ }
81
+ function validateTraceItems(actor) {
82
+ if (actor !== undefined &&
83
+ (!isRecord(actor) || !Array.isArray(actor.items) || actor.items.length > 100000)) {
84
+ throw new Error("ANALYSIS_SOURCE_INVALID");
85
+ }
86
+ const eventIds = new Set();
87
+ for (const item of isRecord(actor) ? actor.items : []) {
88
+ if (!isRecord(item) ||
89
+ typeof item.id !== "string" ||
90
+ item.id.length === 0 ||
91
+ item.id.length > 256 ||
92
+ eventIds.has(item.id) ||
93
+ typeof item.kind !== "string" ||
94
+ typeof item.title !== "string" ||
95
+ (item.text !== undefined && typeof item.text !== "string"))
96
+ throw new Error("ANALYSIS_SOURCE_INVALID");
97
+ eventIds.add(item.id);
98
+ }
99
+ }
80
100
  /** Select once from retained source. Models receive no filesystem or network resolver. */
81
101
  export async function captureEvidence(prepared, bundleBytes, requested = {}) {
82
102
  const limits = { ...EVIDENCE_LIMITS, ...requested };
@@ -92,7 +112,7 @@ export async function captureEvidence(prepared, bundleBytes, requested = {}) {
92
112
  const current = await readBoundedFile(prepared, RUN_BUNDLE_FILE, limits.sourceBytes);
93
113
  if (!current || !current.equals(bundleBytes))
94
114
  throw new Error("ANALYSIS_SOURCE_CHANGED");
95
- const bundle = parseSource(prepared, bundleBytes);
115
+ const bundle = await readSource(prepared, bundleBytes, limits.sourceBytes);
96
116
  const captureVersion = ACTION_CAPTURE_VERSION;
97
117
  if (bundle.streams.some((stream) => stream.liveActor !== undefined || UNFINISHED_STREAM_STATUSES.has(stream.status))) {
98
118
  throw new Error("ANALYSIS_RUN_UNFINISHED");
@@ -393,7 +413,7 @@ function packEvidence(selected, packings, textByEntry, evidence, images) {
393
413
  export async function validateAnalysisEvidence(prepared, artifact, bundleBytes) {
394
414
  if (artifact.captureVersion !== undefined && artifact.captureVersion !== ACTION_CAPTURE_VERSION)
395
415
  throw new Error("ANALYSIS_CAPTURE_VERSION_INVALID");
396
- const bundle = parseSource(prepared, bundleBytes);
416
+ const bundle = await readSource(prepared, bundleBytes, EVIDENCE_LIMITS.sourceBytes);
397
417
  if (artifact.runId !== bundle.runId || artifact.sourceRunSha256 !== sha256(bundleBytes))
398
418
  throw new Error("ANALYSIS_SOURCE_CHANGED");
399
419
  const included = new Set(artifact.coverage.includedStreamIds);
@@ -84,7 +84,14 @@ export async function runStudyCommand(args) {
84
84
  // the keys its plan reads. This comes after the option refusals above, which read no key, and
85
85
  // before the route's CLI setup and prepareStudy, which do.
86
86
  if (resolveStudyDryRun(config, args.options.dryRun, true) === false) {
87
- const announced = announcedKeyNames(config, args.options);
87
+ const planned = planStudy(config, { cwd: args.options.cwd, dryRun: false });
88
+ // The plan's warnings, such as a bundle estimated past a limit, before anything is created.
89
+ // The study file's own were printed above.
90
+ if (planned.ok)
91
+ for (const warning of planned.planned.plan.warnings ?? [])
92
+ if (!resolved.warnings.includes(warning))
93
+ warnAndQueue(args.command, args.io, warning);
94
+ const announced = announcedKeyNames(config, args.options, planned);
88
95
  await discoverCliKeys({
89
96
  io: args.io,
90
97
  cwd: args.options.cwd,
@@ -150,10 +157,9 @@ export async function runStudyCommand(args) {
150
157
  * change who runs, not which keys. Undefined, so every fill is printed, when a scorer is declared,
151
158
  * since its host code may read any key, and when the study does not plan.
152
159
  */
153
- function announcedKeyNames(config, options) {
160
+ function announcedKeyNames(config, options, planned) {
154
161
  if (options.scorer !== undefined || config.review?.scorer !== undefined)
155
162
  return undefined;
156
- const planned = planStudy(config, { cwd: options.cwd, dryRun: false });
157
163
  return planned.ok ? keyNamesOf(planned.planned.plan) : undefined;
158
164
  }
159
165
  /** Each route's CLI setup. It returns undefined when it has already written its own result. */
@@ -1,3 +1,4 @@
1
+ import { crc32, inflateSync } from "node:zlib";
1
2
  import { PNG } from "pngjs";
2
3
  const PNG_SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]);
3
4
  const PNG_IHDR_LENGTH = 13;
@@ -145,12 +146,90 @@ export function screenshotEvidenceError(relativePath, bytes) {
145
146
  if (chunks?.some((chunk) => chunk.type === "IEND" && chunk.end - chunk.start !== 12)) {
146
147
  return "PNG IEND chunk must be empty";
147
148
  }
149
+ return chunks === null || !pngStructureHolds(bytes, chunks)
150
+ ? "could not decode PNG evidence"
151
+ : null;
152
+ }
153
+ /** The bit depths PNG allows for each color type. */
154
+ const PNG_BIT_DEPTHS = {
155
+ 0: [1, 2, 4, 8, 16],
156
+ 2: [8, 16],
157
+ 3: [1, 2, 4, 8],
158
+ 4: [8, 16],
159
+ 6: [8, 16],
160
+ };
161
+ const PNG_CHANNELS = { 0: 1, 2: 3, 3: 1, 4: 2, 6: 4 };
162
+ /** The fewest bytes the decoder reads from a chunk; a shorter one fails to decode. */
163
+ const PNG_MIN_CHUNK_BYTES = { gAMA: 4 };
164
+ /**
165
+ * Whether a PNG that passed pngDecodeRefusal decodes, read from its structure: the file ends at an
166
+ * empty IEND, every chunk's CRC matches, IHDR comes once with fields PNG defines, the image data is
167
+ * one run of IDAT chunks, and it inflates, with zlib's native code, to exactly the
168
+ * bytes its rows need with nothing after the stream, each row starting with a filter type from 0
169
+ * to 4. For an image without a palette that is all a decode establishes, at about an eighth of
170
+ * the time of pngjs, which unfilters every pixel in JavaScript; a run holds a screenshot for each
171
+ * turn. A palette image is also decoded, since each pixel must index a palette that comes before
172
+ * the image data; in any other image a palette chunk is only a suggestion, wherever it is.
173
+ */
174
+ function pngStructureHolds(bytes, chunks) {
175
+ const last = chunks.at(-1);
176
+ if (last?.type !== "IEND" || last.end !== bytes.length)
177
+ return false;
178
+ const crcHolds = (chunk) => crc32(bytes.subarray(chunk.start + 4, chunk.end - 4)) === bytes.readUInt32BE(chunk.end - 4);
179
+ if (!chunks.every(crcHolds))
180
+ return false;
181
+ const types = chunks.map((chunk) => chunk.type);
182
+ const firstIdat = types.indexOf("IDAT");
183
+ const idats = chunks.filter((chunk) => chunk.type === "IDAT");
184
+ if (types.lastIndexOf("IHDR") !== 0 ||
185
+ firstIdat < 0 ||
186
+ types.lastIndexOf("IDAT") !== firstIdat + idats.length - 1)
187
+ return false;
188
+ const [width, height] = [bytes.readUInt32BE(16), bytes.readUInt32BE(20)];
189
+ const [depth, colorType, compression, filter] = [bytes[24], bytes[25], bytes[26], bytes[27]];
190
+ if (!PNG_BIT_DEPTHS[colorType]?.includes(depth) || compression !== 0 || filter !== 0)
191
+ return false;
192
+ // The decoder sizes a row in 32-bit arithmetic and refuses one wider; no screenshot is.
193
+ const rowBits = width * PNG_CHANNELS[colorType] * depth;
194
+ if (rowBits + 7 > 0x7fffffff)
195
+ return false;
196
+ const trnsBytes = colorType === 0 ? 2 : colorType === 2 ? 6 : 0;
197
+ const tooShort = (chunk) => chunk.end - chunk.start - 12 <
198
+ (chunk.type === "tRNS" ? trnsBytes : (PNG_MIN_CHUNK_BYTES[chunk.type] ?? 0));
199
+ if (chunks.some(tooShort))
200
+ return false;
201
+ const rowBytes = 1 + Math.ceil(rowBits / 8);
202
+ const data = Buffer.concat(idats.map((chunk) => bytes.subarray(chunk.start + 8, chunk.end - 4)));
203
+ let rows;
148
204
  try {
149
- const decoded = PNG.sync.read(bytes, { checkCRC: true });
150
- return pngDimensionsError(decoded.width, decoded.height);
205
+ // A stream that would inflate past the rows the image needs stops there, so a small file
206
+ // cannot claim gigabytes.
207
+ const options = {
208
+ maxOutputLength: rowBytes * height,
209
+ info: true,
210
+ };
211
+ const inflated = inflateSync(data, options);
212
+ if (inflated.engine.bytesWritten !== data.length)
213
+ return false;
214
+ rows = inflated.buffer;
151
215
  }
152
216
  catch {
153
- return "could not decode PNG evidence";
217
+ return false;
218
+ }
219
+ if (rows.length !== rowBytes * height)
220
+ return false;
221
+ for (let row = 0; row < rows.length; row += rowBytes)
222
+ if (rows[row] > 4)
223
+ return false;
224
+ return colorType !== 3 || pngDecodes(bytes);
225
+ }
226
+ function pngDecodes(bytes) {
227
+ try {
228
+ PNG.sync.read(bytes, { checkCRC: true });
229
+ return true;
230
+ }
231
+ catch {
232
+ return false;
154
233
  }
155
234
  }
156
235
  export function assertScreenshotEvidence(relativePath, bytes) {
@@ -0,0 +1,12 @@
1
+ import type { RunBundle } from "../run/bundle.js";
2
+ /** A source file's path and bytes. */
3
+ interface TextFile {
4
+ path: string;
5
+ bytes: Buffer;
6
+ }
7
+ export declare function jsonBytes(value: unknown): Buffer;
8
+ export declare function transformText(file: TextFile, imagePaths: ReadonlySet<string>): Buffer;
9
+ export declare function assertRewritableNumbers(json: string): void;
10
+ /** The loaded source bundle as the copy records it: screenshots declared and labelled blurred. */
11
+ export declare function derivedBundle(bundle: RunBundle, imagePaths: ReadonlySet<string>): RunBundle;
12
+ export {};
@@ -0,0 +1,130 @@
1
+ // How a redacted copy rewrites a run's text: screenshots declared blurred, sandbox ids redacted,
2
+ // inline images and secret-shaped text refused. src/feedback/export-bundle.ts copies the files.
3
+ import path from "node:path";
4
+ import { parse as parseYaml } from "yaml";
5
+ import { ACTOR_TRACE_SCHEMA } from "../actors/contract.js";
6
+ import { containsSensitive, redactSandboxIds } from "../evidence/redaction.js";
7
+ import { decodeEscapes, readingsOf } from "../evidence/encoded-text.js";
8
+ import { readPlainText } from "../evidence/plain-text.js";
9
+ export function jsonBytes(value) {
10
+ return Buffer.from(`${JSON.stringify(value, null, 2)}\n`);
11
+ }
12
+ function assertNoInlineRaster(text) {
13
+ // Decode the escapes verify decodes (decodeEscapes) before checking. Unknown embedded bytes are
14
+ // refused: copying a text extension must not smuggle untransformed pixels.
15
+ if (readingsOf(text).some(containsSensitive))
16
+ throw new Error("Decoded text contains a secret-shaped value or private path; export refused.");
17
+ const decoded = decodeEscapes(text);
18
+ if (/data\s*:\s*(?:image\/|application\/|text\/|[^,\s]*;base64,|,)/i.test(decoded) ||
19
+ /iVBORw0KGgo|\/9j\/[A-Za-z0-9+/]|R0lGOD|UklGR[A-Za-z0-9+/]{4}|<svg\b/i.test(decoded)) {
20
+ throw new Error("Text contains an inline image/data payload; embedded raster redaction is unsupported.");
21
+ }
22
+ }
23
+ function decodeText(file) {
24
+ const decoded = readPlainText(file.bytes);
25
+ if (!decoded.ok)
26
+ throw new Error(decoded.reason === "invalid-utf8"
27
+ ? "Source text contains invalid UTF-8 or binary bytes."
28
+ : "Source text contains binary control bytes.");
29
+ assertNoInlineRaster(decoded.text);
30
+ return decoded.text;
31
+ }
32
+ function updateScreenshotDeclarations(value, imagePaths) {
33
+ if (Array.isArray(value))
34
+ return value.map((entry) => updateScreenshotDeclarations(entry, imagePaths));
35
+ if (value === null || typeof value !== "object")
36
+ return value;
37
+ const object = value;
38
+ const result = Object.create(null);
39
+ for (const [key, entry] of Object.entries(object))
40
+ result[key] = updateScreenshotDeclarations(entry, imagePaths);
41
+ if (object.schema === ACTOR_TRACE_SCHEMA &&
42
+ object.redaction !== null &&
43
+ typeof object.redaction === "object") {
44
+ const original = object.redaction;
45
+ if (original.screenshots === "raw" || original.screenshots === "blurred") {
46
+ result.redaction = {
47
+ ...original,
48
+ screenshots: "blurred",
49
+ notes: `${typeof original.notes === "string" ? original.notes : ""} Exported copy: screenshots blurred during export; original capture posture: ${original.screenshots}. See derivation.json.`,
50
+ };
51
+ }
52
+ if (Array.isArray(result.items)) {
53
+ result.items = result.items.map((item) => {
54
+ if (item === null || typeof item !== "object")
55
+ return item;
56
+ const entry = item;
57
+ if (entry.screenshotRef === undefined)
58
+ return item;
59
+ if (entry.screenshotRef === null || typeof entry.screenshotRef !== "object")
60
+ throw new Error("Actor screenshot reference is malformed.");
61
+ const ref = entry.screenshotRef;
62
+ if (typeof ref.path !== "string" || !imagePaths.has(ref.path)) {
63
+ throw new Error("Actor screenshot reference must resolve to a PNG transformed from this run's inventory.");
64
+ }
65
+ return { ...entry, screenshotRef: { ...ref, redaction: "blurred" } };
66
+ });
67
+ }
68
+ }
69
+ return result;
70
+ }
71
+ export function transformText(file, imagePaths) {
72
+ const text = decodeText(file);
73
+ const extension = path.extname(file.path).toLowerCase();
74
+ if (extension === ".json") {
75
+ const parsed = JSON.parse(text);
76
+ // Parsing also resolves escape sequences in string values before inspection.
77
+ assertNoInlineRaster(JSON.stringify(parsed));
78
+ // The copy names no sandbox: an id is local operational evidence, and the copy is shared.
79
+ const transformed = redactSandboxIds(updateScreenshotDeclarations(parsed, imagePaths));
80
+ if (JSON.stringify(transformed) === JSON.stringify(parsed))
81
+ return file.bytes;
82
+ assertRewritableNumbers(text);
83
+ return jsonBytes(transformed);
84
+ }
85
+ if (extension === ".ndjson" || extension === ".jsonl") {
86
+ const lines = text.split("\n");
87
+ const transformed = lines.map((line) => {
88
+ if (line.trim().length === 0)
89
+ return line;
90
+ const parsed = JSON.parse(line);
91
+ assertNoInlineRaster(JSON.stringify(parsed));
92
+ const updated = JSON.stringify(redactSandboxIds(updateScreenshotDeclarations(parsed, imagePaths)));
93
+ if (updated === JSON.stringify(parsed))
94
+ return line;
95
+ assertRewritableNumbers(line);
96
+ return updated;
97
+ });
98
+ return Buffer.from(transformed.join("\n"));
99
+ }
100
+ if (extension === ".yaml" || extension === ".yml") {
101
+ assertNoInlineRaster(JSON.stringify(parseYaml(text, { maxAliasCount: 100, logLevel: "silent" })) ?? "");
102
+ }
103
+ return file.bytes;
104
+ }
105
+ export function assertRewritableNumbers(json) {
106
+ // Opaque adapter JSON stays byte-identical. A known trace that actually needs
107
+ // rewriting must not silently round an identifier outside JS's integer range.
108
+ const unquoted = json.replace(/"(?:\\.|[^"\\])*"/g, '""');
109
+ for (const match of unquoted.matchAll(/-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?/g)) {
110
+ const number = Number(match[0]);
111
+ if (!Number.isFinite(number) || (Number.isInteger(number) && !Number.isSafeInteger(number))) {
112
+ throw new Error("A trace requiring redaction contains an unsafe numeric value; represent large identifiers as strings before capture.");
113
+ }
114
+ }
115
+ }
116
+ /** The loaded source bundle as the copy records it: screenshots declared and labelled blurred. */
117
+ export function derivedBundle(bundle, imagePaths) {
118
+ const transformed = redactSandboxIds(updateScreenshotDeclarations(bundle, imagePaths));
119
+ transformed.redaction = {
120
+ status: "passed",
121
+ notes: `${bundle.redaction.notes} Exported copy: screenshots were blurred during export; the original capture is retained separately. See derivation.json.`,
122
+ };
123
+ for (const stream of transformed.streams) {
124
+ for (const artifact of stream.artifacts) {
125
+ if (artifact.kind === "screenshot")
126
+ artifact.label = artifact.label.replace(/\((raw|blurred)\)/g, "(blurred at export)");
127
+ }
128
+ }
129
+ return transformed;
130
+ }