humanish 0.134.0 → 0.136.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. package/CHANGELOG.md +30 -0
  2. package/dist/actors/computer-use/loop/session.js +7 -2
  3. package/dist/actors/scripted-browser/actor.js +3 -3
  4. package/dist/actors/scripted-browser/steps.d.ts +9 -2
  5. package/dist/actors/scripted-browser/steps.js +17 -12
  6. package/dist/analysis/automatic-config.js +5 -3
  7. package/dist/analysis/evidence-sources.js +4 -7
  8. package/dist/analysis/evidence.js +4 -4
  9. package/dist/analysis/execute.js +2 -1
  10. package/dist/analysis/service.js +2 -1
  11. package/dist/analysis/source.d.ts +7 -4
  12. package/dist/analysis/source.js +22 -4
  13. package/dist/cli/commands/setup.js +7 -10
  14. package/dist/evidence/encoded-text.d.ts +13 -3
  15. package/dist/evidence/encoded-text.js +88 -25
  16. package/dist/evidence/image.d.ts +2 -2
  17. package/dist/evidence/image.js +47 -32
  18. package/dist/observer/data.d.ts +7 -3
  19. package/dist/observer/page-data.d.ts +6 -0
  20. package/dist/observer/page-data.js +34 -7
  21. package/dist/observer/participant-items.d.ts +15 -1
  22. package/dist/observer/participant-items.js +4 -3
  23. package/dist/observer/render.js +2 -4
  24. package/dist/observer/run-routes.js +4 -4
  25. package/dist/observer-app.html +4 -4
  26. package/dist/routes/computer-use/fanout-bundle.js +3 -3
  27. package/dist/routes/computer-use/participant-execution.js +1 -1
  28. package/dist/routes/computer-use/participant-runs.js +2 -2
  29. package/dist/routes/computer-use/single-bundle.js +3 -3
  30. package/dist/routes/scripted/setup.js +62 -17
  31. package/dist/routes/shared-world/bundle.js +3 -3
  32. package/dist/routes/shared-world/participant-specs.js +2 -2
  33. package/dist/routes/terminal/bundle.js +2 -2
  34. package/dist/run/bundle.d.ts +5 -1
  35. package/dist/run/locate.d.ts +13 -11
  36. package/dist/run/locate.js +20 -13
  37. package/dist/run/paths.js +38 -28
  38. package/dist/run/run.js +4 -3
  39. package/dist/run/stream-trace.d.ts +2 -2
  40. package/dist/run/stream-trace.js +24 -11
  41. package/dist/run/streams.d.ts +18 -3
  42. package/dist/run-study.d.ts +3 -3
  43. package/dist/run-study.js +3 -3
  44. package/dist/study/bundle-estimate.d.ts +21 -3
  45. package/dist/study/bundle-estimate.js +65 -45
  46. package/dist/study/config.d.ts +4 -3
  47. package/dist/study/config.js +8 -5
  48. package/dist/study/parse/actors.d.ts +6 -0
  49. package/dist/study/parse/actors.js +14 -0
  50. package/dist/study/parse/subject-state.js +4 -4
  51. package/dist/study/participant-assignment.js +2 -2
  52. package/dist/study/persona.d.ts +7 -1
  53. package/dist/study/persona.js +13 -5
  54. package/dist/study/plan-base.js +4 -3
  55. package/dist/tui-app.js +23 -23
  56. package/dist/verify/screenshot-reads.js +1 -1
  57. package/docs/architecture/observer.md +24 -7
  58. package/docs/architecture/state-driven-executor.md +4 -3
  59. package/docs/contracts/run-bundle.md +3 -0
  60. package/docs/contracts/study-analysis.md +7 -5
  61. package/docs/decisions/0002-observer-is-one-self-contained-file.md +3 -3
  62. package/package.json +1 -1
@@ -2,7 +2,7 @@ import { e2bDesktopTemplate } from "../../substrates/e2b/sandbox.js";
2
2
  import { receivingPublication } from "../../comms/receiving-runtime.js";
3
3
  import { actorEnding } from "../../actors/stop-cause.js";
4
4
  import { cuaGoalSource } from "../../actors/goal-source.js";
5
- import { redactText } from "../../evidence/redaction.js";
5
+ import { redactEncodedText } from "../../evidence/encoded-text.js";
6
6
  import { REVIEW_SCHEMA, bundleArtifacts, bundleHead, } from "../../run/bundle.js";
7
7
  import { aggregateTaskFunnels, formatParticipantOutcomes, formatRunTaskFunnel, gapsListClause, skippedSentence, withCuaReviewProvenance, } from "../../run/outcomes.js";
8
8
  import { providerResourcesForOutcome, publicSafeAppUrlLabel, streakGaps } from "./bundle-parts.js";
@@ -186,7 +186,7 @@ function fanoutFeedbackCandidates(args) {
186
186
  participantId: spec.planned.id,
187
187
  streamId: spec.streamId,
188
188
  personaId: spec.persona.id,
189
- goal: redactText(spec.evidenceInstructions ?? spec.instructions),
189
+ goal: redactEncodedText(spec.evidenceInstructions ?? spec.instructions),
190
190
  ...(outcome?.session === undefined ? {} : { session: outcome.session }),
191
191
  ...(outcome?.session === undefined
192
192
  ? {}
@@ -270,7 +270,7 @@ export function buildCuaFanoutBundle(args) {
270
270
  // this writer produced. The only adopter-side workaround was scanner evasion.
271
271
  //
272
272
  // The instructions the model actually receives are untouched; only the persisted copy changes.
273
- goal: redactText(specs[0].evidenceInstructions ?? specs[0].instructions),
273
+ goal: redactEncodedText(specs[0].evidenceInstructions ?? specs[0].instructions),
274
274
  source: `study:${plan.studyId}`,
275
275
  sourceDigest: specs[0].persona.promptDigest,
276
276
  },
@@ -26,7 +26,7 @@ export function makeParticipantWriteScreenshot(artifactRoot, spec, screenshots)
26
26
  const rel = path.posix.join(relPrefix, name);
27
27
  // Evidence holds image data only, whatever the executor's frames carried.
28
28
  const frame = stripPngMetadataChunks(bytes);
29
- assertScreenshotEvidence(rel, frame);
29
+ await assertScreenshotEvidence(rel, frame);
30
30
  await writeContainedOutputFile(artifactRoot, path.join(...dirParts, name), frame);
31
31
  screenshots.push(rel);
32
32
  return rel;
@@ -6,7 +6,7 @@ import { isLocalBrowserStudy, LOCAL_BROWSER_LIFETIME_MS, } from "../../substrate
6
6
  import { resolveParticipant } from "../../run/participant.js";
7
7
  import {} from "../../study/types.js";
8
8
  import { scrubPersonaBrief } from "../../study/persona.js";
9
- import { redactText } from "../../evidence/redaction.js";
9
+ import { redactEncodedText } from "../../evidence/encoded-text.js";
10
10
  import {} from "../../run/bundle.js";
11
11
  import { participantAssignment } from "../../study/participant-assignment.js";
12
12
  import { resolveCommittedPersonas } from "../../study/persona-resolve.js";
@@ -237,7 +237,7 @@ export function sanitizeParticipantRuns(runs, scrub) {
237
237
  for (const spec of runs) {
238
238
  if (spec.evidenceAssignment)
239
239
  spec.evidenceAssignment = participantAssignment(spec.evidenceAssignment, scrub);
240
- spec.evidenceInstructions = redactText(scrub(spec.instructions));
240
+ spec.evidenceInstructions = redactEncodedText(scrub(spec.instructions));
241
241
  spec.persona = scrubPersonaBrief(spec.persona, scrub);
242
242
  }
243
243
  }
@@ -1,5 +1,5 @@
1
1
  import { participantAssignment } from "../../study/participant-assignment.js";
2
- import { redactText } from "../../evidence/redaction.js";
2
+ import { redactEncodedText } from "../../evidence/encoded-text.js";
3
3
  import { REVIEW_SCHEMA, bundleArtifacts, bundleHead, } from "../../run/bundle.js";
4
4
  import {} from "../../run/streams.js";
5
5
  import { aggregateTaskFunnels, formatParticipantOutcomes, formatRunTaskFunnel, participantSummary, withCuaReviewProvenance, } from "../../run/outcomes.js";
@@ -385,7 +385,7 @@ export function buildSingleParticipantBundle(args) {
385
385
  scenario: {
386
386
  id: `cua-${args.studyId}`,
387
387
  title: args.studyTitle ?? `Computer-use run: ${args.studyId}`,
388
- goal: redactText(args.mission),
388
+ goal: redactEncodedText(args.mission),
389
389
  source: `study:${args.studyId}`,
390
390
  sourceDigest: args.persona.promptDigest,
391
391
  },
@@ -425,7 +425,7 @@ export function buildSingleParticipantBundle(args) {
425
425
  participantId: args.participantId ?? "lane-01",
426
426
  streamId: SINGLE.streamId,
427
427
  personaId: args.persona.id,
428
- goal: redactText(args.mission),
428
+ goal: redactEncodedText(args.mission),
429
429
  ...(args.session === undefined ? {} : { session: args.session }),
430
430
  ...(args.traceArtifactPath === undefined
431
431
  ? {}
@@ -8,6 +8,7 @@ import { resolveBrowserCommand } from "../../actors/scripted-browser/browser-com
8
8
  import { publicSafeToken } from "../../actors/scripted-browser/journey.js";
9
9
  import { prepareSelectedOutputDirectory } from "../../run/contained-output.js";
10
10
  import { RunSecrets } from "../../run/secrets.js";
11
+ import { redactEncodedText } from "../../evidence/encoded-text.js";
11
12
  import { evidenceAppUrlOf } from "./plan.js";
12
13
  import { resolveScriptedScenario } from "./scenario.js";
13
14
  import {} from "./types.js";
@@ -55,6 +56,9 @@ export async function prepareScriptedRun(plan, input) {
55
56
  result: failed("HUMANISH_SCRIPTED_SCENARIO_INVALID", scenario.message),
56
57
  };
57
58
  }
59
+ const steps = recordedStepIds(scenario.journey.steps, secrets);
60
+ if (!steps.ok)
61
+ return { ok: false, result: failed("HUMANISH_SCRIPTED_SCENARIO_INVALID", steps.message) };
58
62
  // The goal, title and step labels are recorded; a step's path, selector, value and expected text
59
63
  // drive the browser and stay as written.
60
64
  const journey = {
@@ -62,9 +66,9 @@ export async function prepareScriptedRun(plan, input) {
62
66
  scenarioId: idHoldsKnownValue(scenario.journey.scenarioId, secrets)
63
67
  ? "scenario"
64
68
  : scenario.journey.scenarioId,
65
- goal: secrets.scrub(scenario.journey.goal),
66
- scenarioTitle: secrets.scrub(scenario.journey.scenarioTitle),
67
- steps: recordedStepIds(scenario.journey.steps, secrets),
69
+ goal: recordedText(scenario.journey.goal, secrets),
70
+ scenarioTitle: recordedText(scenario.journey.scenarioTitle, secrets),
71
+ steps: steps.steps,
68
72
  };
69
73
  // The plan lists E2B_API_KEY and the subject env only for a live clone.
70
74
  const { requirements } = plan;
@@ -127,6 +131,13 @@ export async function prepareScriptedRun(plan, input) {
127
131
  },
128
132
  };
129
133
  }
134
+ /**
135
+ * Scenario text the run records, without the run's known values and then without keys as written
136
+ * or encoded, since verify decodes the files that record it.
137
+ */
138
+ function recordedText(text, secrets) {
139
+ return redactEncodedText(secrets.scrub(text));
140
+ }
130
141
  /**
131
142
  * Whether an id holds a known value as written or as the parser's id token of it, lower-cased with
132
143
  * other characters as `-`, which the scrub would not find.
@@ -139,22 +150,56 @@ function idHoldsKnownValue(id, secrets) {
139
150
  }));
140
151
  }
141
152
  /**
142
- * The steps with their labels scrubbed and their ids safe to record. A step id names the step's
143
- * screenshot file and is recorded beside it, and the parser derives an omitted id from the label,
144
- * lower-cased and cut to 80 characters, so the scrub may not find a value in it. A step whose
145
- * label or id holds a known value is renamed by its position instead.
153
+ * The steps with their labels redacted and their ids safe to record, or why the run is refused. A
154
+ * step id names the step's screenshot file and is recorded beside it, and the parser derives an
155
+ * omitted id from the label, lower-cased and cut to 80 characters, so the scrub may not find a
156
+ * value in it. A step whose label changes under recordedText, or whose id holds a known value, is
157
+ * renamed by its position instead.
146
158
  */
147
159
  function recordedStepIds(steps, secrets) {
148
160
  const taken = new Set(steps.map((step) => step.id));
149
- return steps.map((step, index) => {
150
- const label = secrets.scrub(step.label);
151
- if (label === step.label && !idHoldsKnownValue(step.id, secrets))
152
- return { ...step, label };
153
- const position = `step-${String(index + 1).padStart(2, "0")}`;
154
- let id = position;
155
- for (let suffix = 2; taken.has(id); suffix += 1)
156
- id = `${position}-${suffix}`;
161
+ const recorded = [];
162
+ for (const [index, step] of steps.entries()) {
163
+ const label = recordedText(step.label, secrets);
164
+ if (label === step.label && !idHoldsKnownValue(step.id, secrets)) {
165
+ recorded.push({ ...step, label });
166
+ continue;
167
+ }
168
+ const id = positionId(index, taken, secrets);
169
+ if (id === undefined)
170
+ return {
171
+ ok: false,
172
+ message: `The label or id of scenario step ${index + 1} holds a key, a local path or a value this run keeps out of its files, so the run renames the step by its position, and each id that position gives holds one of those values too. Remove it from the step's label and id.`,
173
+ };
157
174
  taken.add(id);
158
- return { ...step, id, label };
159
- });
175
+ recorded.push({ ...step, id, label });
176
+ }
177
+ return { ok: true, steps: recorded };
178
+ }
179
+ /**
180
+ * The id a renamed step takes by its position, such as step-03 for the third step, then step-03-2,
181
+ * step-03-3 and on when another step has it. A known value can be one of these ids, so one that
182
+ * holds a known value is passed over. Every id holds the position id, so a position id that holds a
183
+ * known value gives undefined at once. Otherwise taken.size + 1 ids that hold no known value are
184
+ * tried, which includes a free one, as long as known values pass over at most
185
+ * MAX_KNOWN_VALUE_SKIPS of them; a value such as `-01-` that every numbered id holds gives undefined.
186
+ */
187
+ function positionId(index, taken, secrets) {
188
+ const position = `step-${String(index + 1).padStart(2, "0")}`;
189
+ if (idHoldsKnownValue(position, secrets))
190
+ return undefined;
191
+ let skips = 0;
192
+ for (let suffix = 1; suffix <= taken.size + 1 + skips; suffix += 1) {
193
+ const id = suffix === 1 ? position : `${position}-${suffix}`;
194
+ if (taken.has(id))
195
+ continue;
196
+ if (!idHoldsKnownValue(id, secrets))
197
+ return id;
198
+ if (skips === MAX_KNOWN_VALUE_SKIPS)
199
+ return undefined;
200
+ skips += 1;
201
+ }
202
+ return undefined;
160
203
  }
204
+ // Known values are few, and a numbered id holds one only when the value matches its digits.
205
+ const MAX_KNOWN_VALUE_SKIPS = 32;
@@ -1,7 +1,7 @@
1
1
  // Builds the run bundle for a concurrent shared-world run: per-participant simulations, streams and
2
2
  // events, the sharedWorld evidence block and the review summary.
3
3
  import { receivingPublication } from "../../comms/receiving-runtime.js";
4
- import { redactText } from "../../evidence/redaction.js";
4
+ import { redactEncodedText } from "../../evidence/encoded-text.js";
5
5
  import { planeStateOf } from "./plan.js";
6
6
  import { REVIEW_SCHEMA, bundleArtifacts, bundleHead, } from "../../run/bundle.js";
7
7
  import { SHARED_WORLD_SCHEMA, concurrencyFacts, sharedWorldParticipantKeys, } from "../../run/shared-world-evidence.js";
@@ -362,7 +362,7 @@ export function buildConcurrentSharedWorldBundle(args) {
362
362
  scenario: {
363
363
  id: `concurrent-shared-world-${plan.studyId}`,
364
364
  title: plan.title ?? `Concurrent shared-world: ${plan.studyId}`,
365
- goal: redactText(actorSpecs[0]?.evidenceInstructions ??
365
+ goal: redactEncodedText(actorSpecs[0]?.evidenceInstructions ??
366
366
  actorSpecs[0]?.instructions ??
367
367
  "Concurrent shared-world interaction."),
368
368
  source: `study:${plan.studyId}`,
@@ -423,7 +423,7 @@ function sharedWorldFeedbackCandidates(args) {
423
423
  participantId: spec.planned.id,
424
424
  streamId: spec.streamId,
425
425
  personaId: spec.persona.id,
426
- goal: redactText(spec.evidenceInstructions ?? spec.instructions),
426
+ goal: redactEncodedText(spec.evidenceInstructions ?? spec.instructions),
427
427
  ...(outcome?.session === undefined
428
428
  ? {}
429
429
  : { session: outcome.session, traceArtifactPath: spec.traceArtifactPath }),
@@ -10,7 +10,7 @@ import { startLiveTraceFlush } from "../computer-use/live-flush.js";
10
10
  import { desktopParticipantDeps, } from "../computer-use/participant-deps.js";
11
11
  import { resolveCommittedPersonasForCwd } from "../../study/persona-resolve.js";
12
12
  import { participantAssignment } from "../../study/participant-assignment.js";
13
- import { redactText } from "../../evidence/redaction.js";
13
+ import { redactEncodedText } from "../../evidence/encoded-text.js";
14
14
  import { sandboxHeadroomMs } from "../../substrates/e2b/lifetime.js";
15
15
  // prose-check: model prompt (the participant model reads this, not a person)
16
16
  const DEFAULT_MISSION = "You are one of MANY users hitting a shared web application at the same time. The browser is already open at the app. Accomplish your role's task, then stop.";
@@ -178,7 +178,7 @@ export async function buildParticipantSpecs(participants, cwd, scrubKnownValues)
178
178
  for (const spec of actorSpecs) {
179
179
  if (spec.evidenceAssignment)
180
180
  spec.evidenceAssignment = participantAssignment(spec.evidenceAssignment, scrubKnownValues);
181
- spec.evidenceInstructions = redactText(scrubKnownValues(spec.instructions));
181
+ spec.evidenceInstructions = redactEncodedText(scrubKnownValues(spec.instructions));
182
182
  spec.persona = scrubPersonaBrief(spec.persona, scrubKnownValues);
183
183
  }
184
184
  return actorSpecs;
@@ -1,4 +1,4 @@
1
- import { redactText } from "../../evidence/redaction.js";
1
+ import { redactEncodedText } from "../../evidence/encoded-text.js";
2
2
  import { participantAssignment } from "../../study/participant-assignment.js";
3
3
  import { REVIEW_SCHEMA, bundleArtifacts, bundleHead, } from "../../run/bundle.js";
4
4
  import {} from "../../run/judge.js";
@@ -250,7 +250,7 @@ function terminalRunBundle(args, parts) {
250
250
  // The author mission is public-safe committed study text. It is recorded plaintext as the goal,
251
251
  // redacted defensively before persisting (it never carries a secret, but the harness never
252
252
  // trusts that). The full composed prompt is bound by digest, not text.
253
- goal: redactText(args.mission),
253
+ goal: redactEncodedText(args.mission),
254
254
  source: `study:${args.studyId}`,
255
255
  sourceDigest: args.persona.promptDigest,
256
256
  },
@@ -4,7 +4,7 @@ import type { ExecutionOutcome } from "./judge.js";
4
4
  import type { SharedWorldEvidence } from "./shared-world-evidence.js";
5
5
  import { type RunStudyProvenance } from "./study-provenance.js";
6
6
  import type { RunInterruptSignal } from "./status.js";
7
- import type { RunSimulationStatus, RunStream, RunStreamKind, StoredRunStream } from "./streams.js";
7
+ import type { DisplayedRunStream, RunSimulationStatus, RunStream, RunStreamKind, StoredRunStream } from "./streams.js";
8
8
  /**
9
9
  * The run bundle version this release writes: each stream's trace, and each running participant's
10
10
  * items so far, are files run.json indexes, and each participant's per-request lists are only in
@@ -395,6 +395,10 @@ export interface RunBundle {
395
395
  export type StoredRunBundle = Omit<RunBundle, "streams"> & {
396
396
  streams: StoredRunStream[];
397
397
  };
398
+ /** A loaded bundle as a reader that shows the run has it (displayedTraces): DisplayedRunStreams. */
399
+ export type DisplayedRunBundle = Omit<RunBundle, "streams"> & {
400
+ streams: DisplayedRunStream[];
401
+ };
398
402
  /**
399
403
  * How a run ended. `finished`: Run.finish published the final bundle with the route result's `ok`
400
404
  * and its execution outcome (`judgeExecution` in src/run/judge.ts). `interrupted`: the CLI's signal
@@ -1,7 +1,7 @@
1
1
  import { type PreparedRunArtifactPaths } from "./paths.js";
2
2
  import { type ContainedRefusal, type PreparedSelectedOutputDirectory } from "./contained-output.js";
3
3
  import type { RunPointer } from "./results.js";
4
- import { type RunBundle } from "./bundle.js";
4
+ import { type DisplayedRunBundle, type RunBundle } from "./bundle.js";
5
5
  import { type BoundedFileResult } from "./evidence-files.js";
6
6
  /** The run input that resolves through latest.json rather than naming a run directory. */
7
7
  export declare const LATEST_RUN_ALIAS = "latest";
@@ -55,15 +55,17 @@ export declare function loadRunBundle(cwdInput: string, runInput: string): Promi
55
55
  bundlePath: string;
56
56
  runDir: string;
57
57
  } | null>;
58
- /**
59
- * loadRunBundle for a caller that already holds prepared paths. Revalidates them first. With
60
- * `display`, a participant whose trace the loader refuses, such as one past the read limit, is
61
- * kept as displayedTraces shows it, for a page that shows the run.
62
- */
63
- export declare function loadRunBundlePrepared(cwdInput: string, runPaths: PreparedRunArtifactPaths, options?: {
64
- display?: boolean;
65
- }): Promise<{
66
- bundle: RunBundle;
58
+ /** A run's bundle, run.json's path relative to the working directory, and the run folder. */
59
+ type LoadedRun<T> = {
60
+ bundle: T;
67
61
  bundlePath: string;
68
62
  runDir: string;
69
- } | null>;
63
+ };
64
+ /** loadRunBundle for a caller that already holds prepared paths. Revalidates them first. */
65
+ export declare function loadRunBundlePrepared(cwdInput: string, runPaths: PreparedRunArtifactPaths): Promise<LoadedRun<RunBundle> | null>;
66
+ /**
67
+ * loadRunBundlePrepared for a page that shows the run: a participant whose trace the loader
68
+ * refuses, such as one past the read limit, is kept as displayedTraces shows it.
69
+ */
70
+ export declare function loadDisplayedRunBundle(cwdInput: string, runPaths: PreparedRunArtifactPaths): Promise<LoadedRun<DisplayedRunBundle> | null>;
71
+ export {};
@@ -1,7 +1,7 @@
1
1
  import path from "node:path";
2
2
  import { bindExistingRunArtifactPaths, isSafeRunIdSegment, resolveExistingRunDirectory, resolveLatestRunDirectory, RUNS_RELATIVE_ROOT, validatePreparedRunArtifactPaths, } from "./paths.js";
3
3
  import { assertPreparedSelectedOutputDirectory, bindExistingManagedHumanishOutputDirectory, ContainedReadRefusedError, LATEST_POINTER_MAX_BYTES, readContainedRegularFile, RUN_ARTIFACT_MAX_BYTES, } from "./contained-output.js";
4
- import { RUN_BUNDLE_FILE } from "./bundle.js";
4
+ import { RUN_BUNDLE_FILE, } from "./bundle.js";
5
5
  import { isRunBundle, isRunPointer } from "./bundle-shape.js";
6
6
  import { displayedTraces, loadRunTraces, loadStreamTraces } from "./stream-trace.js";
7
7
  import { readUnchangedFile } from "./evidence-files.js";
@@ -119,29 +119,36 @@ export async function loadRunBundle(cwdInput, runInput) {
119
119
  }
120
120
  return loadRunBundlePrepared(cwd, runPaths);
121
121
  }
122
+ /** loadRunBundle for a caller that already holds prepared paths. Revalidates them first. */
123
+ export async function loadRunBundlePrepared(cwdInput, runPaths) {
124
+ const run = await readPreparedRunJson(cwdInput, runPaths);
125
+ // A trace that does not match its index throws StreamTraceRefusedError, naming the participant.
126
+ return run === null ? null : { ...run, bundle: await loadRunTraces(runPaths, run.bundle) };
127
+ }
122
128
  /**
123
- * loadRunBundle for a caller that already holds prepared paths. Revalidates them first. With
124
- * `display`, a participant whose trace the loader refuses, such as one past the read limit, is
125
- * kept as displayedTraces shows it, for a page that shows the run.
129
+ * loadRunBundlePrepared for a page that shows the run: a participant whose trace the loader
130
+ * refuses, such as one past the read limit, is kept as displayedTraces shows it.
126
131
  */
127
- export async function loadRunBundlePrepared(cwdInput, runPaths, options = {}) {
132
+ export async function loadDisplayedRunBundle(cwdInput, runPaths) {
133
+ const run = await readPreparedRunJson(cwdInput, runPaths);
134
+ if (run === null)
135
+ return null;
136
+ return { ...run, bundle: displayedTraces(await loadStreamTraces(runPaths, run.bundle)) };
137
+ }
138
+ /** The prepared run's run.json when it is a run bundle, after revalidating the paths; else null. */
139
+ async function readPreparedRunJson(cwdInput, runPaths) {
128
140
  const cwd = path.resolve(cwdInput);
129
141
  await validatePreparedRunArtifactPaths(runPaths);
130
- const bundlePath = path.join(runPaths.absoluteRunRoot, RUN_BUNDLE_FILE);
131
142
  const read = await readRunJsonIfExists(runPaths, RUN_BUNDLE_FILE);
132
143
  // A run.json it refuses is no reason to call the run absent: the caller reports why.
133
144
  if (read.status === "refused")
134
145
  throw new ContainedReadRefusedError(RUN_BUNDLE_FILE, read);
135
146
  const bundle = read.status === "read" ? read.value : null;
136
- if (!isRunBundle(bundle)) {
147
+ if (!isRunBundle(bundle))
137
148
  return null;
138
- }
139
149
  return {
140
- // A trace that does not match its index throws StreamTraceRefusedError, naming the participant.
141
- bundle: options.display === true
142
- ? displayedTraces(await loadStreamTraces(runPaths, bundle))
143
- : await loadRunTraces(runPaths, bundle),
144
- bundlePath: path.relative(cwd, bundlePath),
150
+ bundle,
151
+ bundlePath: path.relative(cwd, path.join(runPaths.absoluteRunRoot, RUN_BUNDLE_FILE)),
145
152
  runDir: runPaths.absoluteRunRoot,
146
153
  };
147
154
  }
package/dist/run/paths.js CHANGED
@@ -1,5 +1,6 @@
1
1
  import { lstat, mkdir, readdir, realpath, rmdir } from "node:fs/promises";
2
2
  import path from "node:path";
3
+ import { mapWithConcurrency } from "./concurrency.js";
3
4
  import { isNodeError } from "./type-guards.js";
4
5
  export const RUNS_RELATIVE_ROOT = path.join(".humanish", "runs");
5
6
  const LATEST_RUN_RELATIVE_PATH = path.join(RUNS_RELATIVE_ROOT, "latest.json");
@@ -262,35 +263,44 @@ export async function assertRegularFileOrMissing(filePath, subject = "humanish s
262
263
  throw error;
263
264
  }
264
265
  }
265
- async function assertNoSymlinkDescendants(directory) {
266
- const entries = await readdir(directory, { withFileTypes: true });
267
- for (const entry of entries) {
268
- const child = path.join(directory, entry.name);
269
- // A run directory can be written concurrently with this walk: the atomic writer creates a
270
- // `.humanish-write-*.tmp` sibling and renames it, and a run now also refreshes its own status
271
- // record on a cadence. An entry that vanished between readdir and lstat therefore proves
272
- // nothing except that it is gone, and a path that no longer exists cannot be a symlink escape:
273
- // skip it rather than failing the containment check on a benign race.
274
- let stats;
275
- try {
276
- stats = await lstat(child);
277
- }
278
- catch (error) {
279
- if (isNodeError(error) && error.code === "ENOENT") {
280
- continue;
281
- }
282
- throw error;
283
- }
284
- if (stats.isSymbolicLink()) {
285
- throw new Error("humanish run directories must not contain symbolic links.");
286
- }
287
- if (stats.isDirectory()) {
288
- await assertNoSymlinkDescendants(child);
289
- }
290
- else if (!stats.isFile() || stats.nlink > 1) {
291
- throw new Error("humanish run leaves must be single-link regular files, not hardlinks or special files.");
292
- }
266
+ // Entries the storage check lists or lstats at once. A run folder holds one entry per screenshot,
267
+ // about 70,000 for 24 participants working 8 hours, and one call at a time left the process idle
268
+ // between calls.
269
+ const STORAGE_CHECK_CONCURRENCY = 32;
270
+ /** Every entry under `root` is a directory or a single-link regular file, checked a level at a time. */
271
+ async function assertNoSymlinkDescendants(root) {
272
+ let directories = [root];
273
+ while (directories.length > 0) {
274
+ const listed = await mapWithConcurrency(directories, STORAGE_CHECK_CONCURRENCY, async (dir) => (await readdir(dir, { withFileTypes: true })).map((entry) => path.join(dir, entry.name)));
275
+ const children = listed.flat();
276
+ const kinds = await mapWithConcurrency(children, STORAGE_CHECK_CONCURRENCY, storageEntryKind);
277
+ directories = children.filter((_, index) => kinds[index] === "directory");
278
+ }
279
+ }
280
+ async function storageEntryKind(child) {
281
+ // A run directory can be written concurrently with this walk: the atomic writer creates a
282
+ // `.humanish-write-*.tmp` sibling and renames it, and a run also refreshes its own status record
283
+ // on a cadence. An entry that vanished between readdir and lstat therefore proves nothing except
284
+ // that it is gone, and a path that no longer exists cannot be a symlink escape: skip it rather
285
+ // than failing the containment check on a benign race.
286
+ let stats;
287
+ try {
288
+ stats = await lstat(child);
289
+ }
290
+ catch (error) {
291
+ if (isNodeError(error) && error.code === "ENOENT")
292
+ return "gone";
293
+ throw error;
294
+ }
295
+ if (stats.isSymbolicLink()) {
296
+ throw new Error("humanish run directories must not contain symbolic links.");
297
+ }
298
+ if (stats.isDirectory())
299
+ return "directory";
300
+ if (!stats.isFile() || stats.nlink > 1) {
301
+ throw new Error("humanish run leaves must be single-link regular files, not hardlinks or special files.");
293
302
  }
303
+ return "file";
294
304
  }
295
305
  async function capturePreparedRunArtifactPaths(paths, runRoot) {
296
306
  const physicalRunRoot = await realpath(runRoot);
package/dist/run/run.js CHANGED
@@ -3,6 +3,7 @@
3
3
  // their own evidence files; this module owns when the run starts, how it is published, and how it
4
4
  // is closed on every exit.
5
5
  import { randomBytes } from "node:crypto";
6
+ import { redactEncodedText } from "../evidence/encoded-text.js";
6
7
  import { redactText } from "../evidence/redaction.js";
7
8
  import { buildObserverData } from "../observer/data.js";
8
9
  import { splitObserverData } from "../observer/participant-items.js";
@@ -160,14 +161,14 @@ function runPublisher(args) {
160
161
  const { options, runId, createdAt, paths, runStatus, now, admit, participantsRan } = args;
161
162
  const observer = observerTarget(options);
162
163
  // The study's warnings join whatever events the route wrote, once, in every write. A warning can
163
- // quote the study file, which may name a sandbox URL a participant needs, so each goes through
164
- // the redaction a run failure gets.
164
+ // quote the study file, which may name a sandbox URL a participant needs or hold an encoded key,
165
+ // so each goes through the redaction study text gets.
165
166
  const studyWarnings = (options.warnings ?? []).map((message, index) => ({
166
167
  id: `event-study-warning-${String(index + 1).padStart(3, "0")}`,
167
168
  at: createdAt,
168
169
  level: "warn",
169
170
  type: "study.warning",
170
- message: redactText(message),
171
+ message: redactEncodedText(message),
171
172
  }));
172
173
  // So do the host suspensions the heartbeat has seen by the time of the write, and the files the
173
174
  // final sandbox id sweep could not read.
@@ -1,4 +1,4 @@
1
- import { type RunBundle, type StoredRunBundle } from "./bundle.js";
1
+ import { type DisplayedRunBundle, type RunBundle, type StoredRunBundle } from "./bundle.js";
2
2
  import { type PreparedOutputRoot } from "./contained-output.js";
3
3
  import { type LiveItemsCursor } from "./live-items.js";
4
4
  import type { LiveItemsIndex, RunStream, StoredRunStream, WithoutRequestLists } from "./streams.js";
@@ -131,7 +131,7 @@ export declare function displayedTraces(loaded: {
131
131
  bundle: RunBundle;
132
132
  refused: readonly StreamTraceRefusedError[];
133
133
  live?: readonly LiveItemsRead[];
134
- }): RunBundle;
134
+ }): DisplayedRunBundle;
135
135
  /**
136
136
  * What `reduce` returns for each stream of the bundle, loaded as loadRunTraces loads it, in stream
137
137
  * order; the first refusal throws. Streams are loaded one at a time and nothing of a stream's files
@@ -178,7 +178,7 @@ function containedReader(runPaths) {
178
178
  }
179
179
  /**
180
180
  * The stream from the bytes of its trace file: its actor with the file's items, and its artifacts
181
- * followed by the file's screenshot entries, without the index.
181
+ * followed by the file's screenshot entries, without the index. Its live actor is as it was.
182
182
  */
183
183
  function joinTrace(stream, bytes) {
184
184
  const index = stream.trace;
@@ -246,12 +246,19 @@ function assertIndex(stream) {
246
246
  throw new StreamTraceRefusedError(stream.id, "run.json holds items beside its trace index");
247
247
  }
248
248
  /**
249
- * A stream of run.json typed as a loaded one, for the streams the loader joins no trace into: one
250
- * without an index, which v1 wrote with its actor's lists, and a refused one, kept as run.json
251
- * holds it. From v4 a refused stream's actor has none of the lists, and its `conversation` lacks
252
- * the `requests` ActorTrace declares; the loader's `refused` names each such stream.
249
+ * The joined stream as a loaded one, once its actor's conversation, if any, holds the requests
250
+ * list ActorTrace types as present. The writer writes the list with every conversation, in
251
+ * run.json before v4 and in the trace file from v4, so a stream without it was edited, and a reader
252
+ * that counts the requests would throw. Items a v1 run.json or a humanish.live-actor.v1 live actor
253
+ * holds inline are taken as they are.
253
254
  */
254
- const unjoined = (stream) => stream;
255
+ function asLoaded(stream) {
256
+ const conversation = stream.actor?.conversation;
257
+ if (conversation !== undefined &&
258
+ !(isRecord(conversation) && Array.isArray(conversation.requests)))
259
+ throw new StreamTraceRefusedError(stream.id, "its actor records a conversation without the requests list");
260
+ return stream;
261
+ }
255
262
  /** A live actor whose items run.json holds inline: humanish.live-actor.v1, or one with no schema. */
256
263
  const isInlineLiveActor = (live) => live.schema === LIVE_ACTOR_V1_SCHEMA || live.schema === undefined;
257
264
  /** A live file that could not be read, which a writer that moved on may have removed. */
@@ -345,7 +352,7 @@ async function readLiveTail(read, stream, index, after) {
345
352
  */
346
353
  async function readStreamTrace(read, stream, maxBytes, liveAfter) {
347
354
  const files = [];
348
- let joined = unjoined(stream);
355
+ let joined = stream;
349
356
  if (stream.trace !== undefined) {
350
357
  assertIndex(stream);
351
358
  if (stream.trace.bytes > maxBytes)
@@ -359,7 +366,11 @@ async function readStreamTrace(read, stream, maxBytes, liveAfter) {
359
366
  const live = await readLiveItems(read, joined, maxBytes, liveAfter);
360
367
  if (live.file !== undefined)
361
368
  files.push(live.file);
362
- return { stream: live.stream, files, ...(live.read === undefined ? {} : { live: live.read }) };
369
+ return {
370
+ stream: asLoaded(live.stream),
371
+ files,
372
+ ...(live.read === undefined ? {} : { live: live.read }),
373
+ };
363
374
  }
364
375
  /**
365
376
  * The stream as a v1 run.json held it: its actor with every item and every artifact entry. A
@@ -500,7 +511,9 @@ async function loadEveryStream(read, bundle, maxBytes, liveAfter) {
500
511
  if (!(error instanceof StreamTraceRefusedError))
501
512
  throw error;
502
513
  refused.push(error);
503
- streams.push(unjoined(stream));
514
+ // Kept as run.json holds it, typed as loaded: from v2 its actor has no items, and from v4
515
+ // none of the lists. The loader's `refused` names it, and displayedTraces shows it so.
516
+ streams.push(stream);
504
517
  }
505
518
  }
506
519
  return { bundle: { ...bundle, streams }, refused, files, live };
@@ -523,12 +536,12 @@ export function displayedTraces(loaded) {
523
536
  if (reason === undefined)
524
537
  return stream;
525
538
  const { liveActor: _live, trace: _index, actor, ...rest } = stream;
526
- return unjoined({
539
+ return {
527
540
  ...rest,
528
541
  ...(actor === undefined ? {} : { actor: { ...withoutRequestLists(actor), items: [] } }),
529
542
  artifacts: stream.artifacts.filter((artifact) => !isScreenshot(artifact)),
530
543
  traceUnavailable: reason,
531
- });
544
+ };
532
545
  });
533
546
  const events = loaded.bundle.streams
534
547
  .filter((stream) => reasons.has(stream.id))
@@ -327,11 +327,26 @@ export type WithoutRequestLists<T> = Omit<T, "providerRequests" | "tokenUsage" |
327
327
  [K in "conversation" & keyof T]?: Omit<NonNullable<T[K]>, "requests"> | undefined;
328
328
  };
329
329
  /**
330
- * A stream as run.json holds it, in any version: its actor and live actor WithoutRequestLists.
331
- * loadStreamTraces joins the lists from the trace file.
330
+ * `T` WithoutRequestLists and with `items` optional: a v1 run.json holds the items inline, and from
331
+ * v2 (an actor) or v3 (a live actor) a file holds them.
332
+ */
333
+ type StoredTrace<T extends {
334
+ items: unknown;
335
+ }> = WithoutRequestLists<Omit<T, "items">> & Partial<Pick<T, "items">>;
336
+ /**
337
+ * A stream as run.json holds it, in any version: its actor and live actor without the per-request
338
+ * lists, and with items only in a v1 run.json. loadStreamTraces joins both from the files.
332
339
  */
333
340
  export type StoredRunStream = Omit<RunStream, "actor" | "liveActor"> & {
341
+ actor?: StoredTrace<ActorTrace>;
342
+ liveActor?: StoredTrace<NonNullable<RunStream["liveActor"]>>;
343
+ };
344
+ /**
345
+ * A stream as a reader that shows the run has it (displayedTraces in src/run/stream-trace.ts):
346
+ * loaded, or with `traceUnavailable` when the loader refused its trace. A refused stream's actor
347
+ * keeps run.json's summary, which from v4 has none of the per-request lists, so the type has none.
348
+ */
349
+ export type DisplayedRunStream = Omit<RunStream, "actor"> & {
334
350
  actor?: WithoutRequestLists<ActorTrace>;
335
- liveActor?: WithoutRequestLists<NonNullable<RunStream["liveActor"]>>;
336
351
  };
337
352
  export {};
@@ -21,9 +21,9 @@ export declare function runStudyWith(config: StudyConfig, options: InternalRunSt
21
21
  /**
22
22
  * The package's study runner (`src/index.ts`): runStudyWith with the public options only. A
23
23
  * config that skipped parseStudy is checked first (libraryConfigRefusal): a humanish.lab.v2 config
24
- * or one with a humanish 0.110 field, no `actor`, or a `route` the config does not take. That, or a
25
- * field RunStudyOptions no longer has, is refused in the route's own result envelope before
26
- * anything runs.
24
+ * or one with a humanish 0.110 field, no `actor`, a `route` the config does not take, or an
25
+ * `actor.model` verify would flag. That, or a field RunStudyOptions no longer has, is refused in the
26
+ * route's own result envelope before anything runs.
27
27
  */
28
28
  export declare function runStudy(config: StudyConfig, options: RunStudyOptions): Promise<StudyOutcome>;
29
29
  /** A study planned once: the route's refusal, or the run of its plan. */