humanish 0.133.1 → 0.135.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. package/CHANGELOG.md +28 -0
  2. package/dist/actors/computer-use/loop/actions.d.ts +2 -0
  3. package/dist/actors/computer-use/loop/actions.js +11 -5
  4. package/dist/actors/computer-use/loop/backstop.d.ts +23 -2
  5. package/dist/actors/computer-use/loop/backstop.js +40 -7
  6. package/dist/actors/computer-use/loop/session.d.ts +3 -0
  7. package/dist/actors/computer-use/loop/session.js +3 -0
  8. package/dist/actors/computer-use/loop/streak.d.ts +18 -0
  9. package/dist/actors/computer-use/loop/streak.js +47 -0
  10. package/dist/actors/computer-use/loop/types.d.ts +5 -1
  11. package/dist/actors/computer-use/loop/watch.d.ts +22 -4
  12. package/dist/actors/computer-use/loop/watch.js +83 -27
  13. package/dist/actors/computer-use/loop.js +18 -6
  14. package/dist/actors/contract.d.ts +3 -1
  15. package/dist/analysis/automatic-config.js +5 -3
  16. package/dist/analysis/execute.js +2 -1
  17. package/dist/analysis/service.js +5 -3
  18. package/dist/analysis/source.d.ts +6 -0
  19. package/dist/analysis/source.js +15 -13
  20. package/dist/cli/commands/run-signals.js +6 -2
  21. package/dist/evidence/encoded-text.d.ts +13 -3
  22. package/dist/evidence/encoded-text.js +88 -25
  23. package/dist/observer/page-data.d.ts +6 -0
  24. package/dist/observer/page-data.js +34 -7
  25. package/dist/observer/participant-items.d.ts +15 -1
  26. package/dist/observer/participant-items.js +4 -3
  27. package/dist/observer/render.js +5 -3
  28. package/dist/observer/run-routes.js +4 -4
  29. package/dist/observer-app.html +4 -4
  30. package/dist/routes/computer-use/bundle-parts.d.ts +9 -0
  31. package/dist/routes/computer-use/bundle-parts.js +24 -4
  32. package/dist/routes/computer-use/fanout-bundle.js +8 -6
  33. package/dist/routes/computer-use/participant-runs.js +2 -2
  34. package/dist/routes/computer-use/single-bundle.js +5 -5
  35. package/dist/routes/scripted/setup.js +62 -17
  36. package/dist/routes/shared-world/bundle.js +3 -3
  37. package/dist/routes/shared-world/participant-specs.js +2 -2
  38. package/dist/routes/terminal/bundle.js +2 -2
  39. package/dist/run/locate.d.ts +8 -2
  40. package/dist/run/locate.js +10 -4
  41. package/dist/run/run.js +22 -5
  42. package/dist/run/sandbox-ids.d.ts +4 -3
  43. package/dist/run/sandbox-ids.js +8 -6
  44. package/dist/run/stream-trace.d.ts +8 -2
  45. package/dist/run/stream-trace.js +13 -7
  46. package/dist/run-study.d.ts +3 -3
  47. package/dist/run-study.js +3 -3
  48. package/dist/study/bundle-estimate.d.ts +0 -2
  49. package/dist/study/bundle-estimate.js +33 -33
  50. package/dist/study/config.d.ts +4 -3
  51. package/dist/study/config.js +8 -5
  52. package/dist/study/parse/actors.d.ts +6 -0
  53. package/dist/study/parse/actors.js +14 -0
  54. package/dist/study/parse/subject-state.js +4 -4
  55. package/dist/study/participant-assignment.js +2 -2
  56. package/dist/study/persona.d.ts +7 -1
  57. package/dist/study/persona.js +13 -5
  58. package/dist/study/plan-base.js +4 -3
  59. package/dist/verify/artifacts.d.ts +7 -1
  60. package/dist/verify/artifacts.js +15 -4
  61. package/dist/verify/verify.js +29 -8
  62. package/docs/architecture/actor-contract.md +3 -1
  63. package/docs/architecture/observer.md +24 -7
  64. package/docs/architecture/state-driven-executor.md +7 -4
  65. package/docs/contracts/run-bundle.md +16 -2
  66. package/docs/contracts/study-analysis.md +8 -5
  67. package/docs/decisions/0002-observer-is-one-self-contained-file.md +3 -3
  68. package/package.json +1 -1
@@ -1,6 +1,15 @@
1
+ import type { CuaLoopResult } from "../../actors/computer-use/loop.js";
1
2
  import { type RunProviderResource, type RunSubjectProvenance } from "../../run/bundle.js";
2
3
  import { type ParticipantIds } from "../../run/participant-records.js";
3
4
  import { type ParticipantRunOutcome } from "./types.js";
5
+ /**
6
+ * The review's gap for the streaks a participant's trace records: turns that repeated a recent
7
+ * action on an unchanged screen past noProgressSteps, which did not end the session of a
8
+ * participant held to its scheduled end. Its session passes, so no other gap names them. One line
9
+ * names the first streak and counts the rest: a page that changes every minute, watched 1 s at a
10
+ * time, records a streak a minute, and a fan-out would list each.
11
+ */
12
+ export declare function streakGaps(participantId: string, session: CuaLoopResult | undefined): string[];
4
13
  /** The human-readable state story appended to the provenance event (and review.md via it). */
5
14
  export declare function describeSubjectState(state: RunSubjectProvenance["state"], dryRun: boolean): string;
6
15
  /** Human-readable provenance line for the single-participant subject.provenance event of a clone
@@ -1,14 +1,34 @@
1
1
  // The parts both computer-use bundle shapes share: the subject-state story and provenance line,
2
- // provider-resource records, and the public-safe app URL label, the URL digest and the
3
- // subject-phase event id suffix. The builders are bundle.ts (the dispatcher and the judge),
4
- // single-bundle.ts and fanout-bundle.ts. Feedback candidates from participant reports are in
5
- // participant-feedback.ts, which the shared-world bundle reads too.
2
+ // provider-resource records, the review's gap for a held participant's streak, and the public-safe
3
+ // app URL label, the URL digest and the subject-phase event id suffix. The builders are bundle.ts
4
+ // (the dispatcher and the judge), single-bundle.ts and fanout-bundle.ts. Feedback candidates from
5
+ // participant reports are in participant-feedback.ts, which the shared-world bundle reads too.
6
+ import { STREAK_TITLE } from "../../actors/computer-use/loop/streak.js";
6
7
  import { scanEncodedText } from "../../evidence/encoded-text.js";
7
8
  import { digestText } from "../../evidence/redaction.js";
8
9
  import {} from "../../run/bundle.js";
9
10
  import { participantResourceIds } from "../../run/participant-records.js";
10
11
  import {} from "./types.js";
11
12
  import { plural } from "../../run/text.js";
13
+ /**
14
+ * The review's gap for the streaks a participant's trace records: turns that repeated a recent
15
+ * action on an unchanged screen past noProgressSteps, which did not end the session of a
16
+ * participant held to its scheduled end. Its session passes, so no other gap names them. One line
17
+ * names the first streak and counts the rest: a page that changes every minute, watched 1 s at a
18
+ * time, records a streak a minute, and a fan-out would list each.
19
+ */
20
+ export function streakGaps(participantId, session) {
21
+ const texts = (session?.trace.items ?? []).flatMap((item) => item.kind === "notice" && item.title === STREAK_TITLE && item.text !== undefined
22
+ ? [item.text]
23
+ : []);
24
+ const [first] = texts;
25
+ if (first === undefined)
26
+ return [];
27
+ const more = texts.length > 1
28
+ ? ` It had ${plural(texts.length - 1, "more such streak")} after that; its trace records each.`
29
+ : "";
30
+ return [`${participantId}: ${first}${more}`];
31
+ }
12
32
  /** The human-readable state story appended to the provenance event (and review.md via it). */
13
33
  export function describeSubjectState(state, dryRun) {
14
34
  switch (state.provenance) {
@@ -2,10 +2,10 @@ import { e2bDesktopTemplate } from "../../substrates/e2b/sandbox.js";
2
2
  import { receivingPublication } from "../../comms/receiving-runtime.js";
3
3
  import { actorEnding } from "../../actors/stop-cause.js";
4
4
  import { cuaGoalSource } from "../../actors/goal-source.js";
5
- import { redactText } from "../../evidence/redaction.js";
5
+ import { redactEncodedText } from "../../evidence/encoded-text.js";
6
6
  import { REVIEW_SCHEMA, bundleArtifacts, bundleHead, } from "../../run/bundle.js";
7
7
  import { aggregateTaskFunnels, formatParticipantOutcomes, formatRunTaskFunnel, gapsListClause, skippedSentence, withCuaReviewProvenance, } from "../../run/outcomes.js";
8
- import { providerResourcesForOutcome, publicSafeAppUrlLabel } from "./bundle-parts.js";
8
+ import { providerResourcesForOutcome, publicSafeAppUrlLabel, streakGaps } from "./bundle-parts.js";
9
9
  import { subjectName } from "../../run/subject-name.js";
10
10
  import { participantFeedbackCandidates } from "./participant-feedback.js";
11
11
  import { participantFactsOf } from "./participant-facts.js";
@@ -70,18 +70,20 @@ function fanoutSummary(args, facts) {
70
70
  }
71
71
  /**
72
72
  * The review's gaps: one line for the participants a host suspension likely failed, then a line
73
- * for each other participant that did not pass.
73
+ * for each other participant that did not pass, then a line for each held participant's streaks.
74
74
  */
75
75
  function fanoutGaps(args, judgment) {
76
76
  const suspended = new Set(args.hostSuspension?.participantIds ?? []);
77
77
  const others = judgment.participants.flatMap((participant, index) => participant.gapLine === undefined || suspended.has(args.specs[index].planned.id)
78
78
  ? []
79
79
  : [participant.gapLine]);
80
+ const streaks = args.specs.flatMap((spec, index) => streakGaps(spec.planned.id, args.outcomes?.[index]?.session));
80
81
  if (suspended.size === 0)
81
- return others;
82
+ return [...others, ...streaks];
82
83
  return [
83
84
  `${[...suspended].join(", ")}: failed after the host was suspended, likely its effect. Each participant's record keeps its own error.`,
84
85
  ...others,
86
+ ...streaks,
85
87
  ];
86
88
  }
87
89
  function fanoutReview(args, streams, judgment) {
@@ -184,7 +186,7 @@ function fanoutFeedbackCandidates(args) {
184
186
  participantId: spec.planned.id,
185
187
  streamId: spec.streamId,
186
188
  personaId: spec.persona.id,
187
- goal: redactText(spec.evidenceInstructions ?? spec.instructions),
189
+ goal: redactEncodedText(spec.evidenceInstructions ?? spec.instructions),
188
190
  ...(outcome?.session === undefined ? {} : { session: outcome.session }),
189
191
  ...(outcome?.session === undefined
190
192
  ? {}
@@ -268,7 +270,7 @@ export function buildCuaFanoutBundle(args) {
268
270
  // this writer produced. The only adopter-side workaround was scanner evasion.
269
271
  //
270
272
  // The instructions the model actually receives are untouched; only the persisted copy changes.
271
- goal: redactText(specs[0].evidenceInstructions ?? specs[0].instructions),
273
+ goal: redactEncodedText(specs[0].evidenceInstructions ?? specs[0].instructions),
272
274
  source: `study:${plan.studyId}`,
273
275
  sourceDigest: specs[0].persona.promptDigest,
274
276
  },
@@ -6,7 +6,7 @@ import { isLocalBrowserStudy, LOCAL_BROWSER_LIFETIME_MS, } from "../../substrate
6
6
  import { resolveParticipant } from "../../run/participant.js";
7
7
  import {} from "../../study/types.js";
8
8
  import { scrubPersonaBrief } from "../../study/persona.js";
9
- import { redactText } from "../../evidence/redaction.js";
9
+ import { redactEncodedText } from "../../evidence/encoded-text.js";
10
10
  import {} from "../../run/bundle.js";
11
11
  import { participantAssignment } from "../../study/participant-assignment.js";
12
12
  import { resolveCommittedPersonas } from "../../study/persona-resolve.js";
@@ -237,7 +237,7 @@ export function sanitizeParticipantRuns(runs, scrub) {
237
237
  for (const spec of runs) {
238
238
  if (spec.evidenceAssignment)
239
239
  spec.evidenceAssignment = participantAssignment(spec.evidenceAssignment, scrub);
240
- spec.evidenceInstructions = redactText(scrub(spec.instructions));
240
+ spec.evidenceInstructions = redactEncodedText(scrub(spec.instructions));
241
241
  spec.persona = scrubPersonaBrief(spec.persona, scrub);
242
242
  }
243
243
  }
@@ -1,12 +1,12 @@
1
1
  import { participantAssignment } from "../../study/participant-assignment.js";
2
- import { redactText } from "../../evidence/redaction.js";
2
+ import { redactEncodedText } from "../../evidence/encoded-text.js";
3
3
  import { REVIEW_SCHEMA, bundleArtifacts, bundleHead, } from "../../run/bundle.js";
4
4
  import {} from "../../run/streams.js";
5
5
  import { aggregateTaskFunnels, formatParticipantOutcomes, formatRunTaskFunnel, participantSummary, withCuaReviewProvenance, } from "../../run/outcomes.js";
6
6
  import { participantCaption } from "../../run/participant-caption.js";
7
7
  import { actorEnding } from "../../actors/stop-cause.js";
8
8
  import { cuaGoalSource, ranUntilClockEnd } from "../../actors/goal-source.js";
9
- import { describeSubjectState, phaseEventIdSuffix, publicSafeAppUrlLabel, subjectProvenanceMessage, } from "./bundle-parts.js";
9
+ import { describeSubjectState, phaseEventIdSuffix, publicSafeAppUrlLabel, streakGaps, subjectProvenanceMessage, } from "./bundle-parts.js";
10
10
  import { participantFeedbackCandidates } from "./participant-feedback.js";
11
11
  import { judgeParticipantRecords } from "../../run/judge.js";
12
12
  import { buildRunCostSummary } from "../../run/cost-summary.js";
@@ -348,7 +348,7 @@ function singleReview(args, view, stream) {
348
348
  ...(credibilityNote === undefined ? {} : { credibilityNote }),
349
349
  })}`,
350
350
  gaps: args.session || args.sessionError !== undefined
351
- ? []
351
+ ? streakGaps(args.participantId ?? "lane-01", args.session)
352
352
  : args.inProgress === true
353
353
  ? ["Live desktop session is still running."]
354
354
  : ["Live desktop session not yet run (dry run only)."],
@@ -385,7 +385,7 @@ export function buildSingleParticipantBundle(args) {
385
385
  scenario: {
386
386
  id: `cua-${args.studyId}`,
387
387
  title: args.studyTitle ?? `Computer-use run: ${args.studyId}`,
388
- goal: redactText(args.mission),
388
+ goal: redactEncodedText(args.mission),
389
389
  source: `study:${args.studyId}`,
390
390
  sourceDigest: args.persona.promptDigest,
391
391
  },
@@ -425,7 +425,7 @@ export function buildSingleParticipantBundle(args) {
425
425
  participantId: args.participantId ?? "lane-01",
426
426
  streamId: SINGLE.streamId,
427
427
  personaId: args.persona.id,
428
- goal: redactText(args.mission),
428
+ goal: redactEncodedText(args.mission),
429
429
  ...(args.session === undefined ? {} : { session: args.session }),
430
430
  ...(args.traceArtifactPath === undefined
431
431
  ? {}
@@ -8,6 +8,7 @@ import { resolveBrowserCommand } from "../../actors/scripted-browser/browser-com
8
8
  import { publicSafeToken } from "../../actors/scripted-browser/journey.js";
9
9
  import { prepareSelectedOutputDirectory } from "../../run/contained-output.js";
10
10
  import { RunSecrets } from "../../run/secrets.js";
11
+ import { redactEncodedText } from "../../evidence/encoded-text.js";
11
12
  import { evidenceAppUrlOf } from "./plan.js";
12
13
  import { resolveScriptedScenario } from "./scenario.js";
13
14
  import {} from "./types.js";
@@ -55,6 +56,9 @@ export async function prepareScriptedRun(plan, input) {
55
56
  result: failed("HUMANISH_SCRIPTED_SCENARIO_INVALID", scenario.message),
56
57
  };
57
58
  }
59
+ const steps = recordedStepIds(scenario.journey.steps, secrets);
60
+ if (!steps.ok)
61
+ return { ok: false, result: failed("HUMANISH_SCRIPTED_SCENARIO_INVALID", steps.message) };
58
62
  // The goal, title and step labels are recorded; a step's path, selector, value and expected text
59
63
  // drive the browser and stay as written.
60
64
  const journey = {
@@ -62,9 +66,9 @@ export async function prepareScriptedRun(plan, input) {
62
66
  scenarioId: idHoldsKnownValue(scenario.journey.scenarioId, secrets)
63
67
  ? "scenario"
64
68
  : scenario.journey.scenarioId,
65
- goal: secrets.scrub(scenario.journey.goal),
66
- scenarioTitle: secrets.scrub(scenario.journey.scenarioTitle),
67
- steps: recordedStepIds(scenario.journey.steps, secrets),
69
+ goal: recordedText(scenario.journey.goal, secrets),
70
+ scenarioTitle: recordedText(scenario.journey.scenarioTitle, secrets),
71
+ steps: steps.steps,
68
72
  };
69
73
  // The plan lists E2B_API_KEY and the subject env only for a live clone.
70
74
  const { requirements } = plan;
@@ -127,6 +131,13 @@ export async function prepareScriptedRun(plan, input) {
127
131
  },
128
132
  };
129
133
  }
134
+ /**
135
+ * Scenario text the run records, without the run's known values and then without keys as written
136
+ * or encoded, since verify decodes the files that record it.
137
+ */
138
+ function recordedText(text, secrets) {
139
+ return redactEncodedText(secrets.scrub(text));
140
+ }
130
141
  /**
131
142
  * Whether an id holds a known value as written or as the parser's id token of it, lower-cased with
132
143
  * other characters as `-`, which the scrub would not find.
@@ -139,22 +150,56 @@ function idHoldsKnownValue(id, secrets) {
139
150
  }));
140
151
  }
141
152
  /**
142
- * The steps with their labels scrubbed and their ids safe to record. A step id names the step's
143
- * screenshot file and is recorded beside it, and the parser derives an omitted id from the label,
144
- * lower-cased and cut to 80 characters, so the scrub may not find a value in it. A step whose
145
- * label or id holds a known value is renamed by its position instead.
153
+ * The steps with their labels redacted and their ids safe to record, or why the run is refused. A
154
+ * step id names the step's screenshot file and is recorded beside it, and the parser derives an
155
+ * omitted id from the label, lower-cased and cut to 80 characters, so the scrub may not find a
156
+ * value in it. A step whose label changes under recordedText, or whose id holds a known value, is
157
+ * renamed by its position instead.
146
158
  */
147
159
  function recordedStepIds(steps, secrets) {
148
160
  const taken = new Set(steps.map((step) => step.id));
149
- return steps.map((step, index) => {
150
- const label = secrets.scrub(step.label);
151
- if (label === step.label && !idHoldsKnownValue(step.id, secrets))
152
- return { ...step, label };
153
- const position = `step-${String(index + 1).padStart(2, "0")}`;
154
- let id = position;
155
- for (let suffix = 2; taken.has(id); suffix += 1)
156
- id = `${position}-${suffix}`;
161
+ const recorded = [];
162
+ for (const [index, step] of steps.entries()) {
163
+ const label = recordedText(step.label, secrets);
164
+ if (label === step.label && !idHoldsKnownValue(step.id, secrets)) {
165
+ recorded.push({ ...step, label });
166
+ continue;
167
+ }
168
+ const id = positionId(index, taken, secrets);
169
+ if (id === undefined)
170
+ return {
171
+ ok: false,
172
+ message: `The label or id of scenario step ${index + 1} holds a key, a local path or a value this run keeps out of its files, so the run renames the step by its position, and each id that position gives holds one of those values too. Remove it from the step's label and id.`,
173
+ };
157
174
  taken.add(id);
158
- return { ...step, id, label };
159
- });
175
+ recorded.push({ ...step, id, label });
176
+ }
177
+ return { ok: true, steps: recorded };
178
+ }
179
+ /**
180
+ * The id a renamed step takes by its position, such as step-03 for the third step, then step-03-2,
181
+ * step-03-3 and on when another step has it. A known value can be one of these ids, so one that
182
+ * holds a known value is passed over. Every id holds the position id, so a position id that holds a
183
+ * known value gives undefined at once. Otherwise taken.size + 1 ids that hold no known value are
184
+ * tried, which includes a free one, as long as known values pass over at most
185
+ * MAX_KNOWN_VALUE_SKIPS of them; a value such as `-01-` that every numbered id holds gives undefined.
186
+ */
187
+ function positionId(index, taken, secrets) {
188
+ const position = `step-${String(index + 1).padStart(2, "0")}`;
189
+ if (idHoldsKnownValue(position, secrets))
190
+ return undefined;
191
+ let skips = 0;
192
+ for (let suffix = 1; suffix <= taken.size + 1 + skips; suffix += 1) {
193
+ const id = suffix === 1 ? position : `${position}-${suffix}`;
194
+ if (taken.has(id))
195
+ continue;
196
+ if (!idHoldsKnownValue(id, secrets))
197
+ return id;
198
+ if (skips === MAX_KNOWN_VALUE_SKIPS)
199
+ return undefined;
200
+ skips += 1;
201
+ }
202
+ return undefined;
160
203
  }
204
+ // Known values are few, and a numbered id holds one only when the value matches its digits.
205
+ const MAX_KNOWN_VALUE_SKIPS = 32;
@@ -1,7 +1,7 @@
1
1
  // Builds the run bundle for a concurrent shared-world run: per-participant simulations, streams and
2
2
  // events, the sharedWorld evidence block and the review summary.
3
3
  import { receivingPublication } from "../../comms/receiving-runtime.js";
4
- import { redactText } from "../../evidence/redaction.js";
4
+ import { redactEncodedText } from "../../evidence/encoded-text.js";
5
5
  import { planeStateOf } from "./plan.js";
6
6
  import { REVIEW_SCHEMA, bundleArtifacts, bundleHead, } from "../../run/bundle.js";
7
7
  import { SHARED_WORLD_SCHEMA, concurrencyFacts, sharedWorldParticipantKeys, } from "../../run/shared-world-evidence.js";
@@ -362,7 +362,7 @@ export function buildConcurrentSharedWorldBundle(args) {
362
362
  scenario: {
363
363
  id: `concurrent-shared-world-${plan.studyId}`,
364
364
  title: plan.title ?? `Concurrent shared-world: ${plan.studyId}`,
365
- goal: redactText(actorSpecs[0]?.evidenceInstructions ??
365
+ goal: redactEncodedText(actorSpecs[0]?.evidenceInstructions ??
366
366
  actorSpecs[0]?.instructions ??
367
367
  "Concurrent shared-world interaction."),
368
368
  source: `study:${plan.studyId}`,
@@ -423,7 +423,7 @@ function sharedWorldFeedbackCandidates(args) {
423
423
  participantId: spec.planned.id,
424
424
  streamId: spec.streamId,
425
425
  personaId: spec.persona.id,
426
- goal: redactText(spec.evidenceInstructions ?? spec.instructions),
426
+ goal: redactEncodedText(spec.evidenceInstructions ?? spec.instructions),
427
427
  ...(outcome?.session === undefined
428
428
  ? {}
429
429
  : { session: outcome.session, traceArtifactPath: spec.traceArtifactPath }),
@@ -10,7 +10,7 @@ import { startLiveTraceFlush } from "../computer-use/live-flush.js";
10
10
  import { desktopParticipantDeps, } from "../computer-use/participant-deps.js";
11
11
  import { resolveCommittedPersonasForCwd } from "../../study/persona-resolve.js";
12
12
  import { participantAssignment } from "../../study/participant-assignment.js";
13
- import { redactText } from "../../evidence/redaction.js";
13
+ import { redactEncodedText } from "../../evidence/encoded-text.js";
14
14
  import { sandboxHeadroomMs } from "../../substrates/e2b/lifetime.js";
15
15
  // prose-check: model prompt (the participant model reads this, not a person)
16
16
  const DEFAULT_MISSION = "You are one of MANY users hitting a shared web application at the same time. The browser is already open at the app. Accomplish your role's task, then stop.";
@@ -178,7 +178,7 @@ export async function buildParticipantSpecs(participants, cwd, scrubKnownValues)
178
178
  for (const spec of actorSpecs) {
179
179
  if (spec.evidenceAssignment)
180
180
  spec.evidenceAssignment = participantAssignment(spec.evidenceAssignment, scrubKnownValues);
181
- spec.evidenceInstructions = redactText(scrubKnownValues(spec.instructions));
181
+ spec.evidenceInstructions = redactEncodedText(scrubKnownValues(spec.instructions));
182
182
  spec.persona = scrubPersonaBrief(spec.persona, scrubKnownValues);
183
183
  }
184
184
  return actorSpecs;
@@ -1,4 +1,4 @@
1
- import { redactText } from "../../evidence/redaction.js";
1
+ import { redactEncodedText } from "../../evidence/encoded-text.js";
2
2
  import { participantAssignment } from "../../study/participant-assignment.js";
3
3
  import { REVIEW_SCHEMA, bundleArtifacts, bundleHead, } from "../../run/bundle.js";
4
4
  import {} from "../../run/judge.js";
@@ -250,7 +250,7 @@ function terminalRunBundle(args, parts) {
250
250
  // The author mission is public-safe committed study text. It is recorded plaintext as the goal,
251
251
  // redacted defensively before persisting (it never carries a secret, but the harness never
252
252
  // trusts that). The full composed prompt is bound by digest, not text.
253
- goal: redactText(args.mission),
253
+ goal: redactEncodedText(args.mission),
254
254
  source: `study:${args.studyId}`,
255
255
  sourceDigest: args.persona.promptDigest,
256
256
  },
@@ -55,8 +55,14 @@ export declare function loadRunBundle(cwdInput: string, runInput: string): Promi
55
55
  bundlePath: string;
56
56
  runDir: string;
57
57
  } | null>;
58
- /** loadRunBundle for a caller that already holds prepared paths. Revalidates them first. */
59
- export declare function loadRunBundlePrepared(cwdInput: string, runPaths: PreparedRunArtifactPaths): Promise<{
58
+ /**
59
+ * loadRunBundle for a caller that already holds prepared paths. Revalidates them first. With
60
+ * `display`, a participant whose trace the loader refuses, such as one past the read limit, is
61
+ * kept as displayedTraces shows it, for a page that shows the run.
62
+ */
63
+ export declare function loadRunBundlePrepared(cwdInput: string, runPaths: PreparedRunArtifactPaths, options?: {
64
+ display?: boolean;
65
+ }): Promise<{
60
66
  bundle: RunBundle;
61
67
  bundlePath: string;
62
68
  runDir: string;
@@ -3,7 +3,7 @@ import { bindExistingRunArtifactPaths, isSafeRunIdSegment, resolveExistingRunDir
3
3
  import { assertPreparedSelectedOutputDirectory, bindExistingManagedHumanishOutputDirectory, ContainedReadRefusedError, LATEST_POINTER_MAX_BYTES, readContainedRegularFile, RUN_ARTIFACT_MAX_BYTES, } from "./contained-output.js";
4
4
  import { RUN_BUNDLE_FILE } from "./bundle.js";
5
5
  import { isRunBundle, isRunPointer } from "./bundle-shape.js";
6
- import { loadRunTraces } from "./stream-trace.js";
6
+ import { displayedTraces, loadRunTraces, loadStreamTraces } from "./stream-trace.js";
7
7
  import { readUnchangedFile } from "./evidence-files.js";
8
8
  /** The run input that resolves through latest.json rather than naming a run directory. */
9
9
  export const LATEST_RUN_ALIAS = "latest";
@@ -119,8 +119,12 @@ export async function loadRunBundle(cwdInput, runInput) {
119
119
  }
120
120
  return loadRunBundlePrepared(cwd, runPaths);
121
121
  }
122
- /** loadRunBundle for a caller that already holds prepared paths. Revalidates them first. */
123
- export async function loadRunBundlePrepared(cwdInput, runPaths) {
122
+ /**
123
+ * loadRunBundle for a caller that already holds prepared paths. Revalidates them first. With
124
+ * `display`, a participant whose trace the loader refuses, such as one past the read limit, is
125
+ * kept as displayedTraces shows it, for a page that shows the run.
126
+ */
127
+ export async function loadRunBundlePrepared(cwdInput, runPaths, options = {}) {
124
128
  const cwd = path.resolve(cwdInput);
125
129
  await validatePreparedRunArtifactPaths(runPaths);
126
130
  const bundlePath = path.join(runPaths.absoluteRunRoot, RUN_BUNDLE_FILE);
@@ -134,7 +138,9 @@ export async function loadRunBundlePrepared(cwdInput, runPaths) {
134
138
  }
135
139
  return {
136
140
  // A trace that does not match its index throws StreamTraceRefusedError, naming the participant.
137
- bundle: await loadRunTraces(runPaths, bundle),
141
+ bundle: options.display === true
142
+ ? displayedTraces(await loadStreamTraces(runPaths, bundle))
143
+ : await loadRunTraces(runPaths, bundle),
138
144
  bundlePath: path.relative(cwd, bundlePath),
139
145
  runDir: runPaths.absoluteRunRoot,
140
146
  };
package/dist/run/run.js CHANGED
@@ -3,6 +3,7 @@
3
3
  // their own evidence files; this module owns when the run starts, how it is published, and how it
4
4
  // is closed on every exit.
5
5
  import { randomBytes } from "node:crypto";
6
+ import { redactEncodedText } from "../evidence/encoded-text.js";
6
7
  import { redactText } from "../evidence/redaction.js";
7
8
  import { buildObserverData } from "../observer/data.js";
8
9
  import { splitObserverData } from "../observer/participant-items.js";
@@ -148,24 +149,35 @@ async function writeRunJson(paths, liveItems, bundle) {
148
149
  });
149
150
  return { ...publicBundle, streams };
150
151
  }
152
+ /** The warn event that names the files the final sandbox id sweep could not read. */
153
+ const unsweptEvent = (message, at) => ({
154
+ id: "event-sandbox-ids-unresolved",
155
+ at: new Date(at).toISOString(),
156
+ level: "warn",
157
+ type: "sandbox-ids.unresolved",
158
+ message,
159
+ });
151
160
  function runPublisher(args) {
152
161
  const { options, runId, createdAt, paths, runStatus, now, admit, participantsRan } = args;
153
162
  const observer = observerTarget(options);
154
163
  // The study's warnings join whatever events the route wrote, once, in every write. A warning can
155
- // quote the study file, which may name a sandbox URL a participant needs, so each goes through
156
- // the redaction a run failure gets.
164
+ // quote the study file, which may name a sandbox URL a participant needs or hold an encoded key,
165
+ // so each goes through the redaction study text gets.
157
166
  const studyWarnings = (options.warnings ?? []).map((message, index) => ({
158
167
  id: `event-study-warning-${String(index + 1).padStart(3, "0")}`,
159
168
  at: createdAt,
160
169
  level: "warn",
161
170
  type: "study.warning",
162
- message: redactText(message),
171
+ message: redactEncodedText(message),
163
172
  }));
164
- // So do the host suspensions the heartbeat has seen by the time of the write.
173
+ // So do the host suspensions the heartbeat has seen by the time of the write, and the files the
174
+ // final sandbox id sweep could not read.
175
+ let unswept = [];
165
176
  const withRunEvents = (events) => {
166
177
  const owned = [
167
178
  ...studyWarnings,
168
179
  ...hostSuspensionEvents(runStatus.suspensions(), Date.parse(createdAt)),
180
+ ...unswept,
169
181
  ];
170
182
  return [...events.filter((event) => !owned.some((mine) => mine.id === event.id)), ...owned];
171
183
  };
@@ -231,7 +243,12 @@ function runPublisher(args) {
231
243
  await writeBundleFiles(bundle, finishedOutcome(outcome), (publicBundle) => runStatus.finish(runStatusOutcome(publicBundle)));
232
244
  // The final run.json indexes each finished trace, so the live files are journal no reader needs.
233
245
  await liveItems.remove();
234
- await scrubRunSandboxIds(paths);
246
+ const unread = await scrubRunSandboxIds(paths);
247
+ if (unread !== undefined) {
248
+ unswept = [unsweptEvent(unread, now())];
249
+ // The sweep reads the files the final write left, so run.json is written again to record it.
250
+ await writeBundleFiles(bundle, finishedOutcome(outcome), async () => { });
251
+ }
235
252
  await writePointer();
236
253
  let current = { ok: outcome.ok, execution: outcome.execution };
237
254
  // An Observer that did not render, judged under the route's policy and written to run.json
@@ -45,10 +45,11 @@ export declare function scrubSandboxIdBytes(file: string, bytes: Buffer, ids: re
45
45
  * receipts, with the label in place of the id. Writers such as a participant's actor.json can
46
46
  * quote an SDK error that names the sandbox. Run.finish calls it before analysis reads the run.
47
47
  * Best effort for writes and listings: verify grades a file this misses. A text file it refuses
48
- * to read, or a journal it refuses, throws once the sweep ends, naming each, since such a file
49
- * may still hold an id.
48
+ * to read, such as a trace past RUN_ARTIFACT_MAX_BYTES, may still hold an id, so the sweep
49
+ * returns a warning that names each such file and why, for the caller to record; the run goes
50
+ * on. A journal it refuses throws, since without its ids no file can be swept.
50
51
  */
51
- export declare function scrubRunSandboxIds(paths: PreparedRunArtifactPaths): Promise<void>;
52
+ export declare function scrubRunSandboxIds(paths: PreparedRunArtifactPaths): Promise<string | undefined>;
52
53
  /**
53
54
  * A run's result as it may be returned or printed: the same object when it names no sandbox,
54
55
  * otherwise its public view, scrubbed of the ids the run's receipts journal.
@@ -197,16 +197,17 @@ const MAX_SWEPT_ENTRIES = 10_000;
197
197
  * receipts, with the label in place of the id. Writers such as a participant's actor.json can
198
198
  * quote an SDK error that names the sandbox. Run.finish calls it before analysis reads the run.
199
199
  * Best effort for writes and listings: verify grades a file this misses. A text file it refuses
200
- * to read, or a journal it refuses, throws once the sweep ends, naming each, since such a file
201
- * may still hold an id.
200
+ * to read, such as a trace past RUN_ARTIFACT_MAX_BYTES, may still hold an id, so the sweep
201
+ * returns a warning that names each such file and why, for the caller to record; the run goes
202
+ * on. A journal it refuses throws, since without its ids no file can be swept.
202
203
  */
203
204
  export async function scrubRunSandboxIds(paths) {
204
205
  const ids = await readRunSandboxIds(paths);
205
206
  if (ids.length === 0)
206
- return;
207
+ return undefined;
207
208
  const valid = await validatePreparedRunRootIdentity(paths).then(() => true, () => false);
208
209
  if (!valid)
209
- return;
210
+ return undefined;
210
211
  const refused = [];
211
212
  const scrub = async (file) => {
212
213
  const read = await readContainedRegularFile(paths, file, RUN_ARTIFACT_MAX_BYTES);
@@ -242,8 +243,9 @@ export async function scrubRunSandboxIds(paths) {
242
243
  await walk("").catch(() => {
243
244
  // A directory that changed while it was listed ends the sweep; verify names what it missed.
244
245
  });
245
- if (refused.length > 0)
246
- throw new Error(`The sandbox id sweep could not read every run file, so these may still name a sandbox: ${refused.join("; ")}.`);
246
+ return refused.length === 0
247
+ ? undefined
248
+ : `The sandbox id sweep could not read every run file, so these may still name a sandbox: ${refused.join("; ")}.`;
247
249
  }
248
250
  /**
249
251
  * A run's result as it may be returned or printed: the same object when it names no sandbox,
@@ -11,10 +11,16 @@ export declare class StreamTraceRefusedError extends Error {
11
11
  constructor(streamId: string, detail: string, advice?: string);
12
12
  }
13
13
  /**
14
- * A trace or live file whose index records more than the loader reads of one file. The loader reads
15
- * no file past the limit, so a copy of the file is refused too.
14
+ * A trace or live file whose index records more than the loader reads of one file, `maxBytes`. The
15
+ * loader reads no file past the limit, so a copy of the file is refused too.
16
16
  */
17
17
  export declare class StreamTraceTooLargeError extends StreamTraceRefusedError {
18
+ /** The file's run-relative path, as its index records it. */
19
+ readonly file: string;
20
+ constructor(streamId: string, index: {
21
+ path: string;
22
+ bytes: number;
23
+ }, maxBytes: number);
18
24
  }
19
25
  /** `trace` without the per-request lists; every other field as it is. */
20
26
  export declare function withoutRequestLists<T extends object>(trace: T): WithoutRequestLists<T>;
@@ -28,13 +28,17 @@ export class StreamTraceRefusedError extends Error {
28
28
  /** Why the loader reads no trace or live file whose index records more than `maxBytes`. */
29
29
  const pastLimitDetail = (index, maxBytes) => `run.json records ${index.path} at ${index.bytes} bytes, more than the ${maxBytes} bytes humanish reads of one file`;
30
30
  /**
31
- * A trace or live file whose index records more than the loader reads of one file. The loader reads
32
- * no file past the limit, so a copy of the file is refused too.
31
+ * A trace or live file whose index records more than the loader reads of one file, `maxBytes`. The
32
+ * loader reads no file past the limit, so a copy of the file is refused too.
33
33
  */
34
34
  export class StreamTraceTooLargeError extends StreamTraceRefusedError {
35
+ /** The file's run-relative path, as its index records it. */
36
+ file;
37
+ constructor(streamId, index, maxBytes) {
38
+ super(streamId, pastLimitDetail(index, maxBytes), "Restoring a copy does not help; open the file with another tool, such as jq, to read its items.");
39
+ this.file = index.path;
40
+ }
35
41
  }
36
- /** The refusal of a trace or live file whose index records more than `maxBytes`. */
37
- const tooLarge = (streamId, index, maxBytes) => new StreamTraceTooLargeError(streamId, pastLimitDetail(index, maxBytes), "Restoring a copy does not help; open the file with another tool, such as jq, to read its items.");
38
42
  const sha256 = (bytes) => createHash("sha256").update(bytes).digest("hex");
39
43
  const isScreenshot = (artifact) => artifact.kind === "screenshot";
40
44
  /**
@@ -139,7 +143,9 @@ export function splitStreamTraces(bundle) {
139
143
  throw new Error(`Two participants' traces would both be written to ${filePath}.`);
140
144
  claimed.add(traceFileKey(filePath));
141
145
  const screenshots = stream.artifacts.filter(isScreenshot);
142
- const bytes = Buffer.from(`${JSON.stringify({ ...actor, artifacts: screenshots }, null, 2)}\n`);
146
+ // Compact: every reader refuses a file past RUN_ARTIFACT_MAX_BYTES, and indenting made a
147
+ // trace 1.36 times its compact size on 236 recorded computer-use traces.
148
+ const bytes = Buffer.from(`${JSON.stringify({ ...actor, artifacts: screenshots })}\n`);
143
149
  files.push({ path: filePath, bytes });
144
150
  const { items, ...summary } = actor;
145
151
  return {
@@ -274,7 +280,7 @@ async function readLiveItems(read, stream, maxBytes, liveAfter) {
274
280
  const after = liveAfter?.(index);
275
281
  if (index.bytes > maxBytes) {
276
282
  if (after === undefined || !cursorFits(index, after))
277
- throw tooLarge(stream.id, index, maxBytes);
283
+ throw new StreamTraceTooLargeError(stream.id, index, maxBytes);
278
284
  // The caller keeps the prefix it holds, read when the file was under the limit.
279
285
  return {
280
286
  stream: { ...stream, liveActor: { ...summary, items: [] } },
@@ -343,7 +349,7 @@ async function readStreamTrace(read, stream, maxBytes, liveAfter) {
343
349
  if (stream.trace !== undefined) {
344
350
  assertIndex(stream);
345
351
  if (stream.trace.bytes > maxBytes)
346
- throw tooLarge(stream.id, stream.trace, maxBytes);
352
+ throw new StreamTraceTooLargeError(stream.id, stream.trace, maxBytes);
347
353
  const file = await read(stream.trace.path, maxBytes);
348
354
  if ("refused" in file)
349
355
  throw new StreamTraceRefusedError(stream.id, file.refused);
@@ -21,9 +21,9 @@ export declare function runStudyWith(config: StudyConfig, options: InternalRunSt
21
21
  /**
22
22
  * The package's study runner (`src/index.ts`): runStudyWith with the public options only. A
23
23
  * config that skipped parseStudy is checked first (libraryConfigRefusal): a humanish.lab.v2 config
24
- * or one with a humanish 0.110 field, no `actor`, or a `route` the config does not take. That, or a
25
- * field RunStudyOptions no longer has, is refused in the route's own result envelope before
26
- * anything runs.
24
+ * or one with a humanish 0.110 field, no `actor`, a `route` the config does not take, or an
25
+ * `actor.model` verify would flag. That, or a field RunStudyOptions no longer has, is refused in the
26
+ * route's own result envelope before anything runs.
27
27
  */
28
28
  export declare function runStudy(config: StudyConfig, options: RunStudyOptions): Promise<StudyOutcome>;
29
29
  /** A study planned once: the route's refusal, or the run of its plan. */
package/dist/run-study.js CHANGED
@@ -34,9 +34,9 @@ export async function runStudyWith(config, options, deps = {}) {
34
34
  /**
35
35
  * The package's study runner (`src/index.ts`): runStudyWith with the public options only. A
36
36
  * config that skipped parseStudy is checked first (libraryConfigRefusal): a humanish.lab.v2 config
37
- * or one with a humanish 0.110 field, no `actor`, or a `route` the config does not take. That, or a
38
- * field RunStudyOptions no longer has, is refused in the route's own result envelope before
39
- * anything runs.
37
+ * or one with a humanish 0.110 field, no `actor`, a `route` the config does not take, or an
38
+ * `actor.model` verify would flag. That, or a field RunStudyOptions no longer has, is refused in the
39
+ * route's own result envelope before anything runs.
40
40
  */
41
41
  export async function runStudy(config, options) {
42
42
  const rejected = libraryConfigRefusal(config);
@@ -16,8 +16,6 @@ export interface BundleEstimate {
16
16
  readonly runFileBytes: number;
17
17
  /** The largest participant trace file, which analysis also reads whole. */
18
18
  readonly largestTraceBytes: number;
19
- /** Every participant's trace file together, whose items the Observer page joins in one string. */
20
- readonly tracesBytes: number;
21
19
  /** observer/index.html, which verify reads whole. */
22
20
  readonly observerPageBytes: number;
23
21
  /** Every file, screenshots included. */