humanish 0.133.1 → 0.135.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. package/CHANGELOG.md +28 -0
  2. package/dist/actors/computer-use/loop/actions.d.ts +2 -0
  3. package/dist/actors/computer-use/loop/actions.js +11 -5
  4. package/dist/actors/computer-use/loop/backstop.d.ts +23 -2
  5. package/dist/actors/computer-use/loop/backstop.js +40 -7
  6. package/dist/actors/computer-use/loop/session.d.ts +3 -0
  7. package/dist/actors/computer-use/loop/session.js +3 -0
  8. package/dist/actors/computer-use/loop/streak.d.ts +18 -0
  9. package/dist/actors/computer-use/loop/streak.js +47 -0
  10. package/dist/actors/computer-use/loop/types.d.ts +5 -1
  11. package/dist/actors/computer-use/loop/watch.d.ts +22 -4
  12. package/dist/actors/computer-use/loop/watch.js +83 -27
  13. package/dist/actors/computer-use/loop.js +18 -6
  14. package/dist/actors/contract.d.ts +3 -1
  15. package/dist/analysis/automatic-config.js +5 -3
  16. package/dist/analysis/execute.js +2 -1
  17. package/dist/analysis/service.js +5 -3
  18. package/dist/analysis/source.d.ts +6 -0
  19. package/dist/analysis/source.js +15 -13
  20. package/dist/cli/commands/run-signals.js +6 -2
  21. package/dist/evidence/encoded-text.d.ts +13 -3
  22. package/dist/evidence/encoded-text.js +88 -25
  23. package/dist/observer/page-data.d.ts +6 -0
  24. package/dist/observer/page-data.js +34 -7
  25. package/dist/observer/participant-items.d.ts +15 -1
  26. package/dist/observer/participant-items.js +4 -3
  27. package/dist/observer/render.js +5 -3
  28. package/dist/observer/run-routes.js +4 -4
  29. package/dist/observer-app.html +4 -4
  30. package/dist/routes/computer-use/bundle-parts.d.ts +9 -0
  31. package/dist/routes/computer-use/bundle-parts.js +24 -4
  32. package/dist/routes/computer-use/fanout-bundle.js +8 -6
  33. package/dist/routes/computer-use/participant-runs.js +2 -2
  34. package/dist/routes/computer-use/single-bundle.js +5 -5
  35. package/dist/routes/scripted/setup.js +62 -17
  36. package/dist/routes/shared-world/bundle.js +3 -3
  37. package/dist/routes/shared-world/participant-specs.js +2 -2
  38. package/dist/routes/terminal/bundle.js +2 -2
  39. package/dist/run/locate.d.ts +8 -2
  40. package/dist/run/locate.js +10 -4
  41. package/dist/run/run.js +22 -5
  42. package/dist/run/sandbox-ids.d.ts +4 -3
  43. package/dist/run/sandbox-ids.js +8 -6
  44. package/dist/run/stream-trace.d.ts +8 -2
  45. package/dist/run/stream-trace.js +13 -7
  46. package/dist/run-study.d.ts +3 -3
  47. package/dist/run-study.js +3 -3
  48. package/dist/study/bundle-estimate.d.ts +0 -2
  49. package/dist/study/bundle-estimate.js +33 -33
  50. package/dist/study/config.d.ts +4 -3
  51. package/dist/study/config.js +8 -5
  52. package/dist/study/parse/actors.d.ts +6 -0
  53. package/dist/study/parse/actors.js +14 -0
  54. package/dist/study/parse/subject-state.js +4 -4
  55. package/dist/study/participant-assignment.js +2 -2
  56. package/dist/study/persona.d.ts +7 -1
  57. package/dist/study/persona.js +13 -5
  58. package/dist/study/plan-base.js +4 -3
  59. package/dist/verify/artifacts.d.ts +7 -1
  60. package/dist/verify/artifacts.js +15 -4
  61. package/dist/verify/verify.js +29 -8
  62. package/docs/architecture/actor-contract.md +3 -1
  63. package/docs/architecture/observer.md +24 -7
  64. package/docs/architecture/state-driven-executor.md +7 -4
  65. package/docs/contracts/run-bundle.md +16 -2
  66. package/docs/contracts/study-analysis.md +8 -5
  67. package/docs/decisions/0002-observer-is-one-self-contained-file.md +3 -3
  68. package/package.json +1 -1
@@ -16,13 +16,14 @@
16
16
  // observer-data.json leaves the lists out too and was 0.94 to 1.12 of run.json's size on the live
17
17
  // runs measured, so it passes 32 MiB only after run.json passes 16 MiB, and the estimate does not
18
18
  // check it.
19
- // The Observer page and the Observer server join every participant's items in one string before
20
- // gzip. A bundle export renders the Observer page, so it fails there too once --max-bytes is
21
- // raised past its 25 MiB default, which refuses such a run first. Writing run.json joins none: the
22
- // sandbox id scrub (publicSandboxView) works one participant at a time. V8's longest string is
23
- // 536,870,888 characters on Node 24. A trace file's bytes are at least the characters of its
24
- // items as compact JSON, so the estimate compares the traces' bytes and may warn a little early.
25
- import { constants } from "node:buffer";
19
+ // A page past 32 MiB grades the run local_only. humanish export then needs --local-only, and a
20
+ // bundle export fails even with it, since its copy must verify as share_ready.
21
+ // The Observer feed, verify, analysis and the run.json writer each hold one participant's items in
22
+ // a string at a time. The page holds all of them gzipped in base64, about 0.18 characters for each
23
+ // trace byte, so it reaches V8's longest string, 536,870,888 characters, near 3 GB of traces.
24
+ // humanish observe on 25 participants with 30 MiB traces wrote a 138 MiB page at a peak of
25
+ // 3.8 GiB, so memory may run out first. Neither limit was measured past that run, and the estimate
26
+ // checks neither.
26
27
  import { EVIDENCE_LIMITS } from "../analysis/analysis-limits.js";
27
28
  import { RUN_ARTIFACT_MAX_BYTES } from "../run/contained-output.js";
28
29
  import { formatDuration } from "../run/projection.js";
@@ -32,9 +33,15 @@ import { runEntryLimit } from "../verify/scan-limit.js";
32
33
  // the 0.125.0 smoke test and the 2026-10-12 live checks, 10 participants with 1 to 28 turns and 137
33
34
  // model requests. Dry: runs of 1, 2, 24 and 100 participants, for the Observer app and the run's
34
35
  // own files. Pace, session length and screenshots: the live computer-use runs kept on the
35
- // maintainer's machine. A trace's rate a turn on the public site/public/runs/lobby-0927 run
36
- // (8 participants, 20 to 78 turns) was 1,579 bytes plus 91 for the page, under the live fit.
37
- // The run.json and page data rates were measured again on those runs written as v4.
36
+ // maintainer's machine. The run.json and page data rates were measured again on those runs
37
+ // written as v4.
38
+ // The trace rates were measured on trace files written indented. The writer now writes them as
39
+ // compact JSON, so each trace rate below is the indented figure divided by 1.36, the ratio of
40
+ // indented to compact bytes over 236 kept computer-use traces (1.17 to 1.49 each), and the page's
41
+ // share of the traces is multiplied by it. A trace's rate a turn on the public
42
+ // site/public/runs/lobby-0927 run (8 participants, 20 to 78 turns) was 1,579 bytes plus 91 for the
43
+ // page indented, under the live fit. Its participant of 78 turns writes a trace of 149,637 bytes
44
+ // indented and 107,897 compact.
38
45
  const RATES = Object.freeze({
39
46
  /** run.json bytes for the run itself: its header, study, review and cost. Live fit: 4,381. */
40
47
  runBytes: 4_400,
@@ -44,21 +51,21 @@ const RATES = Object.freeze({
44
51
  * bytes and those of two 30,647 to 38,948, beside 20,900 and 37,400 at these rates.
45
52
  */
46
53
  participantBytes: 16_500,
47
- /** A trace file's bytes before its turns: the actor summary. Live fit: 5,378. */
48
- participantTraceBytes: 5_400,
54
+ /** A trace file's bytes before its turns: the actor summary. Live fit: 5,378 indented. */
55
+ participantTraceBytes: 4_000,
49
56
  /**
50
57
  * Trace bytes a model turn adds: its items, its screenshot entry and its request's usage, plus 91
51
- * for the `page` each screenshot item now carries. Live fit: 1,965 a turn. Participants with 4
52
- * or more turns ranged from 1,140 to 4,532; the busiest made 311 items in 15 turns, most of them
53
- * UI actions. Analysis refuses the whole run when one trace passes its limit, so each trace is
54
- * counted at the busiest participant's rate.
58
+ * for the `page` each screenshot item now carries, all indented. Live fit: 1,965 a turn.
59
+ * Participants with 4 or more turns ranged from 1,140 to 4,532; the busiest made 311 items in 15
60
+ * turns, most of them UI actions. Analysis refuses the whole run when one trace passes its limit,
61
+ * so each trace is counted at the busiest participant's rate, 4,700 indented.
55
62
  */
56
- turnTraceBytes: 4_700,
63
+ turnTraceBytes: 3_500,
57
64
  /**
58
65
  * Trace bytes a dwell frame adds: 395 for its item and entry on the kept runs with dwell frames,
59
- * plus 91 for the page.
66
+ * plus 91 for the page, indented.
60
67
  */
61
- frameTraceBytes: 500,
68
+ frameTraceBytes: 400,
62
69
  /**
63
70
  * Seconds between model turns: the median of the 27 of 185 kept live participants that worked 5
64
71
  * minutes or more was 10.4.
@@ -83,12 +90,12 @@ const RATES = Object.freeze({
83
90
  */
84
91
  observerPageDataShare: 0.85,
85
92
  /**
86
- * The page's gzipped base64 items, as a share of all the traces' bytes: 0.087 to 0.114 on the
87
- * live runs of two participants, 0.097 for the busiest participant alone (about 475 bytes a
88
- * turn), 0.142 to 0.184 on runs of one participant and 4 to 11 turns, and 0.095 on 0.124.0's
89
- * synthetic 8-hour run.
93
+ * The page's gzipped base64 items, as a share of all the traces' bytes indented: 0.087 to 0.114
94
+ * on the live runs of two participants, 0.097 for the busiest participant alone (about 475 bytes
95
+ * a turn), 0.142 to 0.184 on runs of one participant and 4 to 11 turns, and 0.095 on 0.124.0's
96
+ * synthetic 8-hour run. The estimate counted 0.12 of indented traces.
90
97
  */
91
- observerPageItemsShare: 0.12,
98
+ observerPageItemsShare: 0.16,
92
99
  /** A desktop screenshot: the median of 3,990 kept live PNG frames was 107,895 bytes. */
93
100
  screenshotBytes: 110_000,
94
101
  /**
@@ -163,7 +170,6 @@ export function estimateBundle(participants, sessionMs) {
163
170
  return {
164
171
  runFileBytes,
165
172
  largestTraceBytes: largestTrace,
166
- tracesBytes: traces,
167
173
  observerPageBytes,
168
174
  // observer-data.json is about run.json's size.
169
175
  totalBytes: observerPageBytes + screenshots * RATES.screenshotBytes + traces + 2 * runFileBytes,
@@ -186,7 +192,6 @@ export function estimateBundle(participants, sessionMs) {
186
192
  export function bundleLimitReason(estimate) {
187
193
  const runFile = estimate.runFileBytes > EVIDENCE_LIMITS.sourceBytes;
188
194
  const page = estimate.observerPageBytes > RUN_ARTIFACT_MAX_BYTES;
189
- const joined = estimate.tracesBytes > constants.MAX_STRING_LENGTH;
190
195
  const entries = estimate.entries > estimate.entryLimit;
191
196
  const passed = [
192
197
  ...(runFile
@@ -201,12 +206,7 @@ export function bundleLimitReason(estimate) {
201
206
  : []),
202
207
  ...(page
203
208
  ? [
204
- `The Observer page, observer/index.html, may grow to about ${mib(estimate.observerPageBytes)}, past the ${mib(RUN_ARTIFACT_MAX_BYTES)} verify reads in one file.`,
205
- ]
206
- : []),
207
- ...(joined
208
- ? [
209
- `All participants' traces together may grow to about ${mib(estimate.tracesBytes)}, past the ${mib(constants.MAX_STRING_LENGTH)} that fits in one string. The Observer page puts all their items in one string, so the page would not build. A bundle export allowed past its 25 MiB default renders that page, so it would fail too.`,
209
+ `The Observer page, observer/index.html, may grow to about ${mib(estimate.observerPageBytes)}, past the ${mib(RUN_ARTIFACT_MAX_BYTES)} verify reads in one file, so verify would grade the run local_only: humanish export would refuse it without --local-only, and a bundle export would fail.`,
210
210
  ]
211
211
  : []),
212
212
  ...(entries
@@ -236,7 +236,7 @@ export function bundleLimitReason(estimate) {
236
236
  ...(runFile ? ["shorten the mission (actor.mission)"] : []),
237
237
  ];
238
238
  // A trace is one participant's, so splitting the study helps only the run's own files.
239
- if (runFile || page || joined || entries || changes.length === 0)
239
+ if (runFile || page || entries || changes.length === 0)
240
240
  changes.push("split the participants across studies");
241
241
  const advice = changes.length < 2
242
242
  ? changes.join("")
@@ -30,9 +30,10 @@ interface ConfigRefusal {
30
30
  }
31
31
  /**
32
32
  * Why runStudy refuses a config a library caller passes, or undefined: a humanish.lab.v2 config,
33
- * one that still sets a StudyConfig field of humanish 0.110, one with no `actor` object, and one
34
- * whose `route` is not the route its subject and actor take, with parseStudy's message. A planner
35
- * reads only the v3 fields, so a budget left in `execution.caps` would otherwise run uncapped.
33
+ * one that still sets a StudyConfig field of humanish 0.110, one with no `actor` object, one
34
+ * whose `route` is not the route its subject and actor take, and one whose `actor.model` verify
35
+ * would flag, with parseStudy's message. A planner reads only the v3 fields, so a budget left in
36
+ * `execution.caps` would otherwise run uncapped, and passes the model on as written.
36
37
  */
37
38
  export declare function libraryConfigRefusal(config: StudyConfig): ConfigRefusal | undefined;
38
39
  export {};
@@ -30,7 +30,7 @@ import { isLocalBrowserStudy, localBrowserDefaults, localBrowserUnsupportedReaso
30
30
  import { isRecord } from "../run/type-guards.js";
31
31
  import { compositionReason } from "./composition-rules.js";
32
32
  import { V2_SCHEMA, V2_UNSUPPORTED_MESSAGE, v2FieldMoves } from "./migrate/v2.js";
33
- import { parseActorFields, parseActorType, parseParticipantEntries } from "./parse/actors.js";
33
+ import { actorModelReason, parseActorFields, parseActorType, parseParticipantEntries, } from "./parse/actors.js";
34
34
  import { parseComms, recipientParticipantId } from "./parse/comms.js";
35
35
  import { parseCaps, parseDefaults, parseExecution, parsePolicies, parseReview, } from "./parse/execution.js";
36
36
  import { ACTOR_REQUIRED, readStudyFront, routeNameReason, } from "./parse/front.js";
@@ -96,9 +96,10 @@ function routeMismatchReason(config) {
96
96
  }
97
97
  /**
98
98
  * Why runStudy refuses a config a library caller passes, or undefined: a humanish.lab.v2 config,
99
- * one that still sets a StudyConfig field of humanish 0.110, one with no `actor` object, and one
100
- * whose `route` is not the route its subject and actor take, with parseStudy's message. A planner
101
- * reads only the v3 fields, so a budget left in `execution.caps` would otherwise run uncapped.
99
+ * one that still sets a StudyConfig field of humanish 0.110, one with no `actor` object, one
100
+ * whose `route` is not the route its subject and actor take, and one whose `actor.model` verify
101
+ * would flag, with parseStudy's message. A planner reads only the v3 fields, so a budget left in
102
+ * `execution.caps` would otherwise run uncapped, and passes the model on as written.
102
103
  */
103
104
  export function libraryConfigRefusal(config) {
104
105
  const raw = config;
@@ -114,7 +115,9 @@ export function libraryConfigRefusal(config) {
114
115
  };
115
116
  if (!isRecord(raw.actor))
116
117
  return { code: "HUMANISH_STUDY_INVALID", message: ACTOR_REQUIRED };
117
- const message = routeNameReason(raw.route) ?? routeMismatchReason(config);
118
+ const message = routeNameReason(raw.route) ??
119
+ routeMismatchReason(config) ??
120
+ actorModelReason(raw.actor.model, "actor");
118
121
  return message === undefined ? undefined : { code: "HUMANISH_STUDY_INVALID", message };
119
122
  }
120
123
  // Each section in a fixed order, so a file with more than one error always reports the same one
@@ -6,6 +6,12 @@ export declare function parseActorType(entry: Record<string, unknown>, path: str
6
6
  ok: true;
7
7
  value: string;
8
8
  } | StudyParseFailure;
9
+ /**
10
+ * Why an actor's `model` is refused, or undefined. The provider is asked for the model and its trace
11
+ * records the name, so redacting it would ask for another model. Text verify would flag, as written
12
+ * or encoded, is refused instead. runStudy asks this of a config built without parseStudy too.
13
+ */
14
+ export declare function actorModelReason(raw: unknown, path: string): string | undefined;
9
15
  /** An actor's keys other than `type`. */
10
16
  type ActorFields = Omit<StudyActor, "type">;
11
17
  /**
@@ -51,6 +51,17 @@ function parsePersonaId(raw, path) {
51
51
  }
52
52
  return { ok: true, value: persona };
53
53
  }
54
+ /**
55
+ * Why an actor's `model` is refused, or undefined. The provider is asked for the model and its trace
56
+ * records the name, so redacting it would ask for another model. Text verify would flag, as written
57
+ * or encoded, is refused instead. runStudy asks this of a config built without parseStudy too.
58
+ */
59
+ export function actorModelReason(raw, path) {
60
+ const model = str(raw);
61
+ if (model === undefined || !scanEncodedText(model).sensitive)
62
+ return undefined;
63
+ return `\`${path}.model\` looks like a secret or a local path, and the run records it. Set the model id your provider lists, or leave \`${path}.model\` out to use the default.`;
64
+ }
54
65
  /**
55
66
  * Every actor key other than `type`. The parser reads them after the participant entries, so a file
56
67
  * with errors in both reports the entry first.
@@ -65,6 +76,9 @@ export function parseActorFields(entry, path) {
65
76
  const mission = str(entry.mission);
66
77
  if (mission)
67
78
  actor.mission = mission;
79
+ const modelReason = actorModelReason(entry.model, path);
80
+ if (modelReason)
81
+ return invalid(modelReason);
68
82
  const model = str(entry.model);
69
83
  if (model)
70
84
  actor.model = model;
@@ -1,7 +1,7 @@
1
1
  // The state and env channels of a served subject (clone or local-tree): `subject.envValues`,
2
2
  // the structural parse of `subject.state`, and subjectStateInvalidReason, the semantic check
3
3
  // parseStudy and the computer-use route share.
4
- import { containsSensitive } from "../../evidence/redaction.js";
4
+ import { scanEncodedText } from "../../evidence/encoded-text.js";
5
5
  import { ENV_NAME_PATTERN, invalid, posInt, str, strList } from "./values.js";
6
6
  import { isRecord } from "../../run/type-guards.js";
7
7
  /**
@@ -29,9 +29,9 @@ export function parseEnvValues(raw) {
29
29
  if (value === undefined) {
30
30
  return invalid(`\`subject.envValues.${name}\` must be a string, number, or boolean.`);
31
31
  }
32
- // Reuse the redaction module's own detector rather than inventing a second opinion about what
33
- // a secret looks like; the two must never disagree about the same string.
34
- if (containsSensitive(value)) {
32
+ // verify blocks a run whose files scanEncodedText reads as sensitive, so this refuses what that
33
+ // scan reads: a secret or local path as written, or decoded from base64 or an escape.
34
+ if (scanEncodedText(value).sensitive) {
35
35
  return invalid(`\`subject.envValues.${name}\` looks like a secret or a local path, and envValues are committed with the study and recorded in evidence. List the name in \`subject.env\` instead: its value then comes from your environment and is never stored.`);
36
36
  }
37
37
  envValues[name] = value;
@@ -1,9 +1,9 @@
1
- import { redactText } from "../evidence/redaction.js";
1
+ import { redactEncodedText } from "../evidence/encoded-text.js";
2
2
  /** Project only authored participant-facing fields. Never pass runtime-composed instructions:
3
3
  * they can contain inbox URLs or multiplayer grants. Pick fields explicitly so a task's hidden
4
4
  * success criteria cannot cross this evidence boundary, even from an untyped library caller. */
5
5
  export function participantAssignment(assignment, scrubKnownValues = (text) => text) {
6
- const sanitize = (text) => redactText(scrubKnownValues(text));
6
+ const sanitize = (text) => redactEncodedText(scrubKnownValues(text));
7
7
  return {
8
8
  mission: sanitize(assignment.mission),
9
9
  ...(assignment.focus === undefined ? {} : { focus: sanitize(assignment.focus) }),
@@ -37,7 +37,13 @@ export declare function parseResolvedPersona(raw: unknown, fallback: {
37
37
  }, warnings?: string[]): ResolvedPersona;
38
38
  export declare function personaToDirectives(persona: ResolvedPersona): PersonaDirectives;
39
39
  export declare function renderPersonaPromptSection(persona: ResolvedPersona): string;
40
- /** Only the authored persona section; excludes task criteria, runtime grants and inbox URLs. */
40
+ /**
41
+ * Only the authored persona section; excludes task criteria, runtime grants and inbox URLs. With
42
+ * `scrub`, the run's known values come out first, then keys as written or encoded. Without it, only
43
+ * keys as written come out. A route that learns its known values later passes this brief to
44
+ * scrubPersonaBrief, and an encoded key replaced here would cut a known value that holds one, so
45
+ * that scrub could no longer find the value whole.
46
+ */
41
47
  export declare function personaBrief(persona: ResolvedPersona, scrub?: (text: string) => string): NonNullable<ActorPersonaRef["brief"]>;
42
48
  export declare function scrubPersonaBrief(persona: ActorPersonaRef, scrub: (text: string) => string): ActorPersonaRef;
43
49
  export {};
@@ -1,3 +1,4 @@
1
+ import { redactEncodedText } from "../evidence/encoded-text.js";
1
2
  import { digestText, redactText } from "../evidence/redaction.js";
2
3
  import { isRecord } from "../run/type-guards.js";
3
4
  export const PERSONA_BACKGROUND_MAX_BYTES = 32 * 1024;
@@ -136,8 +137,9 @@ export function personaToDirectives(persona) {
136
137
  ...(patience ? [`patience:${patience}`] : []),
137
138
  ...(skill ? [`skill:${skill}`] : []),
138
139
  ];
140
+ // A live participant's trace records traitsApplied as it is, and the need is free text.
139
141
  if (accessibilityNeeds)
140
- traitsApplied.push(`accessibility:${accessibilityNeeds}`);
142
+ traitsApplied.push(`accessibility:${redactEncodedText(accessibilityNeeds)}`);
141
143
  if (persona.constraints.length)
142
144
  traitsApplied.push(`constraints:${persona.constraints.length}`);
143
145
  return {
@@ -166,9 +168,15 @@ export function renderPersonaPromptSection(persona) {
166
168
  .filter(Boolean)
167
169
  .join("\n\n");
168
170
  }
169
- /** Only the authored persona section; excludes task criteria, runtime grants and inbox URLs. */
170
- export function personaBrief(persona, scrub = (text) => text) {
171
- const original = renderPersonaPromptSection(persona), text = redactText(scrub(original));
171
+ /**
172
+ * Only the authored persona section; excludes task criteria, runtime grants and inbox URLs. With
173
+ * `scrub`, the run's known values come out first, then keys as written or encoded. Without it, only
174
+ * keys as written come out. A route that learns its known values later passes this brief to
175
+ * scrubPersonaBrief, and an encoded key replaced here would cut a known value that holds one, so
176
+ * that scrub could no longer find the value whole.
177
+ */
178
+ export function personaBrief(persona, scrub) {
179
+ const original = renderPersonaPromptSection(persona), text = scrub === undefined ? redactText(original) : redactEncodedText(scrub(original));
172
180
  return {
173
181
  compilerVersion: PERSONA_COMPILER_VERSION,
174
182
  text,
@@ -180,7 +188,7 @@ export function personaBrief(persona, scrub = (text) => text) {
180
188
  export function scrubPersonaBrief(persona, scrub) {
181
189
  if (!persona.brief)
182
190
  return persona;
183
- const text = redactText(scrub(persona.brief.text));
191
+ const text = redactEncodedText(scrub(persona.brief.text));
184
192
  return {
185
193
  ...persona,
186
194
  brief: {
@@ -1,7 +1,7 @@
1
1
  // What every route planner shares: the plan fields common to all routes, the frozen residual config,
2
2
  // and the requirement and subject helpers.
3
3
  import { DEFAULT_OPENAI_CU_MODEL } from "../actors/computer-use/openai-provider.js";
4
- import { redactText } from "../evidence/redaction.js";
4
+ import { redactEncodedText } from "../evidence/encoded-text.js";
5
5
  // A YAML alias can make a parsed record contain itself (an inline persona that names its own
6
6
  // anchor), and structuredClone keeps the cycle, so each object is frozen once.
7
7
  function deepFreeze(value, frozen = new WeakSet()) {
@@ -112,8 +112,9 @@ export function planBase(config, input) {
112
112
  return {
113
113
  studyId: config.id,
114
114
  // Every bundle writer takes the title from here, and review.md, run.json and the Observer show
115
- // it, so a key or sandbox URL in it would make verify grade the run blocked.
116
- ...(config.title === undefined ? {} : { title: redactText(config.title) }),
115
+ // it, so a key or sandbox URL in it, as written or encoded, would make verify grade the run
116
+ // blocked.
117
+ ...(config.title === undefined ? {} : { title: redactEncodedText(config.title) }),
117
118
  residual: residualOf(config),
118
119
  dryRun: input.dryRun,
119
120
  ...(analysis === undefined
@@ -22,7 +22,13 @@ export interface UnscannedArtifacts {
22
22
  * and where it stopped, each in its own sentence. Undefined when the scan read everything.
23
23
  */
24
24
  export declare function unscannedMessage(unscanned: UnscannedArtifacts): string | undefined;
25
- export declare function missingLocalEvidenceArtifacts(runPaths: PreparedRunArtifactPaths, bundle: RunBundle): Promise<string[]>;
25
+ /**
26
+ * Why each local evidence path the bundle requires is missing or unusable. A path in
27
+ * `pastReadLimit` is a trace verify found larger than it reads, which verify lists as unscanned so
28
+ * the run grades local_only. It meets a requirement only while it is still past the limit when
29
+ * this check reads it, and never a screenshot or recording requirement.
30
+ */
31
+ export declare function missingLocalEvidenceArtifacts(runPaths: PreparedRunArtifactPaths, bundle: RunBundle, pastReadLimit: ReadonlySet<string>): Promise<string[]>;
26
32
  /**
27
33
  * The PNG frames that actor traces register as stream screenshots, relative to the run root.
28
34
  * RAW_SCREENSHOTS grades these, so the public-safety scan does not report them as unread. Only
@@ -47,7 +47,13 @@ function listed(paths) {
47
47
  const more = paths.length - MAX_LISTED_UNSCANNED;
48
48
  return `${paths.slice(0, MAX_LISTED_UNSCANNED).join(", ")}${more > 0 ? ` and ${more} more` : ""}`;
49
49
  }
50
- export async function missingLocalEvidenceArtifacts(runPaths, bundle) {
50
+ /**
51
+ * Why each local evidence path the bundle requires is missing or unusable. A path in
52
+ * `pastReadLimit` is a trace verify found larger than it reads, which verify lists as unscanned so
53
+ * the run grades local_only. It meets a requirement only while it is still past the limit when
54
+ * this check reads it, and never a screenshot or recording requirement.
55
+ */
56
+ export async function missingLocalEvidenceArtifacts(runPaths, bundle, pastReadLimit) {
51
57
  const recordings = new Map(bundle.streams.flatMap((stream) => stream.recording ? [[stream.recording.path, stream.recording]] : []));
52
58
  const requiredPaths = new Map();
53
59
  const addRequiredPath = (artifactPath, options = {}) => {
@@ -110,12 +116,15 @@ export async function missingLocalEvidenceArtifacts(runPaths, bundle) {
110
116
  }
111
117
  }
112
118
  // Reads overlap, so one frame's file system calls run while another frame's PNG is checked.
113
- const found = await mapWithConcurrency([...requiredPaths], EVIDENCE_READ_CONCURRENCY, ([artifactPath, requirements]) => missingArtifact(runPaths, artifactPath, requirements, recordings.get(artifactPath)));
119
+ const found = await mapWithConcurrency([...requiredPaths], EVIDENCE_READ_CONCURRENCY, ([artifactPath, requirements]) => missingArtifact(runPaths, artifactPath, { ...requirements, pastReadLimit: pastReadLimit.has(artifactPath) }, recordings.get(artifactPath)));
114
120
  return found.filter((entry) => entry !== null);
115
121
  }
116
122
  // Evidence files verify reads at once. Each read holds at most RUN_ARTIFACT_MAX_BYTES.
117
123
  const EVIDENCE_READ_CONCURRENCY = 8;
118
- /** Why one required path fails its requirements, or null when it meets them. */
124
+ /**
125
+ * Why one required path fails its requirements, or null when it meets them. `pastReadLimit` says
126
+ * verify listed the path as a trace past the read limit, so a file still past it is present.
127
+ */
119
128
  async function missingArtifact(runPaths, artifactPath, requirements, recording) {
120
129
  if (recording) {
121
130
  const handle = await openContainedRegularFile(runPaths, artifactPath);
@@ -136,7 +145,9 @@ async function missingArtifact(runPaths, artifactPath, requirements, recording)
136
145
  ? await readScreenshot(runPaths, artifactPath)
137
146
  : await readSafeRunArtifact(runPaths, artifactPath);
138
147
  if (read.state === "limit")
139
- return `${artifactPath} (larger than the ${RUN_ARTIFACT_MAX_BYTES} bytes verify reads)`;
148
+ return requirements.pastReadLimit && !requirements.screenshot
149
+ ? null
150
+ : `${artifactPath} (larger than the ${RUN_ARTIFACT_MAX_BYTES} bytes verify reads)`;
140
151
  if (read.state === "unavailable" ||
141
152
  (("size" in read ? read.size : read.bytes.length) === 0 && !requirements.allowEmpty))
142
153
  return artifactPath;
@@ -7,9 +7,9 @@ import { containsSensitiveJson } from "../evidence/sensitive-json.js";
7
7
  import { keyedSandboxIds, readRunSandboxIds } from "../run/sandbox-ids.js";
8
8
  import { validatePreparedRunArtifactPaths } from "../run/paths.js";
9
9
  import { isReadRunBundleSchema, RUN_BUNDLE_FILE, RUN_BUNDLE_SCHEMA, RUN_BUNDLE_V1_SCHEMA, RUN_BUNDLE_V2_SCHEMA, RUN_BUNDLE_V3_SCHEMA, } from "../run/bundle.js";
10
- import { ContainedReadRefusedError, refusalText } from "../run/contained-output.js";
10
+ import { ContainedReadRefusedError, readContainedRegularFile, refusalText, RUN_ARTIFACT_MAX_BYTES, } from "../run/contained-output.js";
11
11
  import { isCleanupResult, isRunBundle } from "../run/bundle-shape.js";
12
- import { loadStreamTraces } from "../run/stream-trace.js";
12
+ import { loadStreamTraces, StreamTraceTooLargeError } from "../run/stream-trace.js";
13
13
  import { readRunJsonIfExists, readRunTextIfExists, resolveRunPath, runJsonValue, } from "../run/locate.js";
14
14
  import { SANDBOX_RECEIPTS_ARTIFACT } from "../run/sandbox-receipts.js";
15
15
  import { isRecord } from "../run/type-guards.js";
@@ -79,7 +79,8 @@ export async function verifyResolvedRun(cwd, runInput, runPaths) {
79
79
  const bundlePath = path.join(runPaths.absoluteRunRoot, RUN_BUNDLE_FILE);
80
80
  const bundleRead = await readRunJsonIfExists(runPaths, RUN_BUNDLE_FILE);
81
81
  // Every check below reads the streams with their traces loaded; a trace that does not match its
82
- // index fails the evidence-artifacts check and its stream is checked without its items.
82
+ // index fails the evidence-artifacts check, and its stream, like one past the read limit, is
83
+ // checked without its items.
83
84
  const traces = await loadTraces(runPaths, runJsonValue(bundleRead));
84
85
  const { bundle, recorded, refusedTraces } = traces;
85
86
  const cleanupRead = await readRunJsonIfExists(runPaths, "cleanup.json");
@@ -102,9 +103,7 @@ export async function verifyResolvedRun(cwd, runInput, runPaths) {
102
103
  unscannedArtifacts.files.push(SANDBOX_RECEIPTS_ARTIFACT);
103
104
  }
104
105
  const publicSafetyFindings = await scanRunPublicSafetyArtifacts(runPaths, derivedPublicSafetyFindings, {
105
- recordingPaths: new Set(bundle
106
- ? bundle.streams.flatMap((stream) => (stream.recording ? [stream.recording.path] : []))
107
- : []),
106
+ recordingPaths: recordingPaths(bundle),
108
107
  screenshotPaths: bundle ? streamScreenshotPaths(bundle) : new Set(),
109
108
  // A run from before 0.110 with no receipts still names its ids in run.json.
110
109
  sandboxIds: [...new Set([...journaledSandboxIds, ...keyedSandboxIds(recorded)])],
@@ -127,7 +126,7 @@ export async function verifyResolvedRun(cwd, runInput, runPaths) {
127
126
  : `public-safety findings: ${publicSafetyFindings.slice(0, 5).join(", ")}`,
128
127
  });
129
128
  const missingEvidenceArtifacts = bundle
130
- ? await missingLocalEvidenceArtifacts(runPaths, bundle)
129
+ ? await missingLocalEvidenceArtifacts(runPaths, bundle, traces.pastReadLimit)
131
130
  : [];
132
131
  const redactedShapeFrames = bundle
133
132
  ? await redactedShapeFramePaths(runPaths, bundle)
@@ -155,6 +154,7 @@ export async function verifyResolvedRun(cwd, runInput, runPaths) {
155
154
  : { warnings: [], unfinished: undefined };
156
155
  const warnings = bundle
157
156
  ? [
157
+ ...traces.unreadTraces,
158
158
  ...rawScreenshotPostureWarnings(bundle, redactedShapeFrames),
159
159
  ...undeclaredSubjectStateWarnings(bundle),
160
160
  ...desktopGeometryWarnings(bundle),
@@ -238,6 +238,8 @@ async function loadTraces(runPaths, stored) {
238
238
  recorded: stored,
239
239
  refusedTraces: [],
240
240
  refusedStreams: new Set(),
241
+ unreadTraces: [],
242
+ pastReadLimit: new Set(),
241
243
  ...scanned,
242
244
  pages: [],
243
245
  };
@@ -250,15 +252,34 @@ async function loadTraces(runPaths, stored) {
250
252
  scanned[scan === "sensitive" ? "sensitive" : "opaque"].push(file);
251
253
  },
252
254
  });
255
+ // A file whose index records more than the read limit is not read. One past the limit on disk
256
+ // too is listed as unscanned, so its run grades local_only; one under it does not match its index.
257
+ const unread = [];
258
+ for (const error of refused)
259
+ if (error instanceof StreamTraceTooLargeError && (await pastReadLimit(runPaths, error.file))) {
260
+ unread.push(error);
261
+ scanned.opaque.push(error.file);
262
+ }
253
263
  return {
254
264
  bundle,
255
265
  recorded: bundle,
256
- refusedTraces: refused.map((error) => error.message),
266
+ refusedTraces: refused
267
+ .filter((error) => !unread.some((past) => past === error))
268
+ .map((error) => error.message),
257
269
  refusedStreams: new Set(refused.map((error) => error.streamId)),
270
+ unreadTraces: unread.map((error) => error.message),
271
+ pastReadLimit: new Set(unread.map((error) => error.file)),
258
272
  ...scanned,
259
273
  pages: pageFindings(bundle),
260
274
  };
261
275
  }
276
+ /** Whether the run file at `file` holds more than RUN_ARTIFACT_MAX_BYTES, read only when it does not. */
277
+ async function pastReadLimit(runPaths, file) {
278
+ const read = await readContainedRegularFile(runPaths, file, RUN_ARTIFACT_MAX_BYTES);
279
+ return read.status === "refused" && read.reason === "too-large";
280
+ }
281
+ /** The continuous recordings the bundle registers, which CONTINUOUS_MEDIA grades. */
282
+ const recordingPaths = (bundle) => new Set(bundle?.streams.flatMap((stream) => (stream.recording ? [stream.recording.path] : [])) ?? []);
262
283
  /** The trace scan's results, beside the walk's: each file named once, then each page finding. */
263
284
  function addTraceScan(traces, findings, unscanned) {
264
285
  for (const file of traces.sensitive)
@@ -168,7 +168,9 @@ export interface ActorTraceItem {
168
168
  screenshotRef?: { path: string; redaction: "blurred" | "ocr_scrubbed" | "none" };
169
169
  text?: string; // redacted
170
170
  page?: { origin: string; path: string }; // computer-use screenshot items; redacted, no query
171
- // computer-use: the wait ui_action during which a watch of a turn that only waited ended
171
+ // computer-use: the wait ui_action during which a watch of a turn that only waited ended, or a
172
+ // "watched the screen" notice for a watch no wait carries (a held participant's turn of only
173
+ // screenshots, or the watch after its repeated action past the no-progress limit)
172
174
  watch?: {
173
175
  limitMs: number;
174
176
  askedMs: number; // what the turn's waits asked for; above limitMs past the watch's growth cap
@@ -39,10 +39,22 @@ account participant's `executionProfile` without its receipts. `events.ndjson` i
39
39
  contract that live adapters update while a run is active.
40
40
 
41
41
  `observer/index.html` carries both parts: the data slot holds observer-data.json's
42
- content, and `<script id="observer-items" type="application/octet-stream"
43
- data-encoding="gzip">` holds the items and full artifact lists (`humanish.observer-items.v1`, by stream id)
44
- gzip-compressed in base64, which a `file://` page decodes without a fetch. Real
45
- trace items compress about eight times. The page joins the two before it renders
42
+ content, and `<script id="observer-items" type="application/json">` holds the items
43
+ and full artifact lists by stream id (`humanish.observer-items.v2`) as
44
+ `{ schema, runId, streams }`, where `streams` lists one entry for each participant: its
45
+ JSON with its stream id, gzip-compressed in base64, which a `file://` page decodes
46
+ without a fetch. The ids stay inside the compressed entries because verify's scan
47
+ reads the element's plain text. There, an id such as `GITHUB_TOKEN` before a long
48
+ base64 value reads as a credential, and `https://` with a later id shaped like an
49
+ address at an E2B host reads as one E2B URL across the base64 between them. Each participant is packed
50
+ (`src/observer/page-data.ts`) and decoded (`observer/lib/data.ts`) on its own, so no
51
+ string holds two participants' items. A run of 25 participants with
52
+ 30 MiB traces, 786 MB together, past V8's longest string of 536,870,888 characters,
53
+ renders to a 138 MiB page. The page itself is one string, at about 0.18 characters
54
+ for each trace byte, so it passes that limit near 3 GB of traces, and memory may run
55
+ out first; neither has been measured past 25 participants. Real trace items compress
56
+ about eight times. The page
57
+ joins the two before it renders
46
58
  (`joinObserverData` in `src/observer/participant-items.ts`), so every view reads the
47
59
  same data as before. A stream whose screenshot entries moved records how many in
48
60
  `screenshotArtifacts`, and the page renders nothing from items that are another
@@ -83,9 +95,14 @@ public-safe metadata.
83
95
 
84
96
  The browser polls `observer-feed.json` with `no-store` caching: the server's
85
97
  projection of the run as `{ data, items }`, observer-data.json's content and the
86
- items, from one read of the run per poll. While a participant runs, its entry in
87
- `items` carries `live`, a cursor `<position>.<generation>.<items>.<bytes>` naming its
88
- live file (`live/<position>.<generation>.ndjson`) and how far the list goes. The page
98
+ items packed as the page packs them (`observerFeedBody`), from one read of the run
99
+ per poll. The feed compresses each entry at gzip level 1 on the libuv thread pool, one
100
+ participant at a time, so the server answers other requests while it builds a poll.
101
+ Every poll sends every finished participant's items again: on the 25-participant run
102
+ above a poll takes 17 to 20 seconds, past the 15 seconds the page waits, so that page
103
+ never updates and says the server stopped answering. While a participant runs, its entry in `items` carries `live`, a cursor
104
+ `<position>.<generation>.<items>.<bytes>` naming its live file
105
+ (`live/<position>.<generation>.ndjson`) and how far the list goes. The page
89
106
  sends the cursors of its last body back as `?after=<cursor>,<cursor>`, and the server
90
107
  then reads and sends only the items past each one, marking that entry with
91
108
  `liveAfter`, the cursor they follow (`src/observer/feed-cursors.ts`). A cursor for an
@@ -100,10 +100,13 @@ action**, not a fatal error:
100
100
  `budget_reached` after material progress), a `CuaAbortError` as `harness_error`,
101
101
  and any genuine non-`CommandExitError` adapter fault as `actor_error`.
102
102
  - **No infinite loop.** A skipped action changes nothing on screen, so it is not
103
- progress. A run that keeps failing every action makes no progress and still
104
- terminates through the existing idle / no-progress backstop
105
- (`gave_up`). The resilience only converts a _single flaky command_ from fatal
106
- to survivable; it never masks a genuinely stuck run.
103
+ progress. A run that keeps repeating a failing action makes no progress and
104
+ ends through the no-progress backstop (`gave_up`) at 20 such turns in a row.
105
+ A participant held to its scheduled end keeps its session through repeats on
106
+ an unchanged screen, which is how it waits on a page that changes only on
107
+ reload, but 20 repeats in a row that never reached the desktop end it the
108
+ same way. The resilience only converts a _single flaky command_ from fatal
109
+ to survivable.
107
110
 
108
111
  Public-safety: the notice text carries only the substrate's own stderr (tailed +
109
112
  whitespace-collapsed) and a numeric exit code, and is still run through the loop's
@@ -20,7 +20,7 @@ writer (`src/run/run.ts`, through `splitStreamTraces` in `src/run/stream-trace.t
20
20
  each stream with an `actor`:
21
21
 
22
22
  - the trace file: the actor trace (`humanish.actor-trace.v1`) with every item, plus `artifacts`,
23
- the stream's screenshot entries. Its path is the stream's actor trace artifact (`actor.json` for
23
+ the stream's screenshot entries, as one line of compact JSON. Its path is the stream's actor trace artifact (`actor.json` for
24
24
  one participant, `actors/<stream>.json` for more, `actor-<surface>.json` on the scripted route),
25
25
  or `actors/<n>.json`, `n` its position in `streams[]` (from 1), for a stream with none or whose
26
26
  trace artifact an earlier stream claimed. Names of that form are never reused as a stream's own
@@ -65,7 +65,12 @@ names, is refused before any file is read. The writer claims names the same way.
65
65
  must have an index: an actor with no index, with
66
66
  or without inline `items`, is refused, and so is one with both an index and inline `items`.
67
67
  verify reports a refused trace in the `local evidence artifacts exist` check, so the run is
68
- `blocked`. verify also scans the bytes of each trace it loaded with the public-safety text scan,
68
+ `blocked`. A trace or live file whose index records more than the limit is reported that way too,
69
+ unless the file on disk is a single-link regular file verify can open that holds more than 32 MiB.
70
+ verify lists such a file under `UNSCANNED_ARTIFACT`, so the run is `local_only`. It adds the
71
+ refusal to its warnings and checks that participant without its items. Where the run cites the
72
+ file as evidence, verify counts it as present if it is still past 32 MiB when that check reads it;
73
+ a citation as a screenshot or recording is still checked as one, so the run is `blocked`. verify also scans the bytes of each trace it loaded with the public-safety text scan,
69
74
  apart from its walk of the run folder, so no path exemption the walk makes applies to a trace.
70
75
  run.json's digest, which an analysis record binds, covers each index and so each trace.
71
76
 
@@ -148,6 +153,9 @@ projection is served.
148
153
  observer-data.json is built from the loaded bundle. From 0.124.0 it leaves out each participant's
149
154
  trace items, which stay in the traces, and from this release it is `humanish.observer-data.v3`,
150
155
  which also leaves out each participant's per-request lists (docs/architecture/observer.md).
156
+ The Observer page and `observer-feed.json` carry the items as `humanish.observer-items.v2`, each
157
+ participant's entry gzip-compressed on its own, so `humanish observe` renders a run whose traces
158
+ together pass V8's longest string. It was measured on 25 participants with 30 MiB traces.
151
159
 
152
160
  ## Per-request lists
153
161
 
@@ -371,6 +379,12 @@ An Observer that does not render is an `evidence` failure known only after
371
379
  show the run without it. Messages pass the same redaction and sandbox id scrub
372
380
  as the rest of the bundle.
373
381
 
382
+ After that write, `Run.finish` replaces each raw sandbox id `sandbox-receipts.ndjson` names in the
383
+ run's other text files with its label (`scrubRunSandboxIds` in `src/run/sandbox-ids.ts`). A file
384
+ the sweep cannot read, such as a trace past the 32 MiB read limit, may still name a sandbox. The run
385
+ still finishes: `Run.finish` writes `run.json` again with a `sandbox-ids.unresolved` warn event that
386
+ names each such file and why.
387
+
374
388
  When a signal stops a live run, the handler writes `interrupted` into the last
375
389
  flushed `run.json`, and every later write of that run keeps it. A run stopped
376
390
  before its first flush has no `run.json`; its `status.json` is the record.