humanish 0.133.1 → 0.134.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. package/CHANGELOG.md +13 -0
  2. package/dist/actors/computer-use/loop/actions.d.ts +2 -0
  3. package/dist/actors/computer-use/loop/actions.js +11 -5
  4. package/dist/actors/computer-use/loop/backstop.d.ts +23 -2
  5. package/dist/actors/computer-use/loop/backstop.js +40 -7
  6. package/dist/actors/computer-use/loop/session.d.ts +3 -0
  7. package/dist/actors/computer-use/loop/session.js +3 -0
  8. package/dist/actors/computer-use/loop/streak.d.ts +18 -0
  9. package/dist/actors/computer-use/loop/streak.js +47 -0
  10. package/dist/actors/computer-use/loop/types.d.ts +5 -1
  11. package/dist/actors/computer-use/loop/watch.d.ts +22 -4
  12. package/dist/actors/computer-use/loop/watch.js +83 -27
  13. package/dist/actors/computer-use/loop.js +18 -6
  14. package/dist/actors/contract.d.ts +3 -1
  15. package/dist/analysis/service.js +3 -2
  16. package/dist/analysis/source.d.ts +6 -0
  17. package/dist/analysis/source.js +15 -13
  18. package/dist/cli/commands/run-signals.js +6 -2
  19. package/dist/observer/render.js +5 -3
  20. package/dist/routes/computer-use/bundle-parts.d.ts +9 -0
  21. package/dist/routes/computer-use/bundle-parts.js +24 -4
  22. package/dist/routes/computer-use/fanout-bundle.js +5 -3
  23. package/dist/routes/computer-use/single-bundle.js +2 -2
  24. package/dist/run/locate.d.ts +8 -2
  25. package/dist/run/locate.js +10 -4
  26. package/dist/run/run.js +18 -2
  27. package/dist/run/sandbox-ids.d.ts +4 -3
  28. package/dist/run/sandbox-ids.js +8 -6
  29. package/dist/run/stream-trace.d.ts +8 -2
  30. package/dist/run/stream-trace.js +13 -7
  31. package/dist/study/bundle-estimate.js +23 -17
  32. package/dist/verify/artifacts.d.ts +7 -1
  33. package/dist/verify/artifacts.js +15 -4
  34. package/dist/verify/verify.js +29 -8
  35. package/docs/architecture/actor-contract.md +3 -1
  36. package/docs/architecture/state-driven-executor.md +7 -4
  37. package/docs/contracts/run-bundle.md +13 -2
  38. package/docs/contracts/study-analysis.md +4 -2
  39. package/package.json +1 -1
@@ -11,7 +11,7 @@ import { loadRunBundlePrepared, resolveRunPath } from "../run/locate.js";
11
11
  import { physicalCwdOf, runIdOf, resolvePhysicalCwd, validatePreparedRunRootIdentity, } from "../run/paths.js";
12
12
  import { isRunStatusRecord, RUN_STATUS_FILE } from "../run/status.js";
13
13
  import { captureEvidence, EVIDENCE_LIMITS } from "./evidence.js";
14
- import { AnalysisSourceRefusedError, runFileTooLarge } from "./source.js";
14
+ import { AnalysisSourceRefusedError, runFileTooLarge, throwAsAnalysisRefusal } from "./source.js";
15
15
  import { pathMissing, readBoundedFile, readBoundedFileResult } from "../run/evidence-files.js";
16
16
  import { estimateAnalysisAdmission, preferLargerAnalysisOutput, runAnalysis, ANALYSIS_PROMPT_VERSION, } from "./execute.js";
17
17
  import { appendAnalysisCorrection, assertAnalysisPublicationCapacity, listAnalyses, writeAnalysis, } from "./store.js";
@@ -155,7 +155,8 @@ export async function readCompletedAnalysisSource(cwd, prepared) {
155
155
  const verified = await verifyRunPrepared(cwd, runIdOf(prepared), prepared);
156
156
  if (!verified.ok && verified.recordingOk !== true)
157
157
  throw new Error("ANALYSIS_VERIFY_FAILED");
158
- const loaded = await loadRunBundlePrepared(cwd, prepared);
158
+ // Verify passes a run whose trace is past the read limit, which analysis does not read.
159
+ const loaded = await loadRunBundlePrepared(cwd, prepared).catch(throwAsAnalysisRefusal);
159
160
  if (!loaded || loaded.bundle.streams.length === 0)
160
161
  throw new Error("ANALYSIS_NO_PARTICIPANTS");
161
162
  if (loaded.bundle.mode !== "live")
@@ -26,3 +26,9 @@ export declare function parseAnalysisSource(prepared: PreparedRunArtifactPaths,
26
26
  * ANALYSIS_SOURCE_TOO_LARGE, and any other trace the loader refuses as ANALYSIS_SOURCE_INVALID.
27
27
  */
28
28
  export declare function readParticipantTraces<T>(prepared: PreparedRunArtifactPaths, bundle: StoredRunBundle, keep: (stream: RunStream, index: number) => T): Promise<T[]>;
29
+ /**
30
+ * The trace loader's refusal as analysis gives it, ANALYSIS_SOURCE_TOO_LARGE for a trace past the
31
+ * read limit and ANALYSIS_SOURCE_INVALID for any other; another error as it is. The loader's
32
+ * message names the participant, the file, why, and what to do.
33
+ */
34
+ export declare function throwAsAnalysisRefusal(error: unknown): never;
@@ -78,20 +78,22 @@ export function parseAnalysisSource(prepared, bytes) {
78
78
  * ANALYSIS_SOURCE_TOO_LARGE, and any other trace the loader refuses as ANALYSIS_SOURCE_INVALID.
79
79
  */
80
80
  export async function readParticipantTraces(prepared, bundle, keep) {
81
- try {
82
- return await reduceRunTraces(prepared, bundle, (stream, index) => {
83
- validateTraceItems(stream);
84
- return keep(stream, index);
85
- });
86
- }
87
- catch (error) {
88
- // The loader's refusal names the participant, the file, why, and what to do.
89
- if (error instanceof StreamTraceRefusedError)
90
- throw new AnalysisSourceRefusedError(error instanceof StreamTraceTooLargeError
91
- ? "ANALYSIS_SOURCE_TOO_LARGE"
92
- : "ANALYSIS_SOURCE_INVALID", error.message);
81
+ return reduceRunTraces(prepared, bundle, (stream, index) => {
82
+ validateTraceItems(stream);
83
+ return keep(stream, index);
84
+ }).catch(throwAsAnalysisRefusal);
85
+ }
86
+ /**
87
+ * The trace loader's refusal as analysis gives it, ANALYSIS_SOURCE_TOO_LARGE for a trace past the
88
+ * read limit and ANALYSIS_SOURCE_INVALID for any other; another error as it is. The loader's
89
+ * message names the participant, the file, why, and what to do.
90
+ */
91
+ export function throwAsAnalysisRefusal(error) {
92
+ if (!(error instanceof StreamTraceRefusedError))
93
93
  throw error;
94
- }
94
+ throw new AnalysisSourceRefusedError(error instanceof StreamTraceTooLargeError
95
+ ? "ANALYSIS_SOURCE_TOO_LARGE"
96
+ : "ANALYSIS_SOURCE_INVALID", error.message);
95
97
  }
96
98
  function validateTraceItems(stream) {
97
99
  const actor = stream.actor;
@@ -134,8 +134,12 @@ async function stopActiveRuns(io, signal, stopping, options) {
134
134
  // not swept; verify grades a file that names a receipt's id local_only.
135
135
  Promise.all(stopping
136
136
  .filter((stop) => stop.report !== null)
137
- .map((stop) => scrubRunSandboxIds(stop.run.paths).catch((error) => {
138
- // A file the sweep refused to read may still name a sandbox; say which.
137
+ .map((stop) => scrubRunSandboxIds(stop.run.paths).then(
138
+ // A file the sweep refused to read may still name a sandbox; say which.
139
+ (unread) => {
140
+ if (unread !== undefined)
141
+ say(io, `humanish: ${signal}: ${unread}`);
142
+ }, (error) => {
139
143
  if (error instanceof Error)
140
144
  say(io, `humanish: ${signal}: ${error.message}`);
141
145
  }))));
@@ -90,11 +90,13 @@ export async function renderObserver(cwdInput, runInput, options = {}) {
90
90
  ? "HUMANISH_RUN_NOT_FOUND"
91
91
  : "HUMANISH_INVALID_RUN_BUNDLE", verified.error?.message ?? "Run bundle failed verification.");
92
92
  }
93
- const loaded = await loadRunBundlePrepared(selectedPhysicalCwd, preparedRunPaths);
93
+ // Verify passes a trace past the read limit, so its participant shows without its items.
94
+ const loaded = await loadRunBundlePrepared(selectedPhysicalCwd, preparedRunPaths, {
95
+ display: true,
96
+ });
94
97
  await validatePreparedRunArtifactPaths(preparedRunPaths);
95
- if (!loaded) {
98
+ if (!loaded)
96
99
  return runNotFound(cwd, runInput);
97
- }
98
100
  if (loaded.bundle.runId !== selection.runId ||
99
101
  (await realpath(loaded.runDir)) !== preparedRunPaths.physicalRunRoot) {
100
102
  throw new Error("Observer output directory does not match the selected run.");
@@ -1,6 +1,15 @@
1
+ import type { CuaLoopResult } from "../../actors/computer-use/loop.js";
1
2
  import { type RunProviderResource, type RunSubjectProvenance } from "../../run/bundle.js";
2
3
  import { type ParticipantIds } from "../../run/participant-records.js";
3
4
  import { type ParticipantRunOutcome } from "./types.js";
5
+ /**
6
+ * The review's gap for the streaks a participant's trace records: turns that repeated a recent
7
+ * action on an unchanged screen past noProgressSteps, which did not end the session of a
8
+ * participant held to its scheduled end. Its session passes, so no other gap names them. One line
9
+ * names the first streak and counts the rest: a page that changes every minute, watched 1 s at a
10
+ * time, records a streak a minute, and a fan-out would list each.
11
+ */
12
+ export declare function streakGaps(participantId: string, session: CuaLoopResult | undefined): string[];
4
13
  /** The human-readable state story appended to the provenance event (and review.md via it). */
5
14
  export declare function describeSubjectState(state: RunSubjectProvenance["state"], dryRun: boolean): string;
6
15
  /** Human-readable provenance line for the single-participant subject.provenance event of a clone
@@ -1,14 +1,34 @@
1
1
  // The parts both computer-use bundle shapes share: the subject-state story and provenance line,
2
- // provider-resource records, and the public-safe app URL label, the URL digest and the
3
- // subject-phase event id suffix. The builders are bundle.ts (the dispatcher and the judge),
4
- // single-bundle.ts and fanout-bundle.ts. Feedback candidates from participant reports are in
5
- // participant-feedback.ts, which the shared-world bundle reads too.
2
+ // provider-resource records, the review's gap for a held participant's streak, and the public-safe
3
+ // app URL label, the URL digest and the subject-phase event id suffix. The builders are bundle.ts
4
+ // (the dispatcher and the judge), single-bundle.ts and fanout-bundle.ts. Feedback candidates from
5
+ // participant reports are in participant-feedback.ts, which the shared-world bundle reads too.
6
+ import { STREAK_TITLE } from "../../actors/computer-use/loop/streak.js";
6
7
  import { scanEncodedText } from "../../evidence/encoded-text.js";
7
8
  import { digestText } from "../../evidence/redaction.js";
8
9
  import {} from "../../run/bundle.js";
9
10
  import { participantResourceIds } from "../../run/participant-records.js";
10
11
  import {} from "./types.js";
11
12
  import { plural } from "../../run/text.js";
13
+ /**
14
+ * The review's gap for the streaks a participant's trace records: turns that repeated a recent
15
+ * action on an unchanged screen past noProgressSteps, which did not end the session of a
16
+ * participant held to its scheduled end. Its session passes, so no other gap names them. One line
17
+ * names the first streak and counts the rest: a page that changes every minute, watched 1 s at a
18
+ * time, records a streak a minute, and a fan-out would list each.
19
+ */
20
+ export function streakGaps(participantId, session) {
21
+ const texts = (session?.trace.items ?? []).flatMap((item) => item.kind === "notice" && item.title === STREAK_TITLE && item.text !== undefined
22
+ ? [item.text]
23
+ : []);
24
+ const [first] = texts;
25
+ if (first === undefined)
26
+ return [];
27
+ const more = texts.length > 1
28
+ ? ` It had ${plural(texts.length - 1, "more such streak")} after that; its trace records each.`
29
+ : "";
30
+ return [`${participantId}: ${first}${more}`];
31
+ }
12
32
  /** The human-readable state story appended to the provenance event (and review.md via it). */
13
33
  export function describeSubjectState(state, dryRun) {
14
34
  switch (state.provenance) {
@@ -5,7 +5,7 @@ import { cuaGoalSource } from "../../actors/goal-source.js";
5
5
  import { redactText } from "../../evidence/redaction.js";
6
6
  import { REVIEW_SCHEMA, bundleArtifacts, bundleHead, } from "../../run/bundle.js";
7
7
  import { aggregateTaskFunnels, formatParticipantOutcomes, formatRunTaskFunnel, gapsListClause, skippedSentence, withCuaReviewProvenance, } from "../../run/outcomes.js";
8
- import { providerResourcesForOutcome, publicSafeAppUrlLabel } from "./bundle-parts.js";
8
+ import { providerResourcesForOutcome, publicSafeAppUrlLabel, streakGaps } from "./bundle-parts.js";
9
9
  import { subjectName } from "../../run/subject-name.js";
10
10
  import { participantFeedbackCandidates } from "./participant-feedback.js";
11
11
  import { participantFactsOf } from "./participant-facts.js";
@@ -70,18 +70,20 @@ function fanoutSummary(args, facts) {
70
70
  }
71
71
  /**
72
72
  * The review's gaps: one line for the participants a host suspension likely failed, then a line
73
- * for each other participant that did not pass.
73
+ * for each other participant that did not pass, then a line for each held participant's streaks.
74
74
  */
75
75
  function fanoutGaps(args, judgment) {
76
76
  const suspended = new Set(args.hostSuspension?.participantIds ?? []);
77
77
  const others = judgment.participants.flatMap((participant, index) => participant.gapLine === undefined || suspended.has(args.specs[index].planned.id)
78
78
  ? []
79
79
  : [participant.gapLine]);
80
+ const streaks = args.specs.flatMap((spec, index) => streakGaps(spec.planned.id, args.outcomes?.[index]?.session));
80
81
  if (suspended.size === 0)
81
- return others;
82
+ return [...others, ...streaks];
82
83
  return [
83
84
  `${[...suspended].join(", ")}: failed after the host was suspended, likely its effect. Each participant's record keeps its own error.`,
84
85
  ...others,
86
+ ...streaks,
85
87
  ];
86
88
  }
87
89
  function fanoutReview(args, streams, judgment) {
@@ -6,7 +6,7 @@ import { aggregateTaskFunnels, formatParticipantOutcomes, formatRunTaskFunnel, p
6
6
  import { participantCaption } from "../../run/participant-caption.js";
7
7
  import { actorEnding } from "../../actors/stop-cause.js";
8
8
  import { cuaGoalSource, ranUntilClockEnd } from "../../actors/goal-source.js";
9
- import { describeSubjectState, phaseEventIdSuffix, publicSafeAppUrlLabel, subjectProvenanceMessage, } from "./bundle-parts.js";
9
+ import { describeSubjectState, phaseEventIdSuffix, publicSafeAppUrlLabel, streakGaps, subjectProvenanceMessage, } from "./bundle-parts.js";
10
10
  import { participantFeedbackCandidates } from "./participant-feedback.js";
11
11
  import { judgeParticipantRecords } from "../../run/judge.js";
12
12
  import { buildRunCostSummary } from "../../run/cost-summary.js";
@@ -348,7 +348,7 @@ function singleReview(args, view, stream) {
348
348
  ...(credibilityNote === undefined ? {} : { credibilityNote }),
349
349
  })}`,
350
350
  gaps: args.session || args.sessionError !== undefined
351
- ? []
351
+ ? streakGaps(args.participantId ?? "lane-01", args.session)
352
352
  : args.inProgress === true
353
353
  ? ["Live desktop session is still running."]
354
354
  : ["Live desktop session not yet run (dry run only)."],
@@ -55,8 +55,14 @@ export declare function loadRunBundle(cwdInput: string, runInput: string): Promi
55
55
  bundlePath: string;
56
56
  runDir: string;
57
57
  } | null>;
58
- /** loadRunBundle for a caller that already holds prepared paths. Revalidates them first. */
59
- export declare function loadRunBundlePrepared(cwdInput: string, runPaths: PreparedRunArtifactPaths): Promise<{
58
+ /**
59
+ * loadRunBundle for a caller that already holds prepared paths. Revalidates them first. With
60
+ * `display`, a participant whose trace the loader refuses, such as one past the read limit, is
61
+ * kept as displayedTraces shows it, for a page that shows the run.
62
+ */
63
+ export declare function loadRunBundlePrepared(cwdInput: string, runPaths: PreparedRunArtifactPaths, options?: {
64
+ display?: boolean;
65
+ }): Promise<{
60
66
  bundle: RunBundle;
61
67
  bundlePath: string;
62
68
  runDir: string;
@@ -3,7 +3,7 @@ import { bindExistingRunArtifactPaths, isSafeRunIdSegment, resolveExistingRunDir
3
3
  import { assertPreparedSelectedOutputDirectory, bindExistingManagedHumanishOutputDirectory, ContainedReadRefusedError, LATEST_POINTER_MAX_BYTES, readContainedRegularFile, RUN_ARTIFACT_MAX_BYTES, } from "./contained-output.js";
4
4
  import { RUN_BUNDLE_FILE } from "./bundle.js";
5
5
  import { isRunBundle, isRunPointer } from "./bundle-shape.js";
6
- import { loadRunTraces } from "./stream-trace.js";
6
+ import { displayedTraces, loadRunTraces, loadStreamTraces } from "./stream-trace.js";
7
7
  import { readUnchangedFile } from "./evidence-files.js";
8
8
  /** The run input that resolves through latest.json rather than naming a run directory. */
9
9
  export const LATEST_RUN_ALIAS = "latest";
@@ -119,8 +119,12 @@ export async function loadRunBundle(cwdInput, runInput) {
119
119
  }
120
120
  return loadRunBundlePrepared(cwd, runPaths);
121
121
  }
122
- /** loadRunBundle for a caller that already holds prepared paths. Revalidates them first. */
123
- export async function loadRunBundlePrepared(cwdInput, runPaths) {
122
+ /**
123
+ * loadRunBundle for a caller that already holds prepared paths. Revalidates them first. With
124
+ * `display`, a participant whose trace the loader refuses, such as one past the read limit, is
125
+ * kept as displayedTraces shows it, for a page that shows the run.
126
+ */
127
+ export async function loadRunBundlePrepared(cwdInput, runPaths, options = {}) {
124
128
  const cwd = path.resolve(cwdInput);
125
129
  await validatePreparedRunArtifactPaths(runPaths);
126
130
  const bundlePath = path.join(runPaths.absoluteRunRoot, RUN_BUNDLE_FILE);
@@ -134,7 +138,9 @@ export async function loadRunBundlePrepared(cwdInput, runPaths) {
134
138
  }
135
139
  return {
136
140
  // A trace that does not match its index throws StreamTraceRefusedError, naming the participant.
137
- bundle: await loadRunTraces(runPaths, bundle),
141
+ bundle: options.display === true
142
+ ? displayedTraces(await loadStreamTraces(runPaths, bundle))
143
+ : await loadRunTraces(runPaths, bundle),
138
144
  bundlePath: path.relative(cwd, bundlePath),
139
145
  runDir: runPaths.absoluteRunRoot,
140
146
  };
package/dist/run/run.js CHANGED
@@ -148,6 +148,14 @@ async function writeRunJson(paths, liveItems, bundle) {
148
148
  });
149
149
  return { ...publicBundle, streams };
150
150
  }
151
+ /** The warn event that names the files the final sandbox id sweep could not read. */
152
+ const unsweptEvent = (message, at) => ({
153
+ id: "event-sandbox-ids-unresolved",
154
+ at: new Date(at).toISOString(),
155
+ level: "warn",
156
+ type: "sandbox-ids.unresolved",
157
+ message,
158
+ });
151
159
  function runPublisher(args) {
152
160
  const { options, runId, createdAt, paths, runStatus, now, admit, participantsRan } = args;
153
161
  const observer = observerTarget(options);
@@ -161,11 +169,14 @@ function runPublisher(args) {
161
169
  type: "study.warning",
162
170
  message: redactText(message),
163
171
  }));
164
- // So do the host suspensions the heartbeat has seen by the time of the write.
172
+ // So do the host suspensions the heartbeat has seen by the time of the write, and the files the
173
+ // final sandbox id sweep could not read.
174
+ let unswept = [];
165
175
  const withRunEvents = (events) => {
166
176
  const owned = [
167
177
  ...studyWarnings,
168
178
  ...hostSuspensionEvents(runStatus.suspensions(), Date.parse(createdAt)),
179
+ ...unswept,
169
180
  ];
170
181
  return [...events.filter((event) => !owned.some((mine) => mine.id === event.id)), ...owned];
171
182
  };
@@ -231,7 +242,12 @@ function runPublisher(args) {
231
242
  await writeBundleFiles(bundle, finishedOutcome(outcome), (publicBundle) => runStatus.finish(runStatusOutcome(publicBundle)));
232
243
  // The final run.json indexes each finished trace, so the live files are journal no reader needs.
233
244
  await liveItems.remove();
234
- await scrubRunSandboxIds(paths);
245
+ const unread = await scrubRunSandboxIds(paths);
246
+ if (unread !== undefined) {
247
+ unswept = [unsweptEvent(unread, now())];
248
+ // The sweep reads the files the final write left, so run.json is written again to record it.
249
+ await writeBundleFiles(bundle, finishedOutcome(outcome), async () => { });
250
+ }
235
251
  await writePointer();
236
252
  let current = { ok: outcome.ok, execution: outcome.execution };
237
253
  // An Observer that did not render, judged under the route's policy and written to run.json
@@ -45,10 +45,11 @@ export declare function scrubSandboxIdBytes(file: string, bytes: Buffer, ids: re
45
45
  * receipts, with the label in place of the id. Writers such as a participant's actor.json can
46
46
  * quote an SDK error that names the sandbox. Run.finish calls it before analysis reads the run.
47
47
  * Best effort for writes and listings: verify grades a file this misses. A text file it refuses
48
- * to read, or a journal it refuses, throws once the sweep ends, naming each, since such a file
49
- * may still hold an id.
48
+ * to read, such as a trace past RUN_ARTIFACT_MAX_BYTES, may still hold an id, so the sweep
49
+ * returns a warning that names each such file and why, for the caller to record; the run goes
50
+ * on. A journal it refuses throws, since without its ids no file can be swept.
50
51
  */
51
- export declare function scrubRunSandboxIds(paths: PreparedRunArtifactPaths): Promise<void>;
52
+ export declare function scrubRunSandboxIds(paths: PreparedRunArtifactPaths): Promise<string | undefined>;
52
53
  /**
53
54
  * A run's result as it may be returned or printed: the same object when it names no sandbox,
54
55
  * otherwise its public view, scrubbed of the ids the run's receipts journal.
@@ -197,16 +197,17 @@ const MAX_SWEPT_ENTRIES = 10_000;
197
197
  * receipts, with the label in place of the id. Writers such as a participant's actor.json can
198
198
  * quote an SDK error that names the sandbox. Run.finish calls it before analysis reads the run.
199
199
  * Best effort for writes and listings: verify grades a file this misses. A text file it refuses
200
- * to read, or a journal it refuses, throws once the sweep ends, naming each, since such a file
201
- * may still hold an id.
200
+ * to read, such as a trace past RUN_ARTIFACT_MAX_BYTES, may still hold an id, so the sweep
201
+ * returns a warning that names each such file and why, for the caller to record; the run goes
202
+ * on. A journal it refuses throws, since without its ids no file can be swept.
202
203
  */
203
204
  export async function scrubRunSandboxIds(paths) {
204
205
  const ids = await readRunSandboxIds(paths);
205
206
  if (ids.length === 0)
206
- return;
207
+ return undefined;
207
208
  const valid = await validatePreparedRunRootIdentity(paths).then(() => true, () => false);
208
209
  if (!valid)
209
- return;
210
+ return undefined;
210
211
  const refused = [];
211
212
  const scrub = async (file) => {
212
213
  const read = await readContainedRegularFile(paths, file, RUN_ARTIFACT_MAX_BYTES);
@@ -242,8 +243,9 @@ export async function scrubRunSandboxIds(paths) {
242
243
  await walk("").catch(() => {
243
244
  // A directory that changed while it was listed ends the sweep; verify names what it missed.
244
245
  });
245
- if (refused.length > 0)
246
- throw new Error(`The sandbox id sweep could not read every run file, so these may still name a sandbox: ${refused.join("; ")}.`);
246
+ return refused.length === 0
247
+ ? undefined
248
+ : `The sandbox id sweep could not read every run file, so these may still name a sandbox: ${refused.join("; ")}.`;
247
249
  }
248
250
  /**
249
251
  * A run's result as it may be returned or printed: the same object when it names no sandbox,
@@ -11,10 +11,16 @@ export declare class StreamTraceRefusedError extends Error {
11
11
  constructor(streamId: string, detail: string, advice?: string);
12
12
  }
13
13
  /**
14
- * A trace or live file whose index records more than the loader reads of one file. The loader reads
15
- * no file past the limit, so a copy of the file is refused too.
14
+ * A trace or live file whose index records more than the loader reads of one file, `maxBytes`. The
15
+ * loader reads no file past the limit, so a copy of the file is refused too.
16
16
  */
17
17
  export declare class StreamTraceTooLargeError extends StreamTraceRefusedError {
18
+ /** The file's run-relative path, as its index records it. */
19
+ readonly file: string;
20
+ constructor(streamId: string, index: {
21
+ path: string;
22
+ bytes: number;
23
+ }, maxBytes: number);
18
24
  }
19
25
  /** `trace` without the per-request lists; every other field as it is. */
20
26
  export declare function withoutRequestLists<T extends object>(trace: T): WithoutRequestLists<T>;
@@ -28,13 +28,17 @@ export class StreamTraceRefusedError extends Error {
28
28
  /** Why the loader reads no trace or live file whose index records more than `maxBytes`. */
29
29
  const pastLimitDetail = (index, maxBytes) => `run.json records ${index.path} at ${index.bytes} bytes, more than the ${maxBytes} bytes humanish reads of one file`;
30
30
  /**
31
- * A trace or live file whose index records more than the loader reads of one file. The loader reads
32
- * no file past the limit, so a copy of the file is refused too.
31
+ * A trace or live file whose index records more than the loader reads of one file, `maxBytes`. The
32
+ * loader reads no file past the limit, so a copy of the file is refused too.
33
33
  */
34
34
  export class StreamTraceTooLargeError extends StreamTraceRefusedError {
35
+ /** The file's run-relative path, as its index records it. */
36
+ file;
37
+ constructor(streamId, index, maxBytes) {
38
+ super(streamId, pastLimitDetail(index, maxBytes), "Restoring a copy does not help; open the file with another tool, such as jq, to read its items.");
39
+ this.file = index.path;
40
+ }
35
41
  }
36
- /** The refusal of a trace or live file whose index records more than `maxBytes`. */
37
- const tooLarge = (streamId, index, maxBytes) => new StreamTraceTooLargeError(streamId, pastLimitDetail(index, maxBytes), "Restoring a copy does not help; open the file with another tool, such as jq, to read its items.");
38
42
  const sha256 = (bytes) => createHash("sha256").update(bytes).digest("hex");
39
43
  const isScreenshot = (artifact) => artifact.kind === "screenshot";
40
44
  /**
@@ -139,7 +143,9 @@ export function splitStreamTraces(bundle) {
139
143
  throw new Error(`Two participants' traces would both be written to ${filePath}.`);
140
144
  claimed.add(traceFileKey(filePath));
141
145
  const screenshots = stream.artifacts.filter(isScreenshot);
142
- const bytes = Buffer.from(`${JSON.stringify({ ...actor, artifacts: screenshots }, null, 2)}\n`);
146
+ // Compact: every reader refuses a file past RUN_ARTIFACT_MAX_BYTES, and indenting made a
147
+ // trace 1.36 times its compact size on 236 recorded computer-use traces.
148
+ const bytes = Buffer.from(`${JSON.stringify({ ...actor, artifacts: screenshots })}\n`);
143
149
  files.push({ path: filePath, bytes });
144
150
  const { items, ...summary } = actor;
145
151
  return {
@@ -274,7 +280,7 @@ async function readLiveItems(read, stream, maxBytes, liveAfter) {
274
280
  const after = liveAfter?.(index);
275
281
  if (index.bytes > maxBytes) {
276
282
  if (after === undefined || !cursorFits(index, after))
277
- throw tooLarge(stream.id, index, maxBytes);
283
+ throw new StreamTraceTooLargeError(stream.id, index, maxBytes);
278
284
  // The caller keeps the prefix it holds, read when the file was under the limit.
279
285
  return {
280
286
  stream: { ...stream, liveActor: { ...summary, items: [] } },
@@ -343,7 +349,7 @@ async function readStreamTrace(read, stream, maxBytes, liveAfter) {
343
349
  if (stream.trace !== undefined) {
344
350
  assertIndex(stream);
345
351
  if (stream.trace.bytes > maxBytes)
346
- throw tooLarge(stream.id, stream.trace, maxBytes);
352
+ throw new StreamTraceTooLargeError(stream.id, stream.trace, maxBytes);
347
353
  const file = await read(stream.trace.path, maxBytes);
348
354
  if ("refused" in file)
349
355
  throw new StreamTraceRefusedError(stream.id, file.refused);
@@ -32,9 +32,15 @@ import { runEntryLimit } from "../verify/scan-limit.js";
32
32
  // the 0.125.0 smoke test and the 2026-10-12 live checks, 10 participants with 1 to 28 turns and 137
33
33
  // model requests. Dry: runs of 1, 2, 24 and 100 participants, for the Observer app and the run's
34
34
  // own files. Pace, session length and screenshots: the live computer-use runs kept on the
35
- // maintainer's machine. A trace's rate a turn on the public site/public/runs/lobby-0927 run
36
- // (8 participants, 20 to 78 turns) was 1,579 bytes plus 91 for the page, under the live fit.
37
- // The run.json and page data rates were measured again on those runs written as v4.
35
+ // maintainer's machine. The run.json and page data rates were measured again on those runs
36
+ // written as v4.
37
+ // The trace rates were measured on trace files written indented. The writer now writes them as
38
+ // compact JSON, so each trace rate below is the indented figure divided by 1.36, the ratio of
39
+ // indented to compact bytes over 236 kept computer-use traces (1.17 to 1.49 each), and the page's
40
+ // share of the traces is multiplied by it. A trace's rate a turn on the public
41
+ // site/public/runs/lobby-0927 run (8 participants, 20 to 78 turns) was 1,579 bytes plus 91 for the
42
+ // page indented, under the live fit. Its participant of 78 turns writes a trace of 149,637 bytes
43
+ // indented and 107,897 compact.
38
44
  const RATES = Object.freeze({
39
45
  /** run.json bytes for the run itself: its header, study, review and cost. Live fit: 4,381. */
40
46
  runBytes: 4_400,
@@ -44,21 +50,21 @@ const RATES = Object.freeze({
44
50
  * bytes and those of two 30,647 to 38,948, beside 20,900 and 37,400 at these rates.
45
51
  */
46
52
  participantBytes: 16_500,
47
- /** A trace file's bytes before its turns: the actor summary. Live fit: 5,378. */
48
- participantTraceBytes: 5_400,
53
+ /** A trace file's bytes before its turns: the actor summary. Live fit: 5,378 indented. */
54
+ participantTraceBytes: 4_000,
49
55
  /**
50
56
  * Trace bytes a model turn adds: its items, its screenshot entry and its request's usage, plus 91
51
- * for the `page` each screenshot item now carries. Live fit: 1,965 a turn. Participants with 4
52
- * or more turns ranged from 1,140 to 4,532; the busiest made 311 items in 15 turns, most of them
53
- * UI actions. Analysis refuses the whole run when one trace passes its limit, so each trace is
54
- * counted at the busiest participant's rate.
57
+ * for the `page` each screenshot item now carries, all indented. Live fit: 1,965 a turn.
58
+ * Participants with 4 or more turns ranged from 1,140 to 4,532; the busiest made 311 items in 15
59
+ * turns, most of them UI actions. Analysis refuses the whole run when one trace passes its limit,
60
+ * so each trace is counted at the busiest participant's rate, 4,700 indented.
55
61
  */
56
- turnTraceBytes: 4_700,
62
+ turnTraceBytes: 3_500,
57
63
  /**
58
64
  * Trace bytes a dwell frame adds: 395 for its item and entry on the kept runs with dwell frames,
59
- * plus 91 for the page.
65
+ * plus 91 for the page, indented.
60
66
  */
61
- frameTraceBytes: 500,
67
+ frameTraceBytes: 400,
62
68
  /**
63
69
  * Seconds between model turns: the median of the 27 of 185 kept live participants that worked 5
64
70
  * minutes or more was 10.4.
@@ -83,12 +89,12 @@ const RATES = Object.freeze({
83
89
  */
84
90
  observerPageDataShare: 0.85,
85
91
  /**
86
- * The page's gzipped base64 items, as a share of all the traces' bytes: 0.087 to 0.114 on the
87
- * live runs of two participants, 0.097 for the busiest participant alone (about 475 bytes a
88
- * turn), 0.142 to 0.184 on runs of one participant and 4 to 11 turns, and 0.095 on 0.124.0's
89
- * synthetic 8-hour run.
92
+ * The page's gzipped base64 items, as a share of all the traces' bytes indented: 0.087 to 0.114
93
+ * on the live runs of two participants, 0.097 for the busiest participant alone (about 475 bytes
94
+ * a turn), 0.142 to 0.184 on runs of one participant and 4 to 11 turns, and 0.095 on 0.124.0's
95
+ * synthetic 8-hour run. The estimate counted 0.12 of indented traces.
90
96
  */
91
- observerPageItemsShare: 0.12,
97
+ observerPageItemsShare: 0.16,
92
98
  /** A desktop screenshot: the median of 3,990 kept live PNG frames was 107,895 bytes. */
93
99
  screenshotBytes: 110_000,
94
100
  /**
@@ -22,7 +22,13 @@ export interface UnscannedArtifacts {
22
22
  * and where it stopped, each in its own sentence. Undefined when the scan read everything.
23
23
  */
24
24
  export declare function unscannedMessage(unscanned: UnscannedArtifacts): string | undefined;
25
- export declare function missingLocalEvidenceArtifacts(runPaths: PreparedRunArtifactPaths, bundle: RunBundle): Promise<string[]>;
25
+ /**
26
+ * Why each local evidence path the bundle requires is missing or unusable. A path in
27
+ * `pastReadLimit` is a trace verify found larger than it reads, which verify lists as unscanned so
28
+ * the run grades local_only. It meets a requirement only while it is still past the limit when
29
+ * this check reads it, and never a screenshot or recording requirement.
30
+ */
31
+ export declare function missingLocalEvidenceArtifacts(runPaths: PreparedRunArtifactPaths, bundle: RunBundle, pastReadLimit: ReadonlySet<string>): Promise<string[]>;
26
32
  /**
27
33
  * The PNG frames that actor traces register as stream screenshots, relative to the run root.
28
34
  * RAW_SCREENSHOTS grades these, so the public-safety scan does not report them as unread. Only
@@ -47,7 +47,13 @@ function listed(paths) {
47
47
  const more = paths.length - MAX_LISTED_UNSCANNED;
48
48
  return `${paths.slice(0, MAX_LISTED_UNSCANNED).join(", ")}${more > 0 ? ` and ${more} more` : ""}`;
49
49
  }
50
- export async function missingLocalEvidenceArtifacts(runPaths, bundle) {
50
+ /**
51
+ * Why each local evidence path the bundle requires is missing or unusable. A path in
52
+ * `pastReadLimit` is a trace verify found larger than it reads, which verify lists as unscanned so
53
+ * the run grades local_only. It meets a requirement only while it is still past the limit when
54
+ * this check reads it, and never a screenshot or recording requirement.
55
+ */
56
+ export async function missingLocalEvidenceArtifacts(runPaths, bundle, pastReadLimit) {
51
57
  const recordings = new Map(bundle.streams.flatMap((stream) => stream.recording ? [[stream.recording.path, stream.recording]] : []));
52
58
  const requiredPaths = new Map();
53
59
  const addRequiredPath = (artifactPath, options = {}) => {
@@ -110,12 +116,15 @@ export async function missingLocalEvidenceArtifacts(runPaths, bundle) {
110
116
  }
111
117
  }
112
118
  // Reads overlap, so one frame's file system calls run while another frame's PNG is checked.
113
- const found = await mapWithConcurrency([...requiredPaths], EVIDENCE_READ_CONCURRENCY, ([artifactPath, requirements]) => missingArtifact(runPaths, artifactPath, requirements, recordings.get(artifactPath)));
119
+ const found = await mapWithConcurrency([...requiredPaths], EVIDENCE_READ_CONCURRENCY, ([artifactPath, requirements]) => missingArtifact(runPaths, artifactPath, { ...requirements, pastReadLimit: pastReadLimit.has(artifactPath) }, recordings.get(artifactPath)));
114
120
  return found.filter((entry) => entry !== null);
115
121
  }
116
122
  // Evidence files verify reads at once. Each read holds at most RUN_ARTIFACT_MAX_BYTES.
117
123
  const EVIDENCE_READ_CONCURRENCY = 8;
118
- /** Why one required path fails its requirements, or null when it meets them. */
124
+ /**
125
+ * Why one required path fails its requirements, or null when it meets them. `pastReadLimit` says
126
+ * verify listed the path as a trace past the read limit, so a file still past it is present.
127
+ */
119
128
  async function missingArtifact(runPaths, artifactPath, requirements, recording) {
120
129
  if (recording) {
121
130
  const handle = await openContainedRegularFile(runPaths, artifactPath);
@@ -136,7 +145,9 @@ async function missingArtifact(runPaths, artifactPath, requirements, recording)
136
145
  ? await readScreenshot(runPaths, artifactPath)
137
146
  : await readSafeRunArtifact(runPaths, artifactPath);
138
147
  if (read.state === "limit")
139
- return `${artifactPath} (larger than the ${RUN_ARTIFACT_MAX_BYTES} bytes verify reads)`;
148
+ return requirements.pastReadLimit && !requirements.screenshot
149
+ ? null
150
+ : `${artifactPath} (larger than the ${RUN_ARTIFACT_MAX_BYTES} bytes verify reads)`;
140
151
  if (read.state === "unavailable" ||
141
152
  (("size" in read ? read.size : read.bytes.length) === 0 && !requirements.allowEmpty))
142
153
  return artifactPath;