humanish 0.133.0 → 0.134.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +24 -0
- package/dist/actors/computer-use/loop/actions.d.ts +2 -0
- package/dist/actors/computer-use/loop/actions.js +11 -5
- package/dist/actors/computer-use/loop/backstop.d.ts +23 -2
- package/dist/actors/computer-use/loop/backstop.js +40 -7
- package/dist/actors/computer-use/loop/session.d.ts +3 -0
- package/dist/actors/computer-use/loop/session.js +3 -0
- package/dist/actors/computer-use/loop/streak.d.ts +18 -0
- package/dist/actors/computer-use/loop/streak.js +47 -0
- package/dist/actors/computer-use/loop/types.d.ts +5 -1
- package/dist/actors/computer-use/loop/watch.d.ts +22 -4
- package/dist/actors/computer-use/loop/watch.js +83 -27
- package/dist/actors/computer-use/loop.js +18 -6
- package/dist/actors/contract.d.ts +3 -1
- package/dist/analysis/service.js +3 -2
- package/dist/analysis/source.d.ts +6 -0
- package/dist/analysis/source.js +15 -13
- package/dist/cli/commands/run-signals.js +6 -2
- package/dist/evidence/json-split.d.ts +21 -0
- package/dist/evidence/json-split.js +57 -0
- package/dist/evidence/sensitive-json.js +5 -28
- package/dist/observer/render.js +5 -3
- package/dist/routes/computer-use/bundle-parts.d.ts +9 -0
- package/dist/routes/computer-use/bundle-parts.js +24 -4
- package/dist/routes/computer-use/fanout-bundle.js +5 -3
- package/dist/routes/computer-use/single-bundle.js +2 -2
- package/dist/run/locate.d.ts +8 -2
- package/dist/run/locate.js +10 -4
- package/dist/run/run.js +18 -2
- package/dist/run/sandbox-ids.d.ts +10 -4
- package/dist/run/sandbox-ids.js +73 -11
- package/dist/run/stream-trace.d.ts +8 -2
- package/dist/run/stream-trace.js +13 -7
- package/dist/study/bundle-estimate.d.ts +1 -4
- package/dist/study/bundle-estimate.js +29 -23
- package/dist/verify/artifacts.d.ts +7 -1
- package/dist/verify/artifacts.js +15 -4
- package/dist/verify/verify.js +29 -8
- package/docs/architecture/actor-contract.md +3 -1
- package/docs/architecture/state-driven-executor.md +7 -4
- package/docs/contracts/run-bundle.md +13 -2
- package/docs/contracts/study-analysis.md +4 -2
- package/package.json +1 -1
|
@@ -11,10 +11,16 @@ export declare class StreamTraceRefusedError extends Error {
|
|
|
11
11
|
constructor(streamId: string, detail: string, advice?: string);
|
|
12
12
|
}
|
|
13
13
|
/**
|
|
14
|
-
* A trace or live file whose index records more than the loader reads of one file
|
|
15
|
-
* no file past the limit, so a copy of the file is refused too.
|
|
14
|
+
* A trace or live file whose index records more than the loader reads of one file, `maxBytes`. The
|
|
15
|
+
* loader reads no file past the limit, so a copy of the file is refused too.
|
|
16
16
|
*/
|
|
17
17
|
export declare class StreamTraceTooLargeError extends StreamTraceRefusedError {
|
|
18
|
+
/** The file's run-relative path, as its index records it. */
|
|
19
|
+
readonly file: string;
|
|
20
|
+
constructor(streamId: string, index: {
|
|
21
|
+
path: string;
|
|
22
|
+
bytes: number;
|
|
23
|
+
}, maxBytes: number);
|
|
18
24
|
}
|
|
19
25
|
/** `trace` without the per-request lists; every other field as it is. */
|
|
20
26
|
export declare function withoutRequestLists<T extends object>(trace: T): WithoutRequestLists<T>;
|
package/dist/run/stream-trace.js
CHANGED
|
@@ -28,13 +28,17 @@ export class StreamTraceRefusedError extends Error {
|
|
|
28
28
|
/** Why the loader reads no trace or live file whose index records more than `maxBytes`. */
|
|
29
29
|
const pastLimitDetail = (index, maxBytes) => `run.json records ${index.path} at ${index.bytes} bytes, more than the ${maxBytes} bytes humanish reads of one file`;
|
|
30
30
|
/**
|
|
31
|
-
* A trace or live file whose index records more than the loader reads of one file
|
|
32
|
-
* no file past the limit, so a copy of the file is refused too.
|
|
31
|
+
* A trace or live file whose index records more than the loader reads of one file, `maxBytes`. The
|
|
32
|
+
* loader reads no file past the limit, so a copy of the file is refused too.
|
|
33
33
|
*/
|
|
34
34
|
export class StreamTraceTooLargeError extends StreamTraceRefusedError {
|
|
35
|
+
/** The file's run-relative path, as its index records it. */
|
|
36
|
+
file;
|
|
37
|
+
constructor(streamId, index, maxBytes) {
|
|
38
|
+
super(streamId, pastLimitDetail(index, maxBytes), "Restoring a copy does not help; open the file with another tool, such as jq, to read its items.");
|
|
39
|
+
this.file = index.path;
|
|
40
|
+
}
|
|
35
41
|
}
|
|
36
|
-
/** The refusal of a trace or live file whose index records more than `maxBytes`. */
|
|
37
|
-
const tooLarge = (streamId, index, maxBytes) => new StreamTraceTooLargeError(streamId, pastLimitDetail(index, maxBytes), "Restoring a copy does not help; open the file with another tool, such as jq, to read its items.");
|
|
38
42
|
const sha256 = (bytes) => createHash("sha256").update(bytes).digest("hex");
|
|
39
43
|
const isScreenshot = (artifact) => artifact.kind === "screenshot";
|
|
40
44
|
/**
|
|
@@ -139,7 +143,9 @@ export function splitStreamTraces(bundle) {
|
|
|
139
143
|
throw new Error(`Two participants' traces would both be written to ${filePath}.`);
|
|
140
144
|
claimed.add(traceFileKey(filePath));
|
|
141
145
|
const screenshots = stream.artifacts.filter(isScreenshot);
|
|
142
|
-
|
|
146
|
+
// Compact: every reader refuses a file past RUN_ARTIFACT_MAX_BYTES, and indenting made a
|
|
147
|
+
// trace 1.36 times its compact size on 236 recorded computer-use traces.
|
|
148
|
+
const bytes = Buffer.from(`${JSON.stringify({ ...actor, artifacts: screenshots })}\n`);
|
|
143
149
|
files.push({ path: filePath, bytes });
|
|
144
150
|
const { items, ...summary } = actor;
|
|
145
151
|
return {
|
|
@@ -274,7 +280,7 @@ async function readLiveItems(read, stream, maxBytes, liveAfter) {
|
|
|
274
280
|
const after = liveAfter?.(index);
|
|
275
281
|
if (index.bytes > maxBytes) {
|
|
276
282
|
if (after === undefined || !cursorFits(index, after))
|
|
277
|
-
throw
|
|
283
|
+
throw new StreamTraceTooLargeError(stream.id, index, maxBytes);
|
|
278
284
|
// The caller keeps the prefix it holds, read when the file was under the limit.
|
|
279
285
|
return {
|
|
280
286
|
stream: { ...stream, liveActor: { ...summary, items: [] } },
|
|
@@ -343,7 +349,7 @@ async function readStreamTrace(read, stream, maxBytes, liveAfter) {
|
|
|
343
349
|
if (stream.trace !== undefined) {
|
|
344
350
|
assertIndex(stream);
|
|
345
351
|
if (stream.trace.bytes > maxBytes)
|
|
346
|
-
throw
|
|
352
|
+
throw new StreamTraceTooLargeError(stream.id, stream.trace, maxBytes);
|
|
347
353
|
const file = await read(stream.trace.path, maxBytes);
|
|
348
354
|
if ("refused" in file)
|
|
349
355
|
throw new StreamTraceRefusedError(stream.id, file.refused);
|
|
@@ -16,10 +16,7 @@ export interface BundleEstimate {
|
|
|
16
16
|
readonly runFileBytes: number;
|
|
17
17
|
/** The largest participant trace file, which analysis also reads whole. */
|
|
18
18
|
readonly largestTraceBytes: number;
|
|
19
|
-
/**
|
|
20
|
-
* Every participant's trace file together, whose items the run writer and the Observer page join
|
|
21
|
-
* in one string.
|
|
22
|
-
*/
|
|
19
|
+
/** Every participant's trace file together, whose items the Observer page joins in one string. */
|
|
23
20
|
readonly tracesBytes: number;
|
|
24
21
|
/** observer/index.html, which verify reads whole. */
|
|
25
22
|
readonly observerPageBytes: number;
|
|
@@ -17,11 +17,11 @@
|
|
|
17
17
|
// runs measured, so it passes 32 MiB only after run.json passes 16 MiB, and the estimate does not
|
|
18
18
|
// check it.
|
|
19
19
|
// The Observer page and the Observer server join every participant's items in one string before
|
|
20
|
-
// gzip.
|
|
21
|
-
//
|
|
22
|
-
//
|
|
23
|
-
// on Node 24. A trace file's bytes are at least the characters of its
|
|
24
|
-
// estimate compares the traces' bytes and may warn a little early.
|
|
20
|
+
// gzip. A bundle export renders the Observer page, so it fails there too once --max-bytes is
|
|
21
|
+
// raised past its 25 MiB default, which refuses such a run first. Writing run.json joins none: the
|
|
22
|
+
// sandbox id scrub (publicSandboxView) works one participant at a time. V8's longest string is
|
|
23
|
+
// 536,870,888 characters on Node 24. A trace file's bytes are at least the characters of its
|
|
24
|
+
// items as compact JSON, so the estimate compares the traces' bytes and may warn a little early.
|
|
25
25
|
import { constants } from "node:buffer";
|
|
26
26
|
import { EVIDENCE_LIMITS } from "../analysis/analysis-limits.js";
|
|
27
27
|
import { RUN_ARTIFACT_MAX_BYTES } from "../run/contained-output.js";
|
|
@@ -32,9 +32,15 @@ import { runEntryLimit } from "../verify/scan-limit.js";
|
|
|
32
32
|
// the 0.125.0 smoke test and the 2026-10-12 live checks, 10 participants with 1 to 28 turns and 137
|
|
33
33
|
// model requests. Dry: runs of 1, 2, 24 and 100 participants, for the Observer app and the run's
|
|
34
34
|
// own files. Pace, session length and screenshots: the live computer-use runs kept on the
|
|
35
|
-
// maintainer's machine.
|
|
36
|
-
//
|
|
37
|
-
// The
|
|
35
|
+
// maintainer's machine. The run.json and page data rates were measured again on those runs
|
|
36
|
+
// written as v4.
|
|
37
|
+
// The trace rates were measured on trace files written indented. The writer now writes them as
|
|
38
|
+
// compact JSON, so each trace rate below is the indented figure divided by 1.36, the ratio of
|
|
39
|
+
// indented to compact bytes over 236 kept computer-use traces (1.17 to 1.49 each), and the page's
|
|
40
|
+
// share of the traces is multiplied by it. A trace's rate a turn on the public
|
|
41
|
+
// site/public/runs/lobby-0927 run (8 participants, 20 to 78 turns) was 1,579 bytes plus 91 for the
|
|
42
|
+
// page indented, under the live fit. Its participant of 78 turns writes a trace of 149,637 bytes
|
|
43
|
+
// indented and 107,897 compact.
|
|
38
44
|
const RATES = Object.freeze({
|
|
39
45
|
/** run.json bytes for the run itself: its header, study, review and cost. Live fit: 4,381. */
|
|
40
46
|
runBytes: 4_400,
|
|
@@ -44,21 +50,21 @@ const RATES = Object.freeze({
|
|
|
44
50
|
* bytes and those of two 30,647 to 38,948, beside 20,900 and 37,400 at these rates.
|
|
45
51
|
*/
|
|
46
52
|
participantBytes: 16_500,
|
|
47
|
-
/** A trace file's bytes before its turns: the actor summary. Live fit: 5,378. */
|
|
48
|
-
participantTraceBytes:
|
|
53
|
+
/** A trace file's bytes before its turns: the actor summary. Live fit: 5,378 indented. */
|
|
54
|
+
participantTraceBytes: 4_000,
|
|
49
55
|
/**
|
|
50
56
|
* Trace bytes a model turn adds: its items, its screenshot entry and its request's usage, plus 91
|
|
51
|
-
* for the `page` each screenshot item now carries. Live fit: 1,965 a turn.
|
|
52
|
-
* or more turns ranged from 1,140 to 4,532; the busiest made 311 items in 15
|
|
53
|
-
* UI actions. Analysis refuses the whole run when one trace passes its limit,
|
|
54
|
-
* counted at the busiest participant's rate.
|
|
57
|
+
* for the `page` each screenshot item now carries, all indented. Live fit: 1,965 a turn.
|
|
58
|
+
* Participants with 4 or more turns ranged from 1,140 to 4,532; the busiest made 311 items in 15
|
|
59
|
+
* turns, most of them UI actions. Analysis refuses the whole run when one trace passes its limit,
|
|
60
|
+
* so each trace is counted at the busiest participant's rate, 4,700 indented.
|
|
55
61
|
*/
|
|
56
|
-
turnTraceBytes:
|
|
62
|
+
turnTraceBytes: 3_500,
|
|
57
63
|
/**
|
|
58
64
|
* Trace bytes a dwell frame adds: 395 for its item and entry on the kept runs with dwell frames,
|
|
59
|
-
* plus 91 for the page.
|
|
65
|
+
* plus 91 for the page, indented.
|
|
60
66
|
*/
|
|
61
|
-
frameTraceBytes:
|
|
67
|
+
frameTraceBytes: 400,
|
|
62
68
|
/**
|
|
63
69
|
* Seconds between model turns: the median of the 27 of 185 kept live participants that worked 5
|
|
64
70
|
* minutes or more was 10.4.
|
|
@@ -83,12 +89,12 @@ const RATES = Object.freeze({
|
|
|
83
89
|
*/
|
|
84
90
|
observerPageDataShare: 0.85,
|
|
85
91
|
/**
|
|
86
|
-
* The page's gzipped base64 items, as a share of all the traces' bytes: 0.087 to 0.114
|
|
87
|
-
* live runs of two participants, 0.097 for the busiest participant alone (about 475 bytes
|
|
88
|
-
* turn), 0.142 to 0.184 on runs of one participant and 4 to 11 turns, and 0.095 on 0.124.0's
|
|
89
|
-
* synthetic 8-hour run.
|
|
92
|
+
* The page's gzipped base64 items, as a share of all the traces' bytes indented: 0.087 to 0.114
|
|
93
|
+
* on the live runs of two participants, 0.097 for the busiest participant alone (about 475 bytes
|
|
94
|
+
* a turn), 0.142 to 0.184 on runs of one participant and 4 to 11 turns, and 0.095 on 0.124.0's
|
|
95
|
+
* synthetic 8-hour run. The estimate counted 0.12 of indented traces.
|
|
90
96
|
*/
|
|
91
|
-
observerPageItemsShare: 0.
|
|
97
|
+
observerPageItemsShare: 0.16,
|
|
92
98
|
/** A desktop screenshot: the median of 3,990 kept live PNG frames was 107,895 bytes. */
|
|
93
99
|
screenshotBytes: 110_000,
|
|
94
100
|
/**
|
|
@@ -206,7 +212,7 @@ export function bundleLimitReason(estimate) {
|
|
|
206
212
|
: []),
|
|
207
213
|
...(joined
|
|
208
214
|
? [
|
|
209
|
-
`All participants' traces together may grow to about ${mib(estimate.tracesBytes)}, past the ${mib(constants.MAX_STRING_LENGTH)} that fits in one string.
|
|
215
|
+
`All participants' traces together may grow to about ${mib(estimate.tracesBytes)}, past the ${mib(constants.MAX_STRING_LENGTH)} that fits in one string. The Observer page puts all their items in one string, so the page would not build. A bundle export allowed past its 25 MiB default renders that page, so it would fail too.`,
|
|
210
216
|
]
|
|
211
217
|
: []),
|
|
212
218
|
...(entries
|
|
@@ -22,7 +22,13 @@ export interface UnscannedArtifacts {
|
|
|
22
22
|
* and where it stopped, each in its own sentence. Undefined when the scan read everything.
|
|
23
23
|
*/
|
|
24
24
|
export declare function unscannedMessage(unscanned: UnscannedArtifacts): string | undefined;
|
|
25
|
-
|
|
25
|
+
/**
|
|
26
|
+
* Why each local evidence path the bundle requires is missing or unusable. A path in
|
|
27
|
+
* `pastReadLimit` is a trace verify found larger than it reads, which verify lists as unscanned so
|
|
28
|
+
* the run grades local_only. It meets a requirement only while it is still past the limit when
|
|
29
|
+
* this check reads it, and never a screenshot or recording requirement.
|
|
30
|
+
*/
|
|
31
|
+
export declare function missingLocalEvidenceArtifacts(runPaths: PreparedRunArtifactPaths, bundle: RunBundle, pastReadLimit: ReadonlySet<string>): Promise<string[]>;
|
|
26
32
|
/**
|
|
27
33
|
* The PNG frames that actor traces register as stream screenshots, relative to the run root.
|
|
28
34
|
* RAW_SCREENSHOTS grades these, so the public-safety scan does not report them as unread. Only
|
package/dist/verify/artifacts.js
CHANGED
|
@@ -47,7 +47,13 @@ function listed(paths) {
|
|
|
47
47
|
const more = paths.length - MAX_LISTED_UNSCANNED;
|
|
48
48
|
return `${paths.slice(0, MAX_LISTED_UNSCANNED).join(", ")}${more > 0 ? ` and ${more} more` : ""}`;
|
|
49
49
|
}
|
|
50
|
-
|
|
50
|
+
/**
|
|
51
|
+
* Why each local evidence path the bundle requires is missing or unusable. A path in
|
|
52
|
+
* `pastReadLimit` is a trace verify found larger than it reads, which verify lists as unscanned so
|
|
53
|
+
* the run grades local_only. It meets a requirement only while it is still past the limit when
|
|
54
|
+
* this check reads it, and never a screenshot or recording requirement.
|
|
55
|
+
*/
|
|
56
|
+
export async function missingLocalEvidenceArtifacts(runPaths, bundle, pastReadLimit) {
|
|
51
57
|
const recordings = new Map(bundle.streams.flatMap((stream) => stream.recording ? [[stream.recording.path, stream.recording]] : []));
|
|
52
58
|
const requiredPaths = new Map();
|
|
53
59
|
const addRequiredPath = (artifactPath, options = {}) => {
|
|
@@ -110,12 +116,15 @@ export async function missingLocalEvidenceArtifacts(runPaths, bundle) {
|
|
|
110
116
|
}
|
|
111
117
|
}
|
|
112
118
|
// Reads overlap, so one frame's file system calls run while another frame's PNG is checked.
|
|
113
|
-
const found = await mapWithConcurrency([...requiredPaths], EVIDENCE_READ_CONCURRENCY, ([artifactPath, requirements]) => missingArtifact(runPaths, artifactPath, requirements, recordings.get(artifactPath)));
|
|
119
|
+
const found = await mapWithConcurrency([...requiredPaths], EVIDENCE_READ_CONCURRENCY, ([artifactPath, requirements]) => missingArtifact(runPaths, artifactPath, { ...requirements, pastReadLimit: pastReadLimit.has(artifactPath) }, recordings.get(artifactPath)));
|
|
114
120
|
return found.filter((entry) => entry !== null);
|
|
115
121
|
}
|
|
116
122
|
// Evidence files verify reads at once. Each read holds at most RUN_ARTIFACT_MAX_BYTES.
|
|
117
123
|
const EVIDENCE_READ_CONCURRENCY = 8;
|
|
118
|
-
/**
|
|
124
|
+
/**
|
|
125
|
+
* Why one required path fails its requirements, or null when it meets them. `pastReadLimit` says
|
|
126
|
+
* verify listed the path as a trace past the read limit, so a file still past it is present.
|
|
127
|
+
*/
|
|
119
128
|
async function missingArtifact(runPaths, artifactPath, requirements, recording) {
|
|
120
129
|
if (recording) {
|
|
121
130
|
const handle = await openContainedRegularFile(runPaths, artifactPath);
|
|
@@ -136,7 +145,9 @@ async function missingArtifact(runPaths, artifactPath, requirements, recording)
|
|
|
136
145
|
? await readScreenshot(runPaths, artifactPath)
|
|
137
146
|
: await readSafeRunArtifact(runPaths, artifactPath);
|
|
138
147
|
if (read.state === "limit")
|
|
139
|
-
return
|
|
148
|
+
return requirements.pastReadLimit && !requirements.screenshot
|
|
149
|
+
? null
|
|
150
|
+
: `${artifactPath} (larger than the ${RUN_ARTIFACT_MAX_BYTES} bytes verify reads)`;
|
|
140
151
|
if (read.state === "unavailable" ||
|
|
141
152
|
(("size" in read ? read.size : read.bytes.length) === 0 && !requirements.allowEmpty))
|
|
142
153
|
return artifactPath;
|
package/dist/verify/verify.js
CHANGED
|
@@ -7,9 +7,9 @@ import { containsSensitiveJson } from "../evidence/sensitive-json.js";
|
|
|
7
7
|
import { keyedSandboxIds, readRunSandboxIds } from "../run/sandbox-ids.js";
|
|
8
8
|
import { validatePreparedRunArtifactPaths } from "../run/paths.js";
|
|
9
9
|
import { isReadRunBundleSchema, RUN_BUNDLE_FILE, RUN_BUNDLE_SCHEMA, RUN_BUNDLE_V1_SCHEMA, RUN_BUNDLE_V2_SCHEMA, RUN_BUNDLE_V3_SCHEMA, } from "../run/bundle.js";
|
|
10
|
-
import { ContainedReadRefusedError, refusalText } from "../run/contained-output.js";
|
|
10
|
+
import { ContainedReadRefusedError, readContainedRegularFile, refusalText, RUN_ARTIFACT_MAX_BYTES, } from "../run/contained-output.js";
|
|
11
11
|
import { isCleanupResult, isRunBundle } from "../run/bundle-shape.js";
|
|
12
|
-
import { loadStreamTraces } from "../run/stream-trace.js";
|
|
12
|
+
import { loadStreamTraces, StreamTraceTooLargeError } from "../run/stream-trace.js";
|
|
13
13
|
import { readRunJsonIfExists, readRunTextIfExists, resolveRunPath, runJsonValue, } from "../run/locate.js";
|
|
14
14
|
import { SANDBOX_RECEIPTS_ARTIFACT } from "../run/sandbox-receipts.js";
|
|
15
15
|
import { isRecord } from "../run/type-guards.js";
|
|
@@ -79,7 +79,8 @@ export async function verifyResolvedRun(cwd, runInput, runPaths) {
|
|
|
79
79
|
const bundlePath = path.join(runPaths.absoluteRunRoot, RUN_BUNDLE_FILE);
|
|
80
80
|
const bundleRead = await readRunJsonIfExists(runPaths, RUN_BUNDLE_FILE);
|
|
81
81
|
// Every check below reads the streams with their traces loaded; a trace that does not match its
|
|
82
|
-
// index fails the evidence-artifacts check and its stream
|
|
82
|
+
// index fails the evidence-artifacts check, and its stream, like one past the read limit, is
|
|
83
|
+
// checked without its items.
|
|
83
84
|
const traces = await loadTraces(runPaths, runJsonValue(bundleRead));
|
|
84
85
|
const { bundle, recorded, refusedTraces } = traces;
|
|
85
86
|
const cleanupRead = await readRunJsonIfExists(runPaths, "cleanup.json");
|
|
@@ -102,9 +103,7 @@ export async function verifyResolvedRun(cwd, runInput, runPaths) {
|
|
|
102
103
|
unscannedArtifacts.files.push(SANDBOX_RECEIPTS_ARTIFACT);
|
|
103
104
|
}
|
|
104
105
|
const publicSafetyFindings = await scanRunPublicSafetyArtifacts(runPaths, derivedPublicSafetyFindings, {
|
|
105
|
-
recordingPaths:
|
|
106
|
-
? bundle.streams.flatMap((stream) => (stream.recording ? [stream.recording.path] : []))
|
|
107
|
-
: []),
|
|
106
|
+
recordingPaths: recordingPaths(bundle),
|
|
108
107
|
screenshotPaths: bundle ? streamScreenshotPaths(bundle) : new Set(),
|
|
109
108
|
// A run from before 0.110 with no receipts still names its ids in run.json.
|
|
110
109
|
sandboxIds: [...new Set([...journaledSandboxIds, ...keyedSandboxIds(recorded)])],
|
|
@@ -127,7 +126,7 @@ export async function verifyResolvedRun(cwd, runInput, runPaths) {
|
|
|
127
126
|
: `public-safety findings: ${publicSafetyFindings.slice(0, 5).join(", ")}`,
|
|
128
127
|
});
|
|
129
128
|
const missingEvidenceArtifacts = bundle
|
|
130
|
-
? await missingLocalEvidenceArtifacts(runPaths, bundle)
|
|
129
|
+
? await missingLocalEvidenceArtifacts(runPaths, bundle, traces.pastReadLimit)
|
|
131
130
|
: [];
|
|
132
131
|
const redactedShapeFrames = bundle
|
|
133
132
|
? await redactedShapeFramePaths(runPaths, bundle)
|
|
@@ -155,6 +154,7 @@ export async function verifyResolvedRun(cwd, runInput, runPaths) {
|
|
|
155
154
|
: { warnings: [], unfinished: undefined };
|
|
156
155
|
const warnings = bundle
|
|
157
156
|
? [
|
|
157
|
+
...traces.unreadTraces,
|
|
158
158
|
...rawScreenshotPostureWarnings(bundle, redactedShapeFrames),
|
|
159
159
|
...undeclaredSubjectStateWarnings(bundle),
|
|
160
160
|
...desktopGeometryWarnings(bundle),
|
|
@@ -238,6 +238,8 @@ async function loadTraces(runPaths, stored) {
|
|
|
238
238
|
recorded: stored,
|
|
239
239
|
refusedTraces: [],
|
|
240
240
|
refusedStreams: new Set(),
|
|
241
|
+
unreadTraces: [],
|
|
242
|
+
pastReadLimit: new Set(),
|
|
241
243
|
...scanned,
|
|
242
244
|
pages: [],
|
|
243
245
|
};
|
|
@@ -250,15 +252,34 @@ async function loadTraces(runPaths, stored) {
|
|
|
250
252
|
scanned[scan === "sensitive" ? "sensitive" : "opaque"].push(file);
|
|
251
253
|
},
|
|
252
254
|
});
|
|
255
|
+
// A file whose index records more than the read limit is not read. One past the limit on disk
|
|
256
|
+
// too is listed as unscanned, so its run grades local_only; one under it does not match its index.
|
|
257
|
+
const unread = [];
|
|
258
|
+
for (const error of refused)
|
|
259
|
+
if (error instanceof StreamTraceTooLargeError && (await pastReadLimit(runPaths, error.file))) {
|
|
260
|
+
unread.push(error);
|
|
261
|
+
scanned.opaque.push(error.file);
|
|
262
|
+
}
|
|
253
263
|
return {
|
|
254
264
|
bundle,
|
|
255
265
|
recorded: bundle,
|
|
256
|
-
refusedTraces: refused
|
|
266
|
+
refusedTraces: refused
|
|
267
|
+
.filter((error) => !unread.some((past) => past === error))
|
|
268
|
+
.map((error) => error.message),
|
|
257
269
|
refusedStreams: new Set(refused.map((error) => error.streamId)),
|
|
270
|
+
unreadTraces: unread.map((error) => error.message),
|
|
271
|
+
pastReadLimit: new Set(unread.map((error) => error.file)),
|
|
258
272
|
...scanned,
|
|
259
273
|
pages: pageFindings(bundle),
|
|
260
274
|
};
|
|
261
275
|
}
|
|
276
|
+
/** Whether the run file at `file` holds more than RUN_ARTIFACT_MAX_BYTES, read only when it does not. */
|
|
277
|
+
async function pastReadLimit(runPaths, file) {
|
|
278
|
+
const read = await readContainedRegularFile(runPaths, file, RUN_ARTIFACT_MAX_BYTES);
|
|
279
|
+
return read.status === "refused" && read.reason === "too-large";
|
|
280
|
+
}
|
|
281
|
+
/** The continuous recordings the bundle registers, which CONTINUOUS_MEDIA grades. */
|
|
282
|
+
const recordingPaths = (bundle) => new Set(bundle?.streams.flatMap((stream) => (stream.recording ? [stream.recording.path] : [])) ?? []);
|
|
262
283
|
/** The trace scan's results, beside the walk's: each file named once, then each page finding. */
|
|
263
284
|
function addTraceScan(traces, findings, unscanned) {
|
|
264
285
|
for (const file of traces.sensitive)
|
|
@@ -168,7 +168,9 @@ export interface ActorTraceItem {
|
|
|
168
168
|
screenshotRef?: { path: string; redaction: "blurred" | "ocr_scrubbed" | "none" };
|
|
169
169
|
text?: string; // redacted
|
|
170
170
|
page?: { origin: string; path: string }; // computer-use screenshot items; redacted, no query
|
|
171
|
-
// computer-use: the wait ui_action during which a watch of a turn that only waited ended
|
|
171
|
+
// computer-use: the wait ui_action during which a watch of a turn that only waited ended, or a
|
|
172
|
+
// "watched the screen" notice for a watch no wait carries (a held participant's turn of only
|
|
173
|
+
// screenshots, or the watch after its repeated action past the no-progress limit)
|
|
172
174
|
watch?: {
|
|
173
175
|
limitMs: number;
|
|
174
176
|
askedMs: number; // what the turn's waits asked for; above limitMs past the watch's growth cap
|
|
@@ -100,10 +100,13 @@ action**, not a fatal error:
|
|
|
100
100
|
`budget_reached` after material progress), a `CuaAbortError` as `harness_error`,
|
|
101
101
|
and any genuine non-`CommandExitError` adapter fault as `actor_error`.
|
|
102
102
|
- **No infinite loop.** A skipped action changes nothing on screen, so it is not
|
|
103
|
-
progress. A run that keeps failing
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
103
|
+
progress. A run that keeps repeating a failing action makes no progress and
|
|
104
|
+
ends through the no-progress backstop (`gave_up`) at 20 such turns in a row.
|
|
105
|
+
A participant held to its scheduled end keeps its session through repeats on
|
|
106
|
+
an unchanged screen, which is how it waits on a page that changes only on
|
|
107
|
+
reload, but 20 repeats in a row that never reached the desktop end it the
|
|
108
|
+
same way. The resilience only converts a _single flaky command_ from fatal
|
|
109
|
+
to survivable.
|
|
107
110
|
|
|
108
111
|
Public-safety: the notice text carries only the substrate's own stderr (tailed +
|
|
109
112
|
whitespace-collapsed) and a numeric exit code, and is still run through the loop's
|
|
@@ -20,7 +20,7 @@ writer (`src/run/run.ts`, through `splitStreamTraces` in `src/run/stream-trace.t
|
|
|
20
20
|
each stream with an `actor`:
|
|
21
21
|
|
|
22
22
|
- the trace file: the actor trace (`humanish.actor-trace.v1`) with every item, plus `artifacts`,
|
|
23
|
-
the stream's screenshot entries. Its path is the stream's actor trace artifact (`actor.json` for
|
|
23
|
+
the stream's screenshot entries, as one line of compact JSON. Its path is the stream's actor trace artifact (`actor.json` for
|
|
24
24
|
one participant, `actors/<stream>.json` for more, `actor-<surface>.json` on the scripted route),
|
|
25
25
|
or `actors/<n>.json`, `n` its position in `streams[]` (from 1), for a stream with none or whose
|
|
26
26
|
trace artifact an earlier stream claimed. Names of that form are never reused as a stream's own
|
|
@@ -65,7 +65,12 @@ names, is refused before any file is read. The writer claims names the same way.
|
|
|
65
65
|
must have an index: an actor with no index, with
|
|
66
66
|
or without inline `items`, is refused, and so is one with both an index and inline `items`.
|
|
67
67
|
verify reports a refused trace in the `local evidence artifacts exist` check, so the run is
|
|
68
|
-
`blocked`.
|
|
68
|
+
`blocked`. A trace or live file whose index records more than the limit is reported that way too,
|
|
69
|
+
unless the file on disk is a single-link regular file verify can open that holds more than 32 MiB.
|
|
70
|
+
verify lists such a file under `UNSCANNED_ARTIFACT`, so the run is `local_only`. It adds the
|
|
71
|
+
refusal to its warnings and checks that participant without its items. Where the run cites the
|
|
72
|
+
file as evidence, verify counts it as present if it is still past 32 MiB when that check reads it;
|
|
73
|
+
a citation as a screenshot or recording is still checked as one, so the run is `blocked`. verify also scans the bytes of each trace it loaded with the public-safety text scan,
|
|
69
74
|
apart from its walk of the run folder, so no path exemption the walk makes applies to a trace.
|
|
70
75
|
run.json's digest, which an analysis record binds, covers each index and so each trace.
|
|
71
76
|
|
|
@@ -371,6 +376,12 @@ An Observer that does not render is an `evidence` failure known only after
|
|
|
371
376
|
show the run without it. Messages pass the same redaction and sandbox id scrub
|
|
372
377
|
as the rest of the bundle.
|
|
373
378
|
|
|
379
|
+
After that write, `Run.finish` replaces each raw sandbox id `sandbox-receipts.ndjson` names in the
|
|
380
|
+
run's other text files with its label (`scrubRunSandboxIds` in `src/run/sandbox-ids.ts`). A file
|
|
381
|
+
the sweep cannot read, such as a trace past the 32 MiB read limit, may still name a sandbox. The run
|
|
382
|
+
still finishes: `Run.finish` writes `run.json` again with a `sandbox-ids.unresolved` warn event that
|
|
383
|
+
names each such file and why.
|
|
384
|
+
|
|
374
385
|
When a signal stops a live run, the handler writes `interrupted` into the last
|
|
375
386
|
flushed `run.json`, and every later write of that run keeps it. A run stopped
|
|
376
387
|
before its first flush has no `run.json`; its `status.json` is the record.
|
|
@@ -194,8 +194,10 @@ item's text. A run.json past 16 MiB is refused as `ANALYSIS_SOURCE_TOO_LARGE`, a
|
|
|
194
194
|
a run.json or trace without the fields analysis reads as `ANALYSIS_SOURCE_INVALID`;
|
|
195
195
|
each message names the file, says why, and says no request was sent. Analysis
|
|
196
196
|
verifies the run before it reads a trace, so a trace that does not match its
|
|
197
|
-
index in run.json
|
|
198
|
-
`
|
|
197
|
+
index in run.json fails verify and `humanish analyze` reports
|
|
198
|
+
`ANALYSIS_VERIFY_FAILED`. A trace past 32 MiB leaves the run `local_only`, and
|
|
199
|
+
analysis refuses it as `ANALYSIS_SOURCE_TOO_LARGE`, naming the participant and
|
|
200
|
+
the file. Count and text budgets are
|
|
199
201
|
distributed across included participants, with unused capacity from short
|
|
200
202
|
sessions available to longer ones. Reclaimed capture slots go first to eligible
|
|
201
203
|
participants with fewer admitted images, including those deferred by their
|