humanish 0.133.1 → 0.134.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +13 -0
- package/dist/actors/computer-use/loop/actions.d.ts +2 -0
- package/dist/actors/computer-use/loop/actions.js +11 -5
- package/dist/actors/computer-use/loop/backstop.d.ts +23 -2
- package/dist/actors/computer-use/loop/backstop.js +40 -7
- package/dist/actors/computer-use/loop/session.d.ts +3 -0
- package/dist/actors/computer-use/loop/session.js +3 -0
- package/dist/actors/computer-use/loop/streak.d.ts +18 -0
- package/dist/actors/computer-use/loop/streak.js +47 -0
- package/dist/actors/computer-use/loop/types.d.ts +5 -1
- package/dist/actors/computer-use/loop/watch.d.ts +22 -4
- package/dist/actors/computer-use/loop/watch.js +83 -27
- package/dist/actors/computer-use/loop.js +18 -6
- package/dist/actors/contract.d.ts +3 -1
- package/dist/analysis/service.js +3 -2
- package/dist/analysis/source.d.ts +6 -0
- package/dist/analysis/source.js +15 -13
- package/dist/cli/commands/run-signals.js +6 -2
- package/dist/observer/render.js +5 -3
- package/dist/routes/computer-use/bundle-parts.d.ts +9 -0
- package/dist/routes/computer-use/bundle-parts.js +24 -4
- package/dist/routes/computer-use/fanout-bundle.js +5 -3
- package/dist/routes/computer-use/single-bundle.js +2 -2
- package/dist/run/locate.d.ts +8 -2
- package/dist/run/locate.js +10 -4
- package/dist/run/run.js +18 -2
- package/dist/run/sandbox-ids.d.ts +4 -3
- package/dist/run/sandbox-ids.js +8 -6
- package/dist/run/stream-trace.d.ts +8 -2
- package/dist/run/stream-trace.js +13 -7
- package/dist/study/bundle-estimate.js +23 -17
- package/dist/verify/artifacts.d.ts +7 -1
- package/dist/verify/artifacts.js +15 -4
- package/dist/verify/verify.js +29 -8
- package/docs/architecture/actor-contract.md +3 -1
- package/docs/architecture/state-driven-executor.md +7 -4
- package/docs/contracts/run-bundle.md +13 -2
- package/docs/contracts/study-analysis.md +4 -2
- package/package.json +1 -1
package/dist/analysis/service.js
CHANGED
|
@@ -11,7 +11,7 @@ import { loadRunBundlePrepared, resolveRunPath } from "../run/locate.js";
|
|
|
11
11
|
import { physicalCwdOf, runIdOf, resolvePhysicalCwd, validatePreparedRunRootIdentity, } from "../run/paths.js";
|
|
12
12
|
import { isRunStatusRecord, RUN_STATUS_FILE } from "../run/status.js";
|
|
13
13
|
import { captureEvidence, EVIDENCE_LIMITS } from "./evidence.js";
|
|
14
|
-
import { AnalysisSourceRefusedError, runFileTooLarge } from "./source.js";
|
|
14
|
+
import { AnalysisSourceRefusedError, runFileTooLarge, throwAsAnalysisRefusal } from "./source.js";
|
|
15
15
|
import { pathMissing, readBoundedFile, readBoundedFileResult } from "../run/evidence-files.js";
|
|
16
16
|
import { estimateAnalysisAdmission, preferLargerAnalysisOutput, runAnalysis, ANALYSIS_PROMPT_VERSION, } from "./execute.js";
|
|
17
17
|
import { appendAnalysisCorrection, assertAnalysisPublicationCapacity, listAnalyses, writeAnalysis, } from "./store.js";
|
|
@@ -155,7 +155,8 @@ export async function readCompletedAnalysisSource(cwd, prepared) {
|
|
|
155
155
|
const verified = await verifyRunPrepared(cwd, runIdOf(prepared), prepared);
|
|
156
156
|
if (!verified.ok && verified.recordingOk !== true)
|
|
157
157
|
throw new Error("ANALYSIS_VERIFY_FAILED");
|
|
158
|
-
|
|
158
|
+
// Verify passes a run whose trace is past the read limit, which analysis does not read.
|
|
159
|
+
const loaded = await loadRunBundlePrepared(cwd, prepared).catch(throwAsAnalysisRefusal);
|
|
159
160
|
if (!loaded || loaded.bundle.streams.length === 0)
|
|
160
161
|
throw new Error("ANALYSIS_NO_PARTICIPANTS");
|
|
161
162
|
if (loaded.bundle.mode !== "live")
|
|
@@ -26,3 +26,9 @@ export declare function parseAnalysisSource(prepared: PreparedRunArtifactPaths,
|
|
|
26
26
|
* ANALYSIS_SOURCE_TOO_LARGE, and any other trace the loader refuses as ANALYSIS_SOURCE_INVALID.
|
|
27
27
|
*/
|
|
28
28
|
export declare function readParticipantTraces<T>(prepared: PreparedRunArtifactPaths, bundle: StoredRunBundle, keep: (stream: RunStream, index: number) => T): Promise<T[]>;
|
|
29
|
+
/**
|
|
30
|
+
* The trace loader's refusal as analysis gives it, ANALYSIS_SOURCE_TOO_LARGE for a trace past the
|
|
31
|
+
* read limit and ANALYSIS_SOURCE_INVALID for any other; another error as it is. The loader's
|
|
32
|
+
* message names the participant, the file, why, and what to do.
|
|
33
|
+
*/
|
|
34
|
+
export declare function throwAsAnalysisRefusal(error: unknown): never;
|
package/dist/analysis/source.js
CHANGED
|
@@ -78,20 +78,22 @@ export function parseAnalysisSource(prepared, bytes) {
|
|
|
78
78
|
* ANALYSIS_SOURCE_TOO_LARGE, and any other trace the loader refuses as ANALYSIS_SOURCE_INVALID.
|
|
79
79
|
*/
|
|
80
80
|
export async function readParticipantTraces(prepared, bundle, keep) {
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
81
|
+
return reduceRunTraces(prepared, bundle, (stream, index) => {
|
|
82
|
+
validateTraceItems(stream);
|
|
83
|
+
return keep(stream, index);
|
|
84
|
+
}).catch(throwAsAnalysisRefusal);
|
|
85
|
+
}
|
|
86
|
+
/**
|
|
87
|
+
* The trace loader's refusal as analysis gives it, ANALYSIS_SOURCE_TOO_LARGE for a trace past the
|
|
88
|
+
* read limit and ANALYSIS_SOURCE_INVALID for any other; another error as it is. The loader's
|
|
89
|
+
* message names the participant, the file, why, and what to do.
|
|
90
|
+
*/
|
|
91
|
+
export function throwAsAnalysisRefusal(error) {
|
|
92
|
+
if (!(error instanceof StreamTraceRefusedError))
|
|
93
93
|
throw error;
|
|
94
|
-
|
|
94
|
+
throw new AnalysisSourceRefusedError(error instanceof StreamTraceTooLargeError
|
|
95
|
+
? "ANALYSIS_SOURCE_TOO_LARGE"
|
|
96
|
+
: "ANALYSIS_SOURCE_INVALID", error.message);
|
|
95
97
|
}
|
|
96
98
|
function validateTraceItems(stream) {
|
|
97
99
|
const actor = stream.actor;
|
|
@@ -134,8 +134,12 @@ async function stopActiveRuns(io, signal, stopping, options) {
|
|
|
134
134
|
// not swept; verify grades a file that names a receipt's id local_only.
|
|
135
135
|
Promise.all(stopping
|
|
136
136
|
.filter((stop) => stop.report !== null)
|
|
137
|
-
.map((stop) => scrubRunSandboxIds(stop.run.paths).
|
|
138
|
-
|
|
137
|
+
.map((stop) => scrubRunSandboxIds(stop.run.paths).then(
|
|
138
|
+
// A file the sweep refused to read may still name a sandbox; say which.
|
|
139
|
+
(unread) => {
|
|
140
|
+
if (unread !== undefined)
|
|
141
|
+
say(io, `humanish: ${signal}: ${unread}`);
|
|
142
|
+
}, (error) => {
|
|
139
143
|
if (error instanceof Error)
|
|
140
144
|
say(io, `humanish: ${signal}: ${error.message}`);
|
|
141
145
|
}))));
|
package/dist/observer/render.js
CHANGED
|
@@ -90,11 +90,13 @@ export async function renderObserver(cwdInput, runInput, options = {}) {
|
|
|
90
90
|
? "HUMANISH_RUN_NOT_FOUND"
|
|
91
91
|
: "HUMANISH_INVALID_RUN_BUNDLE", verified.error?.message ?? "Run bundle failed verification.");
|
|
92
92
|
}
|
|
93
|
-
|
|
93
|
+
// Verify passes a trace past the read limit, so its participant shows without its items.
|
|
94
|
+
const loaded = await loadRunBundlePrepared(selectedPhysicalCwd, preparedRunPaths, {
|
|
95
|
+
display: true,
|
|
96
|
+
});
|
|
94
97
|
await validatePreparedRunArtifactPaths(preparedRunPaths);
|
|
95
|
-
if (!loaded)
|
|
98
|
+
if (!loaded)
|
|
96
99
|
return runNotFound(cwd, runInput);
|
|
97
|
-
}
|
|
98
100
|
if (loaded.bundle.runId !== selection.runId ||
|
|
99
101
|
(await realpath(loaded.runDir)) !== preparedRunPaths.physicalRunRoot) {
|
|
100
102
|
throw new Error("Observer output directory does not match the selected run.");
|
|
@@ -1,6 +1,15 @@
|
|
|
1
|
+
import type { CuaLoopResult } from "../../actors/computer-use/loop.js";
|
|
1
2
|
import { type RunProviderResource, type RunSubjectProvenance } from "../../run/bundle.js";
|
|
2
3
|
import { type ParticipantIds } from "../../run/participant-records.js";
|
|
3
4
|
import { type ParticipantRunOutcome } from "./types.js";
|
|
5
|
+
/**
|
|
6
|
+
* The review's gap for the streaks a participant's trace records: turns that repeated a recent
|
|
7
|
+
* action on an unchanged screen past noProgressSteps, which did not end the session of a
|
|
8
|
+
* participant held to its scheduled end. Its session passes, so no other gap names them. One line
|
|
9
|
+
* names the first streak and counts the rest: a page that changes every minute, watched 1 s at a
|
|
10
|
+
* time, records a streak a minute, and a fan-out would list each.
|
|
11
|
+
*/
|
|
12
|
+
export declare function streakGaps(participantId: string, session: CuaLoopResult | undefined): string[];
|
|
4
13
|
/** The human-readable state story appended to the provenance event (and review.md via it). */
|
|
5
14
|
export declare function describeSubjectState(state: RunSubjectProvenance["state"], dryRun: boolean): string;
|
|
6
15
|
/** Human-readable provenance line for the single-participant subject.provenance event of a clone
|
|
@@ -1,14 +1,34 @@
|
|
|
1
1
|
// The parts both computer-use bundle shapes share: the subject-state story and provenance line,
|
|
2
|
-
// provider-resource records,
|
|
3
|
-
// subject-phase event id suffix. The builders are bundle.ts
|
|
4
|
-
// single-bundle.ts and fanout-bundle.ts. Feedback candidates from
|
|
5
|
-
// participant-feedback.ts, which the shared-world bundle reads too.
|
|
2
|
+
// provider-resource records, the review's gap for a held participant's streak, and the public-safe
|
|
3
|
+
// app URL label, the URL digest and the subject-phase event id suffix. The builders are bundle.ts
|
|
4
|
+
// (the dispatcher and the judge), single-bundle.ts and fanout-bundle.ts. Feedback candidates from
|
|
5
|
+
// participant reports are in participant-feedback.ts, which the shared-world bundle reads too.
|
|
6
|
+
import { STREAK_TITLE } from "../../actors/computer-use/loop/streak.js";
|
|
6
7
|
import { scanEncodedText } from "../../evidence/encoded-text.js";
|
|
7
8
|
import { digestText } from "../../evidence/redaction.js";
|
|
8
9
|
import {} from "../../run/bundle.js";
|
|
9
10
|
import { participantResourceIds } from "../../run/participant-records.js";
|
|
10
11
|
import {} from "./types.js";
|
|
11
12
|
import { plural } from "../../run/text.js";
|
|
13
|
+
/**
|
|
14
|
+
* The review's gap for the streaks a participant's trace records: turns that repeated a recent
|
|
15
|
+
* action on an unchanged screen past noProgressSteps, which did not end the session of a
|
|
16
|
+
* participant held to its scheduled end. Its session passes, so no other gap names them. One line
|
|
17
|
+
* names the first streak and counts the rest: a page that changes every minute, watched 1 s at a
|
|
18
|
+
* time, records a streak a minute, and a fan-out would list each.
|
|
19
|
+
*/
|
|
20
|
+
export function streakGaps(participantId, session) {
|
|
21
|
+
const texts = (session?.trace.items ?? []).flatMap((item) => item.kind === "notice" && item.title === STREAK_TITLE && item.text !== undefined
|
|
22
|
+
? [item.text]
|
|
23
|
+
: []);
|
|
24
|
+
const [first] = texts;
|
|
25
|
+
if (first === undefined)
|
|
26
|
+
return [];
|
|
27
|
+
const more = texts.length > 1
|
|
28
|
+
? ` It had ${plural(texts.length - 1, "more such streak")} after that; its trace records each.`
|
|
29
|
+
: "";
|
|
30
|
+
return [`${participantId}: ${first}${more}`];
|
|
31
|
+
}
|
|
12
32
|
/** The human-readable state story appended to the provenance event (and review.md via it). */
|
|
13
33
|
export function describeSubjectState(state, dryRun) {
|
|
14
34
|
switch (state.provenance) {
|
|
@@ -5,7 +5,7 @@ import { cuaGoalSource } from "../../actors/goal-source.js";
|
|
|
5
5
|
import { redactText } from "../../evidence/redaction.js";
|
|
6
6
|
import { REVIEW_SCHEMA, bundleArtifacts, bundleHead, } from "../../run/bundle.js";
|
|
7
7
|
import { aggregateTaskFunnels, formatParticipantOutcomes, formatRunTaskFunnel, gapsListClause, skippedSentence, withCuaReviewProvenance, } from "../../run/outcomes.js";
|
|
8
|
-
import { providerResourcesForOutcome, publicSafeAppUrlLabel } from "./bundle-parts.js";
|
|
8
|
+
import { providerResourcesForOutcome, publicSafeAppUrlLabel, streakGaps } from "./bundle-parts.js";
|
|
9
9
|
import { subjectName } from "../../run/subject-name.js";
|
|
10
10
|
import { participantFeedbackCandidates } from "./participant-feedback.js";
|
|
11
11
|
import { participantFactsOf } from "./participant-facts.js";
|
|
@@ -70,18 +70,20 @@ function fanoutSummary(args, facts) {
|
|
|
70
70
|
}
|
|
71
71
|
/**
|
|
72
72
|
* The review's gaps: one line for the participants a host suspension likely failed, then a line
|
|
73
|
-
* for each other participant that did not pass.
|
|
73
|
+
* for each other participant that did not pass, then a line for each held participant's streaks.
|
|
74
74
|
*/
|
|
75
75
|
function fanoutGaps(args, judgment) {
|
|
76
76
|
const suspended = new Set(args.hostSuspension?.participantIds ?? []);
|
|
77
77
|
const others = judgment.participants.flatMap((participant, index) => participant.gapLine === undefined || suspended.has(args.specs[index].planned.id)
|
|
78
78
|
? []
|
|
79
79
|
: [participant.gapLine]);
|
|
80
|
+
const streaks = args.specs.flatMap((spec, index) => streakGaps(spec.planned.id, args.outcomes?.[index]?.session));
|
|
80
81
|
if (suspended.size === 0)
|
|
81
|
-
return others;
|
|
82
|
+
return [...others, ...streaks];
|
|
82
83
|
return [
|
|
83
84
|
`${[...suspended].join(", ")}: failed after the host was suspended, likely its effect. Each participant's record keeps its own error.`,
|
|
84
85
|
...others,
|
|
86
|
+
...streaks,
|
|
85
87
|
];
|
|
86
88
|
}
|
|
87
89
|
function fanoutReview(args, streams, judgment) {
|
|
@@ -6,7 +6,7 @@ import { aggregateTaskFunnels, formatParticipantOutcomes, formatRunTaskFunnel, p
|
|
|
6
6
|
import { participantCaption } from "../../run/participant-caption.js";
|
|
7
7
|
import { actorEnding } from "../../actors/stop-cause.js";
|
|
8
8
|
import { cuaGoalSource, ranUntilClockEnd } from "../../actors/goal-source.js";
|
|
9
|
-
import { describeSubjectState, phaseEventIdSuffix, publicSafeAppUrlLabel, subjectProvenanceMessage, } from "./bundle-parts.js";
|
|
9
|
+
import { describeSubjectState, phaseEventIdSuffix, publicSafeAppUrlLabel, streakGaps, subjectProvenanceMessage, } from "./bundle-parts.js";
|
|
10
10
|
import { participantFeedbackCandidates } from "./participant-feedback.js";
|
|
11
11
|
import { judgeParticipantRecords } from "../../run/judge.js";
|
|
12
12
|
import { buildRunCostSummary } from "../../run/cost-summary.js";
|
|
@@ -348,7 +348,7 @@ function singleReview(args, view, stream) {
|
|
|
348
348
|
...(credibilityNote === undefined ? {} : { credibilityNote }),
|
|
349
349
|
})}`,
|
|
350
350
|
gaps: args.session || args.sessionError !== undefined
|
|
351
|
-
?
|
|
351
|
+
? streakGaps(args.participantId ?? "lane-01", args.session)
|
|
352
352
|
: args.inProgress === true
|
|
353
353
|
? ["Live desktop session is still running."]
|
|
354
354
|
: ["Live desktop session not yet run (dry run only)."],
|
package/dist/run/locate.d.ts
CHANGED
|
@@ -55,8 +55,14 @@ export declare function loadRunBundle(cwdInput: string, runInput: string): Promi
|
|
|
55
55
|
bundlePath: string;
|
|
56
56
|
runDir: string;
|
|
57
57
|
} | null>;
|
|
58
|
-
/**
|
|
59
|
-
|
|
58
|
+
/**
|
|
59
|
+
* loadRunBundle for a caller that already holds prepared paths. Revalidates them first. With
|
|
60
|
+
* `display`, a participant whose trace the loader refuses, such as one past the read limit, is
|
|
61
|
+
* kept as displayedTraces shows it, for a page that shows the run.
|
|
62
|
+
*/
|
|
63
|
+
export declare function loadRunBundlePrepared(cwdInput: string, runPaths: PreparedRunArtifactPaths, options?: {
|
|
64
|
+
display?: boolean;
|
|
65
|
+
}): Promise<{
|
|
60
66
|
bundle: RunBundle;
|
|
61
67
|
bundlePath: string;
|
|
62
68
|
runDir: string;
|
package/dist/run/locate.js
CHANGED
|
@@ -3,7 +3,7 @@ import { bindExistingRunArtifactPaths, isSafeRunIdSegment, resolveExistingRunDir
|
|
|
3
3
|
import { assertPreparedSelectedOutputDirectory, bindExistingManagedHumanishOutputDirectory, ContainedReadRefusedError, LATEST_POINTER_MAX_BYTES, readContainedRegularFile, RUN_ARTIFACT_MAX_BYTES, } from "./contained-output.js";
|
|
4
4
|
import { RUN_BUNDLE_FILE } from "./bundle.js";
|
|
5
5
|
import { isRunBundle, isRunPointer } from "./bundle-shape.js";
|
|
6
|
-
import { loadRunTraces } from "./stream-trace.js";
|
|
6
|
+
import { displayedTraces, loadRunTraces, loadStreamTraces } from "./stream-trace.js";
|
|
7
7
|
import { readUnchangedFile } from "./evidence-files.js";
|
|
8
8
|
/** The run input that resolves through latest.json rather than naming a run directory. */
|
|
9
9
|
export const LATEST_RUN_ALIAS = "latest";
|
|
@@ -119,8 +119,12 @@ export async function loadRunBundle(cwdInput, runInput) {
|
|
|
119
119
|
}
|
|
120
120
|
return loadRunBundlePrepared(cwd, runPaths);
|
|
121
121
|
}
|
|
122
|
-
/**
|
|
123
|
-
|
|
122
|
+
/**
|
|
123
|
+
* loadRunBundle for a caller that already holds prepared paths. Revalidates them first. With
|
|
124
|
+
* `display`, a participant whose trace the loader refuses, such as one past the read limit, is
|
|
125
|
+
* kept as displayedTraces shows it, for a page that shows the run.
|
|
126
|
+
*/
|
|
127
|
+
export async function loadRunBundlePrepared(cwdInput, runPaths, options = {}) {
|
|
124
128
|
const cwd = path.resolve(cwdInput);
|
|
125
129
|
await validatePreparedRunArtifactPaths(runPaths);
|
|
126
130
|
const bundlePath = path.join(runPaths.absoluteRunRoot, RUN_BUNDLE_FILE);
|
|
@@ -134,7 +138,9 @@ export async function loadRunBundlePrepared(cwdInput, runPaths) {
|
|
|
134
138
|
}
|
|
135
139
|
return {
|
|
136
140
|
// A trace that does not match its index throws StreamTraceRefusedError, naming the participant.
|
|
137
|
-
bundle:
|
|
141
|
+
bundle: options.display === true
|
|
142
|
+
? displayedTraces(await loadStreamTraces(runPaths, bundle))
|
|
143
|
+
: await loadRunTraces(runPaths, bundle),
|
|
138
144
|
bundlePath: path.relative(cwd, bundlePath),
|
|
139
145
|
runDir: runPaths.absoluteRunRoot,
|
|
140
146
|
};
|
package/dist/run/run.js
CHANGED
|
@@ -148,6 +148,14 @@ async function writeRunJson(paths, liveItems, bundle) {
|
|
|
148
148
|
});
|
|
149
149
|
return { ...publicBundle, streams };
|
|
150
150
|
}
|
|
151
|
+
/** The warn event that names the files the final sandbox id sweep could not read. */
|
|
152
|
+
const unsweptEvent = (message, at) => ({
|
|
153
|
+
id: "event-sandbox-ids-unresolved",
|
|
154
|
+
at: new Date(at).toISOString(),
|
|
155
|
+
level: "warn",
|
|
156
|
+
type: "sandbox-ids.unresolved",
|
|
157
|
+
message,
|
|
158
|
+
});
|
|
151
159
|
function runPublisher(args) {
|
|
152
160
|
const { options, runId, createdAt, paths, runStatus, now, admit, participantsRan } = args;
|
|
153
161
|
const observer = observerTarget(options);
|
|
@@ -161,11 +169,14 @@ function runPublisher(args) {
|
|
|
161
169
|
type: "study.warning",
|
|
162
170
|
message: redactText(message),
|
|
163
171
|
}));
|
|
164
|
-
// So do the host suspensions the heartbeat has seen by the time of the write
|
|
172
|
+
// So do the host suspensions the heartbeat has seen by the time of the write, and the files the
|
|
173
|
+
// final sandbox id sweep could not read.
|
|
174
|
+
let unswept = [];
|
|
165
175
|
const withRunEvents = (events) => {
|
|
166
176
|
const owned = [
|
|
167
177
|
...studyWarnings,
|
|
168
178
|
...hostSuspensionEvents(runStatus.suspensions(), Date.parse(createdAt)),
|
|
179
|
+
...unswept,
|
|
169
180
|
];
|
|
170
181
|
return [...events.filter((event) => !owned.some((mine) => mine.id === event.id)), ...owned];
|
|
171
182
|
};
|
|
@@ -231,7 +242,12 @@ function runPublisher(args) {
|
|
|
231
242
|
await writeBundleFiles(bundle, finishedOutcome(outcome), (publicBundle) => runStatus.finish(runStatusOutcome(publicBundle)));
|
|
232
243
|
// The final run.json indexes each finished trace, so the live files are journal no reader needs.
|
|
233
244
|
await liveItems.remove();
|
|
234
|
-
await scrubRunSandboxIds(paths);
|
|
245
|
+
const unread = await scrubRunSandboxIds(paths);
|
|
246
|
+
if (unread !== undefined) {
|
|
247
|
+
unswept = [unsweptEvent(unread, now())];
|
|
248
|
+
// The sweep reads the files the final write left, so run.json is written again to record it.
|
|
249
|
+
await writeBundleFiles(bundle, finishedOutcome(outcome), async () => { });
|
|
250
|
+
}
|
|
235
251
|
await writePointer();
|
|
236
252
|
let current = { ok: outcome.ok, execution: outcome.execution };
|
|
237
253
|
// An Observer that did not render, judged under the route's policy and written to run.json
|
|
@@ -45,10 +45,11 @@ export declare function scrubSandboxIdBytes(file: string, bytes: Buffer, ids: re
|
|
|
45
45
|
* receipts, with the label in place of the id. Writers such as a participant's actor.json can
|
|
46
46
|
* quote an SDK error that names the sandbox. Run.finish calls it before analysis reads the run.
|
|
47
47
|
* Best effort for writes and listings: verify grades a file this misses. A text file it refuses
|
|
48
|
-
* to read,
|
|
49
|
-
*
|
|
48
|
+
* to read, such as a trace past RUN_ARTIFACT_MAX_BYTES, may still hold an id, so the sweep
|
|
49
|
+
* returns a warning that names each such file and why, for the caller to record; the run goes
|
|
50
|
+
* on. A journal it refuses throws, since without its ids no file can be swept.
|
|
50
51
|
*/
|
|
51
|
-
export declare function scrubRunSandboxIds(paths: PreparedRunArtifactPaths): Promise<
|
|
52
|
+
export declare function scrubRunSandboxIds(paths: PreparedRunArtifactPaths): Promise<string | undefined>;
|
|
52
53
|
/**
|
|
53
54
|
* A run's result as it may be returned or printed: the same object when it names no sandbox,
|
|
54
55
|
* otherwise its public view, scrubbed of the ids the run's receipts journal.
|
package/dist/run/sandbox-ids.js
CHANGED
|
@@ -197,16 +197,17 @@ const MAX_SWEPT_ENTRIES = 10_000;
|
|
|
197
197
|
* receipts, with the label in place of the id. Writers such as a participant's actor.json can
|
|
198
198
|
* quote an SDK error that names the sandbox. Run.finish calls it before analysis reads the run.
|
|
199
199
|
* Best effort for writes and listings: verify grades a file this misses. A text file it refuses
|
|
200
|
-
* to read,
|
|
201
|
-
*
|
|
200
|
+
* to read, such as a trace past RUN_ARTIFACT_MAX_BYTES, may still hold an id, so the sweep
|
|
201
|
+
* returns a warning that names each such file and why, for the caller to record; the run goes
|
|
202
|
+
* on. A journal it refuses throws, since without its ids no file can be swept.
|
|
202
203
|
*/
|
|
203
204
|
export async function scrubRunSandboxIds(paths) {
|
|
204
205
|
const ids = await readRunSandboxIds(paths);
|
|
205
206
|
if (ids.length === 0)
|
|
206
|
-
return;
|
|
207
|
+
return undefined;
|
|
207
208
|
const valid = await validatePreparedRunRootIdentity(paths).then(() => true, () => false);
|
|
208
209
|
if (!valid)
|
|
209
|
-
return;
|
|
210
|
+
return undefined;
|
|
210
211
|
const refused = [];
|
|
211
212
|
const scrub = async (file) => {
|
|
212
213
|
const read = await readContainedRegularFile(paths, file, RUN_ARTIFACT_MAX_BYTES);
|
|
@@ -242,8 +243,9 @@ export async function scrubRunSandboxIds(paths) {
|
|
|
242
243
|
await walk("").catch(() => {
|
|
243
244
|
// A directory that changed while it was listed ends the sweep; verify names what it missed.
|
|
244
245
|
});
|
|
245
|
-
|
|
246
|
-
|
|
246
|
+
return refused.length === 0
|
|
247
|
+
? undefined
|
|
248
|
+
: `The sandbox id sweep could not read every run file, so these may still name a sandbox: ${refused.join("; ")}.`;
|
|
247
249
|
}
|
|
248
250
|
/**
|
|
249
251
|
* A run's result as it may be returned or printed: the same object when it names no sandbox,
|
|
@@ -11,10 +11,16 @@ export declare class StreamTraceRefusedError extends Error {
|
|
|
11
11
|
constructor(streamId: string, detail: string, advice?: string);
|
|
12
12
|
}
|
|
13
13
|
/**
|
|
14
|
-
* A trace or live file whose index records more than the loader reads of one file
|
|
15
|
-
* no file past the limit, so a copy of the file is refused too.
|
|
14
|
+
* A trace or live file whose index records more than the loader reads of one file, `maxBytes`. The
|
|
15
|
+
* loader reads no file past the limit, so a copy of the file is refused too.
|
|
16
16
|
*/
|
|
17
17
|
export declare class StreamTraceTooLargeError extends StreamTraceRefusedError {
|
|
18
|
+
/** The file's run-relative path, as its index records it. */
|
|
19
|
+
readonly file: string;
|
|
20
|
+
constructor(streamId: string, index: {
|
|
21
|
+
path: string;
|
|
22
|
+
bytes: number;
|
|
23
|
+
}, maxBytes: number);
|
|
18
24
|
}
|
|
19
25
|
/** `trace` without the per-request lists; every other field as it is. */
|
|
20
26
|
export declare function withoutRequestLists<T extends object>(trace: T): WithoutRequestLists<T>;
|
package/dist/run/stream-trace.js
CHANGED
|
@@ -28,13 +28,17 @@ export class StreamTraceRefusedError extends Error {
|
|
|
28
28
|
/** Why the loader reads no trace or live file whose index records more than `maxBytes`. */
|
|
29
29
|
const pastLimitDetail = (index, maxBytes) => `run.json records ${index.path} at ${index.bytes} bytes, more than the ${maxBytes} bytes humanish reads of one file`;
|
|
30
30
|
/**
|
|
31
|
-
* A trace or live file whose index records more than the loader reads of one file
|
|
32
|
-
* no file past the limit, so a copy of the file is refused too.
|
|
31
|
+
* A trace or live file whose index records more than the loader reads of one file, `maxBytes`. The
|
|
32
|
+
* loader reads no file past the limit, so a copy of the file is refused too.
|
|
33
33
|
*/
|
|
34
34
|
export class StreamTraceTooLargeError extends StreamTraceRefusedError {
|
|
35
|
+
/** The file's run-relative path, as its index records it. */
|
|
36
|
+
file;
|
|
37
|
+
constructor(streamId, index, maxBytes) {
|
|
38
|
+
super(streamId, pastLimitDetail(index, maxBytes), "Restoring a copy does not help; open the file with another tool, such as jq, to read its items.");
|
|
39
|
+
this.file = index.path;
|
|
40
|
+
}
|
|
35
41
|
}
|
|
36
|
-
/** The refusal of a trace or live file whose index records more than `maxBytes`. */
|
|
37
|
-
const tooLarge = (streamId, index, maxBytes) => new StreamTraceTooLargeError(streamId, pastLimitDetail(index, maxBytes), "Restoring a copy does not help; open the file with another tool, such as jq, to read its items.");
|
|
38
42
|
const sha256 = (bytes) => createHash("sha256").update(bytes).digest("hex");
|
|
39
43
|
const isScreenshot = (artifact) => artifact.kind === "screenshot";
|
|
40
44
|
/**
|
|
@@ -139,7 +143,9 @@ export function splitStreamTraces(bundle) {
|
|
|
139
143
|
throw new Error(`Two participants' traces would both be written to ${filePath}.`);
|
|
140
144
|
claimed.add(traceFileKey(filePath));
|
|
141
145
|
const screenshots = stream.artifacts.filter(isScreenshot);
|
|
142
|
-
|
|
146
|
+
// Compact: every reader refuses a file past RUN_ARTIFACT_MAX_BYTES, and indenting made a
|
|
147
|
+
// trace 1.36 times its compact size on 236 recorded computer-use traces.
|
|
148
|
+
const bytes = Buffer.from(`${JSON.stringify({ ...actor, artifacts: screenshots })}\n`);
|
|
143
149
|
files.push({ path: filePath, bytes });
|
|
144
150
|
const { items, ...summary } = actor;
|
|
145
151
|
return {
|
|
@@ -274,7 +280,7 @@ async function readLiveItems(read, stream, maxBytes, liveAfter) {
|
|
|
274
280
|
const after = liveAfter?.(index);
|
|
275
281
|
if (index.bytes > maxBytes) {
|
|
276
282
|
if (after === undefined || !cursorFits(index, after))
|
|
277
|
-
throw
|
|
283
|
+
throw new StreamTraceTooLargeError(stream.id, index, maxBytes);
|
|
278
284
|
// The caller keeps the prefix it holds, read when the file was under the limit.
|
|
279
285
|
return {
|
|
280
286
|
stream: { ...stream, liveActor: { ...summary, items: [] } },
|
|
@@ -343,7 +349,7 @@ async function readStreamTrace(read, stream, maxBytes, liveAfter) {
|
|
|
343
349
|
if (stream.trace !== undefined) {
|
|
344
350
|
assertIndex(stream);
|
|
345
351
|
if (stream.trace.bytes > maxBytes)
|
|
346
|
-
throw
|
|
352
|
+
throw new StreamTraceTooLargeError(stream.id, stream.trace, maxBytes);
|
|
347
353
|
const file = await read(stream.trace.path, maxBytes);
|
|
348
354
|
if ("refused" in file)
|
|
349
355
|
throw new StreamTraceRefusedError(stream.id, file.refused);
|
|
@@ -32,9 +32,15 @@ import { runEntryLimit } from "../verify/scan-limit.js";
|
|
|
32
32
|
// the 0.125.0 smoke test and the 2026-10-12 live checks, 10 participants with 1 to 28 turns and 137
|
|
33
33
|
// model requests. Dry: runs of 1, 2, 24 and 100 participants, for the Observer app and the run's
|
|
34
34
|
// own files. Pace, session length and screenshots: the live computer-use runs kept on the
|
|
35
|
-
// maintainer's machine.
|
|
36
|
-
//
|
|
37
|
-
// The
|
|
35
|
+
// maintainer's machine. The run.json and page data rates were measured again on those runs
|
|
36
|
+
// written as v4.
|
|
37
|
+
// The trace rates were measured on trace files written indented. The writer now writes them as
|
|
38
|
+
// compact JSON, so each trace rate below is the indented figure divided by 1.36, the ratio of
|
|
39
|
+
// indented to compact bytes over 236 kept computer-use traces (1.17 to 1.49 each), and the page's
|
|
40
|
+
// share of the traces is multiplied by it. A trace's rate a turn on the public
|
|
41
|
+
// site/public/runs/lobby-0927 run (8 participants, 20 to 78 turns) was 1,579 bytes plus 91 for the
|
|
42
|
+
// page indented, under the live fit. Its participant of 78 turns writes a trace of 149,637 bytes
|
|
43
|
+
// indented and 107,897 compact.
|
|
38
44
|
const RATES = Object.freeze({
|
|
39
45
|
/** run.json bytes for the run itself: its header, study, review and cost. Live fit: 4,381. */
|
|
40
46
|
runBytes: 4_400,
|
|
@@ -44,21 +50,21 @@ const RATES = Object.freeze({
|
|
|
44
50
|
* bytes and those of two 30,647 to 38,948, beside 20,900 and 37,400 at these rates.
|
|
45
51
|
*/
|
|
46
52
|
participantBytes: 16_500,
|
|
47
|
-
/** A trace file's bytes before its turns: the actor summary. Live fit: 5,378. */
|
|
48
|
-
participantTraceBytes:
|
|
53
|
+
/** A trace file's bytes before its turns: the actor summary. Live fit: 5,378 indented. */
|
|
54
|
+
participantTraceBytes: 4_000,
|
|
49
55
|
/**
|
|
50
56
|
* Trace bytes a model turn adds: its items, its screenshot entry and its request's usage, plus 91
|
|
51
|
-
* for the `page` each screenshot item now carries. Live fit: 1,965 a turn.
|
|
52
|
-
* or more turns ranged from 1,140 to 4,532; the busiest made 311 items in 15
|
|
53
|
-
* UI actions. Analysis refuses the whole run when one trace passes its limit,
|
|
54
|
-
* counted at the busiest participant's rate.
|
|
57
|
+
* for the `page` each screenshot item now carries, all indented. Live fit: 1,965 a turn.
|
|
58
|
+
* Participants with 4 or more turns ranged from 1,140 to 4,532; the busiest made 311 items in 15
|
|
59
|
+
* turns, most of them UI actions. Analysis refuses the whole run when one trace passes its limit,
|
|
60
|
+
* so each trace is counted at the busiest participant's rate, 4,700 indented.
|
|
55
61
|
*/
|
|
56
|
-
turnTraceBytes:
|
|
62
|
+
turnTraceBytes: 3_500,
|
|
57
63
|
/**
|
|
58
64
|
* Trace bytes a dwell frame adds: 395 for its item and entry on the kept runs with dwell frames,
|
|
59
|
-
* plus 91 for the page.
|
|
65
|
+
* plus 91 for the page, indented.
|
|
60
66
|
*/
|
|
61
|
-
frameTraceBytes:
|
|
67
|
+
frameTraceBytes: 400,
|
|
62
68
|
/**
|
|
63
69
|
* Seconds between model turns: the median of the 27 of 185 kept live participants that worked 5
|
|
64
70
|
* minutes or more was 10.4.
|
|
@@ -83,12 +89,12 @@ const RATES = Object.freeze({
|
|
|
83
89
|
*/
|
|
84
90
|
observerPageDataShare: 0.85,
|
|
85
91
|
/**
|
|
86
|
-
* The page's gzipped base64 items, as a share of all the traces' bytes: 0.087 to 0.114
|
|
87
|
-
* live runs of two participants, 0.097 for the busiest participant alone (about 475 bytes
|
|
88
|
-
* turn), 0.142 to 0.184 on runs of one participant and 4 to 11 turns, and 0.095 on 0.124.0's
|
|
89
|
-
* synthetic 8-hour run.
|
|
92
|
+
* The page's gzipped base64 items, as a share of all the traces' bytes indented: 0.087 to 0.114
|
|
93
|
+
* on the live runs of two participants, 0.097 for the busiest participant alone (about 475 bytes
|
|
94
|
+
* a turn), 0.142 to 0.184 on runs of one participant and 4 to 11 turns, and 0.095 on 0.124.0's
|
|
95
|
+
* synthetic 8-hour run. The estimate counted 0.12 of indented traces.
|
|
90
96
|
*/
|
|
91
|
-
observerPageItemsShare: 0.
|
|
97
|
+
observerPageItemsShare: 0.16,
|
|
92
98
|
/** A desktop screenshot: the median of 3,990 kept live PNG frames was 107,895 bytes. */
|
|
93
99
|
screenshotBytes: 110_000,
|
|
94
100
|
/**
|
|
@@ -22,7 +22,13 @@ export interface UnscannedArtifacts {
|
|
|
22
22
|
* and where it stopped, each in its own sentence. Undefined when the scan read everything.
|
|
23
23
|
*/
|
|
24
24
|
export declare function unscannedMessage(unscanned: UnscannedArtifacts): string | undefined;
|
|
25
|
-
|
|
25
|
+
/**
|
|
26
|
+
* Why each local evidence path the bundle requires is missing or unusable. A path in
|
|
27
|
+
* `pastReadLimit` is a trace verify found larger than it reads, which verify lists as unscanned so
|
|
28
|
+
* the run grades local_only. It meets a requirement only while it is still past the limit when
|
|
29
|
+
* this check reads it, and never a screenshot or recording requirement.
|
|
30
|
+
*/
|
|
31
|
+
export declare function missingLocalEvidenceArtifacts(runPaths: PreparedRunArtifactPaths, bundle: RunBundle, pastReadLimit: ReadonlySet<string>): Promise<string[]>;
|
|
26
32
|
/**
|
|
27
33
|
* The PNG frames that actor traces register as stream screenshots, relative to the run root.
|
|
28
34
|
* RAW_SCREENSHOTS grades these, so the public-safety scan does not report them as unread. Only
|
package/dist/verify/artifacts.js
CHANGED
|
@@ -47,7 +47,13 @@ function listed(paths) {
|
|
|
47
47
|
const more = paths.length - MAX_LISTED_UNSCANNED;
|
|
48
48
|
return `${paths.slice(0, MAX_LISTED_UNSCANNED).join(", ")}${more > 0 ? ` and ${more} more` : ""}`;
|
|
49
49
|
}
|
|
50
|
-
|
|
50
|
+
/**
|
|
51
|
+
* Why each local evidence path the bundle requires is missing or unusable. A path in
|
|
52
|
+
* `pastReadLimit` is a trace verify found larger than it reads, which verify lists as unscanned so
|
|
53
|
+
* the run grades local_only. It meets a requirement only while it is still past the limit when
|
|
54
|
+
* this check reads it, and never a screenshot or recording requirement.
|
|
55
|
+
*/
|
|
56
|
+
export async function missingLocalEvidenceArtifacts(runPaths, bundle, pastReadLimit) {
|
|
51
57
|
const recordings = new Map(bundle.streams.flatMap((stream) => stream.recording ? [[stream.recording.path, stream.recording]] : []));
|
|
52
58
|
const requiredPaths = new Map();
|
|
53
59
|
const addRequiredPath = (artifactPath, options = {}) => {
|
|
@@ -110,12 +116,15 @@ export async function missingLocalEvidenceArtifacts(runPaths, bundle) {
|
|
|
110
116
|
}
|
|
111
117
|
}
|
|
112
118
|
// Reads overlap, so one frame's file system calls run while another frame's PNG is checked.
|
|
113
|
-
const found = await mapWithConcurrency([...requiredPaths], EVIDENCE_READ_CONCURRENCY, ([artifactPath, requirements]) => missingArtifact(runPaths, artifactPath, requirements, recordings.get(artifactPath)));
|
|
119
|
+
const found = await mapWithConcurrency([...requiredPaths], EVIDENCE_READ_CONCURRENCY, ([artifactPath, requirements]) => missingArtifact(runPaths, artifactPath, { ...requirements, pastReadLimit: pastReadLimit.has(artifactPath) }, recordings.get(artifactPath)));
|
|
114
120
|
return found.filter((entry) => entry !== null);
|
|
115
121
|
}
|
|
116
122
|
// Evidence files verify reads at once. Each read holds at most RUN_ARTIFACT_MAX_BYTES.
|
|
117
123
|
const EVIDENCE_READ_CONCURRENCY = 8;
|
|
118
|
-
/**
|
|
124
|
+
/**
|
|
125
|
+
* Why one required path fails its requirements, or null when it meets them. `pastReadLimit` says
|
|
126
|
+
* verify listed the path as a trace past the read limit, so a file still past it is present.
|
|
127
|
+
*/
|
|
119
128
|
async function missingArtifact(runPaths, artifactPath, requirements, recording) {
|
|
120
129
|
if (recording) {
|
|
121
130
|
const handle = await openContainedRegularFile(runPaths, artifactPath);
|
|
@@ -136,7 +145,9 @@ async function missingArtifact(runPaths, artifactPath, requirements, recording)
|
|
|
136
145
|
? await readScreenshot(runPaths, artifactPath)
|
|
137
146
|
: await readSafeRunArtifact(runPaths, artifactPath);
|
|
138
147
|
if (read.state === "limit")
|
|
139
|
-
return
|
|
148
|
+
return requirements.pastReadLimit && !requirements.screenshot
|
|
149
|
+
? null
|
|
150
|
+
: `${artifactPath} (larger than the ${RUN_ARTIFACT_MAX_BYTES} bytes verify reads)`;
|
|
140
151
|
if (read.state === "unavailable" ||
|
|
141
152
|
(("size" in read ? read.size : read.bytes.length) === 0 && !requirements.allowEmpty))
|
|
142
153
|
return artifactPath;
|