humanish 0.135.0 → 0.136.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/dist/actors/computer-use/loop/session.js +7 -2
- package/dist/actors/scripted-browser/actor.js +3 -3
- package/dist/actors/scripted-browser/steps.d.ts +9 -2
- package/dist/actors/scripted-browser/steps.js +17 -12
- package/dist/analysis/evidence-sources.js +4 -7
- package/dist/analysis/evidence.js +4 -4
- package/dist/analysis/source.d.ts +7 -4
- package/dist/analysis/source.js +22 -4
- package/dist/cli/commands/setup.js +7 -10
- package/dist/evidence/image.d.ts +2 -2
- package/dist/evidence/image.js +47 -32
- package/dist/observer/data.d.ts +7 -3
- package/dist/observer/render.js +2 -4
- package/dist/routes/computer-use/participant-execution.js +1 -1
- package/dist/run/bundle.d.ts +5 -1
- package/dist/run/locate.d.ts +13 -11
- package/dist/run/locate.js +20 -13
- package/dist/run/paths.js +38 -28
- package/dist/run/stream-trace.d.ts +2 -2
- package/dist/run/stream-trace.js +24 -11
- package/dist/run/streams.d.ts +18 -3
- package/dist/study/bundle-estimate.d.ts +21 -1
- package/dist/study/bundle-estimate.js +55 -29
- package/dist/tui-app.js +23 -23
- package/dist/verify/screenshot-reads.js +1 -1
- package/docs/architecture/state-driven-executor.md +4 -3
- package/docs/contracts/study-analysis.md +3 -2
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -8,6 +8,21 @@ The Unreleased section holds the full notes for the next version until it is tag
|
|
|
8
8
|
|
|
9
9
|
## Unreleased
|
|
10
10
|
|
|
11
|
+
## 0.136.0: verify takes less than half the time on large runs (2026-10-14)
|
|
12
|
+
|
|
13
|
+
humanish 0.136.0 makes `humanish verify` take less than half the time on large runs. It checks a
|
|
14
|
+
run folder for links 32 entries at a time and inflates screenshots on Node's thread pool, 1 MiB of
|
|
15
|
+
rows at a time, and still runs every check it ran before. On a run folder of 15,000 entries verify
|
|
16
|
+
took 3.3 seconds where it took 7.7, and on a run with 3,000 real screenshots 4.2 seconds where it
|
|
17
|
+
took 10.3. A computer-use participant that repeats a base64- or percent-encoded key from its
|
|
18
|
+
instructions no longer makes the run `blocked`. `humanish analyze` names the participant and the
|
|
19
|
+
event id when two run events share an id, and reports `ANALYSIS_SOURCE_INVALID` where it reported
|
|
20
|
+
`ANALYSIS_UNAVAILABLE`. The bundle estimate counts participants' instructions and task goals with
|
|
21
|
+
the mission, `init` prints three next steps in 24 lines, and reading a run refuses a conversation
|
|
22
|
+
without its requests list.
|
|
23
|
+
|
|
24
|
+
[Release notes](https://github.com/danielgwilson/humanish/releases/tag/v0.136.0)
|
|
25
|
+
|
|
11
26
|
## 0.135.0: Encoded keys in a study's own text stay out of the run (2026-10-14)
|
|
12
27
|
|
|
13
28
|
humanish 0.135.0 keeps a base64- or percent-encoded key in a study's own text out of the copies of
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { redactEncodedText } from "../../../evidence/encoded-text.js";
|
|
1
2
|
import { TaskTracker } from "../../../study/tasks.js";
|
|
2
3
|
import { StreakRecord } from "./streak.js";
|
|
3
4
|
import { TraceRecorder } from "./trace.js";
|
|
@@ -124,9 +125,13 @@ export class LoopSession {
|
|
|
124
125
|
}
|
|
125
126
|
// Model-authored narration: literal-scrub known provisioned values, then pattern-redact.
|
|
126
127
|
// A value the model transcribes (a DB password it read on screen) has no shape, so redactText
|
|
127
|
-
// alone cannot catch it; the run's scrubKnownValues, injected as scrubText, closes that.
|
|
128
|
+
// alone cannot catch it; the run's scrubKnownValues, injected as scrubText, closes that. The
|
|
129
|
+
// participant receives its instructions as written, so it can repeat a key the study wrote
|
|
130
|
+
// base64- or percent-encoded, which verify decodes; redactEncodedText removes that one too.
|
|
131
|
+
// The known values are scrubbed again last: a marker written before other text can spell one,
|
|
132
|
+
// as `[REDACTED_SECRET] and` spells `SECRET] and`.
|
|
128
133
|
redactNarration(text) {
|
|
129
|
-
return this.settings.redaction.redactText(this.scrubText(text));
|
|
134
|
+
return this.scrubText(redactEncodedText(this.settings.redaction.redactText(this.scrubText(text))));
|
|
130
135
|
}
|
|
131
136
|
/** Hand a watcher the trace so far with the running usage, without waiting on it. */
|
|
132
137
|
flush() {
|
|
@@ -124,7 +124,7 @@ export async function runScriptedBrowserSessionInPreparedRoot(options, preparedA
|
|
|
124
124
|
browser = await launch({ browserCommand, timeoutMs: options.timeoutMs });
|
|
125
125
|
}
|
|
126
126
|
catch (error) {
|
|
127
|
-
const reason = `Scripted browser launch failed: ${compactBrowserError(error)}`;
|
|
127
|
+
const reason = `Scripted browser launch failed: ${compactBrowserError(error, scrub)}`;
|
|
128
128
|
const capture = await persistScriptedFailureCapture({
|
|
129
129
|
appUrl: options.appUrl,
|
|
130
130
|
artifactRoot: preparedArtifactRoot,
|
|
@@ -196,7 +196,7 @@ async function runScriptedJourney(args) {
|
|
|
196
196
|
const started = Date.now();
|
|
197
197
|
const deadline = started + args.timeoutMs;
|
|
198
198
|
const tracePath = tracePathForBrowserSurface(args.surface);
|
|
199
|
-
const httpProbe = await probeAppUrl(args.appUrl, Math.min(args.timeoutMs, 15_000));
|
|
199
|
+
const httpProbe = await probeAppUrl(args.appUrl, Math.min(args.timeoutMs, 15_000), args.scrub);
|
|
200
200
|
let page = null;
|
|
201
201
|
const steps = [];
|
|
202
202
|
let executedSteps = 0;
|
|
@@ -230,7 +230,7 @@ async function runScriptedJourney(args) {
|
|
|
230
230
|
await assertScriptedOutputRoot(args.artifactRoot);
|
|
231
231
|
timedOut = error instanceof ScriptedJourneyTimeoutError;
|
|
232
232
|
const now = new Date().toISOString();
|
|
233
|
-
const reason = compactBrowserError(error);
|
|
233
|
+
const reason = compactBrowserError(error, args.scrub);
|
|
234
234
|
if (steps.length === 0) {
|
|
235
235
|
steps.push(...buildBlockedBrowserPersonaSteps({
|
|
236
236
|
browserJourney: args.journey,
|
|
@@ -53,10 +53,17 @@ export declare function buildBrowserTrace(args: {
|
|
|
53
53
|
surface: BrowserSurface;
|
|
54
54
|
}): Record<string, unknown>;
|
|
55
55
|
export declare function sanitizeBrowserEvidenceUrl(value: string, urlPolicy?: ScriptedBrowserEvidenceUrlPolicy): string;
|
|
56
|
-
export declare function probeAppUrl(appUrl: string, timeoutMs: number): Promise<{
|
|
56
|
+
export declare function probeAppUrl(appUrl: string, timeoutMs: number, scrubKnown: (text: string) => string): Promise<{
|
|
57
57
|
ok: boolean;
|
|
58
58
|
reason: string;
|
|
59
59
|
status?: number;
|
|
60
60
|
}>;
|
|
61
61
|
export declare function normalizeLocalAppUrl(value: string): string | null;
|
|
62
|
-
|
|
62
|
+
/**
|
|
63
|
+
* A browser error as a step records it, cut to 240 characters. It can quote the step's selector or
|
|
64
|
+
* URL, which the scenario wrote and the browser received as written, so keys are redacted as
|
|
65
|
+
* written and in the encoded forms verify decodes. The run's known values are scrubbed between
|
|
66
|
+
* the two passes, so the encoded pass cannot cut one apart before its scrub reads it. The cut
|
|
67
|
+
* comes last, so it cannot split a value either pass would find.
|
|
68
|
+
*/
|
|
69
|
+
export declare function compactBrowserError(error: unknown, scrubKnown: (text: string) => string): string;
|
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
import { mkdtemp, rm } from "node:fs/promises";
|
|
5
5
|
import os from "node:os";
|
|
6
6
|
import path from "node:path";
|
|
7
|
+
import { redactEncodedText } from "../../evidence/encoded-text.js";
|
|
7
8
|
import { assertScreenshotEvidence, stripPngMetadataChunks } from "../../evidence/image.js";
|
|
8
9
|
import { digestText, redactToSecretLabel } from "../../evidence/redaction.js";
|
|
9
10
|
import { assertPreparedSelectedOutputDirectory, assertSafeOutputPathSegment, prepareContainedOutputFile, prepareSelectedOutputDirectory, readContainedRegularFile, refusalText, RUN_ARTIFACT_MAX_BYTES, writeContainedOutputFile, } from "../../run/contained-output.js";
|
|
@@ -94,7 +95,7 @@ export async function executeBrowserPersonaStep(args) {
|
|
|
94
95
|
const screenshotPath = screenshotPathForBrowserStep(args.surface, args.step);
|
|
95
96
|
await prepareContainedOutputFile(args.absoluteArtifactRoot, screenshotPath);
|
|
96
97
|
const screenshotBytes = await captureScriptedPageScreenshot(args.page);
|
|
97
|
-
assertScreenshotEvidence(screenshotPath, screenshotBytes);
|
|
98
|
+
await assertScreenshotEvidence(screenshotPath, screenshotBytes);
|
|
98
99
|
await writeContainedOutputFile(args.absoluteArtifactRoot, screenshotPath, screenshotBytes);
|
|
99
100
|
const assertions = await evaluateBrowserStepExpectations({
|
|
100
101
|
afterState,
|
|
@@ -233,7 +234,7 @@ export async function captureBlockedStepScreenshot(page, artifactRoot, surface,
|
|
|
233
234
|
}
|
|
234
235
|
try {
|
|
235
236
|
const screenshotBytes = await captureScriptedPageScreenshot(page);
|
|
236
|
-
assertScreenshotEvidence(screenshotPath, screenshotBytes);
|
|
237
|
+
await assertScreenshotEvidence(screenshotPath, screenshotBytes);
|
|
237
238
|
await writeContainedOutputFile(artifactRoot, screenshotPath, screenshotBytes);
|
|
238
239
|
return { screenshotPath, written: true };
|
|
239
240
|
}
|
|
@@ -333,7 +334,7 @@ export function sanitizeBrowserEvidenceUrl(value, urlPolicy = LOOPBACK_EVIDENCE_
|
|
|
333
334
|
return evidenceOrigin;
|
|
334
335
|
}
|
|
335
336
|
}
|
|
336
|
-
export async function probeAppUrl(appUrl, timeoutMs) {
|
|
337
|
+
export async function probeAppUrl(appUrl, timeoutMs, scrubKnown) {
|
|
337
338
|
const controller = new AbortController();
|
|
338
339
|
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
339
340
|
try {
|
|
@@ -349,7 +350,7 @@ export async function probeAppUrl(appUrl, timeoutMs) {
|
|
|
349
350
|
catch (error) {
|
|
350
351
|
return {
|
|
351
352
|
ok: false,
|
|
352
|
-
reason: compactBrowserError(error),
|
|
353
|
+
reason: compactBrowserError(error, scrubKnown),
|
|
353
354
|
};
|
|
354
355
|
}
|
|
355
356
|
finally {
|
|
@@ -375,11 +376,18 @@ export function normalizeLocalAppUrl(value) {
|
|
|
375
376
|
return null;
|
|
376
377
|
}
|
|
377
378
|
}
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
379
|
+
/**
|
|
380
|
+
* A browser error as a step records it, cut to 240 characters. It can quote the step's selector or
|
|
381
|
+
* URL, which the scenario wrote and the browser received as written, so keys are redacted as
|
|
382
|
+
* written and in the encoded forms verify decodes. The run's known values are scrubbed between
|
|
383
|
+
* the two passes, so the encoded pass cannot cut one apart before its scrub reads it. The cut
|
|
384
|
+
* comes last, so it cannot split a value either pass would find.
|
|
385
|
+
*/
|
|
386
|
+
export function compactBrowserError(error, scrubKnown) {
|
|
387
|
+
const text = error instanceof Error ? error.message : String(error);
|
|
388
|
+
return redactEncodedText(scrubKnown(redactToSecretLabel(text)))
|
|
389
|
+
.replace(/\s+/g, " ")
|
|
390
|
+
.slice(0, 240);
|
|
383
391
|
}
|
|
384
392
|
function browserScreenshotBytes(value) {
|
|
385
393
|
if (Buffer.isBuffer(value)) {
|
|
@@ -415,6 +423,3 @@ async function captureScriptedPageScreenshot(page) {
|
|
|
415
423
|
.catch(() => undefined);
|
|
416
424
|
}
|
|
417
425
|
}
|
|
418
|
-
function redactSensitiveText(text) {
|
|
419
|
-
return redactToSecretLabel(text);
|
|
420
|
-
}
|
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
import { receivingAnalysisContext } from "../comms/receiving-evidence.js";
|
|
2
|
-
import {
|
|
2
|
+
import { streamParticipantIdOf } from "../run/participant-records.js";
|
|
3
3
|
import { cuaEndpointSource } from "../actors/goal-source.js";
|
|
4
4
|
import { isRecord } from "../run/type-guards.js";
|
|
5
5
|
import { ACTION_CAPTURE_VERSION, } from "./types.js";
|
|
6
6
|
import { decodesToPlainRelativePath, isEvidencePath } from "../run/evidence-files.js";
|
|
7
|
+
import { isRunEventOf } from "./source.js";
|
|
7
8
|
// How a run bundle becomes analysis sources: each participant's recorded provenance and assignment,
|
|
8
9
|
// the evidence entries its trace offers, and the order in which entries are admitted under the
|
|
9
10
|
// count and byte budgets. captureEvidence selects from these, taking each participant's while its
|
|
@@ -97,8 +98,7 @@ export function participantSource(stream, captureVersion) {
|
|
|
97
98
|
},
|
|
98
99
|
};
|
|
99
100
|
}
|
|
100
|
-
const runEventsOf = (bundle, stream) => bundle.events.filter((
|
|
101
|
-
(entry.streamId === undefined && eventRecordIdOf(entry) === recordIdOf(stream)));
|
|
101
|
+
const runEventsOf = (bundle, stream) => bundle.events.filter((event) => isRunEventOf(stream, event));
|
|
102
102
|
/**
|
|
103
103
|
* How many entries sourceEntries gives of run.json's `stream` once its trace is loaded, counted
|
|
104
104
|
* before the trace is read: the trace loader refuses a trace that does not hold the item count its
|
|
@@ -163,11 +163,8 @@ export function sourceEntries(bundle, stream, captureVersion) {
|
|
|
163
163
|
failed: ["failed", "blocked", "timed_out"].includes(item.status ?? ""),
|
|
164
164
|
});
|
|
165
165
|
}
|
|
166
|
-
|
|
166
|
+
// parseAnalysisSource refuses a run.json in which two of a participant's run events share an id.
|
|
167
167
|
for (const event of runEventsOf(bundle, stream)) {
|
|
168
|
-
if (runEventIds.has(event.id))
|
|
169
|
-
throw new Error("ANALYSIS_SOURCE_EVENT_DUPLICATE");
|
|
170
|
-
runEventIds.add(event.id);
|
|
171
168
|
entries.push({
|
|
172
169
|
eventId: event.id,
|
|
173
170
|
kind: `run_event:${event.type}`,
|
|
@@ -46,8 +46,8 @@ export async function captureEvidence(prepared, bundleBytes, requested = {}) {
|
|
|
46
46
|
// Admission stops at a participant's evidence share. The shares come from run.json's counts, so
|
|
47
47
|
// only the first share of each participant's entries in admission order is kept of its trace.
|
|
48
48
|
const shares = evidenceShares(bundle, limits, captureVersion);
|
|
49
|
-
// A participant past limits.participants has its trace read and checked
|
|
50
|
-
// no entries
|
|
49
|
+
// A participant past limits.participants has its trace read and checked like the others' and
|
|
50
|
+
// gives no entries.
|
|
51
51
|
const sources = await readParticipantTraces(prepared, bundle, (stream, index) => index < limits.participants
|
|
52
52
|
? participantSources(bundle, bundle.streams[index], stream, captureVersion, {
|
|
53
53
|
evidenceShare: shares[index],
|
|
@@ -229,7 +229,7 @@ async function selectCaptures(prepared, packings, sel, omissions) {
|
|
|
229
229
|
const bytes = result.state === "read" ? result.bytes : null;
|
|
230
230
|
sel.returnedImageBytes += bytes?.length ?? 0;
|
|
231
231
|
packing.readBytes += bytes?.length ?? 0;
|
|
232
|
-
if (bytes === null || screenshotEvidenceError(source.capturePath, bytes) !== null) {
|
|
232
|
+
if (bytes === null || (await screenshotEvidenceError(source.capturePath, bytes)) !== null) {
|
|
233
233
|
omissions.add("Some captures were missing, unsafe, oversized, or invalid PNG evidence.");
|
|
234
234
|
continue;
|
|
235
235
|
}
|
|
@@ -453,7 +453,7 @@ export async function validateAnalysisEvidence(prepared, artifact, bundleBytes)
|
|
|
453
453
|
let hash = checkedCaptures.get(source.capturePath);
|
|
454
454
|
if (hash === undefined) {
|
|
455
455
|
const bytes = await readBoundedFile(prepared, source.capturePath, EVIDENCE_LIMITS.imageBytes);
|
|
456
|
-
if (!bytes || screenshotEvidenceError(source.capturePath, bytes) !== null)
|
|
456
|
+
if (!bytes || (await screenshotEvidenceError(source.capturePath, bytes)) !== null)
|
|
457
457
|
throw new Error("ANALYSIS_CAPTURE_UNAVAILABLE");
|
|
458
458
|
checkedImageBytes += bytes.length;
|
|
459
459
|
if (checkedImageBytes > imageByteLimit)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { type StoredRunBundle } from "../run/bundle.js";
|
|
1
|
+
import { type RunEvent, type StoredRunBundle } from "../run/bundle.js";
|
|
2
2
|
import { type PreparedRunArtifactPaths } from "../run/paths.js";
|
|
3
|
-
import type { RunStream } from "../run/streams.js";
|
|
3
|
+
import type { RunStream, StoredRunStream } from "../run/streams.js";
|
|
4
4
|
/**
|
|
5
5
|
* A refusal of the run's recorded source. Its message is the code, as every analysis error's is,
|
|
6
6
|
* and `reason` says which file analysis did not read and why, for the person who asked.
|
|
@@ -11,10 +11,13 @@ export declare class AnalysisSourceRefusedError extends Error {
|
|
|
11
11
|
}
|
|
12
12
|
/** The refusal of a run.json of `size` bytes, more than the `max` analysis reads of it. */
|
|
13
13
|
export declare function runFileTooLarge(size: number | bigint, max: number): AnalysisSourceRefusedError;
|
|
14
|
+
/** Whether `event` is `stream`'s: it names the stream, or no stream and the stream's record. */
|
|
15
|
+
export declare const isRunEventOf: (stream: StoredRunStream, event: RunEvent) => boolean;
|
|
14
16
|
/**
|
|
15
17
|
* run.json's bundle, when it is a run bundle of the prepared run that analysis reads: at most
|
|
16
|
-
* EVIDENCE_LIMITS.sourceBytes, at most EVIDENCE_LIMITS.participants participants,
|
|
17
|
-
* participant and event with the fields analysis reads
|
|
18
|
+
* EVIDENCE_LIMITS.sourceBytes, at most EVIDENCE_LIMITS.participants participants, each
|
|
19
|
+
* participant and event with the fields analysis reads, and no two run events of one participant
|
|
20
|
+
* with one id.
|
|
18
21
|
*/
|
|
19
22
|
export declare function parseAnalysisSource(prepared: PreparedRunArtifactPaths, bytes: Buffer): StoredRunBundle;
|
|
20
23
|
/**
|
package/dist/analysis/source.js
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
// and the refusals it gives for either.
|
|
3
3
|
import { isCommsReceivingEvidence } from "../comms/receiving-evidence.js";
|
|
4
4
|
import { isReadRunBundleSchema } from "../run/bundle.js";
|
|
5
|
+
import { eventRecordIdOf, recordIdOf } from "../run/participant-records.js";
|
|
5
6
|
import { runIdOf } from "../run/paths.js";
|
|
6
7
|
import { reduceRunTraces, StreamTraceRefusedError, StreamTraceTooLargeError, } from "../run/stream-trace.js";
|
|
7
8
|
import { isRecord } from "../run/type-guards.js";
|
|
@@ -24,12 +25,16 @@ const runFileInvalid = (reason) => new AnalysisSourceRefusedError("ANALYSIS_SOUR
|
|
|
24
25
|
const traceInvalid = (streamId, reason) => new AnalysisSourceRefusedError("ANALYSIS_SOURCE_INVALID", `Analysis did not read participant ${streamId}'s trace: ${reason}`);
|
|
25
26
|
/** The refusal of a run.json of `size` bytes, more than the `max` analysis reads of it. */
|
|
26
27
|
export function runFileTooLarge(size, max) {
|
|
27
|
-
return new AnalysisSourceRefusedError("ANALYSIS_SOURCE_TOO_LARGE", `Analysis did not read run.json: it holds ${bytes(size)}, more than the ${limit(max)} analysis reads of it. run.json grows with each participant and the
|
|
28
|
+
return new AnalysisSourceRefusedError("ANALYSIS_SOURCE_TOO_LARGE", `Analysis did not read run.json: it holds ${bytes(size)}, more than the ${limit(max)} analysis reads of it. run.json grows with each participant, its instruction, the mission and the task goals; split the participants across studies to keep it under the limit.`);
|
|
28
29
|
}
|
|
30
|
+
/** Whether `event` is `stream`'s: it names the stream, or no stream and the stream's record. */
|
|
31
|
+
export const isRunEventOf = (stream, event) => event.streamId === stream.id ||
|
|
32
|
+
(event.streamId === undefined && eventRecordIdOf(event) === recordIdOf(stream));
|
|
29
33
|
/**
|
|
30
34
|
* run.json's bundle, when it is a run bundle of the prepared run that analysis reads: at most
|
|
31
|
-
* EVIDENCE_LIMITS.sourceBytes, at most EVIDENCE_LIMITS.participants participants,
|
|
32
|
-
* participant and event with the fields analysis reads
|
|
35
|
+
* EVIDENCE_LIMITS.sourceBytes, at most EVIDENCE_LIMITS.participants participants, each
|
|
36
|
+
* participant and event with the fields analysis reads, and no two run events of one participant
|
|
37
|
+
* with one id.
|
|
33
38
|
*/
|
|
34
39
|
export function parseAnalysisSource(prepared, bytes) {
|
|
35
40
|
if (bytes.length > EVIDENCE_LIMITS.sourceBytes)
|
|
@@ -67,7 +72,20 @@ export function parseAnalysisSource(prepared, bytes) {
|
|
|
67
72
|
typeof event.type !== "string")
|
|
68
73
|
throw runFileInvalid(`event ${index + 1} needs an id, a type and a message.`);
|
|
69
74
|
}
|
|
70
|
-
|
|
75
|
+
const bundle = value;
|
|
76
|
+
for (const stream of bundle.streams) {
|
|
77
|
+
// The place each id was first seen, since an analysis cites a participant's run event by its id.
|
|
78
|
+
const firstPlace = new Map();
|
|
79
|
+
for (const [index, event] of bundle.events.entries()) {
|
|
80
|
+
if (!isRunEventOf(stream, event))
|
|
81
|
+
continue;
|
|
82
|
+
const first = firstPlace.get(event.id);
|
|
83
|
+
if (first !== undefined)
|
|
84
|
+
throw runFileInvalid(`events ${first + 1} and ${index + 1} are participant ${stream.id}'s and share the id ${JSON.stringify(event.id)}, and analysis cites each of a participant's run events by its id. Give one of them another id in run.json; if run.json was not edited, please report this as a bug.`);
|
|
85
|
+
firstPlace.set(event.id, index);
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
return bundle;
|
|
71
89
|
}
|
|
72
90
|
/**
|
|
73
91
|
* What `keep` takes from each stream of run.json's bundle with the stream's trace loaded, in stream
|
|
@@ -395,12 +395,14 @@ function formatInitHuman(result) {
|
|
|
395
395
|
const lines = [title, `cwd: ${result.cwd}`];
|
|
396
396
|
if (result.ok && result.mode === "applied") {
|
|
397
397
|
// Name each file init wrote, so the person or agent who ran it can review exactly those
|
|
398
|
-
// files;
|
|
398
|
+
// files; files it left alone are a count. Neither includes the empty runtime directories under
|
|
399
|
+
// the ignored `.humanish/`: they give the reader nothing to review, leaving them out keeps a
|
|
400
|
+
// first setup with three next steps to 24 lines, and a second run then counts as unchanged
|
|
401
|
+
// the files the first one named. JSON always carries every change.
|
|
399
402
|
const files = (action) => result.changes.filter((change) => change.action === action && change.target !== "runtime");
|
|
400
403
|
const created = files("create");
|
|
401
404
|
const updated = files("update");
|
|
402
|
-
const preserved =
|
|
403
|
-
const directories = result.changes.filter((change) => change.action === "mkdir").length;
|
|
405
|
+
const preserved = files("skip").length;
|
|
404
406
|
lines.push("");
|
|
405
407
|
if (created.length > 0)
|
|
406
408
|
lines.push("created:", ...created.map((change) => ` ${change.path}`));
|
|
@@ -408,13 +410,8 @@ function formatInitHuman(result) {
|
|
|
408
410
|
lines.push("updated:", ...updated.map((change) => ` ${change.path} (${change.reason})`));
|
|
409
411
|
if (created.length === 0 && updated.length === 0)
|
|
410
412
|
lines.push("no files changed");
|
|
411
|
-
if (preserved > 0
|
|
412
|
-
lines.push(
|
|
413
|
-
...(preserved > 0 ? [`${plural(preserved, "existing path")} unchanged`] : []),
|
|
414
|
-
...(directories > 0
|
|
415
|
-
? [`${plural(directories, "ignored directory", "ignored directories")} prepared`]
|
|
416
|
-
: []),
|
|
417
|
-
].join(", "));
|
|
413
|
+
if (preserved > 0)
|
|
414
|
+
lines.push(`${plural(preserved, "existing path")} unchanged`);
|
|
418
415
|
}
|
|
419
416
|
else {
|
|
420
417
|
lines.push("", "changes:", ...result.changes.map(formatInitChange));
|
package/dist/evidence/image.d.ts
CHANGED
|
@@ -20,5 +20,5 @@ export declare function pngDecodeRefusal(bytes: Buffer): string | null;
|
|
|
20
20
|
* were downscaled, and any other chunk is a place to carry bytes no one reviews.
|
|
21
21
|
*/
|
|
22
22
|
export declare function isRedactedFrameShape(bytes: Buffer): boolean;
|
|
23
|
-
export declare function screenshotEvidenceError(relativePath: string, bytes: Buffer): string | null
|
|
24
|
-
export declare function assertScreenshotEvidence(relativePath: string, bytes: Buffer): void
|
|
23
|
+
export declare function screenshotEvidenceError(relativePath: string, bytes: Buffer): Promise<string | null>;
|
|
24
|
+
export declare function assertScreenshotEvidence(relativePath: string, bytes: Buffer): Promise<void>;
|
package/dist/evidence/image.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { crc32,
|
|
1
|
+
import { crc32, createInflate } from "node:zlib";
|
|
2
2
|
import { PNG } from "pngjs";
|
|
3
3
|
const PNG_SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]);
|
|
4
4
|
const PNG_IHDR_LENGTH = 13;
|
|
@@ -128,7 +128,7 @@ export function isRedactedFrameShape(bytes) {
|
|
|
128
128
|
last.end === bytes.length &&
|
|
129
129
|
bytes.readUInt32BE(16) <= SCREENSHOT_MAX_WIDTH_CAP);
|
|
130
130
|
}
|
|
131
|
-
export function screenshotEvidenceError(relativePath, bytes) {
|
|
131
|
+
export async function screenshotEvidenceError(relativePath, bytes) {
|
|
132
132
|
const extension = relativePath.toLowerCase().split(".").pop() ?? "";
|
|
133
133
|
if (extension !== "png") {
|
|
134
134
|
return `unsupported screenshot extension .${extension || "unknown"}; only decoded PNG evidence is supported`;
|
|
@@ -146,7 +146,7 @@ export function screenshotEvidenceError(relativePath, bytes) {
|
|
|
146
146
|
if (chunks?.some((chunk) => chunk.type === "IEND" && chunk.end - chunk.start !== 12)) {
|
|
147
147
|
return "PNG IEND chunk must be empty";
|
|
148
148
|
}
|
|
149
|
-
return chunks === null || !pngStructureHolds(bytes, chunks)
|
|
149
|
+
return chunks === null || !(await pngStructureHolds(bytes, chunks))
|
|
150
150
|
? "could not decode PNG evidence"
|
|
151
151
|
: null;
|
|
152
152
|
}
|
|
@@ -164,14 +164,14 @@ const PNG_MIN_CHUNK_BYTES = { gAMA: 4 };
|
|
|
164
164
|
/**
|
|
165
165
|
* Whether a PNG that passed pngDecodeRefusal decodes, read from its structure: the file ends at an
|
|
166
166
|
* empty IEND, every chunk's CRC matches, IHDR comes once with fields PNG defines, the image data is
|
|
167
|
-
* one run of IDAT chunks, and it inflates
|
|
168
|
-
*
|
|
169
|
-
*
|
|
170
|
-
*
|
|
171
|
-
*
|
|
172
|
-
*
|
|
167
|
+
* one run of IDAT chunks, and it inflates (inflatesToRows) to exactly the bytes its rows need with
|
|
168
|
+
* nothing after the stream, each row starting with a filter type from 0 to 4. For an image without
|
|
169
|
+
* a palette that is all a decode establishes, at about an eighth of the time of pngjs, which
|
|
170
|
+
* unfilters every pixel in JavaScript; a run holds a screenshot for each turn. A palette image is
|
|
171
|
+
* also decoded, since each pixel must index a palette that comes before the image data; in any
|
|
172
|
+
* other image a palette chunk is only a suggestion, wherever it is.
|
|
173
173
|
*/
|
|
174
|
-
function pngStructureHolds(bytes, chunks) {
|
|
174
|
+
async function pngStructureHolds(bytes, chunks) {
|
|
175
175
|
const last = chunks.at(-1);
|
|
176
176
|
if (last?.type !== "IEND" || last.end !== bytes.length)
|
|
177
177
|
return false;
|
|
@@ -200,29 +200,44 @@ function pngStructureHolds(bytes, chunks) {
|
|
|
200
200
|
return false;
|
|
201
201
|
const rowBytes = 1 + Math.ceil(rowBits / 8);
|
|
202
202
|
const data = Buffer.concat(idats.map((chunk) => bytes.subarray(chunk.start + 8, chunk.end - 4)));
|
|
203
|
-
|
|
204
|
-
try {
|
|
205
|
-
// A stream that would inflate past the rows the image needs stops there, so a small file
|
|
206
|
-
// cannot claim gigabytes.
|
|
207
|
-
const options = {
|
|
208
|
-
maxOutputLength: rowBytes * height,
|
|
209
|
-
info: true,
|
|
210
|
-
};
|
|
211
|
-
const inflated = inflateSync(data, options);
|
|
212
|
-
if (inflated.engine.bytesWritten !== data.length)
|
|
213
|
-
return false;
|
|
214
|
-
rows = inflated.buffer;
|
|
215
|
-
}
|
|
216
|
-
catch {
|
|
217
|
-
return false;
|
|
218
|
-
}
|
|
219
|
-
if (rows.length !== rowBytes * height)
|
|
203
|
+
if (!(await inflatesToRows(data, rowBytes, height)))
|
|
220
204
|
return false;
|
|
221
|
-
for (let row = 0; row < rows.length; row += rowBytes)
|
|
222
|
-
if (rows[row] > 4)
|
|
223
|
-
return false;
|
|
224
205
|
return colorType !== 3 || pngDecodes(bytes);
|
|
225
206
|
}
|
|
207
|
+
// The most inflated bytes one check holds at once. Checks overlap on the thread pool, so rows held
|
|
208
|
+
// whole add up across them. A process running 8 checks of 5000x5000 RGBA frames peaked at 832 MB
|
|
209
|
+
// resident with each frame's rows held whole and at 202 MB checked 1 MiB at a time.
|
|
210
|
+
const INFLATE_CHUNK_BYTES = 1024 * 1024;
|
|
211
|
+
/**
|
|
212
|
+
* Whether `data` inflates to exactly `rows` rows of `rowBytes` bytes, each starting with a filter
|
|
213
|
+
* type from 0 to 4, with nothing after the stream. zlib inflates on libuv's thread pool, so checks
|
|
214
|
+
* of several frames overlap. A stream that would inflate past the rows stops there, so a small
|
|
215
|
+
* file cannot claim gigabytes.
|
|
216
|
+
*/
|
|
217
|
+
function inflatesToRows(data, rowBytes, rows) {
|
|
218
|
+
const total = rowBytes * rows;
|
|
219
|
+
return new Promise((resolve) => {
|
|
220
|
+
const inflate = createInflate({
|
|
221
|
+
chunkSize: Math.max(64, Math.min(total, INFLATE_CHUNK_BYTES)),
|
|
222
|
+
});
|
|
223
|
+
let seen = 0;
|
|
224
|
+
inflate.on("data", (chunk) => {
|
|
225
|
+
let filtersHold = true;
|
|
226
|
+
// The first row that starts in this chunk, then each row after it.
|
|
227
|
+
for (let at = (rowBytes - (seen % rowBytes)) % rowBytes; at < chunk.length; at += rowBytes)
|
|
228
|
+
if (chunk[at] > 4)
|
|
229
|
+
filtersHold = false;
|
|
230
|
+
seen += chunk.length;
|
|
231
|
+
if (!filtersHold || seen > total) {
|
|
232
|
+
inflate.destroy();
|
|
233
|
+
resolve(false);
|
|
234
|
+
}
|
|
235
|
+
});
|
|
236
|
+
inflate.on("error", () => resolve(false));
|
|
237
|
+
inflate.on("end", () => resolve(seen === total && inflate.bytesWritten === data.length));
|
|
238
|
+
inflate.end(data);
|
|
239
|
+
});
|
|
240
|
+
}
|
|
226
241
|
function pngDecodes(bytes) {
|
|
227
242
|
try {
|
|
228
243
|
PNG.sync.read(bytes, { checkCRC: true });
|
|
@@ -232,8 +247,8 @@ function pngDecodes(bytes) {
|
|
|
232
247
|
return false;
|
|
233
248
|
}
|
|
234
249
|
}
|
|
235
|
-
export function assertScreenshotEvidence(relativePath, bytes) {
|
|
236
|
-
const error = screenshotEvidenceError(relativePath, bytes);
|
|
250
|
+
export async function assertScreenshotEvidence(relativePath, bytes) {
|
|
251
|
+
const error = await screenshotEvidenceError(relativePath, bytes);
|
|
237
252
|
if (error) {
|
|
238
253
|
throw new Error(`Invalid screenshot evidence ${relativePath}: ${error}`);
|
|
239
254
|
}
|
package/dist/observer/data.d.ts
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
|
+
import type { ActorTrace } from "../actors/contract.js";
|
|
1
2
|
import { type ActorEnding } from "../actors/stop-cause.js";
|
|
2
|
-
import type { RunBundle, RunCostSummary, RunEvent, RunSimulation } from "../run/bundle.js";
|
|
3
|
+
import type { DisplayedRunBundle, RunBundle, RunCostSummary, RunEvent, RunSimulation } from "../run/bundle.js";
|
|
3
4
|
import { type RunDisplay } from "../run/display.js";
|
|
4
|
-
import type { RunStream, RunStreamKind, StoredRunStream } from "../run/streams.js";
|
|
5
|
+
import type { RunStream, RunStreamKind, StoredRunStream, WithoutRequestLists } from "../run/streams.js";
|
|
5
6
|
/**
|
|
6
7
|
* The Observer's data. From v2 observer-data.json leaves out each participant's items, and from v3
|
|
7
8
|
* each participant's per-request lists (REQUEST_LISTS in src/run/stream-trace.ts), which no part
|
|
@@ -97,6 +98,9 @@ export interface ObserverData {
|
|
|
97
98
|
};
|
|
98
99
|
}
|
|
99
100
|
export interface ObserverStream extends StoredRunStream {
|
|
101
|
+
/** The loaded actor, with its items and without its per-request lists, as is the live actor. */
|
|
102
|
+
actor?: WithoutRequestLists<ActorTrace>;
|
|
103
|
+
liveActor?: WithoutRequestLists<NonNullable<RunStream["liveActor"]>>;
|
|
100
104
|
/** Only the attached server may grant provider-origin access for a live desktop iframe. */
|
|
101
105
|
embed?: NonNullable<RunStream["embed"]> & {
|
|
102
106
|
runtimeDesktop?: true;
|
|
@@ -127,7 +131,7 @@ interface ObserverLaneGroup {
|
|
|
127
131
|
* bundle with no outcome in run.json: a run in progress, interrupted, or recorded before run.json
|
|
128
132
|
* carried its outcome.
|
|
129
133
|
*/
|
|
130
|
-
export declare function buildObserverData(bundle:
|
|
134
|
+
export declare function buildObserverData(bundle: DisplayedRunBundle, generatedAt?: string, statusRecord?: unknown): ObserverData;
|
|
131
135
|
/**
|
|
132
136
|
* A saved Observer projection as this version reads it, as v3 and unchecked as before. An
|
|
133
137
|
* observer-data.json written before v2 holds every participant's items, and its per-request lists,
|
package/dist/observer/render.js
CHANGED
|
@@ -11,7 +11,7 @@ import { projectShareCheckedAnalysis } from "../analysis/sharing.js";
|
|
|
11
11
|
import { loadAnalysis } from "../analysis/load.js";
|
|
12
12
|
import { physicalCwdOf, runIdOf, bindExistingRunArtifactPaths, isSafeRunIdSegment, resolveLatestRunDirectory, validatePreparedRunArtifactPaths, } from "../run/paths.js";
|
|
13
13
|
import { writeContainedOutputFile } from "../run/contained-output.js";
|
|
14
|
-
import {
|
|
14
|
+
import { loadDisplayedRunBundle, readRunJsonIfExists, runJsonValue } from "../run/locate.js";
|
|
15
15
|
import { readRunNotes } from "../run/note-files.js";
|
|
16
16
|
import { RUN_STATUS_FILE } from "../run/status.js";
|
|
17
17
|
import { verifyRunPrepared } from "../verify/verify.js";
|
|
@@ -91,9 +91,7 @@ export async function renderObserver(cwdInput, runInput, options = {}) {
|
|
|
91
91
|
: "HUMANISH_INVALID_RUN_BUNDLE", verified.error?.message ?? "Run bundle failed verification.");
|
|
92
92
|
}
|
|
93
93
|
// Verify passes a trace past the read limit, so its participant shows without its items.
|
|
94
|
-
const loaded = await
|
|
95
|
-
display: true,
|
|
96
|
-
});
|
|
94
|
+
const loaded = await loadDisplayedRunBundle(selectedPhysicalCwd, preparedRunPaths);
|
|
97
95
|
await validatePreparedRunArtifactPaths(preparedRunPaths);
|
|
98
96
|
if (!loaded)
|
|
99
97
|
return runNotFound(cwd, runInput);
|
|
@@ -26,7 +26,7 @@ export function makeParticipantWriteScreenshot(artifactRoot, spec, screenshots)
|
|
|
26
26
|
const rel = path.posix.join(relPrefix, name);
|
|
27
27
|
// Evidence holds image data only, whatever the executor's frames carried.
|
|
28
28
|
const frame = stripPngMetadataChunks(bytes);
|
|
29
|
-
assertScreenshotEvidence(rel, frame);
|
|
29
|
+
await assertScreenshotEvidence(rel, frame);
|
|
30
30
|
await writeContainedOutputFile(artifactRoot, path.join(...dirParts, name), frame);
|
|
31
31
|
screenshots.push(rel);
|
|
32
32
|
return rel;
|
package/dist/run/bundle.d.ts
CHANGED
|
@@ -4,7 +4,7 @@ import type { ExecutionOutcome } from "./judge.js";
|
|
|
4
4
|
import type { SharedWorldEvidence } from "./shared-world-evidence.js";
|
|
5
5
|
import { type RunStudyProvenance } from "./study-provenance.js";
|
|
6
6
|
import type { RunInterruptSignal } from "./status.js";
|
|
7
|
-
import type { RunSimulationStatus, RunStream, RunStreamKind, StoredRunStream } from "./streams.js";
|
|
7
|
+
import type { DisplayedRunStream, RunSimulationStatus, RunStream, RunStreamKind, StoredRunStream } from "./streams.js";
|
|
8
8
|
/**
|
|
9
9
|
* The run bundle version this release writes: each stream's trace, and each running participant's
|
|
10
10
|
* items so far, are files run.json indexes, and each participant's per-request lists are only in
|
|
@@ -395,6 +395,10 @@ export interface RunBundle {
|
|
|
395
395
|
export type StoredRunBundle = Omit<RunBundle, "streams"> & {
|
|
396
396
|
streams: StoredRunStream[];
|
|
397
397
|
};
|
|
398
|
+
/** A loaded bundle as a reader that shows the run has it (displayedTraces): DisplayedRunStreams. */
|
|
399
|
+
export type DisplayedRunBundle = Omit<RunBundle, "streams"> & {
|
|
400
|
+
streams: DisplayedRunStream[];
|
|
401
|
+
};
|
|
398
402
|
/**
|
|
399
403
|
* How a run ended. `finished`: Run.finish published the final bundle with the route result's `ok`
|
|
400
404
|
* and its execution outcome (`judgeExecution` in src/run/judge.ts). `interrupted`: the CLI's signal
|
package/dist/run/locate.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { type PreparedRunArtifactPaths } from "./paths.js";
|
|
2
2
|
import { type ContainedRefusal, type PreparedSelectedOutputDirectory } from "./contained-output.js";
|
|
3
3
|
import type { RunPointer } from "./results.js";
|
|
4
|
-
import { type RunBundle } from "./bundle.js";
|
|
4
|
+
import { type DisplayedRunBundle, type RunBundle } from "./bundle.js";
|
|
5
5
|
import { type BoundedFileResult } from "./evidence-files.js";
|
|
6
6
|
/** The run input that resolves through latest.json rather than naming a run directory. */
|
|
7
7
|
export declare const LATEST_RUN_ALIAS = "latest";
|
|
@@ -55,15 +55,17 @@ export declare function loadRunBundle(cwdInput: string, runInput: string): Promi
|
|
|
55
55
|
bundlePath: string;
|
|
56
56
|
runDir: string;
|
|
57
57
|
} | null>;
|
|
58
|
-
/**
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
* kept as displayedTraces shows it, for a page that shows the run.
|
|
62
|
-
*/
|
|
63
|
-
export declare function loadRunBundlePrepared(cwdInput: string, runPaths: PreparedRunArtifactPaths, options?: {
|
|
64
|
-
display?: boolean;
|
|
65
|
-
}): Promise<{
|
|
66
|
-
bundle: RunBundle;
|
|
58
|
+
/** A run's bundle, run.json's path relative to the working directory, and the run folder. */
|
|
59
|
+
type LoadedRun<T> = {
|
|
60
|
+
bundle: T;
|
|
67
61
|
bundlePath: string;
|
|
68
62
|
runDir: string;
|
|
69
|
-
}
|
|
63
|
+
};
|
|
64
|
+
/** loadRunBundle for a caller that already holds prepared paths. Revalidates them first. */
|
|
65
|
+
export declare function loadRunBundlePrepared(cwdInput: string, runPaths: PreparedRunArtifactPaths): Promise<LoadedRun<RunBundle> | null>;
|
|
66
|
+
/**
|
|
67
|
+
* loadRunBundlePrepared for a page that shows the run: a participant whose trace the loader
|
|
68
|
+
* refuses, such as one past the read limit, is kept as displayedTraces shows it.
|
|
69
|
+
*/
|
|
70
|
+
export declare function loadDisplayedRunBundle(cwdInput: string, runPaths: PreparedRunArtifactPaths): Promise<LoadedRun<DisplayedRunBundle> | null>;
|
|
71
|
+
export {};
|