humanish 0.134.0 → 0.136.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +30 -0
- package/dist/actors/computer-use/loop/session.js +7 -2
- package/dist/actors/scripted-browser/actor.js +3 -3
- package/dist/actors/scripted-browser/steps.d.ts +9 -2
- package/dist/actors/scripted-browser/steps.js +17 -12
- package/dist/analysis/automatic-config.js +5 -3
- package/dist/analysis/evidence-sources.js +4 -7
- package/dist/analysis/evidence.js +4 -4
- package/dist/analysis/execute.js +2 -1
- package/dist/analysis/service.js +2 -1
- package/dist/analysis/source.d.ts +7 -4
- package/dist/analysis/source.js +22 -4
- package/dist/cli/commands/setup.js +7 -10
- package/dist/evidence/encoded-text.d.ts +13 -3
- package/dist/evidence/encoded-text.js +88 -25
- package/dist/evidence/image.d.ts +2 -2
- package/dist/evidence/image.js +47 -32
- package/dist/observer/data.d.ts +7 -3
- package/dist/observer/page-data.d.ts +6 -0
- package/dist/observer/page-data.js +34 -7
- package/dist/observer/participant-items.d.ts +15 -1
- package/dist/observer/participant-items.js +4 -3
- package/dist/observer/render.js +2 -4
- package/dist/observer/run-routes.js +4 -4
- package/dist/observer-app.html +4 -4
- package/dist/routes/computer-use/fanout-bundle.js +3 -3
- package/dist/routes/computer-use/participant-execution.js +1 -1
- package/dist/routes/computer-use/participant-runs.js +2 -2
- package/dist/routes/computer-use/single-bundle.js +3 -3
- package/dist/routes/scripted/setup.js +62 -17
- package/dist/routes/shared-world/bundle.js +3 -3
- package/dist/routes/shared-world/participant-specs.js +2 -2
- package/dist/routes/terminal/bundle.js +2 -2
- package/dist/run/bundle.d.ts +5 -1
- package/dist/run/locate.d.ts +13 -11
- package/dist/run/locate.js +20 -13
- package/dist/run/paths.js +38 -28
- package/dist/run/run.js +4 -3
- package/dist/run/stream-trace.d.ts +2 -2
- package/dist/run/stream-trace.js +24 -11
- package/dist/run/streams.d.ts +18 -3
- package/dist/run-study.d.ts +3 -3
- package/dist/run-study.js +3 -3
- package/dist/study/bundle-estimate.d.ts +21 -3
- package/dist/study/bundle-estimate.js +65 -45
- package/dist/study/config.d.ts +4 -3
- package/dist/study/config.js +8 -5
- package/dist/study/parse/actors.d.ts +6 -0
- package/dist/study/parse/actors.js +14 -0
- package/dist/study/parse/subject-state.js +4 -4
- package/dist/study/participant-assignment.js +2 -2
- package/dist/study/persona.d.ts +7 -1
- package/dist/study/persona.js +13 -5
- package/dist/study/plan-base.js +4 -3
- package/dist/tui-app.js +23 -23
- package/dist/verify/screenshot-reads.js +1 -1
- package/docs/architecture/observer.md +24 -7
- package/docs/architecture/state-driven-executor.md +4 -3
- package/docs/contracts/run-bundle.md +3 -0
- package/docs/contracts/study-analysis.md +7 -5
- package/docs/decisions/0002-observer-is-one-self-contained-file.md +3 -3
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -8,6 +8,36 @@ The Unreleased section holds the full notes for the next version until it is tag
|
|
|
8
8
|
|
|
9
9
|
## Unreleased
|
|
10
10
|
|
|
11
|
+
## 0.136.0: verify takes less than half the time on large runs (2026-10-14)
|
|
12
|
+
|
|
13
|
+
humanish 0.136.0 makes `humanish verify` take less than half the time on large runs. It checks a
|
|
14
|
+
run folder for links 32 entries at a time and inflates screenshots on Node's thread pool, 1 MiB of
|
|
15
|
+
rows at a time, and still runs every check it ran before. On a run folder of 15,000 entries verify
|
|
16
|
+
took 3.3 seconds where it took 7.7, and on a run with 3,000 real screenshots 4.2 seconds where it
|
|
17
|
+
took 10.3. A computer-use participant that repeats a base64- or percent-encoded key from its
|
|
18
|
+
instructions no longer makes the run `blocked`. `humanish analyze` names the participant and the
|
|
19
|
+
event id when two run events share an id, and reports `ANALYSIS_SOURCE_INVALID` where it reported
|
|
20
|
+
`ANALYSIS_UNAVAILABLE`. The bundle estimate counts participants' instructions and task goals with
|
|
21
|
+
the mission, `init` prints three next steps in 24 lines, and reading a run refuses a conversation
|
|
22
|
+
without its requests list.
|
|
23
|
+
|
|
24
|
+
[Release notes](https://github.com/danielgwilson/humanish/releases/tag/v0.136.0)
|
|
25
|
+
|
|
26
|
+
## 0.135.0: Encoded keys in a study's own text stay out of the run (2026-10-14)
|
|
27
|
+
|
|
28
|
+
humanish 0.135.0 keeps a base64- or percent-encoded key in a study's own text out of the copies of
|
|
29
|
+
that text the run records. The run redacted keys as written, and verify also decodes those
|
|
30
|
+
encodings, so a study whose title, mission, instructions, persona or scripted scenario held an
|
|
31
|
+
encoded key produced a run graded `blocked`. A participant that repeats such a key in its own words
|
|
32
|
+
still does (#1892). `actor.model`, `subject.envValues` and the analysis question now refuse such a
|
|
33
|
+
value before anything runs, which a study that ran before may now meet. verify's per-file scan also
|
|
34
|
+
finds a key behind an HTML reference and a JSON escape. The Observer page and its live feed carry
|
|
35
|
+
each participant's items on their own, so `humanish observe` writes the page of a run whose traces
|
|
36
|
+
together pass V8's longest string: on a synthetic run of 25 participants with 30 MiB traces it
|
|
37
|
+
exits 0 where it exited 2.
|
|
38
|
+
|
|
39
|
+
[Release notes](https://github.com/danielgwilson/humanish/releases/tag/v0.135.0)
|
|
40
|
+
|
|
11
41
|
## 0.134.0: Participants held to an end keep reloading until it (2026-10-14)
|
|
12
42
|
|
|
13
43
|
humanish 0.134.0 keeps a participant held to a scheduled end running when it reloads a page that
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { redactEncodedText } from "../../../evidence/encoded-text.js";
|
|
1
2
|
import { TaskTracker } from "../../../study/tasks.js";
|
|
2
3
|
import { StreakRecord } from "./streak.js";
|
|
3
4
|
import { TraceRecorder } from "./trace.js";
|
|
@@ -124,9 +125,13 @@ export class LoopSession {
|
|
|
124
125
|
}
|
|
125
126
|
// Model-authored narration: literal-scrub known provisioned values, then pattern-redact.
|
|
126
127
|
// A value the model transcribes (a DB password it read on screen) has no shape, so redactText
|
|
127
|
-
// alone cannot catch it; the run's scrubKnownValues, injected as scrubText, closes that.
|
|
128
|
+
// alone cannot catch it; the run's scrubKnownValues, injected as scrubText, closes that. The
|
|
129
|
+
// participant receives its instructions as written, so it can repeat a key the study wrote
|
|
130
|
+
// base64- or percent-encoded, which verify decodes; redactEncodedText removes that one too.
|
|
131
|
+
// The known values are scrubbed again last: a marker written before other text can spell one,
|
|
132
|
+
// as `[REDACTED_SECRET] and` spells `SECRET] and`.
|
|
128
133
|
redactNarration(text) {
|
|
129
|
-
return this.settings.redaction.redactText(this.scrubText(text));
|
|
134
|
+
return this.scrubText(redactEncodedText(this.settings.redaction.redactText(this.scrubText(text))));
|
|
130
135
|
}
|
|
131
136
|
/** Hand a watcher the trace so far with the running usage, without waiting on it. */
|
|
132
137
|
flush() {
|
|
@@ -124,7 +124,7 @@ export async function runScriptedBrowserSessionInPreparedRoot(options, preparedA
|
|
|
124
124
|
browser = await launch({ browserCommand, timeoutMs: options.timeoutMs });
|
|
125
125
|
}
|
|
126
126
|
catch (error) {
|
|
127
|
-
const reason = `Scripted browser launch failed: ${compactBrowserError(error)}`;
|
|
127
|
+
const reason = `Scripted browser launch failed: ${compactBrowserError(error, scrub)}`;
|
|
128
128
|
const capture = await persistScriptedFailureCapture({
|
|
129
129
|
appUrl: options.appUrl,
|
|
130
130
|
artifactRoot: preparedArtifactRoot,
|
|
@@ -196,7 +196,7 @@ async function runScriptedJourney(args) {
|
|
|
196
196
|
const started = Date.now();
|
|
197
197
|
const deadline = started + args.timeoutMs;
|
|
198
198
|
const tracePath = tracePathForBrowserSurface(args.surface);
|
|
199
|
-
const httpProbe = await probeAppUrl(args.appUrl, Math.min(args.timeoutMs, 15_000));
|
|
199
|
+
const httpProbe = await probeAppUrl(args.appUrl, Math.min(args.timeoutMs, 15_000), args.scrub);
|
|
200
200
|
let page = null;
|
|
201
201
|
const steps = [];
|
|
202
202
|
let executedSteps = 0;
|
|
@@ -230,7 +230,7 @@ async function runScriptedJourney(args) {
|
|
|
230
230
|
await assertScriptedOutputRoot(args.artifactRoot);
|
|
231
231
|
timedOut = error instanceof ScriptedJourneyTimeoutError;
|
|
232
232
|
const now = new Date().toISOString();
|
|
233
|
-
const reason = compactBrowserError(error);
|
|
233
|
+
const reason = compactBrowserError(error, args.scrub);
|
|
234
234
|
if (steps.length === 0) {
|
|
235
235
|
steps.push(...buildBlockedBrowserPersonaSteps({
|
|
236
236
|
browserJourney: args.journey,
|
|
@@ -53,10 +53,17 @@ export declare function buildBrowserTrace(args: {
|
|
|
53
53
|
surface: BrowserSurface;
|
|
54
54
|
}): Record<string, unknown>;
|
|
55
55
|
export declare function sanitizeBrowserEvidenceUrl(value: string, urlPolicy?: ScriptedBrowserEvidenceUrlPolicy): string;
|
|
56
|
-
export declare function probeAppUrl(appUrl: string, timeoutMs: number): Promise<{
|
|
56
|
+
export declare function probeAppUrl(appUrl: string, timeoutMs: number, scrubKnown: (text: string) => string): Promise<{
|
|
57
57
|
ok: boolean;
|
|
58
58
|
reason: string;
|
|
59
59
|
status?: number;
|
|
60
60
|
}>;
|
|
61
61
|
export declare function normalizeLocalAppUrl(value: string): string | null;
|
|
62
|
-
|
|
62
|
+
/**
|
|
63
|
+
* A browser error as a step records it, cut to 240 characters. It can quote the step's selector or
|
|
64
|
+
* URL, which the scenario wrote and the browser received as written, so keys are redacted as
|
|
65
|
+
* written and in the encoded forms verify decodes. The run's known values are scrubbed between
|
|
66
|
+
* the two passes, so the encoded pass cannot cut one apart before its scrub reads it. The cut
|
|
67
|
+
* comes last, so it cannot split a value either pass would find.
|
|
68
|
+
*/
|
|
69
|
+
export declare function compactBrowserError(error: unknown, scrubKnown: (text: string) => string): string;
|
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
import { mkdtemp, rm } from "node:fs/promises";
|
|
5
5
|
import os from "node:os";
|
|
6
6
|
import path from "node:path";
|
|
7
|
+
import { redactEncodedText } from "../../evidence/encoded-text.js";
|
|
7
8
|
import { assertScreenshotEvidence, stripPngMetadataChunks } from "../../evidence/image.js";
|
|
8
9
|
import { digestText, redactToSecretLabel } from "../../evidence/redaction.js";
|
|
9
10
|
import { assertPreparedSelectedOutputDirectory, assertSafeOutputPathSegment, prepareContainedOutputFile, prepareSelectedOutputDirectory, readContainedRegularFile, refusalText, RUN_ARTIFACT_MAX_BYTES, writeContainedOutputFile, } from "../../run/contained-output.js";
|
|
@@ -94,7 +95,7 @@ export async function executeBrowserPersonaStep(args) {
|
|
|
94
95
|
const screenshotPath = screenshotPathForBrowserStep(args.surface, args.step);
|
|
95
96
|
await prepareContainedOutputFile(args.absoluteArtifactRoot, screenshotPath);
|
|
96
97
|
const screenshotBytes = await captureScriptedPageScreenshot(args.page);
|
|
97
|
-
assertScreenshotEvidence(screenshotPath, screenshotBytes);
|
|
98
|
+
await assertScreenshotEvidence(screenshotPath, screenshotBytes);
|
|
98
99
|
await writeContainedOutputFile(args.absoluteArtifactRoot, screenshotPath, screenshotBytes);
|
|
99
100
|
const assertions = await evaluateBrowserStepExpectations({
|
|
100
101
|
afterState,
|
|
@@ -233,7 +234,7 @@ export async function captureBlockedStepScreenshot(page, artifactRoot, surface,
|
|
|
233
234
|
}
|
|
234
235
|
try {
|
|
235
236
|
const screenshotBytes = await captureScriptedPageScreenshot(page);
|
|
236
|
-
assertScreenshotEvidence(screenshotPath, screenshotBytes);
|
|
237
|
+
await assertScreenshotEvidence(screenshotPath, screenshotBytes);
|
|
237
238
|
await writeContainedOutputFile(artifactRoot, screenshotPath, screenshotBytes);
|
|
238
239
|
return { screenshotPath, written: true };
|
|
239
240
|
}
|
|
@@ -333,7 +334,7 @@ export function sanitizeBrowserEvidenceUrl(value, urlPolicy = LOOPBACK_EVIDENCE_
|
|
|
333
334
|
return evidenceOrigin;
|
|
334
335
|
}
|
|
335
336
|
}
|
|
336
|
-
export async function probeAppUrl(appUrl, timeoutMs) {
|
|
337
|
+
export async function probeAppUrl(appUrl, timeoutMs, scrubKnown) {
|
|
337
338
|
const controller = new AbortController();
|
|
338
339
|
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
339
340
|
try {
|
|
@@ -349,7 +350,7 @@ export async function probeAppUrl(appUrl, timeoutMs) {
|
|
|
349
350
|
catch (error) {
|
|
350
351
|
return {
|
|
351
352
|
ok: false,
|
|
352
|
-
reason: compactBrowserError(error),
|
|
353
|
+
reason: compactBrowserError(error, scrubKnown),
|
|
353
354
|
};
|
|
354
355
|
}
|
|
355
356
|
finally {
|
|
@@ -375,11 +376,18 @@ export function normalizeLocalAppUrl(value) {
|
|
|
375
376
|
return null;
|
|
376
377
|
}
|
|
377
378
|
}
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
379
|
+
/**
|
|
380
|
+
* A browser error as a step records it, cut to 240 characters. It can quote the step's selector or
|
|
381
|
+
* URL, which the scenario wrote and the browser received as written, so keys are redacted as
|
|
382
|
+
* written and in the encoded forms verify decodes. The run's known values are scrubbed between
|
|
383
|
+
* the two passes, so the encoded pass cannot cut one apart before its scrub reads it. The cut
|
|
384
|
+
* comes last, so it cannot split a value either pass would find.
|
|
385
|
+
*/
|
|
386
|
+
export function compactBrowserError(error, scrubKnown) {
|
|
387
|
+
const text = error instanceof Error ? error.message : String(error);
|
|
388
|
+
return redactEncodedText(scrubKnown(redactToSecretLabel(text)))
|
|
389
|
+
.replace(/\s+/g, " ")
|
|
390
|
+
.slice(0, 240);
|
|
383
391
|
}
|
|
384
392
|
function browserScreenshotBytes(value) {
|
|
385
393
|
if (Buffer.isBuffer(value)) {
|
|
@@ -415,6 +423,3 @@ async function captureScriptedPageScreenshot(page) {
|
|
|
415
423
|
.catch(() => undefined);
|
|
416
424
|
}
|
|
417
425
|
}
|
|
418
|
-
function redactSensitiveText(text) {
|
|
419
|
-
return redactToSecretLabel(text);
|
|
420
|
-
}
|
|
@@ -3,7 +3,7 @@ import { DEFAULT_ANALYSIS_MAX_OUTPUT_TOKENS, MAX_ANALYSIS_OUTPUT_TOKENS, analysi
|
|
|
3
3
|
import { admissionRule } from "./admission.js";
|
|
4
4
|
import { analysisRequestCount, EVIDENCE_LIMITS } from "./analysis-limits.js";
|
|
5
5
|
import { plural } from "../run/text.js";
|
|
6
|
-
import {
|
|
6
|
+
import { scanEncodedText } from "../evidence/encoded-text.js";
|
|
7
7
|
export const DEFAULT_ANALYSIS_TIMEOUT_MS = 600_000;
|
|
8
8
|
const MAX_ANALYSIS_TIMEOUT_MS = 600_000;
|
|
9
9
|
export const DEFAULT_ANALYSIS_MODEL = "gpt-6-astra";
|
|
@@ -21,6 +21,8 @@ const validTimeout = (value) => typeof value === "number" &&
|
|
|
21
21
|
value <= MAX_ANALYSIS_TIMEOUT_MS;
|
|
22
22
|
/** An absent question is valid; a present one must be a string of at most 4000 characters. */
|
|
23
23
|
const validQuestion = (value) => value === undefined || (typeof value === "string" && value.length <= 4000);
|
|
24
|
+
// The provider receives the question and analysis.json keeps it. verify reads that file as written
|
|
25
|
+
// and decoded, so the question is refused when scanEncodedText reads a secret in it.
|
|
24
26
|
const SENSITIVE_QUESTION = {
|
|
25
27
|
ok: false,
|
|
26
28
|
message: "review.analysis.question contains sensitive text and cannot be sent for analysis.",
|
|
@@ -60,7 +62,7 @@ export function resolveAutomaticAnalysis(raw) {
|
|
|
60
62
|
message: `Codex analysis requires the qualified ${CODEX_ANALYSIS_MODEL} model and timeoutMs 1–${MAX_ANALYSIS_TIMEOUT_MS}. Dollar and output-token caps are unavailable; omit them. The optional question is limited to 4000 characters.`,
|
|
61
63
|
};
|
|
62
64
|
}
|
|
63
|
-
if (question !== null &&
|
|
65
|
+
if (question !== null && scanEncodedText(question).sensitive)
|
|
64
66
|
return SENSITIVE_QUESTION;
|
|
65
67
|
return {
|
|
66
68
|
ok: true,
|
|
@@ -99,7 +101,7 @@ export function resolveAutomaticAnalysis(raw) {
|
|
|
99
101
|
message: `review.analysis requires a positive maxCostUsd up to 1000, a supported analysis model, timeoutMs 1–${MAX_ANALYSIS_TIMEOUT_MS}, maxOutputTokens 256–${MAX_ANALYSIS_OUTPUT_TOKENS}, and an optional question of at most 4000 characters.`,
|
|
100
102
|
};
|
|
101
103
|
}
|
|
102
|
-
if (question !== null &&
|
|
104
|
+
if (question !== null && scanEncodedText(question).sensitive)
|
|
103
105
|
return SENSITIVE_QUESTION;
|
|
104
106
|
return {
|
|
105
107
|
ok: true,
|
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
import { receivingAnalysisContext } from "../comms/receiving-evidence.js";
|
|
2
|
-
import {
|
|
2
|
+
import { streamParticipantIdOf } from "../run/participant-records.js";
|
|
3
3
|
import { cuaEndpointSource } from "../actors/goal-source.js";
|
|
4
4
|
import { isRecord } from "../run/type-guards.js";
|
|
5
5
|
import { ACTION_CAPTURE_VERSION, } from "./types.js";
|
|
6
6
|
import { decodesToPlainRelativePath, isEvidencePath } from "../run/evidence-files.js";
|
|
7
|
+
import { isRunEventOf } from "./source.js";
|
|
7
8
|
// How a run bundle becomes analysis sources: each participant's recorded provenance and assignment,
|
|
8
9
|
// the evidence entries its trace offers, and the order in which entries are admitted under the
|
|
9
10
|
// count and byte budgets. captureEvidence selects from these, taking each participant's while its
|
|
@@ -97,8 +98,7 @@ export function participantSource(stream, captureVersion) {
|
|
|
97
98
|
},
|
|
98
99
|
};
|
|
99
100
|
}
|
|
100
|
-
const runEventsOf = (bundle, stream) => bundle.events.filter((
|
|
101
|
-
(entry.streamId === undefined && eventRecordIdOf(entry) === recordIdOf(stream)));
|
|
101
|
+
const runEventsOf = (bundle, stream) => bundle.events.filter((event) => isRunEventOf(stream, event));
|
|
102
102
|
/**
|
|
103
103
|
* How many entries sourceEntries gives of run.json's `stream` once its trace is loaded, counted
|
|
104
104
|
* before the trace is read: the trace loader refuses a trace that does not hold the item count its
|
|
@@ -163,11 +163,8 @@ export function sourceEntries(bundle, stream, captureVersion) {
|
|
|
163
163
|
failed: ["failed", "blocked", "timed_out"].includes(item.status ?? ""),
|
|
164
164
|
});
|
|
165
165
|
}
|
|
166
|
-
|
|
166
|
+
// parseAnalysisSource refuses a run.json in which two of a participant's run events share an id.
|
|
167
167
|
for (const event of runEventsOf(bundle, stream)) {
|
|
168
|
-
if (runEventIds.has(event.id))
|
|
169
|
-
throw new Error("ANALYSIS_SOURCE_EVENT_DUPLICATE");
|
|
170
|
-
runEventIds.add(event.id);
|
|
171
168
|
entries.push({
|
|
172
169
|
eventId: event.id,
|
|
173
170
|
kind: `run_event:${event.type}`,
|
|
@@ -46,8 +46,8 @@ export async function captureEvidence(prepared, bundleBytes, requested = {}) {
|
|
|
46
46
|
// Admission stops at a participant's evidence share. The shares come from run.json's counts, so
|
|
47
47
|
// only the first share of each participant's entries in admission order is kept of its trace.
|
|
48
48
|
const shares = evidenceShares(bundle, limits, captureVersion);
|
|
49
|
-
// A participant past limits.participants has its trace read and checked
|
|
50
|
-
// no entries
|
|
49
|
+
// A participant past limits.participants has its trace read and checked like the others' and
|
|
50
|
+
// gives no entries.
|
|
51
51
|
const sources = await readParticipantTraces(prepared, bundle, (stream, index) => index < limits.participants
|
|
52
52
|
? participantSources(bundle, bundle.streams[index], stream, captureVersion, {
|
|
53
53
|
evidenceShare: shares[index],
|
|
@@ -229,7 +229,7 @@ async function selectCaptures(prepared, packings, sel, omissions) {
|
|
|
229
229
|
const bytes = result.state === "read" ? result.bytes : null;
|
|
230
230
|
sel.returnedImageBytes += bytes?.length ?? 0;
|
|
231
231
|
packing.readBytes += bytes?.length ?? 0;
|
|
232
|
-
if (bytes === null || screenshotEvidenceError(source.capturePath, bytes) !== null) {
|
|
232
|
+
if (bytes === null || (await screenshotEvidenceError(source.capturePath, bytes)) !== null) {
|
|
233
233
|
omissions.add("Some captures were missing, unsafe, oversized, or invalid PNG evidence.");
|
|
234
234
|
continue;
|
|
235
235
|
}
|
|
@@ -453,7 +453,7 @@ export async function validateAnalysisEvidence(prepared, artifact, bundleBytes)
|
|
|
453
453
|
let hash = checkedCaptures.get(source.capturePath);
|
|
454
454
|
if (hash === undefined) {
|
|
455
455
|
const bytes = await readBoundedFile(prepared, source.capturePath, EVIDENCE_LIMITS.imageBytes);
|
|
456
|
-
if (!bytes || screenshotEvidenceError(source.capturePath, bytes) !== null)
|
|
456
|
+
if (!bytes || (await screenshotEvidenceError(source.capturePath, bytes)) !== null)
|
|
457
457
|
throw new Error("ANALYSIS_CAPTURE_UNAVAILABLE");
|
|
458
458
|
checkedImageBytes += bytes.length;
|
|
459
459
|
if (checkedImageBytes > imageByteLimit)
|
package/dist/analysis/execute.js
CHANGED
|
@@ -4,6 +4,7 @@ import { estimateAnalysisCost } from "./admission.js";
|
|
|
4
4
|
import { createHash, randomUUID } from "node:crypto";
|
|
5
5
|
import { MODEL_RATES } from "../run/pricing.js";
|
|
6
6
|
import { containsSensitive } from "../evidence/redaction.js";
|
|
7
|
+
import { scanEncodedText } from "../evidence/encoded-text.js";
|
|
7
8
|
import { ANALYSIS_ID_PATTERN, ANALYSIS_SCHEMA, } from "./types.js";
|
|
8
9
|
import { INPUT_IMAGE_DATA_URL, createAnalysisProvider, } from "./provider.js";
|
|
9
10
|
import { hashAnalysisValue, analysisResultJsonSchema, validateAnalysisInputMetadata, } from "./validation.js";
|
|
@@ -189,7 +190,7 @@ export function estimateAnalysisAdmission(input, config) {
|
|
|
189
190
|
(typeof config.question !== "string" || config.question.length > 4000))) {
|
|
190
191
|
return denied("analysis_config_invalid");
|
|
191
192
|
}
|
|
192
|
-
if (config.question !== null &&
|
|
193
|
+
if (config.question !== null && scanEncodedText(config.question).sensitive)
|
|
193
194
|
return denied("analysis_question_sensitive");
|
|
194
195
|
const badInput = inputError(input);
|
|
195
196
|
if (badInput)
|
package/dist/analysis/service.js
CHANGED
|
@@ -5,6 +5,7 @@ import { lstat, mkdir, realpath, rmdir } from "node:fs/promises";
|
|
|
5
5
|
import path from "node:path";
|
|
6
6
|
import { renderObserver } from "../observer/render.js";
|
|
7
7
|
import { containsSensitive } from "../evidence/redaction.js";
|
|
8
|
+
import { scanEncodedText } from "../evidence/encoded-text.js";
|
|
8
9
|
import { transientCommsKnownValueScrub } from "../run/transient-comms-secrets.js";
|
|
9
10
|
import { verifyRunPrepared } from "../verify/verify.js";
|
|
10
11
|
import { loadRunBundlePrepared, resolveRunPath } from "../run/locate.js";
|
|
@@ -189,7 +190,7 @@ function analyzeConfigRefusal(config) {
|
|
|
189
190
|
? !validCodexAnalysisConfig(config)
|
|
190
191
|
: !Number.isFinite(config.maxCostUsd) || config.maxCostUsd <= 0 || config.maxCostUsd > 1000)
|
|
191
192
|
return "ANALYSIS_CONFIG_INVALID";
|
|
192
|
-
if (config.question !== null &&
|
|
193
|
+
if (config.question !== null && scanEncodedText(config.question).sensitive)
|
|
193
194
|
return "ANALYSIS_QUESTION_UNSAFE";
|
|
194
195
|
return null;
|
|
195
196
|
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { type StoredRunBundle } from "../run/bundle.js";
|
|
1
|
+
import { type RunEvent, type StoredRunBundle } from "../run/bundle.js";
|
|
2
2
|
import { type PreparedRunArtifactPaths } from "../run/paths.js";
|
|
3
|
-
import type { RunStream } from "../run/streams.js";
|
|
3
|
+
import type { RunStream, StoredRunStream } from "../run/streams.js";
|
|
4
4
|
/**
|
|
5
5
|
* A refusal of the run's recorded source. Its message is the code, as every analysis error's is,
|
|
6
6
|
* and `reason` says which file analysis did not read and why, for the person who asked.
|
|
@@ -11,10 +11,13 @@ export declare class AnalysisSourceRefusedError extends Error {
|
|
|
11
11
|
}
|
|
12
12
|
/** The refusal of a run.json of `size` bytes, more than the `max` analysis reads of it. */
|
|
13
13
|
export declare function runFileTooLarge(size: number | bigint, max: number): AnalysisSourceRefusedError;
|
|
14
|
+
/** Whether `event` is `stream`'s: it names the stream, or no stream and the stream's record. */
|
|
15
|
+
export declare const isRunEventOf: (stream: StoredRunStream, event: RunEvent) => boolean;
|
|
14
16
|
/**
|
|
15
17
|
* run.json's bundle, when it is a run bundle of the prepared run that analysis reads: at most
|
|
16
|
-
* EVIDENCE_LIMITS.sourceBytes, at most EVIDENCE_LIMITS.participants participants,
|
|
17
|
-
* participant and event with the fields analysis reads
|
|
18
|
+
* EVIDENCE_LIMITS.sourceBytes, at most EVIDENCE_LIMITS.participants participants, each
|
|
19
|
+
* participant and event with the fields analysis reads, and no two run events of one participant
|
|
20
|
+
* with one id.
|
|
18
21
|
*/
|
|
19
22
|
export declare function parseAnalysisSource(prepared: PreparedRunArtifactPaths, bytes: Buffer): StoredRunBundle;
|
|
20
23
|
/**
|
package/dist/analysis/source.js
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
// and the refusals it gives for either.
|
|
3
3
|
import { isCommsReceivingEvidence } from "../comms/receiving-evidence.js";
|
|
4
4
|
import { isReadRunBundleSchema } from "../run/bundle.js";
|
|
5
|
+
import { eventRecordIdOf, recordIdOf } from "../run/participant-records.js";
|
|
5
6
|
import { runIdOf } from "../run/paths.js";
|
|
6
7
|
import { reduceRunTraces, StreamTraceRefusedError, StreamTraceTooLargeError, } from "../run/stream-trace.js";
|
|
7
8
|
import { isRecord } from "../run/type-guards.js";
|
|
@@ -24,12 +25,16 @@ const runFileInvalid = (reason) => new AnalysisSourceRefusedError("ANALYSIS_SOUR
|
|
|
24
25
|
const traceInvalid = (streamId, reason) => new AnalysisSourceRefusedError("ANALYSIS_SOURCE_INVALID", `Analysis did not read participant ${streamId}'s trace: ${reason}`);
|
|
25
26
|
/** The refusal of a run.json of `size` bytes, more than the `max` analysis reads of it. */
|
|
26
27
|
export function runFileTooLarge(size, max) {
|
|
27
|
-
return new AnalysisSourceRefusedError("ANALYSIS_SOURCE_TOO_LARGE", `Analysis did not read run.json: it holds ${bytes(size)}, more than the ${limit(max)} analysis reads of it. run.json grows with each participant and the
|
|
28
|
+
return new AnalysisSourceRefusedError("ANALYSIS_SOURCE_TOO_LARGE", `Analysis did not read run.json: it holds ${bytes(size)}, more than the ${limit(max)} analysis reads of it. run.json grows with each participant, its instruction, the mission and the task goals; split the participants across studies to keep it under the limit.`);
|
|
28
29
|
}
|
|
30
|
+
/** Whether `event` is `stream`'s: it names the stream, or no stream and the stream's record. */
|
|
31
|
+
export const isRunEventOf = (stream, event) => event.streamId === stream.id ||
|
|
32
|
+
(event.streamId === undefined && eventRecordIdOf(event) === recordIdOf(stream));
|
|
29
33
|
/**
|
|
30
34
|
* run.json's bundle, when it is a run bundle of the prepared run that analysis reads: at most
|
|
31
|
-
* EVIDENCE_LIMITS.sourceBytes, at most EVIDENCE_LIMITS.participants participants,
|
|
32
|
-
* participant and event with the fields analysis reads
|
|
35
|
+
* EVIDENCE_LIMITS.sourceBytes, at most EVIDENCE_LIMITS.participants participants, each
|
|
36
|
+
* participant and event with the fields analysis reads, and no two run events of one participant
|
|
37
|
+
* with one id.
|
|
33
38
|
*/
|
|
34
39
|
export function parseAnalysisSource(prepared, bytes) {
|
|
35
40
|
if (bytes.length > EVIDENCE_LIMITS.sourceBytes)
|
|
@@ -67,7 +72,20 @@ export function parseAnalysisSource(prepared, bytes) {
|
|
|
67
72
|
typeof event.type !== "string")
|
|
68
73
|
throw runFileInvalid(`event ${index + 1} needs an id, a type and a message.`);
|
|
69
74
|
}
|
|
70
|
-
|
|
75
|
+
const bundle = value;
|
|
76
|
+
for (const stream of bundle.streams) {
|
|
77
|
+
// The place each id was first seen, since an analysis cites a participant's run event by its id.
|
|
78
|
+
const firstPlace = new Map();
|
|
79
|
+
for (const [index, event] of bundle.events.entries()) {
|
|
80
|
+
if (!isRunEventOf(stream, event))
|
|
81
|
+
continue;
|
|
82
|
+
const first = firstPlace.get(event.id);
|
|
83
|
+
if (first !== undefined)
|
|
84
|
+
throw runFileInvalid(`events ${first + 1} and ${index + 1} are participant ${stream.id}'s and share the id ${JSON.stringify(event.id)}, and analysis cites each of a participant's run events by its id. Give one of them another id in run.json; if run.json was not edited, please report this as a bug.`);
|
|
85
|
+
firstPlace.set(event.id, index);
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
return bundle;
|
|
71
89
|
}
|
|
72
90
|
/**
|
|
73
91
|
* What `keep` takes from each stream of run.json's bundle with the stream's trace loaded, in stream
|
|
@@ -395,12 +395,14 @@ function formatInitHuman(result) {
|
|
|
395
395
|
const lines = [title, `cwd: ${result.cwd}`];
|
|
396
396
|
if (result.ok && result.mode === "applied") {
|
|
397
397
|
// Name each file init wrote, so the person or agent who ran it can review exactly those
|
|
398
|
-
// files;
|
|
398
|
+
// files; files it left alone are a count. Neither includes the empty runtime directories under
|
|
399
|
+
// the ignored `.humanish/`: they give the reader nothing to review, leaving them out keeps a
|
|
400
|
+
// first setup with three next steps to 24 lines, and a second run then counts as unchanged
|
|
401
|
+
// the files the first one named. JSON always carries every change.
|
|
399
402
|
const files = (action) => result.changes.filter((change) => change.action === action && change.target !== "runtime");
|
|
400
403
|
const created = files("create");
|
|
401
404
|
const updated = files("update");
|
|
402
|
-
const preserved =
|
|
403
|
-
const directories = result.changes.filter((change) => change.action === "mkdir").length;
|
|
405
|
+
const preserved = files("skip").length;
|
|
404
406
|
lines.push("");
|
|
405
407
|
if (created.length > 0)
|
|
406
408
|
lines.push("created:", ...created.map((change) => ` ${change.path}`));
|
|
@@ -408,13 +410,8 @@ function formatInitHuman(result) {
|
|
|
408
410
|
lines.push("updated:", ...updated.map((change) => ` ${change.path} (${change.reason})`));
|
|
409
411
|
if (created.length === 0 && updated.length === 0)
|
|
410
412
|
lines.push("no files changed");
|
|
411
|
-
if (preserved > 0
|
|
412
|
-
lines.push(
|
|
413
|
-
...(preserved > 0 ? [`${plural(preserved, "existing path")} unchanged`] : []),
|
|
414
|
-
...(directories > 0
|
|
415
|
-
? [`${plural(directories, "ignored directory", "ignored directories")} prepared`]
|
|
416
|
-
: []),
|
|
417
|
-
].join(", "));
|
|
413
|
+
if (preserved > 0)
|
|
414
|
+
lines.push(`${plural(preserved, "existing path")} unchanged`);
|
|
418
415
|
}
|
|
419
416
|
else {
|
|
420
417
|
lines.push("", "changes:", ...result.changes.map(formatInitChange));
|
|
@@ -8,10 +8,12 @@ export declare const latin1RunsAsUtf8: (text: string) => string;
|
|
|
8
8
|
export declare function decodeEscapes(text: string): string;
|
|
9
9
|
/**
|
|
10
10
|
* Every reading of the text a scan matches, each once: as written, decodeEscapes, with percent
|
|
11
|
-
* escapes read as UTF-8, decodeEscapes with transfer escapes expanded,
|
|
11
|
+
* escapes read as UTF-8, decodeEscapes with transfer escapes expanded, the text as written and
|
|
12
12
|
* decoded with each run of U+0080 to U+00FF characters read as UTF-8 bytes, which undoes text that
|
|
13
|
-
* a reader already decoded one byte per character
|
|
14
|
-
*
|
|
13
|
+
* a reader already decoded one byte per character, decodeEscapes with HTML references left as
|
|
14
|
+
* written and that reading with transfer escapes expanded, and the text with only HTML references
|
|
15
|
+
* decoded. Either percent reading can split a value that the other keeps whole, as `%E2%80%80`
|
|
16
|
+
* (U+2000, a space, in UTF-8) does.
|
|
15
17
|
*/
|
|
16
18
|
export declare function readingsOf(text: string): string[];
|
|
17
19
|
/** How scanEncodedText reads text. */
|
|
@@ -34,6 +36,14 @@ export interface EncodedTextScan {
|
|
|
34
36
|
* is opaque unless the caller allows opaque runs.
|
|
35
37
|
*/
|
|
36
38
|
export declare function scanEncodedText(text: string, options?: EncodedTextScanOptions, depth?: number): EncodedTextScan;
|
|
39
|
+
/**
|
|
40
|
+
* redactText, then each whitespace-separated piece that scanEncodedText reads as sensitive replaced
|
|
41
|
+
* by the secret marker, so a key written in base64 or percent-encoded goes as a literal one does.
|
|
42
|
+
* When the pieces read clean one at a time but the text still reads sensitive, as `Bearer` before a
|
|
43
|
+
* percent-encoded token does, the whole text becomes the marker. For study text the run records and
|
|
44
|
+
* the instructions composed from it: on 32 KiB of prose it took 3 ms, and 34 ms with a key in it.
|
|
45
|
+
*/
|
|
46
|
+
export declare function redactEncodedText(text: string): string;
|
|
37
47
|
/**
|
|
38
48
|
* scanEncodedText, cached in this process. The key is the sha256 of the scanned string's UTF-16
|
|
39
49
|
* code units, the options and ENCODED_SCAN_VERSION. Encoding as UTF-16LE keeps every code unit,
|