humanish 0.114.0 → 0.116.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +41 -0
- package/CONTEXT.md +1 -1
- package/README.md +7 -6
- package/TELEMETRY.md +6 -0
- package/dist/actors/closing-report-limits.d.ts +11 -0
- package/dist/actors/closing-report-limits.js +13 -0
- package/dist/actors/closing-report.d.ts +68 -0
- package/dist/actors/closing-report.js +55 -0
- package/dist/actors/codex/app-server-ui.js +11 -4
- package/dist/actors/codex/restricted-participant-policy.d.ts +4 -47
- package/dist/actors/codex/restricted-participant-policy.js +12 -40
- package/dist/actors/computer-use/loop/debrief.d.ts +1 -10
- package/dist/actors/computer-use/loop/debrief.js +68 -70
- package/dist/actors/computer-use/loop/impressions.d.ts +2 -2
- package/dist/actors/computer-use/loop/impressions.js +6 -1
- package/dist/actors/computer-use/loop.d.ts +0 -1
- package/dist/actors/computer-use/loop.js +3 -3
- package/dist/actors/computer-use/openai-conversation.d.ts +1 -1
- package/dist/actors/computer-use/openai-conversation.js +3 -29
- package/dist/actors/computer-use/openai-provider.js +11 -5
- package/dist/actors/contract.d.ts +2 -20
- package/dist/actors/contract.js +0 -13
- package/dist/actors/scripted-browser/steps.js +6 -5
- package/dist/analysis/admission.d.ts +14 -4
- package/dist/analysis/admission.js +14 -9
- package/dist/analysis/analysis-limits.d.ts +35 -0
- package/dist/analysis/analysis-limits.js +41 -0
- package/dist/analysis/execute.d.ts +5 -0
- package/dist/analysis/execute.js +3 -0
- package/dist/analysis/finding-lead.d.ts +27 -0
- package/dist/analysis/finding-lead.js +23 -0
- package/dist/analysis/service.js +5 -3
- package/dist/analysis/validation.js +14 -21
- package/dist/cli/commands/analyze.js +2 -1
- package/dist/cli/commands/notes.js +5 -3
- package/dist/cli/commands/run-signals.js +5 -1
- package/dist/cli/commands/runs.js +2 -1
- package/dist/cli/commands/setup.js +3 -3
- package/dist/cli/commands/study-format.d.ts +5 -2
- package/dist/cli/commands/study-format.js +29 -9
- package/dist/cli/commands/study-route-computer-use.js +2 -2
- package/dist/cli/commands/study-route-scripted.js +1 -1
- package/dist/cli/commands/tui.js +59 -17
- package/dist/cli/doctor.js +9 -2
- package/dist/cli/findings-text.d.ts +5 -0
- package/dist/cli/findings-text.js +169 -0
- package/dist/cli/findings.d.ts +0 -4
- package/dist/cli/findings.js +11 -195
- package/dist/cli/io.js +10 -8
- package/dist/cli/telemetry.js +10 -22
- package/dist/cli/update-check.js +10 -19
- package/dist/cli/user-config.d.ts +11 -0
- package/dist/cli/user-config.js +25 -0
- package/dist/comms/connections.js +4 -3
- package/dist/comms/email-catch.js +7 -31
- package/dist/comms/setup.js +18 -12
- package/dist/evidence/encoded-text.d.ts +12 -11
- package/dist/evidence/encoded-text.js +81 -70
- package/dist/evidence/redaction.d.ts +11 -0
- package/dist/evidence/redaction.js +30 -11
- package/dist/evidence/secret-scrub.d.ts +26 -3
- package/dist/evidence/secret-scrub.js +172 -38
- package/dist/feedback/draft.d.ts +1 -1
- package/dist/feedback/draft.js +13 -16
- package/dist/feedback/export-bundle.js +21 -17
- package/dist/feedback/export.js +32 -46
- package/dist/feedback/feedback.js +31 -26
- package/dist/guest/desktop-keys.js +59 -5
- package/dist/keys/key-resolution.d.ts +9 -0
- package/dist/keys/key-resolution.js +62 -55
- package/dist/keys/key-status.d.ts +2 -0
- package/dist/keys/key-status.js +9 -16
- package/dist/observer/artifact.d.ts +1 -1
- package/dist/observer/http.d.ts +7 -0
- package/dist/observer/http.js +19 -1
- package/dist/observer/notes-route.d.ts +0 -1
- package/dist/observer/notes-route.js +24 -44
- package/dist/observer/pinned-files.js +4 -19
- package/dist/observer/render.js +6 -4
- package/dist/observer/run-routes.js +13 -28
- package/dist/observer/serve.js +2 -1
- package/dist/observer-app.html +7 -7
- package/dist/routes/computer-use/bundle-parts.d.ts +4 -4
- package/dist/routes/computer-use/bundle-parts.js +2 -2
- package/dist/routes/computer-use/bundle.d.ts +4 -5
- package/dist/routes/computer-use/bundle.js +1 -3
- package/dist/routes/computer-use/fanout-records.js +10 -11
- package/dist/routes/computer-use/live-phase.d.ts +1 -1
- package/dist/routes/computer-use/live-phase.js +0 -1
- package/dist/routes/computer-use/participant-execution.d.ts +3 -2
- package/dist/routes/computer-use/participant-runs.d.ts +6 -2
- package/dist/routes/computer-use/participant-runs.js +9 -5
- package/dist/routes/computer-use/plan.d.ts +3 -0
- package/dist/routes/computer-use/plan.js +14 -11
- package/dist/routes/computer-use/rerun-selection.js +2 -2
- package/dist/routes/computer-use/result.js +3 -6
- package/dist/routes/computer-use/setup.d.ts +4 -6
- package/dist/routes/computer-use/setup.js +2 -6
- package/dist/routes/computer-use/single-bundle.d.ts +4 -5
- package/dist/routes/computer-use/single-bundle.js +22 -25
- package/dist/routes/computer-use/subject-projection.d.ts +4 -8
- package/dist/routes/computer-use/subject-projection.js +9 -31
- package/dist/routes/computer-use/types.d.ts +4 -41
- package/dist/routes/scripted/scenario.js +28 -12
- package/dist/routes/scripted/session-result.js +6 -3
- package/dist/routes/shared-world/bundle-records.js +2 -8
- package/dist/routes/shared-world/participant-specs.d.ts +0 -2
- package/dist/routes/shared-world/participant-specs.js +1 -20
- package/dist/routes/shared-world/plan.d.ts +3 -0
- package/dist/routes/shared-world/plan.js +23 -3
- package/dist/routes/shared-world/setup.js +2 -2
- package/dist/routes/terminal/ledger.d.ts +9 -9
- package/dist/routes/terminal/ledger.js +63 -40
- package/dist/routes/terminal/live-finish.js +7 -10
- package/dist/routes/terminal/plan.d.ts +3 -0
- package/dist/routes/terminal/plan.js +8 -5
- package/dist/routes/terminal/token-usage.d.ts +1 -2
- package/dist/routes/terminal/token-usage.js +4 -6
- package/dist/routes/terminal/types.d.ts +9 -1
- package/dist/run/bundle-shape.js +6 -1
- package/dist/run/bundle.d.ts +5 -3
- package/dist/run/contained-output.d.ts +59 -10
- package/dist/run/contained-output.js +160 -68
- package/dist/run/detail.d.ts +2 -1
- package/dist/run/detail.js +12 -7
- package/dist/run/escape-sequences.d.ts +11 -0
- package/dist/run/escape-sequences.js +93 -0
- package/dist/run/evidence-files.d.ts +8 -1
- package/dist/run/evidence-files.js +24 -18
- package/dist/run/locate.d.ts +32 -3
- package/dist/run/locate.js +45 -40
- package/dist/run/note-files.d.ts +26 -45
- package/dist/run/note-files.js +86 -58
- package/dist/run/note-shape.d.ts +35 -0
- package/dist/run/note-shape.js +20 -0
- package/dist/run/notes.d.ts +3 -14
- package/dist/run/notes.js +23 -75
- package/dist/run/outcomes.d.ts +24 -0
- package/dist/run/outcomes.js +19 -0
- package/dist/run/participant-records.d.ts +2 -0
- package/dist/run/participant-records.js +17 -2
- package/dist/run/paths.d.ts +2 -0
- package/dist/run/paths.js +5 -1
- package/dist/run/preflight-receipts.js +13 -8
- package/dist/run/project.js +7 -4
- package/dist/run/reclaim-outcomes.d.ts +8 -6
- package/dist/run/reclaim-outcomes.js +10 -6
- package/dist/run/reclaim.d.ts +5 -0
- package/dist/run/reclaim.js +43 -16
- package/dist/run/run-clock.d.ts +50 -0
- package/dist/run/run-clock.js +66 -0
- package/dist/run/run.js +5 -2
- package/dist/run/sandbox-ids.d.ts +7 -2
- package/dist/run/sandbox-ids.js +27 -11
- package/dist/run/secrets.d.ts +2 -1
- package/dist/run/secrets.js +18 -22
- package/dist/run/stored-runs.js +14 -7
- package/dist/run/transient-comms-secrets.d.ts +6 -5
- package/dist/run/transient-comms-secrets.js +23 -39
- package/dist/run-study.js +22 -13
- package/dist/study/adapter-scorer-loader.js +6 -4
- package/dist/study/discover.js +9 -6
- package/dist/study/init-paths.js +7 -5
- package/dist/study/persona-resolve.d.ts +2 -2
- package/dist/study/persona-resolve.js +9 -5
- package/dist/study/plan-types.d.ts +2 -2
- package/dist/study/plan.d.ts +5 -3
- package/dist/study/plan.js +18 -9
- package/dist/study/preflight-probes.js +11 -3
- package/dist/study/preflight.js +9 -5
- package/dist/study/summary.d.ts +12 -7
- package/dist/study/summary.js +24 -24
- package/dist/study/warnings.d.ts +19 -12
- package/dist/study/warnings.js +39 -17
- package/dist/substrates/e2b/connection.d.ts +12 -0
- package/dist/substrates/e2b/connection.js +35 -0
- package/dist/substrates/e2b/desktop-executor.d.ts +1 -3
- package/dist/substrates/e2b/desktop-executor.js +42 -9
- package/dist/substrates/e2b/lifetime.d.ts +24 -4
- package/dist/substrates/e2b/lifetime.js +41 -4
- package/dist/substrates/e2b/sandbox.d.ts +4 -3
- package/dist/substrates/e2b/sandbox.js +13 -18
- package/dist/substrates/e2b/sdk.d.ts +20 -9
- package/dist/substrates/local/capacity.d.ts +18 -2
- package/dist/substrates/local/capacity.js +21 -2
- package/dist/substrates/local/runtime-host.js +21 -14
- package/dist/substrates/local/runtime.js +3 -11
- package/dist/tui/contract.d.ts +30 -5
- package/dist/tui/contract.js +4 -1
- package/dist/tui-app.js +149 -147
- package/dist/verify/artifacts.d.ts +9 -2
- package/dist/verify/artifacts.js +37 -13
- package/dist/verify/liveness.js +19 -5
- package/dist/verify/verify.js +54 -27
- package/docs/architecture/actor-contract.md +3 -3
- package/docs/architecture/guest-desktop.md +7 -2
- package/docs/architecture/restricted-codex-analysis.md +1 -1
- package/docs/architecture/state-driven-executor.md +5 -3
- package/docs/contracts/run-bundle.md +25 -13
- package/docs/contracts/schemas.md +38 -29
- package/docs/contracts/study-analysis.md +22 -5
- package/docs/product/automatic-analysis.md +7 -1
- package/package.json +8 -7
- package/skills/humanish/SKILL.md +1 -2
|
@@ -2,6 +2,11 @@ import { type PreparedRunArtifactPaths } from "../run/paths.js";
|
|
|
2
2
|
import type { RunBundle } from "../run/bundle.js";
|
|
3
3
|
/** Public-safety and evidence-reference findings stop at this many per list. */
|
|
4
4
|
export declare const MAX_REPORTED_FINDINGS = 50;
|
|
5
|
+
/**
|
|
6
|
+
* The most entries, files and directories, the public-safety scan visits in one run. Bundle export
|
|
7
|
+
* refuses a run with more, so the scan never passes a run whose files it did not all read.
|
|
8
|
+
*/
|
|
9
|
+
export declare const MAX_RUN_ENTRIES = 10000;
|
|
5
10
|
export declare function missingLocalEvidenceArtifacts(runPaths: PreparedRunArtifactPaths, bundle: RunBundle): Promise<string[]>;
|
|
6
11
|
/**
|
|
7
12
|
* The PNG frames that actor traces register as stream screenshots, relative to the run root.
|
|
@@ -44,8 +49,10 @@ interface RegisteredStreamMedia {
|
|
|
44
49
|
}
|
|
45
50
|
/**
|
|
46
51
|
* Scans every run file for secret and path patterns and returns the findings. A file that is not
|
|
47
|
-
* registered stream media and that the scan cannot read as text (readPlainText),
|
|
48
|
-
* all, goes to `unscanned`, so the caller can keep
|
|
52
|
+
* registered stream media and that the scan cannot read as text (readPlainText), cannot read at
|
|
53
|
+
* all, or that holds more than RUN_ARTIFACT_MAX_BYTES goes to `unscanned`, so the caller can keep
|
|
54
|
+
* the run from grading share_ready. So does a directory it cannot list, and the rest of the run
|
|
55
|
+
* after MAX_RUN_ENTRIES entries.
|
|
49
56
|
*/
|
|
50
57
|
export declare function scanRunPublicSafetyArtifacts(runPaths: PreparedRunArtifactPaths, derivedFindings: string[], media: RegisteredStreamMedia, unscanned: string[]): Promise<string[]>;
|
|
51
58
|
export {};
|
package/dist/verify/artifacts.js
CHANGED
|
@@ -6,15 +6,20 @@ import { scanEncodedTextCached } from "../evidence/encoded-text.js";
|
|
|
6
6
|
import { readPlainText } from "../evidence/plain-text.js";
|
|
7
7
|
import { containsSensitive } from "../evidence/redaction.js";
|
|
8
8
|
import { isLocalEvidenceArtifactPath, isRiskyPublicArtifactPath, validatePreparedRunArtifactPaths, } from "../run/paths.js";
|
|
9
|
-
import { openContainedRegularFile } from "../run/contained-output.js";
|
|
9
|
+
import { openContainedRegularFile, RUN_ARTIFACT_MAX_BYTES } from "../run/contained-output.js";
|
|
10
10
|
import { holdsKeyedSandboxId } from "../run/sandbox-ids.js";
|
|
11
11
|
import { SANDBOX_RECEIPTS_ARTIFACT } from "../run/sandbox-receipts.js";
|
|
12
|
-
import { readSafeRunArtifactBytes, readSafeRunArtifactJson } from "../run/locate.js";
|
|
12
|
+
import { readSafeRunArtifact, readSafeRunArtifactBytes, readSafeRunArtifactJson, } from "../run/locate.js";
|
|
13
13
|
import { isRecord } from "../run/type-guards.js";
|
|
14
14
|
import { TERMINAL_EVENTS_ARTIFACT } from "../run/terminal-contract.js";
|
|
15
15
|
import { isZeroEventTerminalTrace } from "./actor.js";
|
|
16
16
|
/** Public-safety and evidence-reference findings stop at this many per list. */
|
|
17
17
|
export const MAX_REPORTED_FINDINGS = 50;
|
|
18
|
+
/**
|
|
19
|
+
* The most entries, files and directories, the public-safety scan visits in one run. Bundle export
|
|
20
|
+
* refuses a run with more, so the scan never passes a run whose files it did not all read.
|
|
21
|
+
*/
|
|
22
|
+
export const MAX_RUN_ENTRIES = 10_000;
|
|
18
23
|
export async function missingLocalEvidenceArtifacts(runPaths, bundle) {
|
|
19
24
|
const recordings = new Map(bundle.streams.flatMap((stream) => stream.recording ? [[stream.recording.path, stream.recording]] : []));
|
|
20
25
|
const requiredPaths = new Map();
|
|
@@ -97,13 +102,18 @@ export async function missingLocalEvidenceArtifacts(runPaths, bundle) {
|
|
|
97
102
|
}
|
|
98
103
|
continue;
|
|
99
104
|
}
|
|
100
|
-
const
|
|
101
|
-
if (
|
|
105
|
+
const artifact = await readSafeRunArtifact(runPaths, artifactPath);
|
|
106
|
+
if (artifact.state === "limit") {
|
|
107
|
+
missing.push(`${artifactPath} (larger than the ${RUN_ARTIFACT_MAX_BYTES} bytes verify reads)`);
|
|
108
|
+
continue;
|
|
109
|
+
}
|
|
110
|
+
if (artifact.state === "unavailable" ||
|
|
111
|
+
(artifact.bytes.length === 0 && !requirements.allowEmpty)) {
|
|
102
112
|
missing.push(artifactPath);
|
|
103
113
|
continue;
|
|
104
114
|
}
|
|
105
115
|
if (requirements.screenshot) {
|
|
106
|
-
const imageError = screenshotEvidenceError(artifactPath, bytes);
|
|
116
|
+
const imageError = screenshotEvidenceError(artifactPath, artifact.bytes);
|
|
107
117
|
if (imageError) {
|
|
108
118
|
missing.push(`${artifactPath} (${imageError})`);
|
|
109
119
|
}
|
|
@@ -280,17 +290,21 @@ export function rawScreenshotStreamIds(bundle, redactedShapeFrames) {
|
|
|
280
290
|
}
|
|
281
291
|
/**
|
|
282
292
|
* Scans every run file for secret and path patterns and returns the findings. A file that is not
|
|
283
|
-
* registered stream media and that the scan cannot read as text (readPlainText),
|
|
284
|
-
* all, goes to `unscanned`, so the caller can keep
|
|
293
|
+
* registered stream media and that the scan cannot read as text (readPlainText), cannot read at
|
|
294
|
+
* all, or that holds more than RUN_ARTIFACT_MAX_BYTES goes to `unscanned`, so the caller can keep
|
|
295
|
+
* the run from grading share_ready. So does a directory it cannot list, and the rest of the run
|
|
296
|
+
* after MAX_RUN_ENTRIES entries.
|
|
285
297
|
*/
|
|
286
298
|
export async function scanRunPublicSafetyArtifacts(runPaths, derivedFindings, media, unscanned) {
|
|
287
299
|
const findings = [];
|
|
288
300
|
await validatePreparedRunArtifactPaths(runPaths);
|
|
289
|
-
await scanRunPublicSafetyDirectory(runPaths, "", findings, derivedFindings, media, unscanned
|
|
301
|
+
await scanRunPublicSafetyDirectory(runPaths, "", findings, derivedFindings, media, unscanned, {
|
|
302
|
+
entries: 0,
|
|
303
|
+
});
|
|
290
304
|
await validatePreparedRunArtifactPaths(runPaths);
|
|
291
305
|
return findings;
|
|
292
306
|
}
|
|
293
|
-
async function scanRunPublicSafetyDirectory(runPaths, relativeDirectory, findings, derivedFindings, media, unscanned) {
|
|
307
|
+
async function scanRunPublicSafetyDirectory(runPaths, relativeDirectory, findings, derivedFindings, media, unscanned, walk) {
|
|
294
308
|
// Each authority has its own finding budget. Derived files must never consume
|
|
295
309
|
// the source scan's budget and make an unscanned recording appear verified.
|
|
296
310
|
if (findings.length >= MAX_REPORTED_FINDINGS && derivedFindings.length >= MAX_REPORTED_FINDINGS) {
|
|
@@ -299,8 +313,18 @@ async function scanRunPublicSafetyDirectory(runPaths, relativeDirectory, finding
|
|
|
299
313
|
const current = relativeDirectory
|
|
300
314
|
? path.join(runPaths.physicalRunRoot, ...relativeDirectory.split("/"))
|
|
301
315
|
: runPaths.physicalRunRoot;
|
|
302
|
-
const entries = await readdir(current).catch(() =>
|
|
316
|
+
const entries = await readdir(current).catch(() => null);
|
|
317
|
+
if (entries === null) {
|
|
318
|
+
unscanned.push(`${relativeDirectory || "."}/ (could not be listed)`);
|
|
319
|
+
return;
|
|
320
|
+
}
|
|
303
321
|
for (const entryName of entries) {
|
|
322
|
+
walk.entries += 1;
|
|
323
|
+
if (walk.entries > MAX_RUN_ENTRIES) {
|
|
324
|
+
if (walk.entries === MAX_RUN_ENTRIES + 1)
|
|
325
|
+
unscanned.push(`${relativeDirectory || "."}/ (the scan stops after ${MAX_RUN_ENTRIES} entries)`);
|
|
326
|
+
return;
|
|
327
|
+
}
|
|
304
328
|
const relativePath = relativeDirectory ? `${relativeDirectory}/${entryName}` : entryName;
|
|
305
329
|
const stats = await lstat(path.join(current, entryName), { bigint: true }).catch(() => null);
|
|
306
330
|
const selectedFindings = !stats?.isDirectory() &&
|
|
@@ -328,7 +352,7 @@ async function scanRunPublicSafetyDirectory(runPaths, relativeDirectory, finding
|
|
|
328
352
|
if (stats.isDirectory()) {
|
|
329
353
|
// A directory named analysis.json is not an owned record. Its children
|
|
330
354
|
// can contain source evidence even after derived findings are saturated.
|
|
331
|
-
await scanRunPublicSafetyDirectory(runPaths, relativePath, findings, derivedFindings, media, unscanned);
|
|
355
|
+
await scanRunPublicSafetyDirectory(runPaths, relativePath, findings, derivedFindings, media, unscanned, walk);
|
|
332
356
|
continue;
|
|
333
357
|
}
|
|
334
358
|
if (selectedFindings.length >= MAX_REPORTED_FINDINGS)
|
|
@@ -342,8 +366,8 @@ async function scanRunPublicSafetyDirectory(runPaths, relativeDirectory, finding
|
|
|
342
366
|
if (media.recordingPaths.has(relativePath) || media.screenshotPaths.has(relativePath)) {
|
|
343
367
|
continue;
|
|
344
368
|
}
|
|
345
|
-
// The scan reads a file by its bytes, not its name. A file that is not text,
|
|
346
|
-
//
|
|
369
|
+
// The scan reads a file by its bytes, not its name. A file that is not text, that could not
|
|
370
|
+
// be read, or that is over the read limit holds bytes the scan never saw.
|
|
347
371
|
const bytes = await readSafeRunArtifactBytes(runPaths, relativePath).catch(() => null);
|
|
348
372
|
const decoded = bytes === null ? undefined : readPlainText(bytes);
|
|
349
373
|
if (decoded === undefined || !decoded.ok) {
|
package/dist/verify/liveness.js
CHANGED
|
@@ -4,7 +4,8 @@
|
|
|
4
4
|
// `ok: true` is not read as "the run finished" or "its sandboxes stopped". It never flips `ok`.
|
|
5
5
|
import { sandboxIdDigest } from "../evidence/redaction.js";
|
|
6
6
|
import path from "node:path";
|
|
7
|
-
import {
|
|
7
|
+
import { refusalText } from "../run/contained-output.js";
|
|
8
|
+
import { readRunJsonIfExists, readRunTextIfExists, runJsonValue } from "../run/locate.js";
|
|
8
9
|
import { isRecord } from "../run/type-guards.js";
|
|
9
10
|
import { RECLAIM_RECEIPT_ARTIFACT } from "../run/reclaim.js";
|
|
10
11
|
import { runLiveness } from "../run/liveness.js";
|
|
@@ -22,7 +23,8 @@ const GONE_STATES = new Set(["killed", "already-gone"]);
|
|
|
22
23
|
*/
|
|
23
24
|
export async function runNotFinished(runPaths, bundle, nowMs = Date.now()) {
|
|
24
25
|
const runId = path.basename(runPaths.absoluteRunRoot);
|
|
25
|
-
|
|
26
|
+
// A status.json it refuses is no usable status, as a missing one is: the bundle decides.
|
|
27
|
+
const status = runJsonValue(await readRunJsonIfExists(runPaths, RUN_STATUS_FILE));
|
|
26
28
|
const { liveness, record } = runLiveness(runId, status, bundle, nowMs);
|
|
27
29
|
if (liveness === "finished")
|
|
28
30
|
return { warnings: [] };
|
|
@@ -56,11 +58,23 @@ export async function runNotFinished(runPaths, bundle, nowMs = Date.now()) {
|
|
|
56
58
|
/** What the reclaim receipt says about this run's sandboxes, and what to run. */
|
|
57
59
|
async function reclaimState(runPaths, runId) {
|
|
58
60
|
const journal = await readRunTextIfExists(runPaths, SANDBOX_RECEIPTS_ARTIFACT);
|
|
61
|
+
const command = `\`${cli(`reclaim --run ${runId}`)}\``;
|
|
62
|
+
// A journal it cannot read hides which sandboxes the run made, so no receipt can cover them.
|
|
63
|
+
if (journal.status === "refused")
|
|
64
|
+
return {
|
|
65
|
+
sandboxes: "unconfirmed",
|
|
66
|
+
text: `Sandboxes unconfirmed: ${refusalText(SANDBOX_RECEIPTS_ARTIFACT, journal)}, so the sandboxes it journaled are not known; check the file, then run ${command}.`,
|
|
67
|
+
};
|
|
59
68
|
// Matched by digest: a reclaim receipt names each sandbox by digest, and one written before it
|
|
60
69
|
// did holds the raw id, which is digested here.
|
|
61
|
-
const journaled = new Set((journal ===
|
|
62
|
-
const
|
|
63
|
-
|
|
70
|
+
const journaled = new Set((journal.status === "missing" ? [] : parseSandboxReceipts(journal.text)).map((receipt) => sandboxIdDigest(receipt.sandboxId)));
|
|
71
|
+
const receiptRead = await readRunJsonIfExists(runPaths, RECLAIM_RECEIPT_ARTIFACT);
|
|
72
|
+
if (receiptRead.status === "refused")
|
|
73
|
+
return {
|
|
74
|
+
sandboxes: "unknown",
|
|
75
|
+
text: `Sandboxes unknown: ${refusalText(RECLAIM_RECEIPT_ARTIFACT, receiptRead)}, and the run journaled ${plural(journaled.size, "sandbox", "sandboxes")}; check the file, then run ${command}.`,
|
|
76
|
+
};
|
|
77
|
+
const receipt = reclaimReceipt(runJsonValue(receiptRead), runId);
|
|
64
78
|
if (receipt === undefined)
|
|
65
79
|
return {
|
|
66
80
|
sandboxes: "unknown",
|
package/dist/verify/verify.js
CHANGED
|
@@ -6,8 +6,10 @@ import { containsSensitive, REDACTED_SANDBOX_ID } from "../evidence/redaction.js
|
|
|
6
6
|
import { keyedSandboxIds, readRunSandboxIds } from "../run/sandbox-ids.js";
|
|
7
7
|
import { validatePreparedRunArtifactPaths } from "../run/paths.js";
|
|
8
8
|
import { RUN_BUNDLE_FILE, RUN_BUNDLE_SCHEMA } from "../run/bundle.js";
|
|
9
|
+
import { ContainedReadRefusedError, refusalText } from "../run/contained-output.js";
|
|
9
10
|
import { isCleanupResult, isRunBundle } from "../run/bundle-shape.js";
|
|
10
|
-
import { readRunJsonIfExists, readRunTextIfExists, resolveRunPath } from "../run/locate.js";
|
|
11
|
+
import { readRunJsonIfExists, readRunTextIfExists, resolveRunPath, runJsonValue, } from "../run/locate.js";
|
|
12
|
+
import { SANDBOX_RECEIPTS_ARTIFACT } from "../run/sandbox-receipts.js";
|
|
11
13
|
import { isRecord } from "../run/type-guards.js";
|
|
12
14
|
import { runNotFoundMessage } from "../run/run-not-found.js";
|
|
13
15
|
import { actorVerdictConsistencyFindings, noEngagementActorFindings, validateCodexAppServerEvidence, validateTerminalProductEvidence, } from "./actor.js";
|
|
@@ -75,26 +77,34 @@ export async function verifyResolvedRun(cwd, runInput, runPaths) {
|
|
|
75
77
|
};
|
|
76
78
|
}
|
|
77
79
|
const bundlePath = path.join(runPaths.absoluteRunRoot, RUN_BUNDLE_FILE);
|
|
78
|
-
const
|
|
79
|
-
const
|
|
80
|
-
const
|
|
81
|
-
const
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
80
|
+
const bundleRead = await readRunJsonIfExists(runPaths, RUN_BUNDLE_FILE);
|
|
81
|
+
const bundle = runJsonValue(bundleRead);
|
|
82
|
+
const cleanupRead = await readRunJsonIfExists(runPaths, "cleanup.json");
|
|
83
|
+
const review = {
|
|
84
|
+
json: await readRunJsonIfExists(runPaths, "review.json"),
|
|
85
|
+
markdown: await readRunTextIfExists(runPaths, "review.md"),
|
|
86
|
+
};
|
|
87
|
+
checks.push(...bundlePresenceChecks(bundleRead, review));
|
|
86
88
|
const derivedPublicSafetyFindings = [];
|
|
87
89
|
const unscannedArtifacts = [];
|
|
88
90
|
const sandboxIdFiles = [];
|
|
91
|
+
let journaledSandboxIds = [];
|
|
92
|
+
try {
|
|
93
|
+
journaledSandboxIds = await readRunSandboxIds(runPaths);
|
|
94
|
+
}
|
|
95
|
+
catch (error) {
|
|
96
|
+
if (!(error instanceof ContainedReadRefusedError))
|
|
97
|
+
throw error;
|
|
98
|
+
// Without the journal's ids, verify cannot clear any file of naming one.
|
|
99
|
+
unscannedArtifacts.push(SANDBOX_RECEIPTS_ARTIFACT);
|
|
100
|
+
}
|
|
89
101
|
const publicSafetyFindings = await scanRunPublicSafetyArtifacts(runPaths, derivedPublicSafetyFindings, {
|
|
90
102
|
recordingPaths: new Set(isRunBundle(bundle)
|
|
91
103
|
? bundle.streams.flatMap((stream) => (stream.recording ? [stream.recording.path] : []))
|
|
92
104
|
: []),
|
|
93
105
|
screenshotPaths: isRunBundle(bundle) ? streamScreenshotPaths(bundle) : new Set(),
|
|
94
106
|
// A run from before 0.110 with no receipts still names its ids in run.json.
|
|
95
|
-
sandboxIds: [
|
|
96
|
-
...new Set([...(await readRunSandboxIds(runPaths)), ...keyedSandboxIds(bundle)]),
|
|
97
|
-
],
|
|
107
|
+
sandboxIds: [...new Set([...journaledSandboxIds, ...keyedSandboxIds(bundle)])],
|
|
98
108
|
sandboxIdFiles,
|
|
99
109
|
}, unscannedArtifacts);
|
|
100
110
|
// JSON escapes can hide a sensitive value from the byte scan while the
|
|
@@ -129,7 +139,7 @@ export async function verifyResolvedRun(cwd, runInput, runPaths) {
|
|
|
129
139
|
? `invalid evidence artifact references: ${invalidEvidenceReferences.join(", ")}`
|
|
130
140
|
: `missing local evidence artifacts: ${missingEvidenceArtifacts.join(", ")}`,
|
|
131
141
|
});
|
|
132
|
-
checks.push(...(await evidenceChecks(runPaths, bundle,
|
|
142
|
+
checks.push(...(await evidenceChecks(runPaths, bundle, cleanupRead)));
|
|
133
143
|
const recordingOk = checks.every((check) => check.ok);
|
|
134
144
|
if (derivedPublicSafetyFindings.length > 0)
|
|
135
145
|
checks.push({
|
|
@@ -219,7 +229,8 @@ const REQUIRED_SECTIONS = [
|
|
|
219
229
|
* Whether run.json, its schema, shape and redaction, and the review artifacts are present. Each
|
|
220
230
|
* check says what it found, so a failing row never prints the rule it enforces.
|
|
221
231
|
*/
|
|
222
|
-
function bundlePresenceChecks(
|
|
232
|
+
function bundlePresenceChecks(bundleRead, review) {
|
|
233
|
+
const bundle = runJsonValue(bundleRead);
|
|
223
234
|
const record = isRecord(bundle) ? bundle : undefined;
|
|
224
235
|
const schema = record?.schema;
|
|
225
236
|
const schemaOk = schema === RUN_BUNDLE_SCHEMA;
|
|
@@ -227,14 +238,26 @@ function bundlePresenceChecks(bundle, review) {
|
|
|
227
238
|
const redaction = isRecord(redactionRecord) ? redactionRecord.status : undefined;
|
|
228
239
|
const missingSections = REQUIRED_SECTIONS.filter((key) => record?.[key] === undefined);
|
|
229
240
|
const missingReview = [
|
|
230
|
-
...(review.json
|
|
231
|
-
|
|
241
|
+
...(review.json.status === "refused"
|
|
242
|
+
? [refusalText("review.json", review.json)]
|
|
243
|
+
: runJsonValue(review.json) === null
|
|
244
|
+
? ["review.json is missing or not valid JSON"]
|
|
245
|
+
: []),
|
|
246
|
+
...(review.markdown.status === "refused"
|
|
247
|
+
? [refusalText("review.md", review.markdown)]
|
|
248
|
+
: review.markdown.status === "missing"
|
|
249
|
+
? ["review.md is missing"]
|
|
250
|
+
: []),
|
|
232
251
|
];
|
|
233
252
|
return [
|
|
234
253
|
{
|
|
235
254
|
name: "run.json exists",
|
|
236
255
|
ok: bundle !== null,
|
|
237
|
-
message:
|
|
256
|
+
message: bundleRead.status === "refused"
|
|
257
|
+
? refusalText(RUN_BUNDLE_FILE, bundleRead)
|
|
258
|
+
: bundle === null
|
|
259
|
+
? "run.json is missing or not valid JSON"
|
|
260
|
+
: "run.json is present",
|
|
238
261
|
},
|
|
239
262
|
{
|
|
240
263
|
name: "run schema",
|
|
@@ -310,7 +333,8 @@ function findingsCheck(name, findings, okMessage, failLabel) {
|
|
|
310
333
|
*/
|
|
311
334
|
const SHAPE_UNCHECKED = "not checked, because run.json failed the shape check";
|
|
312
335
|
/** The bundle-content checks, in report order. A bundle that fails its shape guard has no findings. */
|
|
313
|
-
async function evidenceChecks(runPaths, bundle,
|
|
336
|
+
async function evidenceChecks(runPaths, bundle, cleanupRead) {
|
|
337
|
+
const cleanupJson = runJsonValue(cleanupRead);
|
|
314
338
|
const valid = isRunBundle(bundle) ? bundle : null;
|
|
315
339
|
const pass = (message) => (valid ? message : SHAPE_UNCHECKED);
|
|
316
340
|
const checks = [
|
|
@@ -322,14 +346,17 @@ async function evidenceChecks(runPaths, bundle, cleanupJson) {
|
|
|
322
346
|
findingsCheck("shared-world evidence", valid ? sharedWorldEvidenceFindings(valid) : [], pass("live shared-world evidence is well formed for its mode, with single-plane provenance, digest-only checkpoints and attributionLimits, or the run has none"), "shared-world findings"),
|
|
323
347
|
{
|
|
324
348
|
name: "cleanup receipt",
|
|
325
|
-
ok:
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
349
|
+
ok: cleanupRead.status !== "refused" &&
|
|
350
|
+
(cleanupJson === null || (isCleanupResult(cleanupJson) && cleanupJson.ok)),
|
|
351
|
+
message: cleanupRead.status === "refused"
|
|
352
|
+
? `${refusalText("cleanup.json", cleanupRead)}, so verify cannot say how cleanup went`
|
|
353
|
+
: cleanupJson === null
|
|
354
|
+
? "no cleanup.json; cleanup was not requested"
|
|
355
|
+
: !isCleanupResult(cleanupJson)
|
|
356
|
+
? "cleanup.json is malformed"
|
|
357
|
+
: cleanupJson.ok
|
|
358
|
+
? "cleanup.json records a successful cleanup"
|
|
359
|
+
: "cleanup.json records a failed cleanup",
|
|
333
360
|
},
|
|
334
361
|
findingsCheck("rerun lineage", valid ? rerunLineageFindings(valid) : [], pass("rerun participants link to their prior status and a fan-out rerun event, or this run is not a rerun"), "rerun lineage findings"),
|
|
335
362
|
// Cost is advisory on magnitude and fail-closed on labeling/provenance (claims match mechanism).
|
|
@@ -371,7 +398,7 @@ function buildShareSafety(args) {
|
|
|
371
398
|
});
|
|
372
399
|
}
|
|
373
400
|
if (args.unscannedArtifacts.length > 0) {
|
|
374
|
-
const paths = [...args.unscannedArtifacts].sort();
|
|
401
|
+
const paths = [...new Set(args.unscannedArtifacts)].sort();
|
|
375
402
|
const shown = paths.slice(0, MAX_LISTED_UNSCANNED).join(", ");
|
|
376
403
|
const more = paths.length - MAX_LISTED_UNSCANNED;
|
|
377
404
|
reasons.push({
|
|
@@ -57,7 +57,7 @@ API surface.
|
|
|
57
57
|
1. **Transport-agnostic contract; Codex stays the reference implementation.** The
|
|
58
58
|
contract describes lifecycle and evidence, not transport. An adapter may be a
|
|
59
59
|
subprocess protocol (Codex stdio JSON-RPC, `pi --mode rpc`, `claude -p
|
|
60
|
-
--output-format stream-json`) or an in-process SDK (`pi-agent-core`, Claude
|
|
60
|
+
--output-format stream-json`) or an in-process SDK (`pi-agent-core`, Claude
|
|
61
61
|
Agent SDK, Stagehand). The existing Codex app-server integration is the
|
|
62
62
|
reference adapter. `pi-agent-core` was planned as the first in-process-SDK
|
|
63
63
|
adapter to prove both shapes early; it shipped only as a trace mapper with no
|
|
@@ -325,7 +325,7 @@ the computer-use loop distinguishes two ways to hit the cap:
|
|
|
325
325
|
|
|
326
326
|
- **`budget_reached`**: the deadline was reached after at least one material (non-idle)
|
|
327
327
|
action, or a spend, adapter or token limit ended the session. This maps to `ActorStatus:
|
|
328
|
-
"incomplete"`: the participant did not reach the goal, and the harness did not fail.
|
|
328
|
+
"incomplete"`: the participant did not reach the goal, and the harness did not fail.
|
|
329
329
|
`participantOutcomeOk` is false, the verdict is `fail`, and the CLI exits `2`. The trace
|
|
330
330
|
`reason` and optional `stopCause` say which limit ended the session.
|
|
331
331
|
- **`timed_out`**: the deadline was reached with zero material actions (a hung provider, an
|
|
@@ -422,7 +422,7 @@ returns the full `resolvedPersona` from `parseResolvedPersona`.
|
|
|
422
422
|
Plan:
|
|
423
423
|
|
|
424
424
|
1. **Parse the whole persona** into a `ResolvedPersona`: `{ id, name, summary, goals[],
|
|
425
|
-
traits: { patience, skill, accessibilityNeeds? }, constraints[], sourceDigest }` (map
|
|
425
|
+
traits: { patience, skill, accessibilityNeeds? }, constraints[], sourceDigest }` (map
|
|
426
426
|
`technical_confidence` to `skill`). The shipped shape in `src/study/persona.ts` has no
|
|
427
427
|
`goals`, adds `background`, and makes every trait optional.
|
|
428
428
|
2. **Compile traits into actor-neutral directives**, not prose, via a pure
|
|
@@ -18,8 +18,13 @@ Progress detection uses the same coarse frame signature as the hosted desktop.
|
|
|
18
18
|
The actor receives the existing finite action union. Native tool paths, display,
|
|
19
19
|
Xauthority and guest temporary directory come from the owner. Key names map to a
|
|
20
20
|
closed list before reaching xdotool, whose own command syntax must never receive
|
|
21
|
-
arbitrary participant strings.
|
|
22
|
-
|
|
21
|
+
arbitrary participant strings. The list is `src/guest/desktop-keys.ts`, and the hosted
|
|
22
|
+
E2B desktop translates through it too, so a key name works on both desktops or is
|
|
23
|
+
refused on both. It holds OpenAI's computer-use key names, the names `@e2b/desktop`'s
|
|
24
|
+
`press()` mapped, and every printable ASCII punctuation character as its keysym
|
|
25
|
+
(`?` is `question`). A refused name reaches the participant as a rejected action.
|
|
26
|
+
A native wheel step is not an exact pixel-scroll promise; one action permits at most
|
|
27
|
+
100 wheel steps. The entire drag is checked
|
|
23
28
|
before its first input. A pointer action's held keys must be modifiers (Shift,
|
|
24
29
|
Ctrl, Alt, Super). They go down with one `keydown` before the action and up with
|
|
25
30
|
one `keyup` after it. When an input fails, the `keyup` is still sent once, unless
|
|
@@ -176,7 +176,7 @@ the following, on Linux x64:
|
|
|
176
176
|
runs this check; see "The qualifier" below.
|
|
177
177
|
3. One hosted participant study with the candidate first on `PATH` (for example
|
|
178
178
|
`PATH=<evidence>/npm-<version>/node_modules/.bin:$PATH pnpm humanish run <study>
|
|
179
|
-
--dotenv <file with E2B_API_KEY only>`) reached its goal in the final capture, with a
|
|
179
|
+
--dotenv <file with E2B_API_KEY only>`) reached its goal in the final capture, with a
|
|
180
180
|
verified request receipt, `humanish verify --run <id>` passing and the automatic Codex
|
|
181
181
|
analysis recording the release. Keep `OPENAI_API_KEY` out of that environment so the
|
|
182
182
|
participant stays on ChatGPT-account billing.
|
|
@@ -75,10 +75,12 @@ graphically dense (pixel-art) UI. `stableProgressKey(appState)`:
|
|
|
75
75
|
## A single desktop command failure is a recoverable skipped action
|
|
76
76
|
|
|
77
77
|
`execute(action)` runs against a real substrate, and the real `@e2b/desktop`
|
|
78
|
-
Sandbox **throws** `CommandExitError` on **any** non-zero exit. A
|
|
78
|
+
Sandbox **throws** `CommandExitError` on **any** non-zero exit. A keypress,
|
|
79
79
|
`scroll`, `drag`, `moveMouse`, or `click` can exit non-zero for reasons that have
|
|
80
80
|
nothing to do with the run's health (a `Ctrl+Minus` zoom keypress exiting `2` was
|
|
81
|
-
the reproduced case).
|
|
81
|
+
the reproduced case). xdotool also skips a key name it cannot resolve with a `No such
|
|
82
|
+
key name` warning and exits `0`; the E2B executor turns that warning into an error with
|
|
83
|
+
exit code `0` and xdotool's stderr. The loop treats one such failure as a **recoverable skipped
|
|
82
84
|
action**, not a fatal error:
|
|
83
85
|
|
|
84
86
|
- The per-action `execute()` call is wrapped at the loop boundary, in `runActionBatch`
|
|
@@ -86,7 +88,7 @@ action**, not a fatal error:
|
|
|
86
88
|
uniformly. When the caught error `isCommandExitError`
|
|
87
89
|
(`src/substrates/command-failure.ts`; it matches the SDK class name or any object carrying
|
|
88
90
|
a numeric `exitCode`), the loop records a `notice` item (`status: "error"`, title `action
|
|
89
|
-
skipped: desktop command failed`, text = the public-safe action label + exit code + a
|
|
91
|
+
skipped: desktop command failed`, text = the public-safe action label + exit code + a
|
|
90
92
|
redacted `stderrTail`), does **not** count the action as a material action, and
|
|
91
93
|
**continues**. The next `observe()` hands the model a fresh screenshot/state to adapt to.
|
|
92
94
|
- An executor error declared as `action_rejected` with disposition `not_dispatched` also
|
|
@@ -305,14 +305,16 @@ exactly; this hosted-desktop rule does not change that contract.
|
|
|
305
305
|
`subject` is an optional, additive top-level field: structured provenance for
|
|
306
306
|
what the computer-use, shared-world or scripted route actually drove (code pin plus state story). It
|
|
307
307
|
is absent on pre-existing bundles and on bundles from routes that have not
|
|
308
|
-
adopted it. The field shape, its
|
|
309
|
-
`local-tree`), and the `humanish verify` checks that guard it are the schema doc's
|
|
308
|
+
adopted it. The field shape, its four sources (`clone`, `app-url`,
|
|
309
|
+
`local-tree`, `desktop-cli`), and the `humanish verify` checks that guard it are the schema doc's
|
|
310
310
|
job, not this one: see the `subject` entry under
|
|
311
311
|
[`schemas.md`](schemas.md#run-bundle). In short, `clone` carries a
|
|
312
312
|
`repo`/`commit` pin, `local-tree` carries an `archiveSha256`/`dirty` pin
|
|
313
|
-
instead (a dirty working tree cannot be commit-pinned),
|
|
314
|
-
no code pin at all.
|
|
315
|
-
|
|
313
|
+
instead (a dirty working tree cannot be commit-pinned), `desktop-cli` carries the
|
|
314
|
+
declared `product` name, and `app-url` carries no code pin at all. A computer-use run
|
|
315
|
+
of a desktop-cli study saved before `desktop-cli` was recorded says `app-url`, and
|
|
316
|
+
readers accept it. No path, basename, or other host-machine string ever
|
|
317
|
+
enters this field; identity is digests, a sha, a boolean, counts and the declared product name.
|
|
316
318
|
|
|
317
319
|
## Cost Estimate (advisory)
|
|
318
320
|
|
|
@@ -617,9 +619,13 @@ Each note a person adds while reviewing a run is its own file in the run directo
|
|
|
617
619
|
- `author`: `"you"` in this version.
|
|
618
620
|
- `createdAt`, `editedAt`: ISO-8601. Notes cannot be edited yet, so `editedAt` is `null`.
|
|
619
621
|
|
|
620
|
-
A note file is
|
|
621
|
-
`
|
|
622
|
-
|
|
622
|
+
A note file is created once under its own name, only when no file has that name (`O_CREAT` with
|
|
623
|
+
`O_EXCL`), filled in one write and never rewritten; a taken name gets a new id. No temporary
|
|
624
|
+
file or second link is made, so the run directory passes the single-link check that `verify`,
|
|
625
|
+
serving and adding a note run while a note is added. A reader can find a note while it is being
|
|
626
|
+
written, as an empty or short file; it skips and names that file like any unreadable note, and
|
|
627
|
+
the next read finds the whole note. A writer stopped mid-write leaves such a file, which readers
|
|
628
|
+
keep skipping. Adding a note reads no other note, so notes added at the same time from any
|
|
623
629
|
process are all kept, and there is no lock. Readers list at most 1000 entries of `notes/` and
|
|
624
630
|
read at most 500 notes, each within 16 KiB and refused before reading past that; any other
|
|
625
631
|
entry, or a file that is not a note of the run, is skipped and named. The Observer servers serve
|
|
@@ -719,11 +725,17 @@ a file for secret and path patterns only when the bytes are strict UTF-8 with no
|
|
|
719
725
|
control bytes other than tab, line feed and carriage return, the rule bundle
|
|
720
726
|
export uses for text. `RAW_SCREENSHOTS` and `CONTINUOUS_MEDIA` grade the stream
|
|
721
727
|
screenshots that actor traces reference under `screenshots/`, where the harness
|
|
722
|
-
writes every frame, and the recordings `streams[].recording` registers. Every
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
728
|
+
writes every frame, and the recordings `streams[].recording` registers. Every
|
|
729
|
+
other file the scan cannot read as text, cannot read at all, or that is larger
|
|
730
|
+
than 32 MiB (verify reads at most that much of one file) contributes
|
|
731
|
+
`UNSCANNED_ARTIFACT`. So does a folder the scan cannot list, and the rest of a
|
|
732
|
+
run past its first 10,000 files and folders, the most `export --format bundle`
|
|
733
|
+
takes. The reason's message lists the paths, and it keeps otherwise valid
|
|
734
|
+
evidence `local_only`. This includes images that only feedback candidates,
|
|
735
|
+
adapter artifacts or stream artifact entries cite, and a PNG an actor trace
|
|
736
|
+
references outside `screenshots/`. An adapter artifact larger than 32 MiB grades
|
|
737
|
+
the run `local_only` when nothing cites it, and `blocked` when the bundle cites
|
|
738
|
+
it as evidence, because the evidence check cannot read it. An unregistered
|
|
727
739
|
`.mp4`, and a file or directory whose name contains `\`, are public-safety
|
|
728
740
|
findings and block the run.
|
|
729
741
|
|