humanish 0.110.0 → 0.110.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +15 -12
- package/CHANGELOG.md +26 -0
- package/CONTEXT.md +8 -6
- package/README.md +103 -53
- package/dist/actors/computer-use/loop/ending.js +4 -4
- package/dist/actors/local-agent/claude-participant.d.ts +74 -0
- package/dist/actors/local-agent/claude-participant.js +243 -0
- package/dist/actors/local-agent/claude-session.d.ts +16 -3
- package/dist/actors/local-agent/claude-session.js +64 -23
- package/dist/actors/local-agent/cli.d.ts +17 -0
- package/dist/actors/local-agent/cli.js +111 -24
- package/dist/actors/local-agent/readiness.d.ts +2 -2
- package/dist/actors/local-agent/readiness.js +20 -12
- package/dist/cli/commands/analyze.js +21 -5
- package/dist/cli/commands/observe.js +15 -2
- package/dist/cli/commands/run-signals.d.ts +2 -0
- package/dist/cli/commands/run-signals.js +106 -45
- package/dist/cli/commands/runs.d.ts +5 -0
- package/dist/cli/commands/runs.js +113 -74
- package/dist/cli/commands/setup.js +23 -14
- package/dist/cli/commands/study-route-computer-use.js +9 -3
- package/dist/cli/commands/study-route-open.js +1 -0
- package/dist/cli/commands/study-route-preview.js +1 -0
- package/dist/cli/commands/study-route-scripted.js +2 -0
- package/dist/cli/commands/study-route-shared-world.js +6 -2
- package/dist/cli/commands/study-route-terminal.js +2 -0
- package/dist/cli/commands/watch.js +14 -4
- package/dist/cli/doctor.d.ts +4 -1
- package/dist/cli/doctor.js +51 -10
- package/dist/cli/findings.d.ts +103 -0
- package/dist/cli/findings.js +317 -0
- package/dist/cli/first-run-path.d.ts +32 -11
- package/dist/cli/first-run-path.js +113 -69
- package/dist/cli/invocation.d.ts +7 -0
- package/dist/cli/invocation.js +32 -0
- package/dist/cli/io.d.ts +3 -1
- package/dist/cli/io.js +8 -15
- package/dist/cli/observer-follow.d.ts +20 -0
- package/dist/cli/observer-follow.js +36 -1
- package/dist/cli/orientation.js +16 -14
- package/dist/cli/product-sentence.d.ts +3 -3
- package/dist/cli/product-sentence.js +3 -3
- package/dist/cli/version.d.ts +1 -0
- package/dist/cli/version.js +11 -0
- package/dist/evidence/encoded-text.d.ts +9 -6
- package/dist/evidence/encoded-text.js +84 -19
- package/dist/evidence/redaction.d.ts +2 -0
- package/dist/evidence/redaction.js +123 -21
- package/dist/feedback/export-bundle.js +2 -2
- package/dist/keys/env-file.d.ts +4 -0
- package/dist/keys/env-file.js +17 -0
- package/dist/keys/key-resolution.js +6 -5
- package/dist/observer/data.d.ts +14 -1
- package/dist/observer/data.js +19 -9
- package/dist/observer/exposure.js +1 -1
- package/dist/observer/library.d.ts +4 -0
- package/dist/observer/library.js +21 -17
- package/dist/observer/render.js +10 -3
- package/dist/observer/run-routes.d.ts +2 -0
- package/dist/observer/run-routes.js +51 -35
- package/dist/observer/serve.js +2 -1
- package/dist/observer-app.html +3 -3
- package/dist/routes/computer-use/bundle.js +3 -1
- package/dist/routes/computer-use/fanout-records.js +1 -1
- package/dist/routes/computer-use/participant-execution.js +3 -1
- package/dist/routes/computer-use/participant-model.js +5 -2
- package/dist/routes/computer-use/plan.js +9 -4
- package/dist/routes/computer-use/preflight.js +1 -1
- package/dist/routes/computer-use/result.js +17 -28
- package/dist/routes/computer-use/single-bundle.js +7 -1
- package/dist/routes/scripted/bundle.js +4 -2
- package/dist/routes/scripted/plan.js +2 -2
- package/dist/routes/scripted/result.d.ts +3 -5
- package/dist/routes/scripted/result.js +16 -22
- package/dist/routes/scripted/scenario.js +4 -4
- package/dist/routes/shared-world/bundle-records.js +6 -1
- package/dist/routes/shared-world/bundle.js +2 -0
- package/dist/routes/shared-world/plan.js +6 -2
- package/dist/routes/shared-world/result.js +22 -22
- package/dist/routes/terminal/bundle.js +6 -3
- package/dist/routes/terminal/credentials.d.ts +5 -5
- package/dist/routes/terminal/credentials.js +14 -6
- package/dist/routes/terminal/dry-run.js +8 -12
- package/dist/routes/terminal/ledger.d.ts +3 -3
- package/dist/routes/terminal/ledger.js +6 -6
- package/dist/routes/terminal/live-finish.js +15 -13
- package/dist/routes/terminal/plan.js +12 -4
- package/dist/routes/terminal/result.d.ts +6 -7
- package/dist/routes/terminal/result.js +6 -22
- package/dist/routes/terminal/route.js +5 -4
- package/dist/routes/terminal/runtime-auth.d.ts +6 -0
- package/dist/routes/terminal/runtime-auth.js +5 -0
- package/dist/routes/terminal/session.js +1 -1
- package/dist/run/bundle-shape.d.ts +3 -1
- package/dist/run/bundle-shape.js +25 -1
- package/dist/run/bundle.d.ts +24 -0
- package/dist/run/display.d.ts +59 -0
- package/dist/run/display.js +106 -0
- package/dist/run/dry-run.js +13 -8
- package/dist/run/judge.d.ts +3 -2
- package/dist/run/judge.js +12 -1
- package/dist/run/local-only-files.d.ts +13 -0
- package/dist/run/local-only-files.js +35 -0
- package/dist/run/pricing.js +1 -1
- package/dist/run/projection.d.ts +0 -2
- package/dist/run/projection.js +0 -11
- package/dist/run/reclaim-outcomes.d.ts +89 -0
- package/dist/run/reclaim-outcomes.js +213 -0
- package/dist/run/reclaim.d.ts +23 -17
- package/dist/run/reclaim.js +213 -114
- package/dist/run/results.d.ts +10 -20
- package/dist/run/results.js +2 -2
- package/dist/run/run-index.d.ts +11 -1
- package/dist/run/run-index.js +11 -0
- package/dist/run/run-not-found.js +3 -2
- package/dist/run/run.d.ts +34 -18
- package/dist/run/run.js +166 -67
- package/dist/run/sandbox-creates.d.ts +59 -0
- package/dist/run/sandbox-creates.js +140 -0
- package/dist/run/sandbox-receipts.d.ts +20 -5
- package/dist/run/sandbox-receipts.js +56 -7
- package/dist/run/stats.d.ts +5 -0
- package/dist/run/stats.js +8 -4
- package/dist/run/status.d.ts +6 -9
- package/dist/run/status.js +7 -23
- package/dist/run/stored-runs.d.ts +5 -14
- package/dist/run/stored-runs.js +31 -146
- package/dist/run/streams.d.ts +3 -2
- package/dist/study/agents-section.d.ts +21 -0
- package/dist/study/agents-section.js +86 -0
- package/dist/study/doctor.d.ts +1 -1
- package/dist/study/doctor.js +8 -4
- package/dist/study/init-templates.js +0 -95
- package/dist/study/init.js +45 -37
- package/dist/study/keys.d.ts +12 -0
- package/dist/study/keys.js +32 -0
- package/dist/study/local-browser-host.d.ts +11 -0
- package/dist/study/local-browser-host.js +42 -0
- package/dist/study/parse/actors.js +9 -1
- package/dist/study/parse/subject.js +12 -0
- package/dist/study/parse/url-credentials.d.ts +9 -0
- package/dist/study/parse/url-credentials.js +50 -0
- package/dist/study/persona.d.ts +4 -0
- package/dist/study/persona.js +4 -2
- package/dist/study/preflight-probes.js +2 -1
- package/dist/study/types.d.ts +8 -6
- package/dist/study/url-credentials.d.ts +6 -0
- package/dist/study/url-credentials.js +34 -0
- package/dist/substrates/e2b/peer-install.d.ts +51 -0
- package/dist/substrates/e2b/peer-install.js +161 -0
- package/dist/substrates/e2b/sandbox.d.ts +34 -6
- package/dist/substrates/e2b/sandbox.js +141 -30
- package/dist/substrates/e2b/sdk.d.ts +33 -0
- package/dist/substrates/e2b/sdk.js +21 -21
- package/dist/substrates/local/firecracker-desktop.js +3 -2
- package/dist/substrates/local/runtime-host.js +5 -4
- package/dist/substrates/local/runtime.js +5 -4
- package/dist/substrates/shell.d.ts +2 -0
- package/dist/substrates/shell.js +4 -0
- package/dist/tui/contract.js +5 -4
- package/dist/tui-app.js +133 -133
- package/dist/verify/liveness.d.ts +19 -3
- package/dist/verify/liveness.js +47 -25
- package/dist/verify/verify.d.ts +3 -0
- package/dist/verify/verify.js +8 -3
- package/docs/architecture/actor-contract.md +2 -1
- package/docs/architecture/observer.md +22 -7
- package/docs/architecture/task-protocol-support.md +13 -13
- package/docs/architecture/terminal-product-route.md +15 -13
- package/docs/contracts/run-bundle.md +95 -10
- package/docs/contracts/schemas.md +44 -26
- package/docs/contracts/study-analysis.md +45 -0
- package/docs/contracts/study-costs.md +8 -0
- package/docs/decisions/0001-run-bundle-is-the-source-of-truth.md +3 -0
- package/docs/decisions/0003-contained-paths-and-cleanup-authority.md +9 -3
- package/docs/principles/invariants-and-defaults.md +11 -8
- package/package.json +6 -3
- package/skills/humanish/SKILL.md +38 -20
|
@@ -1,7 +1,23 @@
|
|
|
1
1
|
import type { RunBundle } from "../run/bundle.js";
|
|
2
2
|
import type { PreparedRunArtifactPaths } from "../run/paths.js";
|
|
3
|
+
/** What verify can say about a run that is not finished. */
|
|
4
|
+
export interface UnfinishedRun {
|
|
5
|
+
liveness: "running" | "interrupted";
|
|
6
|
+
/**
|
|
7
|
+
* clean: the run's reclaim receipt records every sandbox gone and a finished search of E2B by
|
|
8
|
+
* the run's tags. unconfirmed: it records a sandbox not confirmed gone, or the run journaled one
|
|
9
|
+
* it does not cover. unknown: there is no reclaim receipt, or it records no finished tag search
|
|
10
|
+
* (written by humanish 0.110.0 or earlier, or by a reclaim that did not finish), so a sandbox whose
|
|
11
|
+
* id never reached a receipt could still run.
|
|
12
|
+
*/
|
|
13
|
+
sandboxes: "clean" | "unconfirmed" | "unknown";
|
|
14
|
+
}
|
|
3
15
|
/**
|
|
4
|
-
* One warning when the run is not finished, by the run index's
|
|
5
|
-
* has a usable one, else a participant still `running` in its
|
|
16
|
+
* One warning, and the unfinished-run facts, when the run is not finished, by the run index's
|
|
17
|
+
* rule: its status record when it has a usable one, else a participant still `running` in its
|
|
18
|
+
* bundle's `simulations[]`. Nothing for a finished run.
|
|
6
19
|
*/
|
|
7
|
-
export declare function
|
|
20
|
+
export declare function runNotFinished(runPaths: PreparedRunArtifactPaths, bundle: RunBundle, nowMs?: number): Promise<{
|
|
21
|
+
warnings: string[];
|
|
22
|
+
unfinished?: UnfinishedRun;
|
|
23
|
+
}>;
|
package/dist/verify/liveness.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
|
-
// `humanish verify`'s RUN_NOT_FINISHED warning
|
|
2
|
-
// and a run killed mid-way leaves an
|
|
3
|
-
//
|
|
1
|
+
// `humanish verify`'s RUN_NOT_FINISHED warning and the unfinished-run facts its one-line output
|
|
2
|
+
// leads with. Verify checks the integrity of what a run wrote, and a run killed mid-way leaves an
|
|
3
|
+
// in-progress bundle that passes those checks. The warning says what verify saw on disk, so
|
|
4
|
+
// `ok: true` is not read as "the run finished" or "its sandboxes stopped". It never flips `ok`.
|
|
4
5
|
import { sandboxIdDigest } from "../evidence/redaction.js";
|
|
5
6
|
import path from "node:path";
|
|
6
7
|
import { readRunJsonIfExists, readRunTextIfExists } from "../run/locate.js";
|
|
@@ -9,19 +10,22 @@ import { RECLAIM_RECEIPT_ARTIFACT } from "../run/reclaim.js";
|
|
|
9
10
|
import { runLiveness } from "../run/run-index.js";
|
|
10
11
|
import { parseSandboxReceipts, SANDBOX_RECEIPTS_ARTIFACT } from "../run/sandbox-receipts.js";
|
|
11
12
|
import { RUN_STATUS_FILE } from "../run/status.js";
|
|
13
|
+
import { plural } from "../run/text.js";
|
|
14
|
+
import { cli } from "../cli/invocation.js";
|
|
12
15
|
/** The stable code every RUN_NOT_FINISHED warning starts with. */
|
|
13
16
|
const RUN_NOT_FINISHED = "RUN_NOT_FINISHED";
|
|
14
17
|
const GONE_STATES = new Set(["killed", "already-gone"]);
|
|
15
18
|
/**
|
|
16
|
-
* One warning when the run is not finished, by the run index's
|
|
17
|
-
* has a usable one, else a participant still `running` in its
|
|
19
|
+
* One warning, and the unfinished-run facts, when the run is not finished, by the run index's
|
|
20
|
+
* rule: its status record when it has a usable one, else a participant still `running` in its
|
|
21
|
+
* bundle's `simulations[]`. Nothing for a finished run.
|
|
18
22
|
*/
|
|
19
|
-
export async function
|
|
23
|
+
export async function runNotFinished(runPaths, bundle, nowMs = Date.now()) {
|
|
20
24
|
const runId = path.basename(runPaths.absoluteRunRoot);
|
|
21
25
|
const status = await readRunJsonIfExists(runPaths, RUN_STATUS_FILE);
|
|
22
26
|
const { liveness, record } = runLiveness(runId, status, bundle, nowMs);
|
|
23
27
|
if (liveness === "finished")
|
|
24
|
-
return [];
|
|
28
|
+
return { warnings: [] };
|
|
25
29
|
const seen = [];
|
|
26
30
|
if (record === undefined) {
|
|
27
31
|
const running = bundle.simulations.filter((simulation) => simulation.status === "running");
|
|
@@ -39,30 +43,48 @@ export async function runNotFinishedWarnings(runPaths, bundle, nowMs = Date.now(
|
|
|
39
43
|
const runningStreams = bundle.streams.filter((stream) => stream.status === "running").length;
|
|
40
44
|
seen.push(`${runningStreams} of ${bundle.streams.length} streams are still running`);
|
|
41
45
|
const reclaim = await reclaimState(runPaths, runId);
|
|
42
|
-
return
|
|
43
|
-
|
|
44
|
-
|
|
46
|
+
return {
|
|
47
|
+
warnings: [
|
|
48
|
+
`${RUN_NOT_FINISHED}: ${seen.join("; ")}. ${reclaim.text} Verify ok covers the integrity of what was written; it does not mean the run finished.`,
|
|
49
|
+
],
|
|
50
|
+
unfinished: { liveness, sandboxes: reclaim.sandboxes },
|
|
51
|
+
};
|
|
45
52
|
}
|
|
46
|
-
/** What the reclaim receipt says about
|
|
53
|
+
/** What the reclaim receipt says about this run's sandboxes, and what to run. */
|
|
47
54
|
async function reclaimState(runPaths, runId) {
|
|
48
55
|
const journal = await readRunTextIfExists(runPaths, SANDBOX_RECEIPTS_ARTIFACT);
|
|
49
56
|
// Matched by digest: a reclaim receipt names each sandbox by digest, and one written before it
|
|
50
57
|
// did holds the raw id, which is digested here.
|
|
51
58
|
const journaled = new Set((journal === null ? [] : parseSandboxReceipts(journal)).map((receipt) => sandboxIdDigest(receipt.sandboxId)));
|
|
52
|
-
const
|
|
53
|
-
const
|
|
54
|
-
if (
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
}
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
59
|
+
const command = `\`${cli(`reclaim --run ${runId}`)}\``;
|
|
60
|
+
const receipt = reclaimReceipt(await readRunJsonIfExists(runPaths, RECLAIM_RECEIPT_ARTIFACT), runId);
|
|
61
|
+
if (receipt === undefined)
|
|
62
|
+
return {
|
|
63
|
+
sandboxes: "unknown",
|
|
64
|
+
text: `Sandboxes unknown: it has no ${RECLAIM_RECEIPT_ARTIFACT} and journaled ${plural(journaled.size, "sandbox", "sandboxes")}; ${command} stops those and searches E2B by this run's tags for any whose id never reached a receipt.`,
|
|
65
|
+
};
|
|
66
|
+
const { outcomes } = receipt;
|
|
67
|
+
const gone = outcomes.filter((outcome) => GONE_STATES.has(outcome.state)).length;
|
|
68
|
+
const reclaimed = new Set(outcomes.map((outcome) => outcome.digest));
|
|
69
|
+
const missing = [...journaled].filter((id) => !reclaimed.has(id)).length;
|
|
70
|
+
const recorded = `${RECLAIM_RECEIPT_ARTIFACT} records ${gone} of ${outcomes.length} sandboxes gone${missing === 0 ? "" : `, and ${missing} journaled sandboxes are not in it`}`;
|
|
71
|
+
if (gone < outcomes.length || missing > 0)
|
|
72
|
+
return {
|
|
73
|
+
sandboxes: "unconfirmed",
|
|
74
|
+
text: `Sandboxes unconfirmed: ${recorded}; ${command} retries.`,
|
|
75
|
+
};
|
|
76
|
+
if (receipt.state === "clean")
|
|
77
|
+
return {
|
|
78
|
+
sandboxes: "clean",
|
|
79
|
+
text: `Sandboxes clean: ${recorded}, and E2B listed none still tagged with this run.`,
|
|
80
|
+
};
|
|
81
|
+
return {
|
|
82
|
+
sandboxes: "unknown",
|
|
83
|
+
text: `Sandboxes unknown: ${recorded}, but it records no finished search of E2B by this run's tags; ${command} searches.`,
|
|
84
|
+
};
|
|
63
85
|
}
|
|
64
|
-
/**
|
|
65
|
-
function
|
|
86
|
+
/** A reclaim receipt written for this run, or undefined when there is none. */
|
|
87
|
+
function reclaimReceipt(receipt, runId) {
|
|
66
88
|
if (!isRecord(receipt) || receipt.runId !== runId || !Array.isArray(receipt.outcomes))
|
|
67
89
|
return undefined;
|
|
68
90
|
const outcomes = [];
|
|
@@ -75,5 +97,5 @@ function reclaimOutcomes(receipt, runId) {
|
|
|
75
97
|
const digest = typeof recorded === "string" ? recorded : sandboxIdDigest(sandboxId);
|
|
76
98
|
outcomes.push({ digest, state });
|
|
77
99
|
}
|
|
78
|
-
return outcomes;
|
|
100
|
+
return typeof receipt.state === "string" ? { state: receipt.state, outcomes } : { outcomes };
|
|
79
101
|
}
|
package/dist/verify/verify.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { type PreparedRunArtifactPaths } from "../run/paths.js";
|
|
2
|
+
import { type UnfinishedRun } from "./liveness.js";
|
|
2
3
|
export declare const VERIFY_SCHEMA = "humanish.verify-result.v1";
|
|
3
4
|
export interface VerifyResult {
|
|
4
5
|
schema: typeof VERIFY_SCHEMA;
|
|
@@ -21,6 +22,8 @@ export interface VerifyResult {
|
|
|
21
22
|
}>;
|
|
22
23
|
};
|
|
23
24
|
warnings: string[];
|
|
25
|
+
/** Present when the run is not finished: its liveness and what is known of its sandboxes. */
|
|
26
|
+
unfinished?: UnfinishedRun;
|
|
24
27
|
error?: {
|
|
25
28
|
code: "HUMANISH_RUN_NOT_FOUND" | "HUMANISH_INVALID_RUN_BUNDLE";
|
|
26
29
|
message: string;
|
package/dist/verify/verify.js
CHANGED
|
@@ -13,12 +13,13 @@ import { runNotFoundMessage } from "../run/run-not-found.js";
|
|
|
13
13
|
import { actorVerdictConsistencyFindings, noEngagementActorFindings, validateCodexAppServerEvidence, validateTerminalProductEvidence, } from "./actor.js";
|
|
14
14
|
import { invalidRunEvidenceReferences, MAX_REPORTED_FINDINGS, missingLocalEvidenceArtifacts, rawScreenshotPostureWarnings, rawScreenshotStreamIds, redactedShapeFramePaths, scanRunPublicSafetyArtifacts, streamScreenshotPaths, } from "./artifacts.js";
|
|
15
15
|
import { flatContextWarnings } from "./context-growth.js";
|
|
16
|
-
import {
|
|
16
|
+
import { runNotFinished } from "./liveness.js";
|
|
17
17
|
import { costAndReceiptFindings } from "./costs.js";
|
|
18
18
|
import { rerunLineageFindings } from "./rerun.js";
|
|
19
19
|
import { sharedWorldEvidenceFindings } from "./shared-world.js";
|
|
20
20
|
import { subjectStateFindings, undeclaredSubjectStateWarnings } from "./subject.js";
|
|
21
21
|
import { plural } from "../run/text.js";
|
|
22
|
+
import { cli } from "../cli/invocation.js";
|
|
22
23
|
export const VERIFY_SCHEMA = "humanish.verify-result.v1";
|
|
23
24
|
// The UNSCANNED_ARTIFACT reason names this many paths and counts the rest.
|
|
24
25
|
const MAX_LISTED_UNSCANNED = 10;
|
|
@@ -137,13 +138,16 @@ export async function verifyResolvedRun(cwd, runInput, runPaths) {
|
|
|
137
138
|
message: "Derived analysis contains sensitive text or unsafe artifact paths; sharing is blocked, original recording remains independently verifiable.",
|
|
138
139
|
});
|
|
139
140
|
const ok = checks.every((check) => check.ok);
|
|
141
|
+
const notFinished = isRunBundle(bundle)
|
|
142
|
+
? await runNotFinished(runPaths, bundle)
|
|
143
|
+
: { warnings: [], unfinished: undefined };
|
|
140
144
|
const warnings = isRunBundle(bundle)
|
|
141
145
|
? [
|
|
142
146
|
...rawScreenshotPostureWarnings(bundle, redactedShapeFrames),
|
|
143
147
|
...undeclaredSubjectStateWarnings(bundle),
|
|
144
148
|
...desktopGeometryWarnings(bundle),
|
|
145
149
|
...flatContextWarnings(bundle),
|
|
146
|
-
...
|
|
150
|
+
...notFinished.warnings,
|
|
147
151
|
]
|
|
148
152
|
: [];
|
|
149
153
|
const shareSafety = isRunBundle(bundle)
|
|
@@ -175,6 +179,7 @@ export async function verifyResolvedRun(cwd, runInput, runPaths) {
|
|
|
175
179
|
checks,
|
|
176
180
|
shareSafety,
|
|
177
181
|
warnings,
|
|
182
|
+
...(notFinished.unfinished === undefined ? {} : { unfinished: notFinished.unfinished }),
|
|
178
183
|
...(ok
|
|
179
184
|
? {}
|
|
180
185
|
: {
|
|
@@ -383,7 +388,7 @@ function buildShareSafety(args) {
|
|
|
383
388
|
];
|
|
384
389
|
reasons.push({
|
|
385
390
|
code: "RAW_SANDBOX_ID",
|
|
386
|
-
message: `Raw sandbox ids appear in ${files.sort().join(", ")}. Runs from 0.110 keep them only in sandbox-receipts.ndjson;
|
|
391
|
+
message: `Raw sandbox ids appear in ${files.sort().join(", ")}. Runs from 0.110 keep them only in sandbox-receipts.ndjson; \`${cli("export --format bundle --redact-screenshots")}\` writes a copy without them.`,
|
|
387
392
|
});
|
|
388
393
|
}
|
|
389
394
|
const rawStreamIds = rawScreenshotStreamIds(args.bundle, args.redactedShapeFrames);
|
|
@@ -444,7 +444,8 @@ traits: { patience, skill, accessibilityNeeds? }, constraints[], sourceDigest }`
|
|
|
444
444
|
receives them as thread/start `baseInstructions` (`threadStartParams` in
|
|
445
445
|
`src/actors/codex/restricted-launch.ts`), on an ephemeral read-only thread whose only
|
|
446
446
|
dynamic tool is `humanish_ui`. A local-agent Claude Code runs one `claude -p`
|
|
447
|
-
stream-json session
|
|
447
|
+
stream-json session restricted to Read in its folder (`claudeParticipantFlags` in
|
|
448
|
+
`src/actors/local-agent/claude-participant.ts`); the instructions open its first user
|
|
448
449
|
message (`promptFor` in `src/actors/local-agent/cli.ts`), and later turns rely on the
|
|
449
450
|
session's memory. `HUMANISH_LOCAL_AGENT_ONE_SHOT` instead spawns one `claude -p` per
|
|
450
451
|
turn, which resends the instructions every turn.
|
|
@@ -93,10 +93,23 @@ Served data may also include `runtime` with `state`, `observedAt`, and
|
|
|
93
93
|
`source: "local-run-status"`. This is a current observation of a contained,
|
|
94
94
|
matching `status.json`; it never changes the run's recorded verdict or participant
|
|
95
95
|
outcomes. A fresh heartbeat means running, an explicitly finalized record means
|
|
96
|
-
finished,
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
and export do not create this
|
|
96
|
+
finished, a record the CLI's signal handler wrote means interrupted, and stale or
|
|
97
|
+
invalid timing means unknown. Missing, malformed, or mismatched records omit the
|
|
98
|
+
observation. A stale heartbeat alone does not prove interruption, and stored PIDs
|
|
99
|
+
are neither probed nor returned. Static rendering and export do not create this
|
|
100
|
+
served-only observation.
|
|
101
|
+
|
|
102
|
+
`run.display` says whether the run passed: `{ state, label, tone, reason? }` from
|
|
103
|
+
`runDisplay` (`src/run/display.ts`), the function every surface that shows a run's
|
|
104
|
+
result calls. `state` is `passed` only when `run.status` (the verdict) is `pass` and
|
|
105
|
+
`run.json`'s `outcome` does not record the run as failed. A run whose execution
|
|
106
|
+
failed under a pass verdict is `failed`, with the first failure in `reason`.
|
|
107
|
+
`blocked`, `timed_out`, `no_verdict`, `interrupted`, `running`, `dry_run` and
|
|
108
|
+
`unknown` are the other states. For a `run.json` without an `outcome`, the served
|
|
109
|
+
Observer and `observe` read `status.json` for liveness and, on a run recorded
|
|
110
|
+
before `run.json` carried an outcome, for its ok. The sidebar's dot is the pass
|
|
111
|
+
accent only for `passed`. observer-data.json written before `display` existed
|
|
112
|
+
keeps the verdict reading.
|
|
100
113
|
|
|
101
114
|
Watch is deliberately distinct from `humanish observe --all`. Watch serves one
|
|
102
115
|
attached run, and the process that created it may inject runtime stream URLs
|
|
@@ -167,9 +180,11 @@ origin-dependent scripts and modules in those artifacts may require independent
|
|
|
167
180
|
hosting. See the [CSP sandbox standard](https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/Content-Security-Policy/sandbox).
|
|
168
181
|
|
|
169
182
|
History entries may include `runtimeState` from the same contained local status
|
|
170
|
-
read as the Observer
|
|
171
|
-
|
|
172
|
-
|
|
183
|
+
read as the Observer, and `display` from the run's Observer data. Their existing
|
|
184
|
+
`status` remains the recorded verdict. The library's pip color and the sidebar's
|
|
185
|
+
label read `display`. Running filters should use runtime state when present,
|
|
186
|
+
preserving the difference between an active study and its provisional evidence
|
|
187
|
+
outcome.
|
|
173
188
|
|
|
174
189
|
## UI Shape
|
|
175
190
|
|
|
@@ -5,20 +5,20 @@ participant; hidden success criteria go only to the observation tracker. A route
|
|
|
5
5
|
that cannot carry both halves refuses the declaration before execution. Removing
|
|
6
6
|
`tasks` is an explicit choice to run a mission-only study, not an automatic fallback.
|
|
7
7
|
|
|
8
|
-
| Execution path | Task support | Mechanism
|
|
9
|
-
| ----------------------------------------------------- | ------------ |
|
|
10
|
-
| CUA, app-url, one participant or fan-out | Supported | The participant prompt composer renders goals; the CUA loop tracks observations
|
|
11
|
-
| CUA, provisioned clone or local-tree, separate worlds | Supported | Same prompt composer and loop
|
|
12
|
-
| CUA, local-app with caller executor/provider | Supported | Same loop; the caller supplies observations
|
|
13
|
-
| CUA, desktop-cli | Supported | Same prompt and tracker; unavailable criterion inputs remain unmeasured
|
|
14
|
-
| Shared-world, provisioned or external-public | Rejected | Actor specs omit the protocol before CUA session dispatch
|
|
15
|
-
| Terminal-product | Rejected | Terminal prompt and result contract do not implement tasks
|
|
16
|
-
| Scripted-browser, local or provisioned | Rejected | Scenario steps drive the participant; no task protocol is consumed
|
|
17
|
-
|
|
|
18
|
-
| Any second or later `actors[]` entry | Rejected | Current runners consume only the first actor; declare every participant under the first actor |
|
|
8
|
+
| Execution path | Task support | Mechanism |
|
|
9
|
+
| ----------------------------------------------------- | ------------ | ------------------------------------------------------------------------------- |
|
|
10
|
+
| CUA, app-url, one participant or fan-out | Supported | The participant prompt composer renders goals; the CUA loop tracks observations |
|
|
11
|
+
| CUA, provisioned clone or local-tree, separate worlds | Supported | Same prompt composer and loop |
|
|
12
|
+
| CUA, local-app with caller executor/provider | Supported | Same loop; the caller supplies observations |
|
|
13
|
+
| CUA, desktop-cli | Supported | Same prompt and tracker; unavailable criterion inputs remain unmeasured |
|
|
14
|
+
| Shared-world, provisioned or external-public | Rejected | Actor specs omit the protocol before CUA session dispatch |
|
|
15
|
+
| Terminal-product | Rejected | Terminal prompt and result contract do not implement tasks |
|
|
16
|
+
| Scripted-browser, local or provisioned | Rejected | Scenario steps drive the participant; no task protocol is consumed |
|
|
17
|
+
| Preview (`this-repo`) | Rejected | Study dispatch does not pass task declarations to the dry-run engine |
|
|
19
18
|
|
|
20
|
-
|
|
21
|
-
|
|
19
|
+
A study has one `actor`, and its participants go in `participants`, so every participant
|
|
20
|
+
gets the same tasks. Both registered CUA actors (`openai-computer-use` and `local-agent`) share
|
|
21
|
+
the CUA session loop. Their task support does not depend on which provider chooses actions.
|
|
22
22
|
Custom session hooks remain caller-owned implementations of that same contract;
|
|
23
23
|
this preflight does not certify arbitrary hook behavior.
|
|
24
24
|
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# Terminal-product route
|
|
2
2
|
|
|
3
3
|
The live terminal-product route has shipped since `0.8.0`, with the in-sandbox runtime,
|
|
4
|
-
|
|
4
|
+
explicit credential placement (an E2B egress proxy by default, or command-scoped), exact-id cleanup proof, an interventions ledger, a cost and
|
|
5
5
|
no-spend ledger, caps, and product scoring and feedback hooks. The
|
|
6
6
|
[original plan](https://github.com/danielgwilson/humanish/blob/main/docs/history/goals/terminal-product-lane/goal.md)
|
|
7
7
|
holds the build order and the safety contract.
|
|
@@ -30,10 +30,10 @@ fail-closed cross-validation, and forward-declared warnings.
|
|
|
30
30
|
| `subject.product` | `{ name, publicSurfaces[], install?, workdir?, upload? }`: the only world the agent sees |
|
|
31
31
|
| `execution.target` | `e2b-terminal` (or absent → implied) |
|
|
32
32
|
| `execution.terminal` | `{ transport: exec-stream, stdin: disabled }` |
|
|
33
|
-
| `execution.runtimeAuth` | `openai-
|
|
33
|
+
| `execution.runtimeAuth` | `openai-egress` (default) or opt-in `openai-env`; names-only durable evidence |
|
|
34
34
|
| `execution.runtime.version` | Optional exact `@openai/codex` version; observed before keyed execution |
|
|
35
35
|
| `actor.model` / `reasoningEffort` | Passed to Codex as `--model` (default `gpt-5.6-sol`) and `-c model_reasoning_effort`; declarations, not observed provider identity |
|
|
36
|
-
| `caps` | `{ maxUsd, maxJobs, maxMinutes }`: the blast-radius budget; `maxUsd > 0` is refused
|
|
36
|
+
| `caps` | `{ maxUsd, maxJobs, maxMinutes }`: the blast-radius budget; `maxUsd > 0` is refused: no adopter cost source exists until issue 347 lands |
|
|
37
37
|
| `policies` | `allowPrivateRepoAccess` / `allowProviderCredentials` / `allowPaymentCredentials` / `allowGitHubMutation`, all default false |
|
|
38
38
|
| `actor.type` | `codex-exec`: a registered terminal actor (`keyPlacement: in-sandbox-command-scoped`) |
|
|
39
39
|
| route | `terminal` → `runTerminalProductStudy` ([`src/routes/terminal/route.ts`](https://github.com/danielgwilson/humanish/blob/main/src/routes/terminal/route.ts)) |
|
|
@@ -86,11 +86,8 @@ product's no-spend boundary still needs separate interpretation.
|
|
|
86
86
|
|
|
87
87
|
## Runtime auth: raw-key placement and remaining provider access
|
|
88
88
|
|
|
89
|
-
`execution.runtimeAuth: openai-
|
|
90
|
-
`
|
|
91
|
-
that was the host source. Child processes can read and use the raw key.
|
|
92
|
-
|
|
93
|
-
Opt in to keeping the raw key outside the sandbox:
|
|
89
|
+
`execution.runtimeAuth: openai-egress` is the default: a study that declares no
|
|
90
|
+
`runtimeAuth` keeps the raw key outside the sandbox, as if it declared:
|
|
94
91
|
|
|
95
92
|
```yaml
|
|
96
93
|
execution:
|
|
@@ -115,7 +112,7 @@ output and errors, including errors during sandbox creation.
|
|
|
115
112
|
This mode supports the default OpenAI endpoint only. humanish explicitly sets Codex's
|
|
116
113
|
built-in `openai` provider and `openai_base_url` to `https://api.openai.com/v1` for
|
|
117
114
|
that invocation. It does not support a custom provider, proxy base URL, or regional
|
|
118
|
-
endpoint under this mode
|
|
115
|
+
endpoint under this mode; a study that needs one declares `openai-env`. E2B
|
|
119
116
|
header rules are a public-beta capability;
|
|
120
117
|
`tests/routes/terminal/runtime-auth.test.ts` checks the local contract against the
|
|
121
118
|
installed Desktop SDK (the lockfile has `@e2b/desktop` 2.4.0, resolving `e2b` 2.49.0).
|
|
@@ -136,6 +133,11 @@ requests can fail. E2B domain allowlists are routing controls rather than strict
|
|
|
136
133
|
destination isolation on shared infrastructure. An existing exact OpenAI host
|
|
137
134
|
rule is rejected instead of silently overwritten.
|
|
138
135
|
|
|
136
|
+
`execution.runtimeAuth: openai-env` is the opt-in alternative. It supplies
|
|
137
|
+
`CODEX_API_KEY` command-scoped to Codex, with `OPENAI_API_KEY` also supplied when
|
|
138
|
+
that was the host source. Child processes can read and use the raw key, and so can
|
|
139
|
+
anything the agent runs or a page it reads asks it to run.
|
|
140
|
+
|
|
139
141
|
Evidence records the selected auth mode and the residual proxy capability.
|
|
140
142
|
Resolved live actor traces use `keyPlacement: external` in `openai-egress`; the
|
|
141
143
|
actor registry continues to describe the default `in-sandbox-command-scoped`
|
|
@@ -197,10 +199,10 @@ deadline.
|
|
|
197
199
|
|
|
198
200
|
## The original command-scoped safety contract
|
|
199
201
|
|
|
200
|
-
The
|
|
201
|
-
On the computer-use route the model's key stays _outside_ the sandbox;
|
|
202
|
-
agent-under-test runs _inside_ with a real `OPENAI_API_KEY`/`CODEX_API_KEY` and
|
|
203
|
-
|
|
202
|
+
The `openai-env` mode **inverts** the credential-placement default of every other E2B route.
|
|
203
|
+
On the computer-use route the model's key stays _outside_ the sandbox; under `openai-env` the
|
|
204
|
+
agent-under-test runs _inside_ with a real `OPENAI_API_KEY`/`CODEX_API_KEY` and is **presumed
|
|
205
|
+
exfiltratable**. The placement rule in invariants-and-defaults.md
|
|
204
206
|
applies: _keys live where the keyed process runs, and nowhere else;
|
|
205
207
|
blast radius is bounded by key scoping and budgets, not by hoping._
|
|
206
208
|
|
|
@@ -70,6 +70,15 @@ artifacts:
|
|
|
70
70
|
review:
|
|
71
71
|
schema: humanish.review.v1
|
|
72
72
|
verdict: "contract_proof_only|pass|fail|blocked|timed_out"
|
|
73
|
+
outcome: # optional, additive: how the run ended (Run Outcome below)
|
|
74
|
+
state: "finished"
|
|
75
|
+
ok: false
|
|
76
|
+
execution:
|
|
77
|
+
succeeded: false
|
|
78
|
+
failures:
|
|
79
|
+
- kind: "harness|provider-cleanup|provider-policy|sandbox-cleanup|evidence|cap|run"
|
|
80
|
+
message: "<public-safe message>"
|
|
81
|
+
warnings: [] # omitted when empty
|
|
73
82
|
redaction:
|
|
74
83
|
status: "passed"
|
|
75
84
|
notes: "<public-safe note>"
|
|
@@ -100,6 +109,59 @@ Persisted `run.json` files must not contain absolute local target paths. Runtime
|
|
|
100
109
|
commands may return the caller's working directory in process-local JSON
|
|
101
110
|
responses, but durable run bundles use the public-safe `[target-cwd]` marker.
|
|
102
111
|
|
|
112
|
+
## Run Outcome
|
|
113
|
+
|
|
114
|
+
`outcome` records how the run ended as an execution. `review.verdict` is what the
|
|
115
|
+
participants experienced; `outcome.ok` is the route result's `ok`, which
|
|
116
|
+
`humanish run --json` prints as `runOk`. The two differ: a terminal run whose agent passed but whose sandbox
|
|
117
|
+
teardown is unproven has verdict `pass` and `ok: false`.
|
|
118
|
+
|
|
119
|
+
```yaml
|
|
120
|
+
# Run.finish published the final bundle.
|
|
121
|
+
outcome:
|
|
122
|
+
state: finished
|
|
123
|
+
ok: true
|
|
124
|
+
execution:
|
|
125
|
+
succeeded: true
|
|
126
|
+
failures: []
|
|
127
|
+
warnings: # omitted when empty
|
|
128
|
+
- kind: sandbox-cleanup
|
|
129
|
+
message: "lane-01: Sandbox release is unconfirmed. Reclaim it by recorded id with `humanish reclaim --run <id>`."
|
|
130
|
+
---
|
|
131
|
+
# The CLI's signal handler stopped the run before it finished.
|
|
132
|
+
outcome:
|
|
133
|
+
state: interrupted
|
|
134
|
+
ok: false
|
|
135
|
+
signal: SIGINT # or SIGTERM, SIGHUP
|
|
136
|
+
at: "<ISO timestamp>"
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
`Run.finish` (`src/run/run.ts`) writes `finished` with the route's `ok` and its
|
|
140
|
+
execution outcome (`judgeExecution` in `src/run/judge.ts`). Each route's policy in
|
|
141
|
+
`OUTCOME_POLICIES` decides which failure kinds fail the run and which only warn.
|
|
142
|
+
An Observer that does not render is an `evidence` failure known only after
|
|
143
|
+
`run.json` is on disk: `FinishedRun.renderObserver` adds it to `outcome` and to
|
|
144
|
+
`status.json` before the route returns, and no Observer page exists that could
|
|
145
|
+
show the run without it. Messages pass the same redaction and sandbox id scrub
|
|
146
|
+
as the rest of the bundle.
|
|
147
|
+
|
|
148
|
+
When a signal stops a live run, the handler writes `interrupted` into the last
|
|
149
|
+
flushed `run.json`, and every later write of that run keeps it. A run stopped
|
|
150
|
+
before its first flush has no `run.json`; its `status.json` is the record.
|
|
151
|
+
|
|
152
|
+
`outcome` is absent while a run is in progress, when its process died without
|
|
153
|
+
writing it, and on runs recorded before the field existed. For those, readers
|
|
154
|
+
take liveness from `status.json` and, for an older finished run, its `ok` and
|
|
155
|
+
`execution` from `status.json`'s `outcome`. That record is the only copy of an
|
|
156
|
+
older run's `ok`; for a run with `outcome` in `run.json`, `status.json` copies it.
|
|
157
|
+
|
|
158
|
+
Every surface that shows whether a run passed calls `runDisplay`
|
|
159
|
+
(`src/run/display.ts`) with the run's liveness, verdict and `ok`: the Observer
|
|
160
|
+
(`observer-data.json` `run.display`), its run library, the served Observer,
|
|
161
|
+
`humanish runs`, `humanish review`, `humanish stats`, the TUI and `review.md`'s
|
|
162
|
+
`outcome` line. A run
|
|
163
|
+
shows as `passed` only when its verdict is `pass` and its `ok` is not false.
|
|
164
|
+
|
|
103
165
|
## Recorded session stop causes
|
|
104
166
|
|
|
105
167
|
An actor trace may include `stopCause` alongside its unchanged `status`,
|
|
@@ -523,7 +585,9 @@ shareSafety:
|
|
|
523
585
|
bundle is safe to promote into a public issue. Public promotion should branch on
|
|
524
586
|
`shareSafety.status`:
|
|
525
587
|
|
|
526
|
-
- `share_ready`: feedback draft commands may render public issue payloads
|
|
588
|
+
- `share_ready`: feedback draft commands may render public issue payloads, and `observe --all --safe`
|
|
589
|
+
serves the run. The folder still holds `sandbox-receipts.ndjson` (raw sandbox ids) and `status.json`
|
|
590
|
+
(the recording pid); no Observer server serves them and bundle export omits them;
|
|
527
591
|
- `local_only`: keep the run local; only supported redaction-only cases can produce a shareable derivative;
|
|
528
592
|
`RAW_SANDBOX_ID` is one: a file other than `sandbox-receipts.ndjson` names one of the run's raw
|
|
529
593
|
sandbox ids, which verify reads from the receipts and from any raw `providerResources[].id` in
|
|
@@ -543,12 +607,19 @@ writing; an older one reads as interrupted. The warning names what verify saw:
|
|
|
543
607
|
- the record's state and its last `updatedAt`, or the running `simulations[]` entries when
|
|
544
608
|
there is no usable record;
|
|
545
609
|
- how many streams are still `running`;
|
|
546
|
-
- what `reclaim-receipt.json` records: how many of the run's
|
|
547
|
-
(killed or already gone), and how many journaled sandboxes it does
|
|
548
|
-
matches its outcomes to the receipts by `sandboxIdDigest`; a reclaim
|
|
549
|
-
before 0.110 names raw ids, which are digested to match;
|
|
550
|
-
- with
|
|
551
|
-
`humanish reclaim --run <id>`.
|
|
610
|
+
- the sandbox state, and what `reclaim-receipt.json` records: how many of the run's
|
|
611
|
+
sandboxes are gone (killed or already gone), and how many journaled sandboxes it does
|
|
612
|
+
not cover. It matches its outcomes to the receipts by `sandboxIdDigest`; a reclaim
|
|
613
|
+
receipt from before 0.110 names raw ids, which are digested to match;
|
|
614
|
+
- with no reclaim receipt, the command that stops the journaled sandboxes and searches
|
|
615
|
+
E2B by the run's owner tags: `humanish reclaim --run <id>`.
|
|
616
|
+
|
|
617
|
+
The result also carries `unfinished: { liveness, sandboxes }`, and the one-line output
|
|
618
|
+
leads with it, for example `verified <id> (interrupted, sandboxes unknown) · share_ready`.
|
|
619
|
+
`sandboxes` is `clean` only when the reclaim receipt records every sandbox gone and its
|
|
620
|
+
`state` is `clean`, which means the search by tag finished. It is `unconfirmed` when the
|
|
621
|
+
receipt records a sandbox not gone or misses a journaled one, and `unknown` otherwise,
|
|
622
|
+
including a receipt from humanish 0.110.0 or earlier, which did not search by tag.
|
|
552
623
|
|
|
553
624
|
The local-evidence check includes screenshots declared only by
|
|
554
625
|
`streams[].actor.items[].screenshotRef` or `streams[].liveActor.items[].screenshotRef`,
|
|
@@ -589,6 +660,16 @@ trace references outside `screenshots/`. An unregistered
|
|
|
589
660
|
`.mp4`, and a file or directory whose name contains `\`, are public-safety
|
|
590
661
|
findings and block the run.
|
|
591
662
|
|
|
663
|
+
The patterns live in `src/evidence/redaction.ts`: provider key formats (OpenAI,
|
|
664
|
+
Anthropic, E2B, GitHub, GitLab, npm, Vercel, AWS, Google, Stripe, Slack,
|
|
665
|
+
SendGrid, Hugging Face), JWTs, bearer and basic `Authorization` values, private
|
|
666
|
+
key headers, database URLs with a password, the password in any URL's user info,
|
|
667
|
+
credential query and fragment parameters with a value of 16 characters or more,
|
|
668
|
+
credential-named upper-case variables set to a value of 16 characters or more
|
|
669
|
+
with a digit and a letter, an AWS secret access key next to its name, E2B URLs, and Linux,
|
|
670
|
+
macOS and Windows local paths. `tests/verify/secret-formats.test.ts` lists the
|
|
671
|
+
formats and the ordinary values that must not match.
|
|
672
|
+
|
|
592
673
|
Before matching, the scan undoes JSON and JS escapes, percent-encoding and HTML
|
|
593
674
|
character references, the decoding bundle export applies (`decodeEscapes`), and
|
|
594
675
|
then JSON whitespace escapes and quoted-printable. It reads inside base64 runs of
|
|
@@ -604,9 +685,13 @@ wrapped across lines, and inside hex runs of 32 characters or more:
|
|
|
604
685
|
|
|
605
686
|
`observer/index.html` is exempt from that base64 rule: serve renders it from
|
|
606
687
|
`run.json`, export regenerates it, and it embeds the Observer's own base64 fonts.
|
|
607
|
-
Its text is still scanned.
|
|
608
|
-
|
|
609
|
-
|
|
688
|
+
Its text is still scanned. A base64 run of up to 4096 characters is also read one
|
|
689
|
+
slash-separated piece at a time, and from just after each slash in its first 64
|
|
690
|
+
characters, so a URL path before an encoded key cannot shift its decoding. Known
|
|
691
|
+
limits: base64 split across separate strings, nesting deeper than three levels,
|
|
692
|
+
base64 that holds a slash itself and starts more than 64 characters into a longer
|
|
693
|
+
run, and encodings a pattern scan cannot undo (such as encryption) still grade
|
|
694
|
+
`share_ready`. `tests/verify/encoding-coverage.test.ts`
|
|
610
695
|
records each case.
|
|
611
696
|
|
|
612
697
|
Real email receiving adds `publication.restrictions: [real-communications]` and
|