humanish 0.109.1 → 0.110.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -0
- package/README.md +1 -1
- package/dist/cli/commands/comms.js +25 -7
- package/dist/cli/commands/observe.js +4 -2
- package/dist/cli/commands/run-command.js +4 -3
- package/dist/cli/commands/runs.js +11 -5
- package/dist/cli/commands/setup.js +19 -5
- package/dist/cli/commands/study-format.js +29 -2
- package/dist/cli/commands/study.js +4 -3
- package/dist/cli/commands/tui.js +4 -3
- package/dist/cli/commands/watch.js +2 -2
- package/dist/cli/doctor.d.ts +3 -1
- package/dist/cli/doctor.js +5 -4
- package/dist/cli/io.d.ts +17 -3
- package/dist/cli/io.js +20 -3
- package/dist/comms/receiving-runtime.js +1 -1
- package/dist/comms/setup.js +1 -1
- package/dist/evidence/redaction.d.ts +7 -3
- package/dist/evidence/redaction.js +68 -24
- package/dist/feedback/export-bundle.js +6 -1
- package/dist/feedback/export.js +3 -0
- package/dist/keys/key-resolution.d.ts +7 -0
- package/dist/keys/key-resolution.js +10 -8
- package/dist/routes/computer-use/live-phase.d.ts +1 -0
- package/dist/routes/computer-use/preflight.js +1 -1
- package/dist/routes/computer-use/types.d.ts +6 -0
- package/dist/routes/scripted/types.d.ts +1 -0
- package/dist/routes/shared-world/route.js +1 -1
- package/dist/routes/shared-world/types.d.ts +2 -0
- package/dist/routes/terminal/credentials.js +1 -1
- package/dist/routes/terminal/live-finish.js +8 -4
- package/dist/routes/terminal/types.d.ts +3 -0
- package/dist/run/bundle-shape.js +1 -0
- package/dist/run/bundle.d.ts +6 -0
- package/dist/run/reclaim.d.ts +3 -0
- package/dist/run/reclaim.js +10 -3
- package/dist/run/results.d.ts +3 -0
- package/dist/run/run.js +14 -8
- package/dist/run/sandbox-ids.d.ts +53 -0
- package/dist/run/sandbox-ids.js +198 -0
- package/dist/run/sandbox-receipts.d.ts +2 -0
- package/dist/run/sandbox-receipts.js +26 -0
- package/dist/run/status.js +5 -2
- package/dist/run/stored-runs.js +10 -1
- package/dist/run-study.js +6 -1
- package/dist/study/doctor.js +2 -2
- package/dist/study/init-templates.js +2 -2
- package/dist/study/types.d.ts +1 -1
- package/dist/verify/artifacts.d.ts +4 -0
- package/dist/verify/artifacts.js +8 -0
- package/dist/verify/liveness.js +8 -4
- package/dist/verify/verify.d.ts +1 -1
- package/dist/verify/verify.js +21 -1
- package/docs/architecture/restricted-codex-analysis.md +1 -1
- package/docs/contracts/run-bundle.md +18 -4
- package/package.json +3 -2
- package/skills/humanish/SKILL.md +2 -2
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
// A run's raw sandbox ids live in one place: sandbox-receipts.ndjson, which reclaim reads to kill
|
|
2
|
+
// them and export omits. Every other record a run writes (run.json, its review and projections, the
|
|
3
|
+
// cleanup and reclaim receipts) and every result or CLI line names a sandbox by marker and digest,
|
|
4
|
+
// so output an agent pastes into an issue never carries an id.
|
|
5
|
+
import { opendir, realpath } from "node:fs/promises";
|
|
6
|
+
import path from "node:path";
|
|
7
|
+
import { readPlainText } from "../evidence/plain-text.js";
|
|
8
|
+
import { collectSandboxIds, redactSandboxIds, sandboxIdDigest, REDACTED_SANDBOX_ID, } from "../evidence/redaction.js";
|
|
9
|
+
import { readContainedRegularFile, writeContainedOutputFile, } from "./contained-output.js";
|
|
10
|
+
import { bindExistingRunArtifactPaths, isSafeRunIdSegment, validatePreparedRunRootIdentity, } from "./paths.js";
|
|
11
|
+
import { appendedSandboxIds, parseSandboxReceipts, SANDBOX_RECEIPTS_ARTIFACT, } from "./sandbox-receipts.js";
|
|
12
|
+
/** How text names a sandbox: the marker with the id's digest. */
|
|
13
|
+
function sandboxIdLabel(id) {
|
|
14
|
+
return `${REDACTED_SANDBOX_ID.slice(0, -1)} ${sandboxIdDigest(id)}]`;
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* How YAML names a sandbox: the label without its brackets, since a plain scalar that starts with
|
|
18
|
+
* `[` reads as a list. Every other byte of the file stays as written.
|
|
19
|
+
*/
|
|
20
|
+
function yamlSandboxIdLabel(id) {
|
|
21
|
+
return `redacted-sandbox-id-${sandboxIdDigest(id)}`;
|
|
22
|
+
}
|
|
23
|
+
/** `text` with every occurrence of each raw id replaced by its label, longest id first. */
|
|
24
|
+
export function scrubSandboxIds(text, ids, label = sandboxIdLabel) {
|
|
25
|
+
const unique = [...new Set(ids)].filter((id) => id.length > 0);
|
|
26
|
+
unique.sort((left, right) => right.length - left.length);
|
|
27
|
+
return unique.reduce((current, id) => current.split(id).join(label(id)), text);
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* `value` as it may be written or returned: ids at the sandbox-id keys become the marker and a
|
|
31
|
+
* digest, and any raw id left in free text or a URL becomes its label.
|
|
32
|
+
*/
|
|
33
|
+
export function publicSandboxView(value, ids) {
|
|
34
|
+
const redacted = redactSandboxIds(value);
|
|
35
|
+
const known = [...ids];
|
|
36
|
+
if (known.length === 0)
|
|
37
|
+
return redacted;
|
|
38
|
+
const text = JSON.stringify(redacted);
|
|
39
|
+
if (text === undefined)
|
|
40
|
+
return redacted;
|
|
41
|
+
const scrubbed = scrubSandboxIds(text, known);
|
|
42
|
+
return (scrubbed === text ? redacted : JSON.parse(scrubbed));
|
|
43
|
+
}
|
|
44
|
+
/** The raw sandbox ids a receipts journal names. */
|
|
45
|
+
function receiptSandboxIds(journal) {
|
|
46
|
+
return parseSandboxReceipts(journal.toString()).map((receipt) => receipt.sandboxId);
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* The raw ids at sandbox-id keys in `value`, such as a run.json recorded before 0.110. A value
|
|
50
|
+
* shorter than an E2B id is left to the key rule, so the text scrub never rewrites a common word.
|
|
51
|
+
*/
|
|
52
|
+
export function keyedSandboxIds(value) {
|
|
53
|
+
return [...collectSandboxIds(value)].filter((id) => id.length >= 8);
|
|
54
|
+
}
|
|
55
|
+
/** The JSON records in one file's text: one document, or one per NDJSON line. */
|
|
56
|
+
function jsonRecords(file, text) {
|
|
57
|
+
// Most run files name no sandbox key, and parsing each one would slow verify on large runs. A
|
|
58
|
+
// key spelled with an escape only shows once parsed.
|
|
59
|
+
if (!/"(?:sandboxId|subjectSandboxId|providerResources|resources)"|\\u/.test(text))
|
|
60
|
+
return [];
|
|
61
|
+
const parse = (part) => {
|
|
62
|
+
try {
|
|
63
|
+
return [JSON.parse(part)];
|
|
64
|
+
}
|
|
65
|
+
catch {
|
|
66
|
+
return [];
|
|
67
|
+
}
|
|
68
|
+
};
|
|
69
|
+
if (/\.json$/i.test(file))
|
|
70
|
+
return parse(text);
|
|
71
|
+
if (/\.(?:ndjson|jsonl)$/i.test(file))
|
|
72
|
+
return text.split("\n").flatMap(parse);
|
|
73
|
+
return [];
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* Every raw sandbox id a run's files name: the receipts', and any at a sandbox-id key in its
|
|
77
|
+
* JSON, which is where a run from before 0.110 that has no receipts still records them.
|
|
78
|
+
*/
|
|
79
|
+
export function inventorySandboxIds(files) {
|
|
80
|
+
const ids = new Set();
|
|
81
|
+
for (const file of files) {
|
|
82
|
+
if (file.path === SANDBOX_RECEIPTS_ARTIFACT) {
|
|
83
|
+
for (const id of receiptSandboxIds(file.bytes))
|
|
84
|
+
ids.add(id);
|
|
85
|
+
continue;
|
|
86
|
+
}
|
|
87
|
+
for (const record of jsonRecords(file.path, file.bytes.toString("utf8")))
|
|
88
|
+
for (const id of keyedSandboxIds(record))
|
|
89
|
+
ids.add(id);
|
|
90
|
+
}
|
|
91
|
+
return [...ids];
|
|
92
|
+
}
|
|
93
|
+
/** Whether a run file's JSON holds a raw id at a sandbox-id key, which no receipt may name. */
|
|
94
|
+
export function holdsKeyedSandboxId(file, text) {
|
|
95
|
+
return jsonRecords(file, text).some((record) => collectSandboxIds(record).size > 0);
|
|
96
|
+
}
|
|
97
|
+
/**
|
|
98
|
+
* The raw sandbox ids a run's receipts journal, with any this process receipted whose append
|
|
99
|
+
* failed; empty when there are none or the journal cannot be read.
|
|
100
|
+
*/
|
|
101
|
+
export async function readRunSandboxIds(root) {
|
|
102
|
+
let journaled = [];
|
|
103
|
+
try {
|
|
104
|
+
const bytes = await readContainedRegularFile(root, SANDBOX_RECEIPTS_ARTIFACT);
|
|
105
|
+
if (bytes)
|
|
106
|
+
journaled = receiptSandboxIds(bytes);
|
|
107
|
+
}
|
|
108
|
+
catch {
|
|
109
|
+
// The ids this process receipted still apply.
|
|
110
|
+
}
|
|
111
|
+
return [...new Set([...journaled, ...appendedSandboxIds(root)])];
|
|
112
|
+
}
|
|
113
|
+
/**
|
|
114
|
+
* `value` as a run writes it: raw sandbox ids stay in the run's sandbox-receipts.ndjson, and the
|
|
115
|
+
* file names each by marker and digest.
|
|
116
|
+
*/
|
|
117
|
+
export async function withPublicSandboxIds(root, value) {
|
|
118
|
+
return publicSandboxView(value, await readRunSandboxIds(root));
|
|
119
|
+
}
|
|
120
|
+
/**
|
|
121
|
+
* The bytes of `file` with each raw id replaced by its label: the same buffer when none occurs, or
|
|
122
|
+
* when the bytes are not the plain text verify and export read as text.
|
|
123
|
+
*/
|
|
124
|
+
export function scrubSandboxIdBytes(file, bytes, ids) {
|
|
125
|
+
if (!ids.some((id) => id.length > 0 && bytes.includes(id)))
|
|
126
|
+
return bytes;
|
|
127
|
+
const decoded = readPlainText(bytes);
|
|
128
|
+
if (!decoded.ok)
|
|
129
|
+
return bytes;
|
|
130
|
+
const label = /\.ya?ml$/i.test(file) ? yamlSandboxIdLabel : sandboxIdLabel;
|
|
131
|
+
return Buffer.from(scrubSandboxIds(decoded.text, ids, label));
|
|
132
|
+
}
|
|
133
|
+
/** The run files the sweep rewrites: the text formats export copies, and the Observer page. */
|
|
134
|
+
const TEXT_FILE = /\.(?:json|ndjson|jsonl|md|txt|log|yaml|yml|csv|html)$/i;
|
|
135
|
+
/** Enough for any run directory; a sweep stops there rather than hold up the run's finish. */
|
|
136
|
+
const MAX_SWEPT_ENTRIES = 10_000;
|
|
137
|
+
/**
|
|
138
|
+
* Rewrite each file of a finished run that names one of its raw sandbox ids, apart from the
|
|
139
|
+
* receipts, with the label in place of the id. Writers such as a participant's actor.json can
|
|
140
|
+
* quote an SDK error that names the sandbox. Run.finish calls it before analysis reads the run.
|
|
141
|
+
* Best effort: verify grades a file this misses.
|
|
142
|
+
*/
|
|
143
|
+
export async function scrubRunSandboxIds(paths) {
|
|
144
|
+
const ids = await readRunSandboxIds(paths);
|
|
145
|
+
if (ids.length === 0)
|
|
146
|
+
return;
|
|
147
|
+
const valid = await validatePreparedRunRootIdentity(paths).then(() => true, () => false);
|
|
148
|
+
if (!valid)
|
|
149
|
+
return;
|
|
150
|
+
const scrub = async (file) => {
|
|
151
|
+
const bytes = await readContainedRegularFile(paths, file);
|
|
152
|
+
if (bytes === null)
|
|
153
|
+
return;
|
|
154
|
+
const scrubbed = scrubSandboxIdBytes(file, bytes, ids);
|
|
155
|
+
if (scrubbed !== bytes)
|
|
156
|
+
await writeContainedOutputFile(paths, file, scrubbed);
|
|
157
|
+
};
|
|
158
|
+
let seen = 0;
|
|
159
|
+
const walk = async (relative) => {
|
|
160
|
+
const directory = path.join(paths.physicalRunRoot, relative);
|
|
161
|
+
// A directory replaced by a link after it was listed resolves elsewhere and is not walked.
|
|
162
|
+
if ((await realpath(directory).catch(() => null)) !== directory)
|
|
163
|
+
return;
|
|
164
|
+
const listing = await opendir(directory).catch(() => null);
|
|
165
|
+
if (listing === null)
|
|
166
|
+
return;
|
|
167
|
+
for await (const entry of listing) {
|
|
168
|
+
// Returning from the loop closes the listing.
|
|
169
|
+
if (++seen > MAX_SWEPT_ENTRIES)
|
|
170
|
+
return;
|
|
171
|
+
const file = relative ? `${relative}/${entry.name}` : entry.name;
|
|
172
|
+
if (entry.isDirectory())
|
|
173
|
+
await walk(file);
|
|
174
|
+
else if (entry.isFile() && file !== SANDBOX_RECEIPTS_ARTIFACT && TEXT_FILE.test(file))
|
|
175
|
+
// A file that cannot be scrubbed stays as written; verify names it.
|
|
176
|
+
await scrub(file).catch(() => { });
|
|
177
|
+
}
|
|
178
|
+
};
|
|
179
|
+
await walk("").catch(() => {
|
|
180
|
+
// A directory that changed while it was listed ends the sweep; verify names what it missed.
|
|
181
|
+
});
|
|
182
|
+
}
|
|
183
|
+
/**
|
|
184
|
+
* A run's result as it may be returned or printed: the same object when it names no sandbox,
|
|
185
|
+
* otherwise its public view, scrubbed of the ids the run's receipts journal.
|
|
186
|
+
*/
|
|
187
|
+
export async function publicRunResult(result, cwd) {
|
|
188
|
+
let ids = [];
|
|
189
|
+
if (typeof result.runId === "string" && isSafeRunIdSegment(result.runId)) {
|
|
190
|
+
try {
|
|
191
|
+
ids = await readRunSandboxIds(await bindExistingRunArtifactPaths(cwd, result.runId));
|
|
192
|
+
}
|
|
193
|
+
catch {
|
|
194
|
+
// A run that never created its directory has no receipts; the keys are still redacted.
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
return publicSandboxView(result, ids);
|
|
198
|
+
}
|
|
@@ -20,6 +20,8 @@ export interface SandboxReceipt {
|
|
|
20
20
|
export interface ParsedSandboxReceipt extends Omit<SandboxReceipt, "provider"> {
|
|
21
21
|
provider: string;
|
|
22
22
|
}
|
|
23
|
+
/** The ids this process has receipted for `root`, whether or not the journal write landed. */
|
|
24
|
+
export declare function appendedSandboxIds(root: PreparedOutputRoot): string[];
|
|
23
25
|
/**
|
|
24
26
|
* Append one receipt. Best-effort by design: the receipt exists to protect the run, so a failed
|
|
25
27
|
* receipt write must never fail the participant. The only cost of a miss is that `reclaim` cannot see
|
|
@@ -4,6 +4,23 @@
|
|
|
4
4
|
import { appendFile } from "node:fs/promises";
|
|
5
5
|
import { prepareContainedOutputFile } from "./contained-output.js";
|
|
6
6
|
export const SANDBOX_RECEIPTS_ARTIFACT = "sandbox-receipts.ndjson";
|
|
7
|
+
/**
|
|
8
|
+
* The ids this process has receipted for each run directory, kept even when the append fails, so
|
|
9
|
+
* the run's other records can still be scrubbed of them. Keyed by the directory's identity, so a
|
|
10
|
+
* run created later at a deleted run's path starts empty, and only the latest runs are kept.
|
|
11
|
+
*/
|
|
12
|
+
const appended = new Map();
|
|
13
|
+
const MAX_REMEMBERED_RUNS = 64;
|
|
14
|
+
function rootKey(root) {
|
|
15
|
+
const [where, { dev, ino, birthtimeNs }] = "physicalRunRoot" in root
|
|
16
|
+
? [root.physicalRunRoot, root.runRootIdentity]
|
|
17
|
+
: [root.physicalPath, root.identity];
|
|
18
|
+
return `${where}\0${dev}:${ino}:${birthtimeNs}`;
|
|
19
|
+
}
|
|
20
|
+
/** The ids this process has receipted for `root`, whether or not the journal write landed. */
|
|
21
|
+
export function appendedSandboxIds(root) {
|
|
22
|
+
return [...(appended.get(rootKey(root)) ?? [])];
|
|
23
|
+
}
|
|
7
24
|
/**
|
|
8
25
|
* Append one receipt. Best-effort by design: the receipt exists to protect the run, so a failed
|
|
9
26
|
* receipt write must never fail the participant. The only cost of a miss is that `reclaim` cannot see
|
|
@@ -11,6 +28,15 @@ export const SANDBOX_RECEIPTS_ARTIFACT = "sandbox-receipts.ndjson";
|
|
|
11
28
|
* artifact write uses; append (not atomic-replace) keeps racing participants' receipts intact.
|
|
12
29
|
*/
|
|
13
30
|
export async function appendSandboxReceipt(root, receipt) {
|
|
31
|
+
const key = rootKey(root);
|
|
32
|
+
const ids = appended.get(key) ?? new Set();
|
|
33
|
+
appended.delete(key);
|
|
34
|
+
appended.set(key, ids.add(receipt.sandboxId));
|
|
35
|
+
for (const oldest of appended.keys()) {
|
|
36
|
+
if (appended.size <= MAX_REMEMBERED_RUNS)
|
|
37
|
+
break;
|
|
38
|
+
appended.delete(oldest);
|
|
39
|
+
}
|
|
14
40
|
try {
|
|
15
41
|
const filePath = await prepareContainedOutputFile(root, SANDBOX_RECEIPTS_ARTIFACT);
|
|
16
42
|
await appendFile(filePath, `${JSON.stringify(receipt)}\n`, "utf8");
|
package/dist/run/status.js
CHANGED
|
@@ -12,6 +12,7 @@
|
|
|
12
12
|
import { isProvenanceField, studyFields } from "./study-provenance.js";
|
|
13
13
|
import { writeContainedOutputFile } from "./contained-output.js";
|
|
14
14
|
import { redactText } from "../evidence/redaction.js";
|
|
15
|
+
import { readRunSandboxIds, scrubSandboxIds } from "./sandbox-ids.js";
|
|
15
16
|
export const RUN_STATUS_SCHEMA = "humanish.run-status.v1";
|
|
16
17
|
/** The file, relative to the run directory. */
|
|
17
18
|
export const RUN_STATUS_FILE = "status.json";
|
|
@@ -106,10 +107,12 @@ export function beginRunStatus(runPaths, options) {
|
|
|
106
107
|
if (finishedRecord === undefined)
|
|
107
108
|
return;
|
|
108
109
|
// The record is public-safe by construction, so each message passes the shape redaction
|
|
109
|
-
// again even though the routes scrubbed it.
|
|
110
|
+
// again even though the routes scrubbed it. A route settles with its own result, so an SDK
|
|
111
|
+
// error here can still name a sandbox; the receipts give the ids to replace.
|
|
112
|
+
const ids = await readRunSandboxIds(runPaths);
|
|
110
113
|
const redacted = (failure) => ({
|
|
111
114
|
kind: failure.kind,
|
|
112
|
-
message: redactText(failure.message),
|
|
115
|
+
message: scrubSandboxIds(redactText(failure.message), ids),
|
|
113
116
|
});
|
|
114
117
|
const execution = {
|
|
115
118
|
succeeded: result.execution.succeeded,
|
package/dist/run/stored-runs.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { REDACTED_SANDBOX_ID, sandboxIdDigest } from "../evidence/redaction.js";
|
|
1
2
|
import { lstat, readdir } from "node:fs/promises";
|
|
2
3
|
import { runNotFoundMessage } from "./run-not-found.js";
|
|
3
4
|
import path from "node:path";
|
|
@@ -30,7 +31,15 @@ function cleanupRefusal(cwd, runInput, checkedAt, error, bundlePath) {
|
|
|
30
31
|
}
|
|
31
32
|
/** What the recorded evidence says about one provider resource; nothing is killed from here. */
|
|
32
33
|
function recordedResourceResult(resource) {
|
|
33
|
-
|
|
34
|
+
// Named by digest: a run recorded before 0.110 holds the raw id here, which is digested.
|
|
35
|
+
const raw = resource.id === REDACTED_SANDBOX_ID ? undefined : resource.id;
|
|
36
|
+
const idDigest = resource.idDigest ?? (raw === undefined ? undefined : sandboxIdDigest(raw));
|
|
37
|
+
const base = {
|
|
38
|
+
provider: resource.provider,
|
|
39
|
+
kind: resource.kind,
|
|
40
|
+
id: REDACTED_SANDBOX_ID,
|
|
41
|
+
...(idDigest === undefined ? {} : { idDigest }),
|
|
42
|
+
};
|
|
34
43
|
if (resource.provider !== "e2b-desktop" || resource.kind !== "sandbox") {
|
|
35
44
|
return {
|
|
36
45
|
...base,
|
package/dist/run-study.js
CHANGED
|
@@ -19,6 +19,7 @@ import {} from "./routes/shared-world/types.js";
|
|
|
19
19
|
import { admitTerminalPlan, terminalStudyRefusal } from "./routes/terminal/route.js";
|
|
20
20
|
import {} from "./routes/terminal/types.js";
|
|
21
21
|
import {} from "./run/bundle.js";
|
|
22
|
+
import { publicRunResult } from "./run/sandbox-ids.js";
|
|
22
23
|
import { isLocalBrowserStudy, localBrowserDefaults } from "./substrates/local/runtime-config.js";
|
|
23
24
|
/**
|
|
24
25
|
* Runs a study on its route. The options are checked against the route and mapped into the route's
|
|
@@ -100,7 +101,11 @@ export async function prepareStudy(config, options, deps = {}) {
|
|
|
100
101
|
// study, so a rebuilt one would leave the participants they start for no finally to close.
|
|
101
102
|
const outcome = await admitted.run(scorer);
|
|
102
103
|
outcome.result.warnings.push(...normalized.warnings);
|
|
103
|
-
|
|
104
|
+
// The result names each sandbox by digest; the raw ids stay in the run's receipts.
|
|
105
|
+
return {
|
|
106
|
+
...outcome,
|
|
107
|
+
result: await publicRunResult(outcome.result, options.cwd),
|
|
108
|
+
};
|
|
104
109
|
}
|
|
105
110
|
finally {
|
|
106
111
|
await vm?.close();
|
package/dist/study/doctor.js
CHANGED
|
@@ -227,7 +227,7 @@ async function realEmailCheck(connection, keys, args) {
|
|
|
227
227
|
message: name === null
|
|
228
228
|
? "The selected email connection is missing or invalid. Open Connections in the TUI."
|
|
229
229
|
: !args.keyPresent(name)
|
|
230
|
-
? `Missing ${name} for the selected email connection. Provide it through process env or --
|
|
230
|
+
? `Missing ${name} for the selected email connection. Provide it through process env or --dotenv. Authentication has not been checked.`
|
|
231
231
|
: "Fresh hosted inbox per participant. Local presence only; run humanish comms check --online to authenticate. Provider permissions/capacity and delivery remain untested.",
|
|
232
232
|
};
|
|
233
233
|
}
|
|
@@ -300,7 +300,7 @@ function subjectEnvChecks(names, args) {
|
|
|
300
300
|
ok: !!args.env[name]?.trim() || args.keyPresent(name),
|
|
301
301
|
message: args.env[name]?.trim() || args.keyPresent(name)
|
|
302
302
|
? "present; value not shown"
|
|
303
|
-
: "missing declared subject environment variable; provide it with --
|
|
303
|
+
: "missing declared subject environment variable; provide it with --dotenv",
|
|
304
304
|
}));
|
|
305
305
|
}
|
|
306
306
|
/** The post-run analysis: the Codex account, or the OpenAI key and the analysis cost limit. */
|
|
@@ -245,9 +245,9 @@ title: Computer-use browser study
|
|
|
245
245
|
description: >-
|
|
246
246
|
A computer-use participant drives your app in a hosted desktop browser and writes its evidence
|
|
247
247
|
to gitignored .humanish/. Starts as a dry run. For a live session, set mode to live
|
|
248
|
-
and pass OPENAI_API_KEY and E2B_API_KEY with --
|
|
248
|
+
and pass OPENAI_API_KEY and E2B_API_KEY with --dotenv. The clone subject below serves your
|
|
249
249
|
repo inside the sandbox, and the env names declared under subject get their values from
|
|
250
|
-
--
|
|
250
|
+
--dotenv without the values being saved. Screenshots keep full fidelity by default; set
|
|
251
251
|
policies.redactScreenshots: true to blur them at capture for a bundle you can share as is.
|
|
252
252
|
Typed text is recorded as its length only.
|
|
253
253
|
route: computer-use
|
package/dist/study/types.d.ts
CHANGED
|
@@ -239,7 +239,7 @@ export interface StudySubject {
|
|
|
239
239
|
serve?: StudySubjectServe;
|
|
240
240
|
/**
|
|
241
241
|
* Env var names the subject app needs, provisioned into the sandbox from the caller's
|
|
242
|
-
* environment (--
|
|
242
|
+
* environment (--dotenv). Names are recorded in evidence; values never are. Consumed
|
|
243
243
|
* on the computer-use clone route.
|
|
244
244
|
*/
|
|
245
245
|
env?: string[];
|
|
@@ -37,6 +37,10 @@ export declare function rawScreenshotStreamIds(bundle: RunBundle, redactedShapeF
|
|
|
37
37
|
interface RegisteredStreamMedia {
|
|
38
38
|
recordingPaths: Set<string>;
|
|
39
39
|
screenshotPaths: Set<string>;
|
|
40
|
+
/** The run's raw sandbox ids, from its receipts; any other file holding one is listed below. */
|
|
41
|
+
sandboxIds?: readonly string[];
|
|
42
|
+
/** Filled by the scan: files other than the receipts that name a raw sandbox id. */
|
|
43
|
+
sandboxIdFiles?: string[];
|
|
40
44
|
}
|
|
41
45
|
/**
|
|
42
46
|
* Scans every run file for secret and path patterns and returns the findings. A file that is not
|
package/dist/verify/artifacts.js
CHANGED
|
@@ -7,6 +7,8 @@ import { readPlainText } from "../evidence/plain-text.js";
|
|
|
7
7
|
import { containsSensitive } from "../evidence/redaction.js";
|
|
8
8
|
import { isLocalEvidenceArtifactPath, isRiskyPublicArtifactPath, validatePreparedRunArtifactPaths, } from "../run/paths.js";
|
|
9
9
|
import { openContainedRegularFile } from "../run/contained-output.js";
|
|
10
|
+
import { holdsKeyedSandboxId } from "../run/sandbox-ids.js";
|
|
11
|
+
import { SANDBOX_RECEIPTS_ARTIFACT } from "../run/sandbox-receipts.js";
|
|
10
12
|
import { readSafeRunArtifactBytes, readSafeRunArtifactJson } from "../run/locate.js";
|
|
11
13
|
import { isRecord } from "../run/type-guards.js";
|
|
12
14
|
import { TERMINAL_EVENTS_ARTIFACT } from "../run/terminal-contract.js";
|
|
@@ -348,6 +350,12 @@ async function scanRunPublicSafetyDirectory(runPaths, relativeDirectory, finding
|
|
|
348
350
|
unscanned.push(relativePath);
|
|
349
351
|
continue;
|
|
350
352
|
}
|
|
353
|
+
// A receipt's id in any other file, or a raw id at a sandbox-id key, which an older run or
|
|
354
|
+
// export copy can hold with no receipt to name it.
|
|
355
|
+
if (relativePath !== SANDBOX_RECEIPTS_ARTIFACT &&
|
|
356
|
+
(media.sandboxIds?.some((id) => decoded.text.includes(id)) ||
|
|
357
|
+
holdsKeyedSandboxId(relativePath, decoded.text)))
|
|
358
|
+
media.sandboxIdFiles?.push(relativePath);
|
|
351
359
|
// serve renders observer/index.html from run.json and export regenerates it, so the on-disk
|
|
352
360
|
// copy reaches neither. It embeds the Observer's own base64 fonts and scripts.
|
|
353
361
|
const scan = scanEncodedTextCached(decoded.text, {
|
package/dist/verify/liveness.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
// `humanish verify`'s RUN_NOT_FINISHED warning. Verify checks the integrity of what a run wrote,
|
|
2
2
|
// and a run killed mid-way leaves an in-progress bundle that passes those checks. The warning says
|
|
3
3
|
// what verify saw on disk, so `ok: true` is not read as "the run finished". It never flips `ok`.
|
|
4
|
+
import { sandboxIdDigest } from "../evidence/redaction.js";
|
|
4
5
|
import path from "node:path";
|
|
5
6
|
import { readRunJsonIfExists, readRunTextIfExists } from "../run/locate.js";
|
|
6
7
|
import { isRecord } from "../run/type-guards.js";
|
|
@@ -45,12 +46,14 @@ export async function runNotFinishedWarnings(runPaths, bundle, nowMs = Date.now(
|
|
|
45
46
|
/** What the reclaim receipt says about the sandboxes this run journaled, or what to run. */
|
|
46
47
|
async function reclaimState(runPaths, runId) {
|
|
47
48
|
const journal = await readRunTextIfExists(runPaths, SANDBOX_RECEIPTS_ARTIFACT);
|
|
48
|
-
|
|
49
|
+
// Matched by digest: a reclaim receipt names each sandbox by digest, and one written before it
|
|
50
|
+
// did holds the raw id, which is digested here.
|
|
51
|
+
const journaled = new Set((journal === null ? [] : parseSandboxReceipts(journal)).map((receipt) => sandboxIdDigest(receipt.sandboxId)));
|
|
49
52
|
const receipt = await readRunJsonIfExists(runPaths, RECLAIM_RECEIPT_ARTIFACT);
|
|
50
53
|
const outcomes = reclaimOutcomes(receipt, runId);
|
|
51
54
|
if (outcomes !== undefined) {
|
|
52
55
|
const gone = outcomes.filter((outcome) => GONE_STATES.has(outcome.state)).length;
|
|
53
|
-
const reclaimed = new Set(outcomes.map((outcome) => outcome.
|
|
56
|
+
const reclaimed = new Set(outcomes.map((outcome) => outcome.digest));
|
|
54
57
|
const missing = [...journaled].filter((id) => !reclaimed.has(id)).length;
|
|
55
58
|
return `${RECLAIM_RECEIPT_ARTIFACT} records ${gone} of ${outcomes.length} sandboxes gone${missing === 0 ? "" : `, and ${missing} journaled sandboxes are not in it`}.`;
|
|
56
59
|
}
|
|
@@ -66,10 +69,11 @@ function reclaimOutcomes(receipt, runId) {
|
|
|
66
69
|
for (const outcome of receipt.outcomes) {
|
|
67
70
|
if (!isRecord(outcome))
|
|
68
71
|
return undefined;
|
|
69
|
-
const { sandboxId, state } = outcome;
|
|
72
|
+
const { sandboxId, sandboxIdDigest: recorded, state } = outcome;
|
|
70
73
|
if (typeof sandboxId !== "string" || typeof state !== "string")
|
|
71
74
|
return undefined;
|
|
72
|
-
|
|
75
|
+
const digest = typeof recorded === "string" ? recorded : sandboxIdDigest(sandboxId);
|
|
76
|
+
outcomes.push({ digest, state });
|
|
73
77
|
}
|
|
74
78
|
return outcomes;
|
|
75
79
|
}
|
package/dist/verify/verify.d.ts
CHANGED
|
@@ -16,7 +16,7 @@ export interface VerifyResult {
|
|
|
16
16
|
shareSafety: {
|
|
17
17
|
status: "share_ready" | "local_only" | "blocked";
|
|
18
18
|
reasons: Array<{
|
|
19
|
-
code: "VERIFY_FAILED" | "PUBLIC_SAFETY_FINDINGS" | "ANALYSIS_UNVERIFIED" | "RAW_SCREENSHOTS" | "CONTINUOUS_MEDIA" | "REAL_COMMUNICATIONS" | "UNSCANNED_ARTIFACT";
|
|
19
|
+
code: "VERIFY_FAILED" | "PUBLIC_SAFETY_FINDINGS" | "ANALYSIS_UNVERIFIED" | "RAW_SCREENSHOTS" | "RAW_SANDBOX_ID" | "CONTINUOUS_MEDIA" | "REAL_COMMUNICATIONS" | "UNSCANNED_ARTIFACT";
|
|
20
20
|
message: string;
|
|
21
21
|
}>;
|
|
22
22
|
};
|
package/dist/verify/verify.js
CHANGED
|
@@ -2,7 +2,8 @@ import path from "node:path";
|
|
|
2
2
|
import { listAnalysisExecutions } from "../analysis/store-executions.js";
|
|
3
3
|
import { loadAnalysis } from "../analysis/load.js";
|
|
4
4
|
import { analysisSharingProblems } from "../analysis/sharing.js";
|
|
5
|
-
import { containsSensitive } from "../evidence/redaction.js";
|
|
5
|
+
import { containsSensitive, REDACTED_SANDBOX_ID } from "../evidence/redaction.js";
|
|
6
|
+
import { keyedSandboxIds, readRunSandboxIds } from "../run/sandbox-ids.js";
|
|
6
7
|
import { validatePreparedRunArtifactPaths } from "../run/paths.js";
|
|
7
8
|
import { RUN_BUNDLE_FILE, RUN_BUNDLE_SCHEMA } from "../run/bundle.js";
|
|
8
9
|
import { isCleanupResult, isRunBundle } from "../run/bundle-shape.js";
|
|
@@ -83,11 +84,17 @@ export async function verifyResolvedRun(cwd, runInput, runPaths) {
|
|
|
83
84
|
}));
|
|
84
85
|
const derivedPublicSafetyFindings = [];
|
|
85
86
|
const unscannedArtifacts = [];
|
|
87
|
+
const sandboxIdFiles = [];
|
|
86
88
|
const publicSafetyFindings = await scanRunPublicSafetyArtifacts(runPaths, derivedPublicSafetyFindings, {
|
|
87
89
|
recordingPaths: new Set(isRunBundle(bundle)
|
|
88
90
|
? bundle.streams.flatMap((stream) => (stream.recording ? [stream.recording.path] : []))
|
|
89
91
|
: []),
|
|
90
92
|
screenshotPaths: isRunBundle(bundle) ? streamScreenshotPaths(bundle) : new Set(),
|
|
93
|
+
// A run from before 0.110 with no receipts still names its ids in run.json.
|
|
94
|
+
sandboxIds: [
|
|
95
|
+
...new Set([...(await readRunSandboxIds(runPaths)), ...keyedSandboxIds(bundle)]),
|
|
96
|
+
],
|
|
97
|
+
sandboxIdFiles,
|
|
91
98
|
}, unscannedArtifacts);
|
|
92
99
|
// JSON escapes can hide a sensitive value from the byte scan while the
|
|
93
100
|
// decoded recording exposes it to Observer, feedback, or analysis input.
|
|
@@ -146,6 +153,7 @@ export async function verifyResolvedRun(cwd, runInput, runPaths) {
|
|
|
146
153
|
publicSafetyFindings,
|
|
147
154
|
unscannedArtifacts,
|
|
148
155
|
redactedShapeFrames,
|
|
156
|
+
sandboxIdFiles,
|
|
149
157
|
})
|
|
150
158
|
: {
|
|
151
159
|
status: "blocked",
|
|
@@ -366,6 +374,18 @@ function buildShareSafety(args) {
|
|
|
366
374
|
message: `The public-safety scan cannot read ${plural(paths.length, "file")} as text that ${paths.length === 1 ? "is not a stream screenshot under screenshots/ or a registered recording" : "are not stream screenshots under screenshots/ or registered recordings"}: ${shown}${more > 0 ? ` and ${more} more` : ""}. Review them before sharing.`,
|
|
367
375
|
});
|
|
368
376
|
}
|
|
377
|
+
// Runs from 0.110 keep raw sandbox ids only in sandbox-receipts.ndjson. An earlier run records
|
|
378
|
+
// them in run.json, and any file that names one is not share-ready as it is.
|
|
379
|
+
const rawResources = (args.bundle.providerResources ?? []).some((resource) => resource.id !== REDACTED_SANDBOX_ID);
|
|
380
|
+
if (rawResources || args.sandboxIdFiles.length > 0) {
|
|
381
|
+
const files = [
|
|
382
|
+
...new Set([...(rawResources ? [RUN_BUNDLE_FILE] : []), ...args.sandboxIdFiles]),
|
|
383
|
+
];
|
|
384
|
+
reasons.push({
|
|
385
|
+
code: "RAW_SANDBOX_ID",
|
|
386
|
+
message: `Raw sandbox ids appear in ${files.sort().join(", ")}. Runs from 0.110 keep them only in sandbox-receipts.ndjson; \`humanish export --format bundle --redact-screenshots\` writes a copy without them.`,
|
|
387
|
+
});
|
|
388
|
+
}
|
|
369
389
|
const rawStreamIds = rawScreenshotStreamIds(args.bundle, args.redactedShapeFrames);
|
|
370
390
|
if (rawStreamIds.length > 0) {
|
|
371
391
|
reasons.push({
|
|
@@ -176,7 +176,7 @@ the following, on Linux x64:
|
|
|
176
176
|
runs this check; see "The qualifier" below.
|
|
177
177
|
3. One hosted participant study with the candidate first on `PATH` (for example
|
|
178
178
|
`PATH=<evidence>/npm-<version>/node_modules/.bin:$PATH pnpm humanish run <study>
|
|
179
|
-
--
|
|
179
|
+
--dotenv <file with E2B_API_KEY only>`) reached its goal in the final capture, with a
|
|
180
180
|
verified request receipt, `humanish verify --run <id>` passing and the automatic Codex
|
|
181
181
|
analysis recording the release. Keep `OPENAI_API_KEY` out of that environment so the
|
|
182
182
|
participant stays on ChatGPT-account billing.
|
|
@@ -525,6 +525,10 @@ bundle is safe to promote into a public issue. Public promotion should branch on
|
|
|
525
525
|
|
|
526
526
|
- `share_ready`: feedback draft commands may render public issue payloads;
|
|
527
527
|
- `local_only`: keep the run local; only supported redaction-only cases can produce a shareable derivative;
|
|
528
|
+
`RAW_SANDBOX_ID` is one: a file other than `sandbox-receipts.ndjson` names one of the run's raw
|
|
529
|
+
sandbox ids, which verify reads from the receipts and from any raw `providerResources[].id` in
|
|
530
|
+
`run.json`, or a JSON file holds a raw id at a sandbox-id key. Runs from 0.110 keep raw ids only in the receipts, so it fires on runs recorded
|
|
531
|
+
before 0.110, whose `run.json` holds them, until they are exported;
|
|
528
532
|
- `blocked`: fix the verification or public-safety failure first.
|
|
529
533
|
|
|
530
534
|
`ok: true` also does not mean the run finished. A run killed mid-way leaves an
|
|
@@ -540,7 +544,9 @@ writing; an older one reads as interrupted. The warning names what verify saw:
|
|
|
540
544
|
there is no usable record;
|
|
541
545
|
- how many streams are still `running`;
|
|
542
546
|
- what `reclaim-receipt.json` records: how many of the run's sandboxes are gone
|
|
543
|
-
(killed or already gone), and how many journaled sandboxes it does not cover
|
|
547
|
+
(killed or already gone), and how many journaled sandboxes it does not cover. It
|
|
548
|
+
matches its outcomes to the receipts by `sandboxIdDigest`; a reclaim receipt from
|
|
549
|
+
before 0.110 names raw ids, which are digested to match;
|
|
544
550
|
- with journaled sandboxes and no reclaim receipt, the command that stops them:
|
|
545
551
|
`humanish reclaim --run <id>`.
|
|
546
552
|
|
|
@@ -634,9 +640,17 @@ finding is true. Its own bytes are not included in its hash inventory. Feedback
|
|
|
634
640
|
commands can subsequently generate new derivative-local artifacts.
|
|
635
641
|
|
|
636
642
|
The derivative names no sandbox. In every JSON and NDJSON file it copies, the
|
|
637
|
-
values at `sandboxId`, `subjectSandboxId
|
|
638
|
-
`[redacted-sandbox-id]
|
|
639
|
-
|
|
643
|
+
values at `sandboxId`, `subjectSandboxId`, each `providerResources[].id` and each
|
|
644
|
+
cleanup.json `resources[].id` read `[redacted-sandbox-id]`, with the id's digest
|
|
645
|
+
beside them. In every text file it copies, Markdown, logs and YAML included, each
|
|
646
|
+
id the source names becomes `[redacted-sandbox-id <digest>]`: the ids in its
|
|
647
|
+
`sandbox-receipts.ndjson`, and any raw id at one of those keys, which is where a
|
|
648
|
+
run recorded before 0.110 with no receipts names them. Only those exact ids are
|
|
649
|
+
replaced, so the rest of each file is unchanged. In YAML the label is
|
|
650
|
+
`redacted-sandbox-id-<digest>`, without brackets, so a value keeps its type.
|
|
651
|
+
An id that appears only in free text, in a run with no receipts, is not known to
|
|
652
|
+
export. `sandbox-receipts.ndjson` is omitted, and the source keeps its raw ids,
|
|
653
|
+
which reclaim reads.
|
|
640
654
|
|
|
641
655
|
PNG files that actor traces reference as stream screenshots are re-encoded as
|
|
642
656
|
blurred thumbnails. A PNG that nothing in `run.json` cites is omitted and
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "humanish",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.110.0",
|
|
4
4
|
"description": "Synthetic user research for apps, CLIs, and agent-facing product flows.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"cli",
|
|
@@ -56,9 +56,10 @@
|
|
|
56
56
|
},
|
|
57
57
|
"scripts": {
|
|
58
58
|
"build": "tsc -p tsconfig.build.json && pnpm --filter humanish-observer build && pnpm --filter humanish-tui build && node scripts/finish-build.mjs",
|
|
59
|
-
"check": "pnpm format:check && pnpm lint && pnpm knip && pnpm prose:check && pnpm vocabulary:check && pnpm site-css:check && pnpm typecheck && pnpm test && pnpm tui:test && pnpm build && pnpm guest-media:startup:proof && pnpm cli:startup:test && pnpm cli:preflight:test && pnpm tui:smoke",
|
|
59
|
+
"check": "pnpm format:check && pnpm lint && pnpm knip && pnpm prose:check && pnpm vocabulary:check && pnpm site-css:check && pnpm typecheck && pnpm test && pnpm tui:test && pnpm build && pnpm guest-media:startup:proof && pnpm cli:startup:test && pnpm cli:preflight:test && pnpm cli:dotenv:test && pnpm tui:smoke",
|
|
60
60
|
"cli:startup:test": "node scripts/terminal-startup-exit-proof.mjs",
|
|
61
61
|
"cli:preflight:test": "node scripts/task-route-preflight-proof.mjs",
|
|
62
|
+
"cli:dotenv:test": "node scripts/dotenv-proof.mjs",
|
|
62
63
|
"humanish": "tsx src/cli.ts",
|
|
63
64
|
"pack:dry-run": "npm pack --dry-run",
|
|
64
65
|
"prepack": "pnpm build",
|
package/skills/humanish/SKILL.md
CHANGED
|
@@ -16,7 +16,7 @@ patient data, or private upstream artifacts.
|
|
|
16
16
|
|
|
17
17
|
Do not edit `.env` or secret files. Do not paste credential values. Use env var
|
|
18
18
|
names only, usually `OPENAI_API_KEY` and `E2B_API_KEY`. For live local runs,
|
|
19
|
-
prefer an explicit ignored env file passed with `--
|
|
19
|
+
prefer an explicit ignored env file passed with `--dotenv <path>`; do not
|
|
20
20
|
assume broad inherited job env is safe. Stop before live provider spend,
|
|
21
21
|
hosted execution, deploys, public tunnels, or GitHub mutation unless the user
|
|
22
22
|
explicitly approves that exact action.
|
|
@@ -442,7 +442,7 @@ Do not paste values into files, prompts, run bundles, issue drafts, or logs.
|
|
|
442
442
|
Load local values only at invocation time:
|
|
443
443
|
|
|
444
444
|
```bash
|
|
445
|
-
npx humanish watch .humanish/studies/local-live.yaml --
|
|
445
|
+
npx humanish watch .humanish/studies/local-live.yaml --dotenv .humanish/local/provider.env
|
|
446
446
|
```
|
|
447
447
|
|
|
448
448
|
When choosing dogfood targets, prefer apps, CLIs, or agent-facing tools with a
|