humanish 0.117.0 → 0.118.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -0
- package/dist/analysis/admission.d.ts +7 -2
- package/dist/analysis/admission.js +12 -3
- package/dist/analysis/automatic-config.d.ts +6 -1
- package/dist/analysis/automatic-config.js +12 -11
- package/dist/analysis/execute.d.ts +19 -7
- package/dist/analysis/execute.js +55 -17
- package/dist/cli/commands/analyze.js +2 -1
- package/dist/cli/commands/study-format.js +8 -11
- package/dist/cli/commands/study-route-computer-use.js +2 -2
- package/dist/cli/commands/study-route-scripted.js +2 -2
- package/dist/cli/commands/study-route-shared-world.js +5 -3
- package/dist/cli/commands/study-route-terminal.js +2 -2
- package/dist/cli/commands/study-run.js +3 -3
- package/dist/cli/commands/study.js +2 -2
- package/dist/cli/doctor.js +11 -1
- package/dist/cli/io.js +3 -2
- package/dist/comms/lease-store.d.ts +1 -1
- package/dist/comms/lease-store.js +3 -2
- package/dist/comms/receiving-types.d.ts +2 -0
- package/dist/comms/receiving-types.js +2 -0
- package/dist/observer-app.html +2 -2
- package/dist/routes/computer-use/fanout-bundle.js +2 -1
- package/dist/routes/computer-use/fanout-records.js +2 -1
- package/dist/routes/computer-use/participant-runs.js +3 -9
- package/dist/routes/computer-use/plan.d.ts +3 -1
- package/dist/routes/computer-use/plan.js +25 -7
- package/dist/routes/computer-use/single-bundle.js +7 -4
- package/dist/routes/computer-use/subject-projection.js +2 -0
- package/dist/routes/scripted/plan.d.ts +4 -1
- package/dist/routes/scripted/plan.js +25 -2
- package/dist/routes/scripted/setup.js +1 -4
- package/dist/routes/scripted/types.d.ts +1 -1
- package/dist/routes/shared-world/participant-specs.js +2 -2
- package/dist/routes/shared-world/plan.d.ts +3 -1
- package/dist/routes/shared-world/plan.js +37 -6
- package/dist/routes/shared-world/route.js +2 -2
- package/dist/run/bundle-shape.js +2 -1
- package/dist/run/bundle.d.ts +5 -3
- package/dist/run/projection.js +7 -2
- package/dist/run/route-shell.js +10 -5
- package/dist/run/subject-name.d.ts +11 -0
- package/dist/run/subject-name.js +11 -0
- package/dist/study/automatic-analysis-plan.d.ts +28 -0
- package/dist/study/automatic-analysis-plan.js +57 -0
- package/dist/study/composition-rules.js +1 -1
- package/dist/study/config.js +6 -20
- package/dist/study/doctor.js +17 -3
- package/dist/study/parse/actors.d.ts +1 -1
- package/dist/study/parse/actors.js +1 -1
- package/dist/study/parse/front.d.ts +1 -1
- package/dist/study/parse/front.js +7 -1
- package/dist/study/plan-base.d.ts +2 -0
- package/dist/study/plan-base.js +3 -0
- package/dist/study/plan-participants.js +2 -5
- package/dist/study/plan-types.d.ts +8 -3
- package/dist/study/plan.d.ts +4 -4
- package/dist/study/plan.js +21 -12
- package/dist/study/preflight-result.js +2 -2
- package/dist/study/preflight.d.ts +2 -1
- package/dist/study/preflight.js +12 -3
- package/dist/study/routing.d.ts +5 -4
- package/dist/study/routing.js +16 -7
- package/dist/study/summary.d.ts +3 -3
- package/dist/study/summary.js +3 -5
- package/dist/study/validation.d.ts +2 -2
- package/dist/study/validation.js +22 -14
- package/dist/substrates/e2b/lifetime.d.ts +75 -4
- package/dist/substrates/e2b/lifetime.js +134 -10
- package/dist/substrates/e2b/subject-sandbox.js +2 -6
- package/dist/tui-app.js +112 -112
- package/dist/verify/artifacts.d.ts +19 -4
- package/dist/verify/artifacts.js +39 -8
- package/dist/verify/verify.js +6 -15
- package/docs/contracts/run-bundle.md +7 -6
- package/docs/contracts/schemas.md +14 -8
- package/docs/contracts/study-analysis.md +2 -2
- package/docs/principles/invariants-and-defaults.md +1 -1
- package/docs/product/automatic-analysis.md +4 -3
- package/package.json +1 -1
|
@@ -7,6 +7,20 @@ export declare const MAX_REPORTED_FINDINGS = 50;
|
|
|
7
7
|
* refuses a run with more, so the scan never passes a run whose files it did not all read.
|
|
8
8
|
*/
|
|
9
9
|
export declare const MAX_RUN_ENTRIES = 10000;
|
|
10
|
+
/** What the public-safety scan did not read. A run with any of it cannot grade share_ready. */
|
|
11
|
+
export interface UnscannedArtifacts {
|
|
12
|
+
/** Run files it could not read as text, by run-relative path. */
|
|
13
|
+
readonly files: string[];
|
|
14
|
+
/** Folders it could not list, by run-relative path ("" is the run folder). */
|
|
15
|
+
readonly unlistedFolders: string[];
|
|
16
|
+
/** The folder it was listing when it reached MAX_RUN_ENTRIES. It read nothing listed after. */
|
|
17
|
+
stoppedIn?: string;
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* The UNSCANNED_ARTIFACT reason: the files the scan could not read, the folders it could not list
|
|
21
|
+
* and where it stopped, each in its own sentence. Undefined when the scan read everything.
|
|
22
|
+
*/
|
|
23
|
+
export declare function unscannedMessage(unscanned: UnscannedArtifacts): string | undefined;
|
|
10
24
|
export declare function missingLocalEvidenceArtifacts(runPaths: PreparedRunArtifactPaths, bundle: RunBundle): Promise<string[]>;
|
|
11
25
|
/**
|
|
12
26
|
* The PNG frames that actor traces register as stream screenshots, relative to the run root.
|
|
@@ -50,9 +64,10 @@ interface RegisteredStreamMedia {
|
|
|
50
64
|
/**
|
|
51
65
|
* Scans every run file for secret and path patterns and returns the findings. A file that is not
|
|
52
66
|
* registered stream media and that the scan cannot read as text (readPlainText), cannot read at
|
|
53
|
-
* all, or that holds more than RUN_ARTIFACT_MAX_BYTES goes to `unscanned`, so the caller can
|
|
54
|
-
* the run from grading share_ready.
|
|
55
|
-
*
|
|
67
|
+
* all, or that holds more than RUN_ARTIFACT_MAX_BYTES goes to `unscanned.files`, so the caller can
|
|
68
|
+
* keep the run from grading share_ready. A directory it cannot list goes to
|
|
69
|
+
* `unscanned.unlistedFolders`, and the directory it was listing when it passed MAX_RUN_ENTRIES
|
|
70
|
+
* entries is `unscanned.stoppedIn`.
|
|
56
71
|
*/
|
|
57
|
-
export declare function scanRunPublicSafetyArtifacts(runPaths: PreparedRunArtifactPaths, derivedFindings: string[], media: RegisteredStreamMedia, unscanned:
|
|
72
|
+
export declare function scanRunPublicSafetyArtifacts(runPaths: PreparedRunArtifactPaths, derivedFindings: string[], media: RegisteredStreamMedia, unscanned: UnscannedArtifacts): Promise<string[]>;
|
|
58
73
|
export {};
|
package/dist/verify/artifacts.js
CHANGED
|
@@ -12,6 +12,7 @@ import { SANDBOX_RECEIPTS_ARTIFACT } from "../run/sandbox-receipts.js";
|
|
|
12
12
|
import { readSafeRunArtifact, readSafeRunArtifactBytes, readSafeRunArtifactJson, } from "../run/locate.js";
|
|
13
13
|
import { isRecord } from "../run/type-guards.js";
|
|
14
14
|
import { TERMINAL_EVENTS_ARTIFACT } from "../run/terminal-contract.js";
|
|
15
|
+
import { plural } from "../run/text.js";
|
|
15
16
|
import { isZeroEventTerminalTrace } from "./actor.js";
|
|
16
17
|
/** Public-safety and evidence-reference findings stop at this many per list. */
|
|
17
18
|
export const MAX_REPORTED_FINDINGS = 50;
|
|
@@ -20,6 +21,35 @@ export const MAX_REPORTED_FINDINGS = 50;
|
|
|
20
21
|
* refuses a run with more, so the scan never passes a run whose files it did not all read.
|
|
21
22
|
*/
|
|
22
23
|
export const MAX_RUN_ENTRIES = 10_000;
|
|
24
|
+
// The UNSCANNED_ARTIFACT reason names this many paths of each kind and counts the rest.
|
|
25
|
+
const MAX_LISTED_UNSCANNED = 10;
|
|
26
|
+
/**
|
|
27
|
+
* The UNSCANNED_ARTIFACT reason: the files the scan could not read, the folders it could not list
|
|
28
|
+
* and where it stopped, each in its own sentence. Undefined when the scan read everything.
|
|
29
|
+
*/
|
|
30
|
+
export function unscannedMessage(unscanned) {
|
|
31
|
+
const files = [...new Set(unscanned.files)].sort();
|
|
32
|
+
const folders = [...new Set(unscanned.unlistedFolders)].sort().map(folderName);
|
|
33
|
+
const sentences = [];
|
|
34
|
+
if (files.length > 0)
|
|
35
|
+
sentences.push(`The public-safety scan cannot read ${plural(files.length, "file")} as text that ${files.length === 1 ? "is not a stream screenshot under screenshots/ or a registered recording" : "are not stream screenshots under screenshots/ or registered recordings"}: ${listed(files)}.`);
|
|
36
|
+
if (folders.length > 0)
|
|
37
|
+
sentences.push(`The public-safety scan could not list ${plural(folders.length, "folder")}, so it read none of the files in ${folders.length === 1 ? "it" : "them"}: ${listed(folders)}.`);
|
|
38
|
+
if (unscanned.stoppedIn !== undefined)
|
|
39
|
+
sentences.push(`The public-safety scan stopped at its limit of ${MAX_RUN_ENTRIES} files and folders while listing ${folderName(unscanned.stoppedIn)}, so it read nothing listed after that point.`);
|
|
40
|
+
if (sentences.length === 0)
|
|
41
|
+
return undefined;
|
|
42
|
+
const onlyFiles = folders.length === 0 && unscanned.stoppedIn === undefined;
|
|
43
|
+
sentences.push(onlyFiles ? "Review them before sharing." : "Review the run folder before sharing.");
|
|
44
|
+
return sentences.join(" ");
|
|
45
|
+
}
|
|
46
|
+
function folderName(relativeDirectory) {
|
|
47
|
+
return relativeDirectory === "" ? "the run folder" : `${relativeDirectory}/`;
|
|
48
|
+
}
|
|
49
|
+
function listed(paths) {
|
|
50
|
+
const more = paths.length - MAX_LISTED_UNSCANNED;
|
|
51
|
+
return `${paths.slice(0, MAX_LISTED_UNSCANNED).join(", ")}${more > 0 ? ` and ${more} more` : ""}`;
|
|
52
|
+
}
|
|
23
53
|
export async function missingLocalEvidenceArtifacts(runPaths, bundle) {
|
|
24
54
|
const recordings = new Map(bundle.streams.flatMap((stream) => stream.recording ? [[stream.recording.path, stream.recording]] : []));
|
|
25
55
|
const requiredPaths = new Map();
|
|
@@ -291,9 +321,10 @@ export function rawScreenshotStreamIds(bundle, redactedShapeFrames) {
|
|
|
291
321
|
/**
|
|
292
322
|
* Scans every run file for secret and path patterns and returns the findings. A file that is not
|
|
293
323
|
* registered stream media and that the scan cannot read as text (readPlainText), cannot read at
|
|
294
|
-
* all, or that holds more than RUN_ARTIFACT_MAX_BYTES goes to `unscanned`, so the caller can
|
|
295
|
-
* the run from grading share_ready.
|
|
296
|
-
*
|
|
324
|
+
* all, or that holds more than RUN_ARTIFACT_MAX_BYTES goes to `unscanned.files`, so the caller can
|
|
325
|
+
* keep the run from grading share_ready. A directory it cannot list goes to
|
|
326
|
+
* `unscanned.unlistedFolders`, and the directory it was listing when it passed MAX_RUN_ENTRIES
|
|
327
|
+
* entries is `unscanned.stoppedIn`.
|
|
297
328
|
*/
|
|
298
329
|
export async function scanRunPublicSafetyArtifacts(runPaths, derivedFindings, media, unscanned) {
|
|
299
330
|
const findings = [];
|
|
@@ -315,14 +346,14 @@ async function scanRunPublicSafetyDirectory(runPaths, relativeDirectory, finding
|
|
|
315
346
|
: runPaths.physicalRunRoot;
|
|
316
347
|
const entries = await readdir(current).catch(() => null);
|
|
317
348
|
if (entries === null) {
|
|
318
|
-
unscanned.push(
|
|
349
|
+
unscanned.unlistedFolders.push(relativeDirectory);
|
|
319
350
|
return;
|
|
320
351
|
}
|
|
321
352
|
for (const entryName of entries) {
|
|
322
353
|
walk.entries += 1;
|
|
323
354
|
if (walk.entries > MAX_RUN_ENTRIES) {
|
|
324
|
-
|
|
325
|
-
|
|
355
|
+
// Every directory the walk returns through comes here once more; the first one stopped it.
|
|
356
|
+
unscanned.stoppedIn ??= relativeDirectory;
|
|
326
357
|
return;
|
|
327
358
|
}
|
|
328
359
|
const relativePath = relativeDirectory ? `${relativeDirectory}/${entryName}` : entryName;
|
|
@@ -371,7 +402,7 @@ async function scanRunPublicSafetyDirectory(runPaths, relativeDirectory, finding
|
|
|
371
402
|
const bytes = await readSafeRunArtifactBytes(runPaths, relativePath).catch(() => null);
|
|
372
403
|
const decoded = bytes === null ? undefined : readPlainText(bytes);
|
|
373
404
|
if (decoded === undefined || !decoded.ok) {
|
|
374
|
-
unscanned.push(relativePath);
|
|
405
|
+
unscanned.files.push(relativePath);
|
|
375
406
|
continue;
|
|
376
407
|
}
|
|
377
408
|
// A receipt's id in any other file, or a raw id at a sandbox-id key, which an older run or
|
|
@@ -389,7 +420,7 @@ async function scanRunPublicSafetyDirectory(runPaths, relativeDirectory, finding
|
|
|
389
420
|
selectedFindings.push(`sensitive text ${relativePath}`);
|
|
390
421
|
}
|
|
391
422
|
else if (scan.opaque) {
|
|
392
|
-
unscanned.push(relativePath);
|
|
423
|
+
unscanned.files.push(relativePath);
|
|
393
424
|
}
|
|
394
425
|
}
|
|
395
426
|
}
|
package/dist/verify/verify.js
CHANGED
|
@@ -13,18 +13,15 @@ import { SANDBOX_RECEIPTS_ARTIFACT } from "../run/sandbox-receipts.js";
|
|
|
13
13
|
import { isRecord } from "../run/type-guards.js";
|
|
14
14
|
import { runNotFoundMessage } from "../run/run-not-found.js";
|
|
15
15
|
import { actorVerdictConsistencyFindings, noEngagementActorFindings, validateCodexAppServerEvidence, validateTerminalProductEvidence, } from "./actor.js";
|
|
16
|
-
import { invalidRunEvidenceReferences, MAX_REPORTED_FINDINGS, missingLocalEvidenceArtifacts, rawScreenshotPostureWarnings, rawScreenshotStreamIds, redactedShapeFramePaths, scanRunPublicSafetyArtifacts, streamScreenshotPaths, } from "./artifacts.js";
|
|
16
|
+
import { invalidRunEvidenceReferences, MAX_REPORTED_FINDINGS, missingLocalEvidenceArtifacts, rawScreenshotPostureWarnings, rawScreenshotStreamIds, redactedShapeFramePaths, scanRunPublicSafetyArtifacts, streamScreenshotPaths, unscannedMessage, } from "./artifacts.js";
|
|
17
17
|
import { flatContextWarnings } from "./context-growth.js";
|
|
18
18
|
import { runNotFinished } from "./liveness.js";
|
|
19
19
|
import { costAndReceiptFindings } from "./costs.js";
|
|
20
20
|
import { rerunLineageFindings } from "./rerun.js";
|
|
21
21
|
import { sharedWorldEvidenceFindings } from "./shared-world.js";
|
|
22
22
|
import { subjectStateFindings, undeclaredSubjectStateWarnings } from "./subject.js";
|
|
23
|
-
import { plural } from "../run/text.js";
|
|
24
23
|
import { cli } from "../cli/invocation.js";
|
|
25
24
|
export const VERIFY_SCHEMA = "humanish.verify-result.v1";
|
|
26
|
-
// The UNSCANNED_ARTIFACT reason names this many paths and counts the rest.
|
|
27
|
-
const MAX_LISTED_UNSCANNED = 10;
|
|
28
25
|
export async function verifyRun(cwdInput, runInput) {
|
|
29
26
|
const cwd = path.resolve(cwdInput);
|
|
30
27
|
let runPaths;
|
|
@@ -86,7 +83,7 @@ export async function verifyResolvedRun(cwd, runInput, runPaths) {
|
|
|
86
83
|
};
|
|
87
84
|
checks.push(...bundlePresenceChecks(bundleRead, review));
|
|
88
85
|
const derivedPublicSafetyFindings = [];
|
|
89
|
-
const unscannedArtifacts = [];
|
|
86
|
+
const unscannedArtifacts = { files: [], unlistedFolders: [] };
|
|
90
87
|
const sandboxIdFiles = [];
|
|
91
88
|
let journaledSandboxIds = [];
|
|
92
89
|
try {
|
|
@@ -96,7 +93,7 @@ export async function verifyResolvedRun(cwd, runInput, runPaths) {
|
|
|
96
93
|
if (!(error instanceof ContainedReadRefusedError))
|
|
97
94
|
throw error;
|
|
98
95
|
// Without the journal's ids, verify cannot clear any file of naming one.
|
|
99
|
-
unscannedArtifacts.push(SANDBOX_RECEIPTS_ARTIFACT);
|
|
96
|
+
unscannedArtifacts.files.push(SANDBOX_RECEIPTS_ARTIFACT);
|
|
100
97
|
}
|
|
101
98
|
const publicSafetyFindings = await scanRunPublicSafetyArtifacts(runPaths, derivedPublicSafetyFindings, {
|
|
102
99
|
recordingPaths: new Set(isRunBundle(bundle)
|
|
@@ -397,15 +394,9 @@ function buildShareSafety(args) {
|
|
|
397
394
|
message: "Continuous screen/audio recordings are retained for local review. Screenshot redaction does not redact this media.",
|
|
398
395
|
});
|
|
399
396
|
}
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
const more = paths.length - MAX_LISTED_UNSCANNED;
|
|
404
|
-
reasons.push({
|
|
405
|
-
code: "UNSCANNED_ARTIFACT",
|
|
406
|
-
message: `The public-safety scan cannot read ${plural(paths.length, "file")} as text that ${paths.length === 1 ? "is not a stream screenshot under screenshots/ or a registered recording" : "are not stream screenshots under screenshots/ or registered recordings"}: ${shown}${more > 0 ? ` and ${more} more` : ""}. Review them before sharing.`,
|
|
407
|
-
});
|
|
408
|
-
}
|
|
397
|
+
const unscanned = unscannedMessage(args.unscannedArtifacts);
|
|
398
|
+
if (unscanned !== undefined)
|
|
399
|
+
reasons.push({ code: "UNSCANNED_ARTIFACT", message: unscanned });
|
|
409
400
|
// Runs from 0.110 keep raw sandbox ids only in sandbox-receipts.ndjson. An earlier run records
|
|
410
401
|
// them in run.json, and any file that names one is not share-ready as it is.
|
|
411
402
|
const rawResources = (args.bundle.providerResources ?? []).some((resource) => resource.id !== REDACTED_SANDBOX_ID);
|
|
@@ -305,15 +305,16 @@ exactly; this hosted-desktop rule does not change that contract.
|
|
|
305
305
|
`subject` is an optional, additive top-level field: structured provenance for
|
|
306
306
|
what the computer-use, shared-world or scripted route actually drove (code pin plus state story). It
|
|
307
307
|
is absent on pre-existing bundles and on bundles from routes that have not
|
|
308
|
-
adopted it. The field shape, its
|
|
309
|
-
`local-tree`, `desktop-cli`), and the `humanish verify` checks that guard it are the
|
|
310
|
-
job, not this one: see the `subject` entry under
|
|
308
|
+
adopted it. The field shape, its five sources (`clone`, `app-url`,
|
|
309
|
+
`local-tree`, `desktop-cli`, `local-app`), and the `humanish verify` checks that guard it are the
|
|
310
|
+
schema doc's job, not this one: see the `subject` entry under
|
|
311
311
|
[`schemas.md`](schemas.md#run-bundle). In short, `clone` carries a
|
|
312
312
|
`repo`/`commit` pin, `local-tree` carries an `archiveSha256`/`dirty` pin
|
|
313
313
|
instead (a dirty working tree cannot be commit-pinned), `desktop-cli` carries the
|
|
314
|
-
declared `product` name, and `app-url`
|
|
315
|
-
of a desktop-cli study saved before
|
|
316
|
-
|
|
314
|
+
declared `product` name, and `app-url` and `local-app` (a running app the caller drives in
|
|
315
|
+
process) carry no code pin at all. A computer-use run of a desktop-cli study saved before
|
|
316
|
+
`desktop-cli` was recorded, or of a local-app study saved before `local-app` was recorded, says
|
|
317
|
+
`app-url`, and readers accept it. No path, basename, or other host-machine string ever
|
|
317
318
|
enters this field; identity is digests, a sha, a boolean, counts and the declared product name.
|
|
318
319
|
|
|
319
320
|
## Cost Estimate (advisory)
|
|
@@ -183,7 +183,7 @@ The rest of this section names keys by their v3 spelling:
|
|
|
183
183
|
owns the live sandbox, auth, cap, evidence, and cleanup lifecycle.
|
|
184
184
|
A `participants` count is the preview route's participant count (simCount),
|
|
185
185
|
and on the computer-use **E2B** route the homogeneous fan-out count (N
|
|
186
|
-
identical participants, each its own E2B desktop: separate worlds,
|
|
186
|
+
identical participants, each its own E2B desktop: separate worlds, at most 100).
|
|
187
187
|
The scripted route's `surfaces` is its surface list (`[desktop]`, the default,
|
|
188
188
|
or `[desktop, mobile]`). The in-process/local-app computer-use route stays
|
|
189
189
|
single participant (no E2B to fan out);
|
|
@@ -197,7 +197,8 @@ The rest of this section names keys by their v3 spelling:
|
|
|
197
197
|
so `--count` does not apply to it, and each entry's `instruction` is the steer;
|
|
198
198
|
an entry's `device` is XOR with a raw `execution.desktop.resolution`. Participant ids
|
|
199
199
|
default `lane-01`..`lane-NN` (`role-01`..`role-NN` for shared-world participants), must be unique, and name per-participant evidence paths
|
|
200
|
-
(`actors/<streamId>.json`, `screenshots/<laneId>/`).
|
|
200
|
+
(`actors/<streamId>.json`, `screenshots/<laneId>/`). At most 100 participants, on the
|
|
201
|
+
shared-world route too. The other
|
|
201
202
|
routes refuse a list (table above). `subject.clone.fanout` is rejected on the
|
|
202
203
|
computer-use, shared-world and scripted routes (declare fan-out with
|
|
203
204
|
`participants`). No current route reads `clone.fanout`;
|
|
@@ -216,9 +217,12 @@ The rest of this section names keys by their v3 spelling:
|
|
|
216
217
|
shape. Ids are checked for collisions after expansion;
|
|
217
218
|
- `execution.concurrency` (computer-use E2B routes, including shared-world): a
|
|
218
219
|
cap on participants in flight at once. When omitted, every participant runs
|
|
219
|
-
simultaneously:
|
|
220
|
-
after any `--count` override
|
|
221
|
-
|
|
220
|
+
simultaneously: both routes resolve it when they plan, from the final participant count
|
|
221
|
+
after any `--count` override. On hosted desktops the plan runs at most
|
|
222
|
+
`HUMANISH_E2B_MAX_CONCURRENT_SANDBOXES` sandboxes at once (20 when unset; a provisioned
|
|
223
|
+
shared world's app sandbox counts against it), so a larger roster runs in waves and the plan
|
|
224
|
+
records a warning naming the setting. A declared value above that limit is refused at plan
|
|
225
|
+
time. Total sessions and spend are identical either way; only wall-clock
|
|
222
226
|
and simultaneity differ. Declaring a value below the participant count runs participants in
|
|
223
227
|
waves and emits a warning saying so, because a green waved run is otherwise
|
|
224
228
|
indistinguishable from the all-live run the author meant. Shared-world studies
|
|
@@ -529,7 +533,7 @@ Core-owned fields:
|
|
|
529
533
|
an older run, else `inferLegacyStudyId`, which reads the `persona.source` or `scenario.source`
|
|
530
534
|
convention `study:<id>`, or `lab:<id>` from runs written before 0.108, and nothing else.
|
|
531
535
|
- `subject` (optional, additive): structured subject provenance,
|
|
532
|
-
`{ source: clone | app-url | local-tree | desktop-cli, repo?, product?, commit?,
|
|
536
|
+
`{ source: clone | app-url | local-tree | desktop-cli | local-app, repo?, product?, commit?,
|
|
533
537
|
archiveSha256?, dirty?, envNames?, state }` where `state` is `{ provenance: seeded |
|
|
534
538
|
unpinned | declared-not-run | undeclared | external-public, seed?: [{ name, when,
|
|
535
539
|
commandDigest, ok?, exitCode?, timedOut?, durationMs? }], externalEnvNames?
|
|
@@ -541,7 +545,9 @@ Core-owned fields:
|
|
|
541
545
|
the archive content digest stands in for it. `product` is the desktop-cli field: the
|
|
542
546
|
`subject.product.name` the study declares, with state `undeclared`. A computer-use
|
|
543
547
|
run of a desktop-cli study saved before this field existed records `app-url` with no
|
|
544
|
-
`product`. `
|
|
548
|
+
`product`. `local-app` is an in-process run of a local-app study, the caller's running app
|
|
549
|
+
with no code pin and state `undeclared`; one saved before that value existed records
|
|
550
|
+
`app-url`. `commandDigest` is the
|
|
545
551
|
sha256-16 of the exact seed command; command text and env values never
|
|
546
552
|
appear. `humanish verify` fails closed when a live `local-tree` bundle carries
|
|
547
553
|
no well-formed `archiveSha256`, in addition to the existing `subject state
|
|
@@ -1676,7 +1682,7 @@ of it fails.
|
|
|
1676
1682
|
| `cuaLaneCount` | the `plan` `StudyEvent`, which lists every participant |
|
|
1677
1683
|
| `resolveSeatUrl` | `parseStudy`, which refuses a participant entry that is not same-origin loopback; build the URL with `new URL(entry, serveUrl)` |
|
|
1678
1684
|
| `cuaLaneValidationReason`, `sharedWorldValidationReason`, `concurrentSharedWorldValidationReason`, `externalPublicSharedWorldValidationReason` | `parseStudy`, which returns the same reason |
|
|
1679
|
-
| `MAX_CUA_LANES` | none; `parseStudy` refuses a roster of more than
|
|
1685
|
+
| `MAX_CUA_LANES` | none; `parseStudy` refuses a roster of more than 100 participants |
|
|
1680
1686
|
|
|
1681
1687
|
## Product-Adapter Extension Seam
|
|
1682
1688
|
|
|
@@ -27,8 +27,8 @@ The default provider is `openai`; its default model is `gpt-6-astra`, with high
|
|
|
27
27
|
captures to OpenAI, without tools, redirects, provider-side response storage, or
|
|
28
28
|
automatic retries. `--question` adds a reviewer question; it never changes the
|
|
29
29
|
participant assignment. For OpenAI API analysis, `--max-cost` is required, including for dry-run
|
|
30
|
-
admission. Admission
|
|
31
|
-
provider bill is not limited by it.
|
|
30
|
+
admission. Admission refuses the request when both its worst case and its expected cost plus a
|
|
31
|
+
10% margin are over it; the provider bill is not limited by it.
|
|
32
32
|
`--timeout-ms` and `--max-output-tokens` bound the request. A bill above the
|
|
33
33
|
worst case or the cap retains valid findings and usage but returns a partial
|
|
34
34
|
result and a nonzero command exit, including when that version is reused.
|
|
@@ -96,7 +96,7 @@ silently drifting from one is not.
|
|
|
96
96
|
| Loopback entry URLs | Public-safety: never drive third-party sites unbidden | `policies.allowPublicTargets` for an owner-declared deployment/preview (a Vercel preview of your own app). Multi-participant public/preview fan-out needs explicit `participants[].target` for every participant, so the adapter-owned topology is declared rather than inferred. Provisioned clone subjects always serve in-sandbox on loopback |
|
|
97
97
|
| Full-fidelity screenshots, local | The common case is watching a run of your own app locally; blur destroys the deliverable. Raw frames live in gitignored `.humanish/` (this repo's CI adds a binary-asset commit scan; downstream projects rely on the scaffolded `.gitignore` and their own review) | `policies.redactScreenshots: true` blurs at capture; `export --format bundle --redact-screenshots` produces a separate verified copy of a completed run |
|
|
98
98
|
| Synthetic, seeded state | Pinned provenance; no real user data in evidence paths | Declared external state, recorded as unpinned in provenance |
|
|
99
|
-
| Single participant | Cost + evidence simplicity | Declared fan-out where the route supports it: `participants: N` (homogeneous), an explicit `participants` list (differentiated persona/device/instruction), or list entries with a `count` that expand into one entry per participant on the computer-use E2B route (separate worlds,
|
|
99
|
+
| Single participant | Cost + evidence simplicity | Declared fan-out where the route supports it: `participants: N` (homogeneous), an explicit `participants` list (differentiated persona/device/instruction), or list entries with a `count` that expand into one entry per participant on the computer-use E2B route (separate worlds, at most 100; `execution.concurrency` and the E2B plan bound concurrent paid participants) |
|
|
100
100
|
| Stock `desktop` template | The stock E2B desktop image is right for most subjects; absent `execution.desktop.template` keeps `Sandbox.create(opts)` byte-stable | `execution.desktop.template` names a custom E2B desktop image (any name/id, no allowlist) for a subject needing baked-in runtimes the stock image lacks (e.g. node/bun/a local Postgres); it is threaded to `Sandbox.create(template, opts)` on every desktop-creating route and recorded in the bundle as `desktopTemplate` (public-safe) |
|
|
101
101
|
| Desktop default URL opener | Preserve the route/image's historical browser/default opener behavior when unset | `execution.desktop.browser` set to `chrome`, `chromium` or `firefox` makes the hosted CUA/shared-world browser choice explicit, fail-closed, and recorded as `desktopBrowser` in the run bundle |
|
|
102
102
|
| Loopback-only observer serving | Run bundles are local by default (public-safety plank 3); the serve/watch surfaces bind `127.0.0.1` unconditionally, and exposing them is publishing-adjacent, so exposure is fail-closed and carries declared friction | `humanish observe --all --expose` behind tunnel-edge auth (ngrok `--oauth google` with `--allow-email`/`--allow-domain` allow rules, or an operator `--public-url` you secure), or `--expose --safe` (open, gated on verify-`share_ready` runs only). `watch --expose` streams a live run behind edge auth (a live run is never `share_ready`, so `--safe` alone is refused). humanish carries no in-process auth; both print computed warnings naming exactly what became reachable |
|
|
@@ -23,12 +23,13 @@ review:
|
|
|
23
23
|
Omitting `review.analysis` uses these defaults. Set `review.analysis: false` to
|
|
24
24
|
run participants without the additional analysis request. An explicit analysis
|
|
25
25
|
mapping using the default OpenAI API provider requires `maxCostUsd`. Admission
|
|
26
|
-
|
|
27
|
-
provider's final bill, and it is separate from participant spending limits. The
|
|
26
|
+
refuses the request when both its worst case and its expected cost plus a 10% margin are over
|
|
27
|
+
it; it does not limit the provider's final bill, and it is separate from participant spending limits. The
|
|
28
28
|
estimate gives two numbers: the expected cost, and the worst case if the analyst
|
|
29
29
|
spends its whole output allowance. `humanish study check` gives the expected cost
|
|
30
30
|
range for the study's participant count, from a run that keeps no evidence to one
|
|
31
|
-
|
|
31
|
+
that keeps the most evidence admission admits under the cap, at most the evidence limits. When
|
|
32
|
+
the cap admits no analysis at all, it says so instead. Use `analyze --dry-run --max-cost <usd>` on retained
|
|
32
33
|
evidence to inspect the expected cost, the worst case and the selected token
|
|
33
34
|
allowance before deliberately choosing a larger budget. Explicit
|
|
34
35
|
`maxOutputTokens` and `--max-output-tokens` limits are honored exactly. The output allowance
|