humanish 0.115.0 → 0.116.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +22 -0
- package/README.md +7 -6
- package/TELEMETRY.md +6 -0
- package/dist/actors/codex/app-server-ui.js +11 -4
- package/dist/actors/computer-use/loop/debrief.js +61 -27
- package/dist/actors/scripted-browser/steps.js +6 -5
- package/dist/analysis/analysis-limits.d.ts +35 -0
- package/dist/analysis/analysis-limits.js +41 -0
- package/dist/analysis/validation.js +14 -21
- package/dist/cli/commands/analyze.js +2 -1
- package/dist/cli/commands/notes.js +2 -1
- package/dist/cli/commands/run-signals.js +5 -1
- package/dist/cli/commands/runs.js +2 -1
- package/dist/cli/commands/setup.js +3 -3
- package/dist/cli/commands/study-format.d.ts +5 -2
- package/dist/cli/commands/study-format.js +29 -9
- package/dist/cli/commands/study-route-computer-use.js +2 -2
- package/dist/cli/commands/study-route-scripted.js +1 -1
- package/dist/cli/commands/tui.js +59 -17
- package/dist/cli/doctor.js +9 -2
- package/dist/cli/findings-text.d.ts +5 -0
- package/dist/cli/findings-text.js +169 -0
- package/dist/cli/findings.d.ts +0 -4
- package/dist/cli/findings.js +5 -167
- package/dist/cli/io.js +6 -1
- package/dist/cli/update-check.js +4 -1
- package/dist/comms/connections.js +4 -3
- package/dist/comms/setup.js +18 -12
- package/dist/evidence/redaction.js +9 -1
- package/dist/evidence/secret-scrub.d.ts +15 -5
- package/dist/evidence/secret-scrub.js +67 -9
- package/dist/feedback/draft.js +1 -1
- package/dist/feedback/export-bundle.js +18 -14
- package/dist/feedback/feedback.js +25 -7
- package/dist/guest/desktop-keys.js +59 -5
- package/dist/keys/key-resolution.d.ts +9 -0
- package/dist/keys/key-resolution.js +60 -48
- package/dist/keys/key-status.d.ts +2 -0
- package/dist/keys/key-status.js +9 -16
- package/dist/observer/pinned-files.js +1 -1
- package/dist/observer/render.js +3 -2
- package/dist/observer-app.html +7 -7
- package/dist/routes/computer-use/bundle-parts.d.ts +4 -4
- package/dist/routes/computer-use/bundle-parts.js +2 -2
- package/dist/routes/computer-use/bundle.d.ts +4 -5
- package/dist/routes/computer-use/bundle.js +1 -3
- package/dist/routes/computer-use/fanout-records.js +10 -11
- package/dist/routes/computer-use/live-phase.d.ts +1 -1
- package/dist/routes/computer-use/live-phase.js +0 -1
- package/dist/routes/computer-use/participant-execution.d.ts +3 -2
- package/dist/routes/computer-use/participant-runs.d.ts +6 -2
- package/dist/routes/computer-use/participant-runs.js +9 -5
- package/dist/routes/computer-use/plan.d.ts +3 -0
- package/dist/routes/computer-use/plan.js +14 -11
- package/dist/routes/computer-use/rerun-selection.js +2 -2
- package/dist/routes/computer-use/result.js +3 -6
- package/dist/routes/computer-use/setup.d.ts +4 -6
- package/dist/routes/computer-use/setup.js +2 -6
- package/dist/routes/computer-use/single-bundle.d.ts +4 -5
- package/dist/routes/computer-use/single-bundle.js +22 -25
- package/dist/routes/computer-use/subject-projection.d.ts +4 -8
- package/dist/routes/computer-use/subject-projection.js +9 -31
- package/dist/routes/computer-use/types.d.ts +4 -41
- package/dist/routes/scripted/scenario.js +28 -12
- package/dist/routes/scripted/session-result.js +6 -3
- package/dist/routes/shared-world/bundle-records.js +2 -8
- package/dist/routes/shared-world/participant-specs.d.ts +0 -2
- package/dist/routes/shared-world/participant-specs.js +1 -20
- package/dist/routes/shared-world/plan.d.ts +3 -0
- package/dist/routes/shared-world/plan.js +23 -3
- package/dist/routes/shared-world/setup.js +2 -2
- package/dist/routes/terminal/ledger.d.ts +9 -9
- package/dist/routes/terminal/ledger.js +63 -40
- package/dist/routes/terminal/live-finish.js +7 -10
- package/dist/routes/terminal/plan.d.ts +3 -0
- package/dist/routes/terminal/plan.js +8 -5
- package/dist/routes/terminal/token-usage.d.ts +1 -2
- package/dist/routes/terminal/token-usage.js +4 -6
- package/dist/routes/terminal/types.d.ts +9 -1
- package/dist/run/bundle-shape.js +6 -1
- package/dist/run/bundle.d.ts +5 -3
- package/dist/run/contained-output.d.ts +53 -5
- package/dist/run/contained-output.js +141 -42
- package/dist/run/detail.d.ts +2 -1
- package/dist/run/detail.js +12 -7
- package/dist/run/escape-sequences.d.ts +11 -0
- package/dist/run/escape-sequences.js +93 -0
- package/dist/run/evidence-files.d.ts +8 -8
- package/dist/run/evidence-files.js +19 -23
- package/dist/run/locate.d.ts +32 -3
- package/dist/run/locate.js +45 -40
- package/dist/run/notes.d.ts +0 -8
- package/dist/run/notes.js +5 -51
- package/dist/run/outcomes.d.ts +24 -0
- package/dist/run/outcomes.js +19 -0
- package/dist/run/paths.d.ts +2 -0
- package/dist/run/paths.js +5 -1
- package/dist/run/preflight-receipts.js +13 -8
- package/dist/run/project.js +7 -4
- package/dist/run/reclaim-outcomes.d.ts +8 -6
- package/dist/run/reclaim-outcomes.js +10 -6
- package/dist/run/reclaim.d.ts +5 -0
- package/dist/run/reclaim.js +43 -16
- package/dist/run/run-clock.d.ts +50 -0
- package/dist/run/run-clock.js +66 -0
- package/dist/run/run.js +5 -2
- package/dist/run/sandbox-ids.d.ts +7 -2
- package/dist/run/sandbox-ids.js +27 -11
- package/dist/run/secrets.d.ts +2 -1
- package/dist/run/secrets.js +18 -22
- package/dist/run/stored-runs.js +14 -7
- package/dist/run/transient-comms-secrets.js +6 -17
- package/dist/run-study.js +22 -13
- package/dist/study/adapter-scorer-loader.js +6 -4
- package/dist/study/discover.js +9 -6
- package/dist/study/init-paths.js +7 -5
- package/dist/study/persona-resolve.d.ts +2 -2
- package/dist/study/persona-resolve.js +9 -5
- package/dist/study/plan-types.d.ts +2 -2
- package/dist/study/plan.d.ts +5 -3
- package/dist/study/plan.js +18 -9
- package/dist/study/preflight-probes.js +11 -3
- package/dist/study/preflight.js +9 -5
- package/dist/study/summary.d.ts +8 -2
- package/dist/study/summary.js +20 -22
- package/dist/study/warnings.d.ts +19 -12
- package/dist/study/warnings.js +38 -17
- package/dist/substrates/e2b/connection.d.ts +12 -0
- package/dist/substrates/e2b/connection.js +35 -0
- package/dist/substrates/e2b/desktop-executor.d.ts +1 -3
- package/dist/substrates/e2b/desktop-executor.js +42 -9
- package/dist/substrates/e2b/lifetime.d.ts +24 -4
- package/dist/substrates/e2b/lifetime.js +41 -4
- package/dist/substrates/e2b/sandbox.d.ts +4 -3
- package/dist/substrates/e2b/sandbox.js +13 -18
- package/dist/substrates/e2b/sdk.d.ts +20 -9
- package/dist/substrates/local/capacity.d.ts +18 -2
- package/dist/substrates/local/capacity.js +21 -2
- package/dist/substrates/local/runtime-host.js +21 -14
- package/dist/substrates/local/runtime.js +3 -11
- package/dist/tui/contract.d.ts +30 -5
- package/dist/tui/contract.js +4 -1
- package/dist/tui-app.js +149 -147
- package/dist/verify/artifacts.d.ts +9 -2
- package/dist/verify/artifacts.js +37 -13
- package/dist/verify/liveness.js +19 -5
- package/dist/verify/verify.js +54 -27
- package/docs/architecture/guest-desktop.md +7 -2
- package/docs/architecture/state-driven-executor.md +4 -2
- package/docs/contracts/run-bundle.md +18 -10
- package/docs/contracts/schemas.md +27 -18
- package/docs/contracts/study-analysis.md +10 -1
- package/docs/product/automatic-analysis.md +7 -1
- package/package.json +2 -2
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
//
|
|
2
|
-
// and what the participants ran
|
|
3
|
-
// bundle
|
|
1
|
+
// How a computer-use run records its subject: each participant's subject and the subject-state
|
|
2
|
+
// marker, from the declared subject and what the participants ran, and the run-level subject
|
|
3
|
+
// aggregated from them. Both bundle shapes and the JSON result write what this module returns.
|
|
4
4
|
import { commandDigestOf } from "../../subject/state.js";
|
|
5
5
|
import { cuaDeclaredState } from "./plan.js";
|
|
6
6
|
import {} from "../../run/bundle.js";
|
|
@@ -25,7 +25,7 @@ export function projectParticipantSubjects(args) {
|
|
|
25
25
|
executed: outcome?.stateStepRecords ?? [],
|
|
26
26
|
});
|
|
27
27
|
return participantSubjectProjection({
|
|
28
|
-
|
|
28
|
+
subject,
|
|
29
29
|
...(publicRepo === undefined ? {} : { publicRepo }),
|
|
30
30
|
subjectEnvNames,
|
|
31
31
|
...(outcome?.subjectCommit === undefined ? {} : { subjectCommit: outcome.subjectCommit }),
|
|
@@ -79,7 +79,7 @@ export function resolveSubjectState(args) {
|
|
|
79
79
|
* commit/dirty (no divergence is possible, unlike the clone route's per-participant
|
|
80
80
|
* in-sandbox commit). */
|
|
81
81
|
function participantSubjectProjection(args) {
|
|
82
|
-
if (args.kind === "clone" && args.publicRepo) {
|
|
82
|
+
if (args.subject.kind === "clone" && args.publicRepo) {
|
|
83
83
|
return {
|
|
84
84
|
source: "clone",
|
|
85
85
|
repo: args.publicRepo,
|
|
@@ -88,7 +88,7 @@ function participantSubjectProjection(args) {
|
|
|
88
88
|
state: args.subjectState,
|
|
89
89
|
};
|
|
90
90
|
}
|
|
91
|
-
if (args.kind === "local-tree") {
|
|
91
|
+
if (args.subject.kind === "local-tree") {
|
|
92
92
|
const archive = args.localTreeArchive;
|
|
93
93
|
return {
|
|
94
94
|
source: "local-tree",
|
|
@@ -100,32 +100,10 @@ function participantSubjectProjection(args) {
|
|
|
100
100
|
state: args.subjectState,
|
|
101
101
|
};
|
|
102
102
|
}
|
|
103
|
-
|
|
104
|
-
}
|
|
105
|
-
/** Narrow a resolved CuaSubjectProjection into the shape buildSingleParticipantBundle's
|
|
106
|
-
* subjectProvenance param wants (provisioned-route sources only; app-url stays undeclared, the
|
|
107
|
-
* default branch that builder already handles). */
|
|
108
|
-
export function subjectProvenanceArg(subject, publicRepo, subjectEnvNames) {
|
|
109
|
-
if (subject.source === "clone" && publicRepo) {
|
|
110
|
-
return {
|
|
111
|
-
source: "clone",
|
|
112
|
-
repo: publicRepo,
|
|
113
|
-
...(subject.commit === undefined ? {} : { commit: subject.commit }),
|
|
114
|
-
envNames: subjectEnvNames,
|
|
115
|
-
state: subject.state,
|
|
116
|
-
};
|
|
103
|
+
if (args.subject.kind === "desktop-cli") {
|
|
104
|
+
return { source: "desktop-cli", product: args.subject.product.name, state: args.subjectState };
|
|
117
105
|
}
|
|
118
|
-
|
|
119
|
-
return {
|
|
120
|
-
source: "local-tree",
|
|
121
|
-
...(subject.archiveSha256 === undefined ? {} : { archiveSha256: subject.archiveSha256 }),
|
|
122
|
-
...(subject.commit === undefined ? {} : { commit: subject.commit }),
|
|
123
|
-
...(subject.dirty === undefined ? {} : { dirty: subject.dirty }),
|
|
124
|
-
envNames: subjectEnvNames,
|
|
125
|
-
state: subject.state,
|
|
126
|
-
};
|
|
127
|
-
}
|
|
128
|
-
return undefined;
|
|
106
|
+
return { source: "app-url", state: args.subjectState };
|
|
129
107
|
}
|
|
130
108
|
/**
|
|
131
109
|
* The run-level subject for the top level and the bundle. Local-tree participants all pack from the
|
|
@@ -161,7 +161,7 @@ export interface CuaParticipantResult {
|
|
|
161
161
|
killed: boolean;
|
|
162
162
|
streamUrlPresent: boolean;
|
|
163
163
|
};
|
|
164
|
-
subject:
|
|
164
|
+
subject: RunSubjectProvenance;
|
|
165
165
|
diagnostics?: CuaDiagnostics;
|
|
166
166
|
/** Set when the participant was skipped (pinned reason string). */
|
|
167
167
|
skippedReason?: string;
|
|
@@ -186,43 +186,6 @@ export interface CuaParticipantSummary {
|
|
|
186
186
|
waves: number;
|
|
187
187
|
}
|
|
188
188
|
export type CuaActorStudyErrorCode = StudyResultErrorCode | "HUMANISH_COMPUTER_USE_FAILED" | "HUMANISH_COMPUTER_USE_KEYS_MISSING" | "HUMANISH_COMPUTER_USE_AGENT_MISSING" | "HUMANISH_COMPUTER_USE_AGENT_SIGNIN_REQUIRED" | "HUMANISH_COMPUTER_USE_SUBJECT_ENV_MISSING" | "HUMANISH_COMPUTER_USE_ACTOR_UNSUPPORTED" | "HUMANISH_COMPUTER_USE_SUBJECT_INVALID" | "HUMANISH_COMPUTER_USE_SUBJECT_UNSAFE" | "HUMANISH_COMPUTER_USE_EXECUTOR_NO_PROVIDER" | "HUMANISH_COMPUTER_USE_LOCAL_APP_NO_EXECUTOR" | "HUMANISH_COMPUTER_USE_LOCAL_DESKTOP_MISSING" | "HUMANISH_COMPUTER_USE_LOCAL_CAPACITY_EXCEEDED" | "HUMANISH_COMPUTER_USE_DESKTOP_OUT_OF_MEMORY" | "HUMANISH_COMPUTER_USE_FANOUT_INVALID" | "HUMANISH_COMPUTER_USE_RERUN_INVALID" | "HUMANISH_COMPUTER_USE_DEVICE_GEOMETRY" | "HUMANISH_COMPUTER_USE_UNPRICED_CAP" | "HUMANISH_COMPUTER_USE_COMMS_CATCH_UNREACHABLE" | "HUMANISH_COMPUTER_USE_COMMS_TOKEN_INVALID" | "HUMANISH_WATCH_ALLOW_REQUIRES_OAUTH" | "HUMANISH_WATCH_OAUTH_REQUIRES_TUNNEL" | "HUMANISH_WATCH_OPTION_CONFLICT" | "HUMANISH_WATCH_TUNNEL_REQUIRES_EXPOSE" | "HUMANISH_WATCH_EXPOSE_REQUIRES_EDGE_AUTH" | "HUMANISH_WATCH_EXPOSE_REQUIRES_LIVE_FOLLOW" | "HUMANISH_WATCH_SAFE_NOT_APPLICABLE" | "HUMANISH_SERVE_TUNNEL_NOT_FOUND" | "HUMANISH_SERVE_TUNNEL_START_FAILED";
|
|
189
|
-
/** Subject provenance projection: what the actor actually drove. */
|
|
190
|
-
export interface CuaSubjectProjection {
|
|
191
|
-
source: "app-url" | "clone" | "local-tree";
|
|
192
|
-
/** Clone-route only: the (possibly redacted) owner/repo slug. */
|
|
193
|
-
repo?: string;
|
|
194
|
-
/** Cloned commit SHA (clone route) or host-side HEAD at pack time (local-tree route, when
|
|
195
|
-
* the packed root was a git work tree). */
|
|
196
|
-
commit?: string;
|
|
197
|
-
/** Local-tree-route only: 64-hex sha256 over the sorted packed-entries list: the content
|
|
198
|
-
* pin for a tree that cannot be commit-pinned. Absent on dry-run (nothing was packed). */
|
|
199
|
-
archiveSha256?: string;
|
|
200
|
-
/** Local-tree-route only: host-side porcelain status at pack time (true when the working
|
|
201
|
-
* tree had uncommitted changes). Absent when the packed root was not a git work tree. */
|
|
202
|
-
dirty?: boolean;
|
|
203
|
-
/** Declared env names provisioned for the subject (values never surface anywhere). */
|
|
204
|
-
envNames?: string[];
|
|
205
|
-
/** The subject's state story (seeded digests / unpinned external / declared-not-run /
|
|
206
|
-
* undeclared): the same block the run bundle records. */
|
|
207
|
-
state: RunSubjectProvenance["state"];
|
|
208
|
-
}
|
|
209
|
-
/** The provisioned-route-only shape threaded through as buildSingleParticipantBundle's
|
|
210
|
-
* subjectProvenance arg (clone or local-tree; an app-url subject stays undeclared, which that
|
|
211
|
-
* builder's own default branch already handles without this type). */
|
|
212
|
-
export type CuaSubjectProvenanceArg = {
|
|
213
|
-
source: "clone";
|
|
214
|
-
repo: string;
|
|
215
|
-
commit?: string;
|
|
216
|
-
envNames: string[];
|
|
217
|
-
state: RunSubjectProvenance["state"];
|
|
218
|
-
} | {
|
|
219
|
-
source: "local-tree";
|
|
220
|
-
archiveSha256?: string;
|
|
221
|
-
commit?: string;
|
|
222
|
-
dirty?: boolean;
|
|
223
|
-
envNames: string[];
|
|
224
|
-
state: RunSubjectProvenance["state"];
|
|
225
|
-
};
|
|
226
189
|
export interface CuaActorStudyResult extends AutomaticAnalysisResult, StudyResultIdentity<"computer-use"> {
|
|
227
190
|
/** True when the Observer verified the bundle, all live participants passed credibility checks
|
|
228
191
|
* (or this is a dry-run), and no declared adapter/scorer verdict failed. */
|
|
@@ -253,7 +216,7 @@ export interface CuaActorStudyResult extends AutomaticAnalysisResult, StudyResul
|
|
|
253
216
|
};
|
|
254
217
|
/** Subject provenance: what the actor actually drove. At N>1 this is the
|
|
255
218
|
* unanimity-gated aggregate (top-level `commit` only when every participant resolved the same one). */
|
|
256
|
-
subject?:
|
|
219
|
+
subject?: RunSubjectProvenance;
|
|
257
220
|
/** The pre-flight participant plan (present once participants resolve; absent on early validation errors). */
|
|
258
221
|
plan?: CuaParticipantPlan;
|
|
259
222
|
/** Per-participant results, always present once participants resolve (length 1 at N=1). */
|
|
@@ -485,8 +448,8 @@ export interface CuaFanoutBundleArgs {
|
|
|
485
448
|
run: BundleRun;
|
|
486
449
|
specs: DesktopParticipantRun[];
|
|
487
450
|
outcomes?: ParticipantRunOutcome[];
|
|
488
|
-
subjects:
|
|
489
|
-
aggregateSubject:
|
|
451
|
+
subjects: RunSubjectProvenance[];
|
|
452
|
+
aggregateSubject: RunSubjectProvenance;
|
|
490
453
|
descriptor: CuaActorDescriptor;
|
|
491
454
|
appUrl: string;
|
|
492
455
|
dryRun: boolean;
|
|
@@ -5,7 +5,7 @@ import { parse as parseYaml } from "yaml";
|
|
|
5
5
|
import path from "node:path";
|
|
6
6
|
import { parseBrowserPersonaJourneyFromScenario } from "../../actors/scripted-browser/journey.js";
|
|
7
7
|
import { digestText } from "../../evidence/redaction.js";
|
|
8
|
-
import { readContainedRegularFile, } from "../../run/contained-output.js";
|
|
8
|
+
import { readContainedRegularFile, PROJECT_FILE_MAX_BYTES, refusalText, } from "../../run/contained-output.js";
|
|
9
9
|
// Same public-safe token shape the study id uses; an id-style `scenario` must match it before
|
|
10
10
|
// it is interpolated into a repo path.
|
|
11
11
|
const SCENARIO_REF_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_.-]*$/;
|
|
@@ -20,6 +20,7 @@ export async function resolveScriptedScenario(projectRoot, ref) {
|
|
|
20
20
|
const trimmed = ref.trim();
|
|
21
21
|
let absolutePath;
|
|
22
22
|
let source;
|
|
23
|
+
let read;
|
|
23
24
|
if (scenarioRefLooksLikePath(trimmed)) {
|
|
24
25
|
absolutePath = path.resolve(projectRoot.physicalPath, trimmed);
|
|
25
26
|
const relative = path.relative(projectRoot.physicalPath, absolutePath);
|
|
@@ -42,22 +43,36 @@ export async function resolveScriptedScenario(projectRoot, ref) {
|
|
|
42
43
|
path.posix.join("humanish", "scenarios", `${trimmed}.yaml`),
|
|
43
44
|
path.posix.join("humanish", "scenarios", `${trimmed}.yml`),
|
|
44
45
|
];
|
|
45
|
-
const found = await
|
|
46
|
-
if (
|
|
46
|
+
const found = await firstScenarioFile(projectRoot, candidates);
|
|
47
|
+
if (found === null) {
|
|
47
48
|
return {
|
|
48
49
|
ok: false,
|
|
49
50
|
message: `scenario "${trimmed}" was not found (looked for ${candidates.join(", ")}).`,
|
|
50
51
|
};
|
|
51
52
|
}
|
|
52
|
-
|
|
53
|
-
|
|
53
|
+
// A candidate that is there and refused stops the search: the next one is not the scenario.
|
|
54
|
+
if (found.read.status === "refused") {
|
|
55
|
+
return {
|
|
56
|
+
ok: false,
|
|
57
|
+
message: `scenario "${trimmed}" could not be read: ${refusalText(found.candidate, found.read)}.`,
|
|
58
|
+
};
|
|
59
|
+
}
|
|
60
|
+
source = found.candidate;
|
|
61
|
+
read = found.read;
|
|
62
|
+
absolutePath = path.join(projectRoot.physicalPath, found.candidate);
|
|
54
63
|
}
|
|
55
64
|
const relativeScenarioPath = path.relative(projectRoot.physicalPath, absolutePath);
|
|
56
|
-
|
|
57
|
-
if (
|
|
65
|
+
read ??= await readContainedRegularFile(projectRoot, relativeScenarioPath, PROJECT_FILE_MAX_BYTES);
|
|
66
|
+
if (read.status === "refused" && read.reason === "too-large") {
|
|
67
|
+
return {
|
|
68
|
+
ok: false,
|
|
69
|
+
message: `scenario "${trimmed}" could not be read: ${refusalText(source, read)}.`,
|
|
70
|
+
};
|
|
71
|
+
}
|
|
72
|
+
if (read.status !== "read") {
|
|
58
73
|
return { ok: false, message: `scenario "${trimmed}" could not be read (${source}).` };
|
|
59
74
|
}
|
|
60
|
-
const text =
|
|
75
|
+
const text = read.bytes.toString("utf8");
|
|
61
76
|
let raw;
|
|
62
77
|
try {
|
|
63
78
|
raw = parseYaml(text);
|
|
@@ -92,11 +107,12 @@ function scenarioRefLooksLikePath(ref) {
|
|
|
92
107
|
ref.includes("\\") ||
|
|
93
108
|
ref.startsWith("."));
|
|
94
109
|
}
|
|
95
|
-
|
|
110
|
+
/** The first candidate that is there, read or refused; null when none is. */
|
|
111
|
+
async function firstScenarioFile(projectRoot, candidates) {
|
|
96
112
|
for (const candidate of candidates) {
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
113
|
+
const read = await readContainedRegularFile(projectRoot, candidate, PROJECT_FILE_MAX_BYTES);
|
|
114
|
+
if (read.status !== "missing")
|
|
115
|
+
return { candidate, read };
|
|
100
116
|
}
|
|
101
117
|
return null;
|
|
102
118
|
}
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// path it names must be a safe path inside the run's artifact root. A result that fails is unsafe,
|
|
3
3
|
// and the route stops without writing a bundle from it.
|
|
4
4
|
import path from "node:path";
|
|
5
|
-
import { readContainedRegularFile } from "../../run/contained-output.js";
|
|
5
|
+
import { readContainedRegularFile, refusalText, RUN_ARTIFACT_MAX_BYTES, } from "../../run/contained-output.js";
|
|
6
6
|
export class UnsafeScriptedSessionResultError extends Error {
|
|
7
7
|
constructor(message) {
|
|
8
8
|
super(message);
|
|
@@ -16,8 +16,11 @@ export async function existingScreenshots(runPaths, result) {
|
|
|
16
16
|
if (!step.screenshotPath) {
|
|
17
17
|
continue;
|
|
18
18
|
}
|
|
19
|
-
const screenshot = await readContainedRegularFile(runPaths, step.screenshotPath);
|
|
20
|
-
|
|
19
|
+
const screenshot = await readContainedRegularFile(runPaths, step.screenshotPath, RUN_ARTIFACT_MAX_BYTES);
|
|
20
|
+
// A screenshot that is there and refused would leave the bundle without a frame it took.
|
|
21
|
+
if (screenshot.status === "refused")
|
|
22
|
+
throw new UnsafeScriptedSessionResultError(`Scripted session screenshot ${refusalText(step.screenshotPath, screenshot)}.`);
|
|
23
|
+
if (screenshot.status === "read" && screenshot.bytes.byteLength > 0) {
|
|
21
24
|
existing.push(step.screenshotPath);
|
|
22
25
|
}
|
|
23
26
|
}
|
|
@@ -6,7 +6,7 @@ import { participantEvent, participantRecord, participantStream, } from "../../r
|
|
|
6
6
|
import { declaredScreenForRender } from "../../substrates/e2b/desktop-geometry.js";
|
|
7
7
|
import { publicSafeRouteLabel } from "./provenance.js";
|
|
8
8
|
import { participantCaption } from "../../run/participant-caption.js";
|
|
9
|
-
import {
|
|
9
|
+
import { participantSummary } from "../../run/outcomes.js";
|
|
10
10
|
function participantView(ctx, spec, index) {
|
|
11
11
|
const { args, external } = ctx;
|
|
12
12
|
const name = participantCaption({ id: spec.planned.id, personaId: spec.persona.id });
|
|
@@ -66,13 +66,7 @@ function sharedWorldSimulation(ctx, spec, index, view) {
|
|
|
66
66
|
mode: "browser-sim",
|
|
67
67
|
progress: inProgress ? 35 : 100,
|
|
68
68
|
currentStep: view.reason,
|
|
69
|
-
summary: session
|
|
70
|
-
? `${name} used the shared app and ${sessionEndingInWords(session.trace)}.`
|
|
71
|
-
: outcome?.sessionError !== undefined
|
|
72
|
-
? `${name} did not finish a session: ${outcome.sessionError}`
|
|
73
|
-
: inProgress
|
|
74
|
-
? `${name} is using the shared app.`
|
|
75
|
-
: `${name} would use the shared app; no session ran.`,
|
|
69
|
+
summary: participantSummary(name, { name: "the shared app" }, { trace: session?.trace, sessionError: outcome?.sessionError, inProgress }),
|
|
76
70
|
startedAt: args.run.createdAt,
|
|
77
71
|
updatedAt: args.run.createdAt,
|
|
78
72
|
});
|
|
@@ -1,10 +1,8 @@
|
|
|
1
|
-
import type { SharedWorldPlan } from "../../study/plan-types.js";
|
|
2
1
|
import type { SharedWorldParticipant } from "../../study/plan-participants.js";
|
|
3
2
|
import type { RunBundle } from "../../run/bundle.js";
|
|
4
3
|
import type { DesktopParticipantRun, ParticipantRunOutcome } from "../computer-use/types.js";
|
|
5
4
|
import { type DesktopParticipantDeps } from "../computer-use/participant-deps.js";
|
|
6
5
|
import type { LiveParticipants, PlaneContext } from "./types.js";
|
|
7
|
-
export declare function defaultSessionTimeoutMs(plan: SharedWorldPlan): number;
|
|
8
6
|
/** Resolve an actor's participant URL against the harness-minted getHost base (entry is a
|
|
9
7
|
* same-origin relative path, validated at parse against serve.url). */
|
|
10
8
|
export declare function resolveActorEntryUrl(baseUrl: string, entry: string | undefined): string;
|
|
@@ -11,26 +11,7 @@ import { desktopParticipantDeps, } from "../computer-use/participant-deps.js";
|
|
|
11
11
|
import { resolveCommittedPersonasForCwd } from "../../study/persona-resolve.js";
|
|
12
12
|
import { participantAssignment } from "../../study/participant-assignment.js";
|
|
13
13
|
import { redactText } from "../../evidence/redaction.js";
|
|
14
|
-
import {
|
|
15
|
-
import { DEFAULT_STATE_STEP_TIMEOUT_MS } from "../../subject/state.js";
|
|
16
|
-
// The default per-participant session budget is derived from the route. On a provisioned route the
|
|
17
|
-
// binding constraint is the subject sandbox (it must outlive every participant: timeoutMs +
|
|
18
|
-
// provisioning + seeding + teardown buffer, and E2B refuses a sandbox over one hour), so the
|
|
19
|
-
// derivation hands each participant the most that cap allows, capped at 15 minutes, floored at the
|
|
20
|
-
// historical 300s so a seed-heavy study never gets less room than it always had. App-url
|
|
21
|
-
// participants have no subject sandbox and default to 30 minutes (participant sandbox: 30m + 10m
|
|
22
|
-
// buffer stays well under the hour). An explicit execution.timeoutMs is never adjusted. The handoff
|
|
23
|
-
// latch scales off this (40%).
|
|
24
|
-
const MAX_DERIVED_SESSION_MS = 15 * 60_000;
|
|
25
|
-
const MIN_DERIVED_SESSION_MS = 300_000;
|
|
26
|
-
const DEFAULT_APP_URL_SESSION_MS = 30 * 60_000;
|
|
27
|
-
export function defaultSessionTimeoutMs(plan) {
|
|
28
|
-
if (plan.plane.kind !== "provisioned")
|
|
29
|
-
return DEFAULT_APP_URL_SESSION_MS;
|
|
30
|
-
const stateBudgetMs = (plan.plane.subject.state.seed ?? []).reduce((sum, step) => sum + (step.timeoutMs ?? DEFAULT_STATE_STEP_TIMEOUT_MS), 0);
|
|
31
|
-
const room = MAX_SANDBOX_MS - SUBJECT_PROVISION_BUDGET_MS - stateBudgetMs - SANDBOX_TIMEOUT_BUFFER_MS;
|
|
32
|
-
return Math.max(MIN_DERIVED_SESSION_MS, Math.min(MAX_DERIVED_SESSION_MS, room));
|
|
33
|
-
}
|
|
14
|
+
import { SANDBOX_TIMEOUT_BUFFER_MS } from "../../substrates/e2b/lifetime.js";
|
|
34
15
|
// prose-check: model prompt (the participant model reads this, not a person)
|
|
35
16
|
const DEFAULT_MISSION = "You are one of MANY users hitting a shared web application at the same time. The browser is already open at the app. Accomplish your role's task, then stop.";
|
|
36
17
|
/** Resolve an actor's participant URL against the harness-minted getHost base (entry is a
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { type CuaActorDescriptor } from "../../actors/registry.js";
|
|
2
2
|
import type { RoutePlanResult, RouteRefusal, SharedWorldPlan } from "../../study/plan-types.js";
|
|
3
3
|
import type { StudyConfig, StudySubjectState } from "../../study/types.js";
|
|
4
|
+
import { type SandboxCeiling } from "../../substrates/e2b/lifetime.js";
|
|
4
5
|
import type { ConcurrentSharedWorldStudyErrorCode } from "./types.js";
|
|
5
6
|
/** The error a shared-world study returns before a run starts. */
|
|
6
7
|
export interface SharedWorldRefusal extends RouteRefusal<"shared-world", ConcurrentSharedWorldStudyErrorCode> {
|
|
@@ -18,6 +19,8 @@ export declare function planSharedWorldStudy(config: StudyConfig, input: {
|
|
|
18
19
|
readonly dryRun: boolean;
|
|
19
20
|
/** Whether deps.runSession is set: a caller's session runner cannot enforce maxOutputTokens. */
|
|
20
21
|
readonly hasRunSession?: boolean;
|
|
22
|
+
/** The longest sandbox lifetime the operator's E2B plan allows (sandboxCeiling). */
|
|
23
|
+
readonly sandboxCeiling: SandboxCeiling;
|
|
21
24
|
}): SharedWorldPlanResult;
|
|
22
25
|
/** The provisioned plane's declared subject state. The external-public plane declares none. */
|
|
23
26
|
export declare function planeStateOf(plan: SharedWorldPlan): StudySubjectState | undefined;
|
|
@@ -12,6 +12,8 @@ import { PUBLIC_TARGET_OWNER_PATTERN } from "../../study/parse/subject.js";
|
|
|
12
12
|
import { REPO_SLUG_PATTERN } from "../../study/parse/values.js";
|
|
13
13
|
import { concurrentSharedWorldValidationReason, desktopMediaValidationReason, externalPublicSharedWorldValidationReason, outputTokenLimitValidationReason, receivingEmailValidationReason, taskProtocolValidationReason, } from "../../study/validation.js";
|
|
14
14
|
import { unpricedCapCheck } from "../../study/requirements.js";
|
|
15
|
+
import { DEFAULT_STATE_STEP_TIMEOUT_MS } from "../../subject/state.js";
|
|
16
|
+
import { SANDBOX_TIMEOUT_BUFFER_MS, SUBJECT_PROVISION_BUDGET_MS, } from "../../substrates/e2b/lifetime.js";
|
|
15
17
|
/** The registered descriptor for a planned actor; planSharedWorldStudy checked the registry. */
|
|
16
18
|
export function sharedWorldDescriptorOf(actorType) {
|
|
17
19
|
const descriptor = actorRegistry[actorType];
|
|
@@ -87,6 +89,9 @@ export function planSharedWorldStudy(config, input) {
|
|
|
87
89
|
// subject.env on an app-url subject for the same reason.
|
|
88
90
|
if (externalPublic && config.subject.env !== undefined)
|
|
89
91
|
return refuse(invalid, "`subject.env` applies only to clone subjects or local-tree subjects (the served app's environment channel).", actor);
|
|
92
|
+
const ceiling = input.sandboxCeiling;
|
|
93
|
+
if (!ceiling.ok)
|
|
94
|
+
return refuse(invalid, ceiling.message, actor);
|
|
90
95
|
const plane = planeOf(config);
|
|
91
96
|
if (plane === undefined)
|
|
92
97
|
throw new Error("shared-world validation admitted a plane it cannot plan");
|
|
@@ -105,9 +110,7 @@ export function planSharedWorldStudy(config, input) {
|
|
|
105
110
|
actor,
|
|
106
111
|
plane,
|
|
107
112
|
concurrency: config.execution?.concurrency ?? plane.participants.length,
|
|
108
|
-
|
|
109
|
-
? {}
|
|
110
|
-
: { sessionTimeoutMs: config.execution.timeoutMs }),
|
|
113
|
+
sessionTimeoutMs: config.execution?.timeoutMs ?? defaultSessionTimeoutMs(plane, ceiling.ms),
|
|
111
114
|
brain,
|
|
112
115
|
caps: planCaps(config),
|
|
113
116
|
requirements: base.dryRun
|
|
@@ -128,6 +131,23 @@ export function planSharedWorldStudy(config, input) {
|
|
|
128
131
|
},
|
|
129
132
|
};
|
|
130
133
|
}
|
|
134
|
+
// The default per-participant session budget is derived from the plane. On a provisioned plane the
|
|
135
|
+
// binding constraint is the subject sandbox (it must outlive every participant: timeoutMs +
|
|
136
|
+
// provisioning + seeding + teardown buffer, under the sandbox ceiling), so the derivation hands each
|
|
137
|
+
// participant the most the ceiling allows, capped at 15 minutes, floored at the historical 300s so
|
|
138
|
+
// a seed-heavy study never gets less room than it always had. App-url participants have no subject
|
|
139
|
+
// sandbox and default to 30 minutes (participant sandbox: 30m + 10m buffer stays under the hour).
|
|
140
|
+
// An explicit execution.timeoutMs is never adjusted. The handoff latch scales off this (40%).
|
|
141
|
+
const MAX_DERIVED_SESSION_MS = 15 * 60_000;
|
|
142
|
+
const MIN_DERIVED_SESSION_MS = 300_000;
|
|
143
|
+
const DEFAULT_APP_URL_SESSION_MS = 30 * 60_000;
|
|
144
|
+
function defaultSessionTimeoutMs(plane, ceilingMs) {
|
|
145
|
+
if (plane.kind !== "provisioned")
|
|
146
|
+
return DEFAULT_APP_URL_SESSION_MS;
|
|
147
|
+
const stateBudgetMs = (plane.subject.state.seed ?? []).reduce((sum, step) => sum + (step.timeoutMs ?? DEFAULT_STATE_STEP_TIMEOUT_MS), 0);
|
|
148
|
+
const room = ceilingMs - SUBJECT_PROVISION_BUDGET_MS - stateBudgetMs - SANDBOX_TIMEOUT_BUFFER_MS;
|
|
149
|
+
return Math.max(MIN_DERIVED_SESSION_MS, Math.min(MAX_DERIVED_SESSION_MS, room));
|
|
150
|
+
}
|
|
131
151
|
/** The provisioned plane's declared subject state. The external-public plane declares none. */
|
|
132
152
|
export function planeStateOf(plan) {
|
|
133
153
|
return plan.plane.kind === "provisioned" ? plan.plane.subject.state : undefined;
|
|
@@ -9,7 +9,7 @@ import { prepareEmailReceiving, prepareExternalComms, receivingSourceOf, subject
|
|
|
9
9
|
import { declaredOriginDigestOf } from "./external-public.js";
|
|
10
10
|
import { packSubjectTree } from "./provisioned.js";
|
|
11
11
|
import { emptyPlaneResults } from "./result.js";
|
|
12
|
-
import { buildParticipantSpecs
|
|
12
|
+
import { buildParticipantSpecs } from "./participant-specs.js";
|
|
13
13
|
import { planeStateOf } from "./plan.js";
|
|
14
14
|
import path from "node:path";
|
|
15
15
|
import { e2bRequestTimeoutMs } from "../../substrates/e2b/lifetime.js";
|
|
@@ -87,7 +87,7 @@ export async function prepareConcurrentRun(study, scope) {
|
|
|
87
87
|
const { run } = started;
|
|
88
88
|
const { runId, createdAt, paths: runPaths, source } = run;
|
|
89
89
|
const artifactRoot = runPaths.absoluteRunRoot;
|
|
90
|
-
const timeoutMs = plan.sessionTimeoutMs
|
|
90
|
+
const timeoutMs = plan.sessionTimeoutMs;
|
|
91
91
|
const requestTimeoutMs = e2bRequestTimeoutMs(env);
|
|
92
92
|
const redactScreenshots = plan.residual.policies?.redactScreenshots === true;
|
|
93
93
|
const timers = deps.detachedTimers ?? {};
|
|
@@ -1,21 +1,21 @@
|
|
|
1
|
-
import type { ActorTokenUsage } from "../../actors/contract.js";
|
|
1
|
+
import type { ActorTokenUsage, ActorTrace } from "../../actors/contract.js";
|
|
2
2
|
import type { StudyCaps } from "../../study/types.js";
|
|
3
3
|
import { type CostCategory } from "../../run/terminal-contract.js";
|
|
4
4
|
import type { CostLine, NoSpendProof, TerminalCostLedger } from "./types.js";
|
|
5
5
|
/**
|
|
6
6
|
* Build the spend ledger from the captured session. The null discipline:
|
|
7
7
|
* - The `provider` line is populated from the actor trace's tokenUsage.costUsd when the trace
|
|
8
|
-
* carries it (a measured value, incl. a measured 0).
|
|
9
|
-
*
|
|
8
|
+
* carries it (a measured value, incl. a measured 0). Otherwise its `usd` is null = not
|
|
9
|
+
* measured, and when the trace's `estimatedCost` prices the tokens, the line carries that
|
|
10
|
+
* price as `estimatedUsd`: the figure run.json's model-tokens line gives, from the same
|
|
11
|
+
* object. Tokens with no rate, and a run with no token count, say so in the note.
|
|
10
12
|
* - product/media/payment are `null` by default: core has no signal for those categories, and
|
|
11
13
|
* only a test can supply one, through `StudyDeps.costProbe`.
|
|
12
14
|
* `injectedLines` lets a test supply known spend for a category,
|
|
13
15
|
* exercising the fail-closed cap enforcement deterministically without a real billable run.
|
|
14
16
|
*/
|
|
15
17
|
export declare function buildCostLedger(args: {
|
|
16
|
-
|
|
17
|
-
/** Measured token counts, when the run produced them but no rate could price them. */
|
|
18
|
-
tokenUsage?: ActorTokenUsage;
|
|
18
|
+
trace: Pick<ActorTrace, "tokenUsage" | "estimatedCost">;
|
|
19
19
|
injectedLines?: Partial<Record<CostCategory, CostLine>>;
|
|
20
20
|
}): TerminalCostLedger;
|
|
21
21
|
/** True when no cost line carries a dollar value, so a no-spend proof has nothing to stand on. */
|
|
@@ -26,9 +26,9 @@ export declare function noSpendLineMeasured(proof: Pick<NoSpendProof, "knownZero
|
|
|
26
26
|
*/
|
|
27
27
|
export declare function noSpendNotEstablished(maxUsd: number): string;
|
|
28
28
|
/**
|
|
29
|
-
* What the ledger measured, in words: the dollar lines it knows, provider tokens it counted
|
|
30
|
-
*
|
|
31
|
-
* an all-null ledger for a proven $0.
|
|
29
|
+
* What the ledger measured, in words: the dollar lines it knows, the provider tokens it counted
|
|
30
|
+
* with their estimate or as unpriced, and the lines it has no signal for. A reader of a maxUsd=0
|
|
31
|
+
* run must not take an all-null ledger for a proven $0.
|
|
32
32
|
*/
|
|
33
33
|
export declare function describeMeasuredSpend(ledger: TerminalCostLedger, tokenUsage?: ActorTokenUsage): string;
|
|
34
34
|
/** Derive the no-spend proof from the ledger. It vouches for known-zero lines and
|
|
@@ -4,48 +4,21 @@ import { COST_CATEGORIES } from "../../run/terminal-contract.js";
|
|
|
4
4
|
/**
|
|
5
5
|
* Build the spend ledger from the captured session. The null discipline:
|
|
6
6
|
* - The `provider` line is populated from the actor trace's tokenUsage.costUsd when the trace
|
|
7
|
-
* carries it (a measured value, incl. a measured 0).
|
|
8
|
-
*
|
|
7
|
+
* carries it (a measured value, incl. a measured 0). Otherwise its `usd` is null = not
|
|
8
|
+
* measured, and when the trace's `estimatedCost` prices the tokens, the line carries that
|
|
9
|
+
* price as `estimatedUsd`: the figure run.json's model-tokens line gives, from the same
|
|
10
|
+
* object. Tokens with no rate, and a run with no token count, say so in the note.
|
|
9
11
|
* - product/media/payment are `null` by default: core has no signal for those categories, and
|
|
10
12
|
* only a test can supply one, through `StudyDeps.costProbe`.
|
|
11
13
|
* `injectedLines` lets a test supply known spend for a category,
|
|
12
14
|
* exercising the fail-closed cap enforcement deterministically without a real billable run.
|
|
13
15
|
*/
|
|
14
16
|
export function buildCostLedger(args) {
|
|
15
|
-
const providerLine = typeof args.tokenCostUsd === "number"
|
|
16
|
-
? {
|
|
17
|
-
usd: args.tokenCostUsd,
|
|
18
|
-
source: "provider-token-usage",
|
|
19
|
-
note: `Provider spend metered from the actor trace tokenUsage.costUsd (${args.tokenCostUsd} USD).`,
|
|
20
|
-
}
|
|
21
|
-
: args.tokenUsage
|
|
22
|
-
? {
|
|
23
|
-
// Tokens counted, no rate to price them. This stays `usd: null` because a guessed
|
|
24
|
-
// dollar figure would be worse than none, but the note carries the measured fact so a
|
|
25
|
-
// reader never mistakes "no charge recorded" for "nothing was consumed".
|
|
26
|
-
usd: null,
|
|
27
|
-
source: "unpriced-token-usage",
|
|
28
|
-
note: `Provider spend unpriced: the run consumed ${describeTokenUsage(args.tokenUsage)}, ` +
|
|
29
|
-
"but the terminal participant records the model as `codex` and humanish has no rate " +
|
|
30
|
-
"for it, so no dollar figure is given. The token count is measured; the price is " +
|
|
31
|
-
"not known. Recorded as null, never guessed as 0.",
|
|
32
|
-
}
|
|
33
|
-
: {
|
|
34
|
-
usd: null,
|
|
35
|
-
source: "unmeasured",
|
|
36
|
-
note: "Provider spend not measured: the actor trace carried no tokenUsage.costUsd in this run, so it is recorded as null and not guessed as 0.",
|
|
37
|
-
};
|
|
38
|
-
const unmeasured = (category) => ({
|
|
39
|
-
usd: null,
|
|
40
|
-
count: null,
|
|
41
|
-
source: "unmeasured",
|
|
42
|
-
note: `${category} spend not measured: humanish has no ${category} spend signal, and a study cannot supply one. Recorded as null, never guessed as 0.`,
|
|
43
|
-
});
|
|
44
17
|
const lines = {
|
|
45
18
|
product: args.injectedLines?.product ?? unmeasured("product"),
|
|
46
19
|
media: args.injectedLines?.media ?? unmeasured("media"),
|
|
47
20
|
payment: args.injectedLines?.payment ?? unmeasured("payment"),
|
|
48
|
-
provider: args.injectedLines?.provider ?? providerLine,
|
|
21
|
+
provider: args.injectedLines?.provider ?? providerLine(args.trace),
|
|
49
22
|
};
|
|
50
23
|
// knownTotalUsd sums only the non-null lines. A null line contributes nothing and is never
|
|
51
24
|
// coerced to 0 (that would let an unmeasured category masquerade as a measured zero).
|
|
@@ -68,6 +41,51 @@ export function buildCostLedger(args) {
|
|
|
68
41
|
fullyMeasured,
|
|
69
42
|
};
|
|
70
43
|
}
|
|
44
|
+
function unmeasured(category) {
|
|
45
|
+
return {
|
|
46
|
+
usd: null,
|
|
47
|
+
count: null,
|
|
48
|
+
source: "unmeasured",
|
|
49
|
+
note: `${category} spend not measured: humanish has no ${category} spend signal, and a study cannot supply one. Recorded as null, never guessed as 0.`,
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
/** The participant's model tokens: a measured charge, an estimate, unpriced, or not counted. */
|
|
53
|
+
function providerLine(trace) {
|
|
54
|
+
const { tokenUsage, estimatedCost } = trace;
|
|
55
|
+
if (typeof tokenUsage?.costUsd === "number")
|
|
56
|
+
return {
|
|
57
|
+
usd: tokenUsage.costUsd,
|
|
58
|
+
source: "provider-token-usage",
|
|
59
|
+
note: `Provider spend metered from the actor trace tokenUsage.costUsd (${tokenUsage.costUsd} USD).`,
|
|
60
|
+
};
|
|
61
|
+
if (tokenUsage === undefined)
|
|
62
|
+
return {
|
|
63
|
+
usd: null,
|
|
64
|
+
source: "unmeasured",
|
|
65
|
+
note: "Provider spend not measured: the actor trace carried no tokenUsage.costUsd in this run, so it is recorded as null and not guessed as 0.",
|
|
66
|
+
};
|
|
67
|
+
if (estimatedCost !== undefined && estimatedCost.estimatedCostUsd !== null)
|
|
68
|
+
return {
|
|
69
|
+
usd: null,
|
|
70
|
+
estimatedUsd: estimatedCost.estimatedCostUsd,
|
|
71
|
+
source: "estimated-token-usage",
|
|
72
|
+
note: `Provider spend estimated at ${estimatedCost.estimatedCostUsd} USD: the run consumed ` +
|
|
73
|
+
`${describeTokenUsage(tokenUsage)}, priced at ${estimatedCost.modelId ?? "the participant model's"} ` +
|
|
74
|
+
`rates as of ${estimatedCost.ratesAsOf}, as run.json's cost summary prices them. No provider ` +
|
|
75
|
+
"charge was measured, so usd stays null and caps.maxUsd does not count the estimate.",
|
|
76
|
+
};
|
|
77
|
+
// Tokens counted, no rate to price them. A guessed dollar figure would be worse than none, but
|
|
78
|
+
// the note carries the measured fact so a reader never mistakes "no charge recorded" for
|
|
79
|
+
// "nothing was consumed".
|
|
80
|
+
return {
|
|
81
|
+
usd: null,
|
|
82
|
+
source: "unpriced-token-usage",
|
|
83
|
+
note: `Provider spend unpriced: the run consumed ${describeTokenUsage(tokenUsage)}, but humanish ` +
|
|
84
|
+
`has no rate for ${estimatedCost?.modelId === undefined ? "the participant's model" : `the model ${estimatedCost.modelId}`}, ` +
|
|
85
|
+
"so no dollar figure is given. The token count is measured; the price is not known. " +
|
|
86
|
+
"Recorded as null, never guessed as 0.",
|
|
87
|
+
};
|
|
88
|
+
}
|
|
71
89
|
/** True when no cost line carries a dollar value, so a no-spend proof has nothing to stand on. */
|
|
72
90
|
export function noSpendLineMeasured(proof) {
|
|
73
91
|
return proof.knownZeroLines.length + proof.knownNonZeroLines.length === 0;
|
|
@@ -80,19 +98,24 @@ export function noSpendNotEstablished(maxUsd) {
|
|
|
80
98
|
return `No-spend proof not established for maxUsd=${maxUsd}: no spend line was measured.`;
|
|
81
99
|
}
|
|
82
100
|
/**
|
|
83
|
-
* What the ledger measured, in words: the dollar lines it knows, provider tokens it counted
|
|
84
|
-
*
|
|
85
|
-
* an all-null ledger for a proven $0.
|
|
101
|
+
* What the ledger measured, in words: the dollar lines it knows, the provider tokens it counted
|
|
102
|
+
* with their estimate or as unpriced, and the lines it has no signal for. A reader of a maxUsd=0
|
|
103
|
+
* run must not take an all-null ledger for a proven $0.
|
|
86
104
|
*/
|
|
87
105
|
export function describeMeasuredSpend(ledger, tokenUsage) {
|
|
88
106
|
const measured = COST_CATEGORIES.filter((c) => ledger.lines[c].usd !== null).map((c) => `${c} ${ledger.lines[c].usd} USD`);
|
|
89
|
-
const
|
|
90
|
-
const
|
|
107
|
+
const provider = ledger.lines.provider;
|
|
108
|
+
const countedProvider = provider.usd === null &&
|
|
109
|
+
(provider.source === "unpriced-token-usage" || provider.source === "estimated-token-usage");
|
|
110
|
+
const unmeasured = COST_CATEGORIES.filter((c) => ledger.lines[c].usd === null && !(c === "provider" && countedProvider));
|
|
111
|
+
const counts = tokenUsage === undefined ? "" : ` (${describeTokenUsage(tokenUsage)})`;
|
|
91
112
|
return [
|
|
92
113
|
measured.length > 0 ? `Measured: ${measured.join(", ")}.` : "",
|
|
93
|
-
|
|
94
|
-
?
|
|
95
|
-
:
|
|
114
|
+
!countedProvider
|
|
115
|
+
? ""
|
|
116
|
+
: provider.estimatedUsd === undefined
|
|
117
|
+
? `Provider tokens were consumed${counts} and are unpriced, not zero.`
|
|
118
|
+
: `Provider tokens${counts} are estimated at ${provider.estimatedUsd} USD, the participant's model cost, which caps.maxUsd does not count.`,
|
|
96
119
|
unmeasured.length > 0 ? `Not measured (null, not claimed zero): ${unmeasured.join(", ")}.` : "",
|
|
97
120
|
]
|
|
98
121
|
.filter((part) => part.length > 0)
|
|
@@ -81,17 +81,13 @@ async function settleLiveLedgers(inputs, trace, normalizedTranscript) {
|
|
|
81
81
|
const { recordLifecycle, lifecycle, commandLog, interventions, terminalEvents } = inputs.recorder;
|
|
82
82
|
// --- Spend ledger + no-spend proof + full caps enforcement (fail-closed). ---
|
|
83
83
|
// The cost ledger is derived, with the null discipline: provider spend from the trace's
|
|
84
|
-
// tokenUsage.costUsd when present
|
|
85
|
-
// default (core has no signal). The costProbe hook lets tests
|
|
86
|
-
// spend to exercise the fail-closed cap without a real billable run.
|
|
84
|
+
// tokenUsage.costUsd when present, else null with the trace's token estimate beside it, and
|
|
85
|
+
// product/media/payment null by default (core has no signal). The costProbe hook lets tests
|
|
86
|
+
// inject known spend to exercise the fail-closed cap without a real billable run.
|
|
87
87
|
const injectedLines = costProbe?.(trace.tokenUsage?.costUsd === undefined ? {} : { tokenCostUsd: trace.tokenUsage.costUsd });
|
|
88
88
|
if (costProbe)
|
|
89
89
|
await validatePreparedRunArtifactPaths(runPaths);
|
|
90
|
-
const cost = buildCostLedger({
|
|
91
|
-
...(trace.tokenUsage?.costUsd === undefined ? {} : { tokenCostUsd: trace.tokenUsage.costUsd }),
|
|
92
|
-
...(trace.tokenUsage === undefined ? {} : { tokenUsage: trace.tokenUsage }),
|
|
93
|
-
...(injectedLines ? { injectedLines } : {}),
|
|
94
|
-
});
|
|
90
|
+
const cost = buildCostLedger({ trace, ...(injectedLines ? { injectedLines } : {}) });
|
|
95
91
|
const noSpendProof = buildNoSpendProof(cost, maxUsd ?? null, trace.tokenUsage);
|
|
96
92
|
const proofVerdict = !noSpendProof.satisfied
|
|
97
93
|
? `No-spend proof not satisfied for maxUsd=${maxUsd ?? "null"}.`
|
|
@@ -138,8 +134,9 @@ export async function finishLiveTerminalSession(inputs) {
|
|
|
138
134
|
const participant = terminalParticipantFacts(trace, session.error);
|
|
139
135
|
const judgment = judgeTerminal({ dryRun: false, participant });
|
|
140
136
|
// The run cost summary, as the computer-use route records it: the sandbox's compute time from
|
|
141
|
-
// its span and observed size, and the participant's tokens
|
|
142
|
-
// of the cap ledger above, whose lines sum
|
|
137
|
+
// its span and observed size, and the participant's tokens at trace.estimatedCost, the price the
|
|
138
|
+
// ledger's provider line also carries. It is not part of the cap ledger above, whose lines sum
|
|
139
|
+
// against caps.maxUsd.
|
|
143
140
|
const desktops = session.runCostDesktops();
|
|
144
141
|
const runCost = buildRunCostSummary({
|
|
145
142
|
participants: [{ trace }],
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { RoutePlanResult, RouteRefusal, TerminalPlan } from "../../study/plan-types.js";
|
|
2
2
|
import type { StudyConfig } from "../../study/types.js";
|
|
3
|
+
import { type SandboxCeiling } from "../../substrates/e2b/lifetime.js";
|
|
3
4
|
import { type TerminalProductStudyResult } from "./types.js";
|
|
4
5
|
/** The error a terminal study returns before a run starts. `actor` names the registered actor. */
|
|
5
6
|
export interface TerminalRefusal extends RouteRefusal<"terminal", NonNullable<TerminalProductStudyResult["error"]>["code"]> {
|
|
@@ -14,4 +15,6 @@ export declare function planTerminalStudy(config: StudyConfig, input: {
|
|
|
14
15
|
readonly dryRun: boolean;
|
|
15
16
|
/** A test's costProbe can measure spend lines, so a positive maxUsd can trip. */
|
|
16
17
|
readonly hasCostProbe?: boolean;
|
|
18
|
+
/** The longest sandbox lifetime the operator's E2B plan allows (sandboxCeiling). */
|
|
19
|
+
readonly sandboxCeiling: SandboxCeiling;
|
|
17
20
|
}): TerminalPlanResult;
|