humanish 0.110.4 → 0.111.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +31 -0
- package/CONTEXT.md +2 -2
- package/dist/actors/codex/restricted-participant-policy.d.ts +2 -0
- package/dist/actors/codex/restricted-participant-policy.js +30 -2
- package/dist/actors/codex/restricted-participant.js +1 -1
- package/dist/actors/computer-use/executor-diagnostic.d.ts +15 -0
- package/dist/actors/computer-use/executor-diagnostic.js +105 -0
- package/dist/actors/computer-use/executor-error.d.ts +18 -3
- package/dist/actors/computer-use/executor-error.js +30 -4
- package/dist/actors/computer-use/loop/actions.d.ts +6 -3
- package/dist/actors/computer-use/loop/actions.js +27 -7
- package/dist/actors/computer-use/loop/ending.js +9 -1
- package/dist/actors/computer-use/loop/types.d.ts +14 -0
- package/dist/actors/computer-use/loop.d.ts +1 -1
- package/dist/actors/computer-use/loop.js +1 -1
- package/dist/actors/computer-use/openai-context.d.ts +8 -2
- package/dist/actors/computer-use/openai-context.js +91 -26
- package/dist/actors/computer-use/openai-provider.js +55 -12
- package/dist/actors/computer-use/openai-requests.d.ts +2 -1
- package/dist/actors/computer-use/openai-requests.js +4 -1
- package/dist/actors/contract.d.ts +7 -1
- package/dist/actors/local-agent/claude-session.js +1 -1
- package/dist/actors/local-agent/cli.js +1 -1
- package/dist/actors/scripted-browser/actor.d.ts +2 -2
- package/dist/browser-control/client.js +5 -5
- package/dist/browser-control/protocol.d.ts +30 -1
- package/dist/browser-control/protocol.js +19 -2
- package/dist/browser-control/transport.d.ts +3 -2
- package/dist/browser-control/transport.js +12 -7
- package/dist/cli/commands/study-route-computer-use.js +2 -2
- package/dist/cli/commands/study-route-shared-world.js +2 -2
- package/dist/cli/commands/study.js +1 -4
- package/dist/cli/doctor.js +11 -0
- package/dist/evidence/encoded-text.d.ts +3 -3
- package/dist/evidence/encoded-text.js +10 -5
- package/dist/evidence/redaction.d.ts +2 -2
- package/dist/evidence/redaction.js +18 -7
- package/dist/guest/browser-tools.js +5 -1
- package/dist/guest/chromium-text.js +42 -18
- package/dist/guest/desktop-executor.js +5 -1
- package/dist/keys/key-resolution.d.ts +3 -0
- package/dist/keys/key-resolution.js +15 -2
- package/dist/routes/computer-use/bundle-parts.d.ts +4 -0
- package/dist/routes/computer-use/bundle-parts.js +7 -2
- package/dist/routes/computer-use/fanout-bundle.js +4 -2
- package/dist/routes/computer-use/local-vm.js +1 -2
- package/dist/routes/computer-use/participant-model.js +7 -2
- package/dist/routes/computer-use/plan.js +7 -10
- package/dist/routes/computer-use/preflight.js +1 -1
- package/dist/routes/computer-use/result.js +1 -1
- package/dist/routes/computer-use/route.js +2 -3
- package/dist/routes/computer-use/setup.js +2 -2
- package/dist/routes/computer-use/types.d.ts +4 -4
- package/dist/routes/preview.js +1 -0
- package/dist/routes/scripted/plan.js +4 -5
- package/dist/routes/scripted/route.js +3 -3
- package/dist/routes/scripted/scenario.d.ts +1 -1
- package/dist/routes/scripted/scenario.js +3 -3
- package/dist/routes/scripted/types.d.ts +2 -2
- package/dist/routes/shared-world/handoff.js +1 -1
- package/dist/routes/shared-world/participant-specs.js +1 -1
- package/dist/routes/shared-world/plan.js +6 -8
- package/dist/routes/shared-world/route.js +2 -2
- package/dist/routes/shared-world/setup.js +1 -0
- package/dist/routes/shared-world/types.d.ts +1 -1
- package/dist/routes/terminal/bundle.d.ts +3 -3
- package/dist/routes/terminal/dry-run.js +1 -0
- package/dist/routes/terminal/ledger.d.ts +2 -2
- package/dist/routes/terminal/lifetime.js +1 -1
- package/dist/routes/terminal/live-finish.js +25 -6
- package/dist/routes/terminal/live-sandbox.js +5 -1
- package/dist/routes/terminal/plan.js +3 -4
- package/dist/routes/terminal/recorder.d.ts +6 -0
- package/dist/routes/terminal/recorder.js +61 -6
- package/dist/routes/terminal/route.js +2 -3
- package/dist/routes/terminal/session.js +2 -1
- package/dist/routes/terminal/token-usage.d.ts +3 -0
- package/dist/routes/terminal/token-usage.js +5 -0
- package/dist/routes/terminal/types.d.ts +3 -2
- package/dist/routes/terminal/types.js +2 -1
- package/dist/run/dry-run.js +1 -0
- package/dist/run/results.d.ts +3 -1
- package/dist/run/run.d.ts +2 -0
- package/dist/run/run.js +16 -2
- package/dist/run/terminal-contract.d.ts +10 -0
- package/dist/run/terminal-contract.js +27 -9
- package/dist/run-study.d.ts +5 -3
- package/dist/run-study.js +7 -11
- package/dist/study/adapter-scorer-loader.d.ts +1 -1
- package/dist/study/adapter-scorer-loader.js +3 -3
- package/dist/study/composition-rules.js +36 -27
- package/dist/study/config.d.ts +29 -13
- package/dist/study/config.js +174 -157
- package/dist/study/discover.d.ts +1 -1
- package/dist/study/discover.js +1 -1
- package/dist/study/doctor.js +4 -5
- package/dist/study/keys.d.ts +250 -2
- package/dist/study/keys.js +32 -62
- package/dist/study/migrate/convert.js +3 -4
- package/dist/study/migrate/v2.d.ts +22 -7
- package/dist/study/migrate/v2.js +173 -23
- package/dist/study/parse/actors.d.ts +11 -32
- package/dist/study/parse/actors.js +11 -156
- package/dist/study/parse/comms.js +6 -2
- package/dist/study/parse/execution.d.ts +9 -4
- package/dist/study/parse/execution.js +4 -39
- package/dist/study/parse/{study-v3.d.ts → front.d.ts} +6 -17
- package/dist/study/parse/{study-v3.js → front.js} +24 -112
- package/dist/study/parse/subject.d.ts +2 -0
- package/dist/study/parse/subject.js +15 -18
- package/dist/study/parse/url-credentials.d.ts +3 -3
- package/dist/study/parse/url-credentials.js +155 -13
- package/dist/study/persona-resolve.js +7 -9
- package/dist/study/plan-base.d.ts +1 -1
- package/dist/study/plan-base.js +5 -7
- package/dist/study/plan-participants.d.ts +2 -2
- package/dist/study/plan-participants.js +3 -3
- package/dist/study/plan-types.d.ts +7 -5
- package/dist/study/plan.js +12 -6
- package/dist/study/preflight-probes.js +2 -2
- package/dist/study/preflight.d.ts +1 -1
- package/dist/study/preflight.js +1 -1
- package/dist/study/routing.d.ts +5 -5
- package/dist/study/routing.js +16 -16
- package/dist/study/run-study-options.d.ts +1 -1
- package/dist/study/run-study-options.js +2 -2
- package/dist/study/study-deps.d.ts +1 -1
- package/dist/study/study-fields.d.ts +4 -25
- package/dist/study/study-fields.js +15 -44
- package/dist/study/summary.js +13 -8
- package/dist/study/types.d.ts +51 -128
- package/dist/study/types.js +0 -1
- package/dist/study/url-credentials.js +3 -4
- package/dist/study/validation.d.ts +2 -10
- package/dist/study/validation.js +23 -53
- package/dist/study/warnings.d.ts +11 -3
- package/dist/study/warnings.js +56 -110
- package/dist/substrates/local/runtime-config.js +11 -13
- package/docs/architecture/browser-control.md +12 -2
- package/docs/architecture/guest-desktop.md +7 -1
- package/docs/architecture/terminal-product-route.md +3 -2
- package/docs/contracts/schemas.md +39 -9
- package/package.json +1 -1
- package/dist/study/parse/study.d.ts +0 -38
- package/dist/study/parse/study.js +0 -225
|
@@ -25,7 +25,6 @@ import { projectParticipantSubjects, subjectProvenanceArg } from "./subject-proj
|
|
|
25
25
|
import { participantSubjectEnv, } from "./types.js";
|
|
26
26
|
import { studyPersonaIds } from "../../study/persona-resolve.js";
|
|
27
27
|
import { studyResultIdentity } from "../../run/study-result.js";
|
|
28
|
-
import { actorOf } from "../../study/study-fields.js";
|
|
29
28
|
/** The physical project, bound before any caller hook runs. */
|
|
30
29
|
async function bindProject(cwd) {
|
|
31
30
|
// Capture the physical project before reading or invoking any caller hook. A supported
|
|
@@ -48,7 +47,7 @@ export async function refuseCuaStudy(options, refusal) {
|
|
|
48
47
|
...studyResultIdentity("computer-use", config.id),
|
|
49
48
|
ok: false,
|
|
50
49
|
cwd: projectRoot.physicalPath,
|
|
51
|
-
actor: refusal.actor ??
|
|
50
|
+
actor: refusal.actor ?? config.actor?.type ?? "",
|
|
52
51
|
appUrl: declaredAppUrl(config),
|
|
53
52
|
dryRun,
|
|
54
53
|
runId: options.runId ?? "not-created",
|
|
@@ -214,6 +213,7 @@ export async function startCuaRun(plan, input, admitted, scope) {
|
|
|
214
213
|
mintRunId: makeCuaRunId,
|
|
215
214
|
mode: dryRun ? "dry-run" : "live",
|
|
216
215
|
study: plan.study,
|
|
216
|
+
warnings: plan.warnings,
|
|
217
217
|
renderReview: renderCuaReviewMarkdown,
|
|
218
218
|
observer: { open: input.open === true, render: seams.renderObserver },
|
|
219
219
|
});
|
|
@@ -52,8 +52,8 @@ export interface ComputerUseRunInput {
|
|
|
52
52
|
cwd: string;
|
|
53
53
|
open?: boolean;
|
|
54
54
|
runId?: string;
|
|
55
|
-
/** CLI `--count` override for the homogeneous fan-out participant count (ignored when a `
|
|
56
|
-
*
|
|
55
|
+
/** CLI `--count` override for the homogeneous fan-out participant count (ignored when a `participants`
|
|
56
|
+
* list is declared; a roster's length is authoritative). */
|
|
57
57
|
countOverride?: number;
|
|
58
58
|
/** Explicitly create a new run containing failed or selected participants from a prior fan-out run. */
|
|
59
59
|
rerun?: {
|
|
@@ -182,7 +182,7 @@ export interface CuaParticipantSummary {
|
|
|
182
182
|
concurrency: number;
|
|
183
183
|
waves: number;
|
|
184
184
|
}
|
|
185
|
-
export type CuaActorStudyErrorCode = "HUMANISH_STUDY_ANALYSIS_INVALID" | "HUMANISH_STUDY_TASKS_UNSUPPORTED" | "HUMANISH_STUDY_OPTION_UNSUPPORTED" | "HUMANISH_STUDY_V2_UNSUPPORTED" | "HUMANISH_COMPUTER_USE_FAILED" | "HUMANISH_COMPUTER_USE_KEYS_MISSING" | "HUMANISH_COMPUTER_USE_AGENT_MISSING" | "HUMANISH_COMPUTER_USE_AGENT_SIGNIN_REQUIRED" | "HUMANISH_COMPUTER_USE_SUBJECT_ENV_MISSING" | "HUMANISH_COMPUTER_USE_ACTOR_UNSUPPORTED" | "HUMANISH_COMPUTER_USE_SUBJECT_INVALID" | "HUMANISH_COMPUTER_USE_SUBJECT_UNSAFE" | "HUMANISH_COMPUTER_USE_EXECUTOR_NO_PROVIDER" | "HUMANISH_COMPUTER_USE_LOCAL_APP_NO_EXECUTOR" | "HUMANISH_COMPUTER_USE_LOCAL_DESKTOP_MISSING" | "HUMANISH_COMPUTER_USE_FANOUT_INVALID" | "HUMANISH_COMPUTER_USE_RERUN_INVALID" | "HUMANISH_COMPUTER_USE_DEVICE_GEOMETRY" | "HUMANISH_RUN_ID_IN_USE" | "HUMANISH_COMPUTER_USE_UNPRICED_CAP" | "HUMANISH_COMPUTER_USE_COMMS_CATCH_UNREACHABLE" | "HUMANISH_COMPUTER_USE_COMMS_TOKEN_INVALID" | "HUMANISH_WATCH_ALLOW_REQUIRES_OAUTH" | "HUMANISH_WATCH_OAUTH_REQUIRES_TUNNEL" | "HUMANISH_WATCH_OPTION_CONFLICT" | "HUMANISH_WATCH_TUNNEL_REQUIRES_EXPOSE" | "HUMANISH_WATCH_EXPOSE_REQUIRES_EDGE_AUTH" | "HUMANISH_WATCH_EXPOSE_REQUIRES_LIVE_FOLLOW" | "HUMANISH_WATCH_SAFE_NOT_APPLICABLE" | "HUMANISH_SERVE_TUNNEL_NOT_FOUND" | "HUMANISH_SERVE_TUNNEL_START_FAILED";
|
|
185
|
+
export type CuaActorStudyErrorCode = "HUMANISH_STUDY_ANALYSIS_INVALID" | "HUMANISH_STUDY_TASKS_UNSUPPORTED" | "HUMANISH_STUDY_OPTION_UNSUPPORTED" | "HUMANISH_STUDY_V2_UNSUPPORTED" | "HUMANISH_STUDY_INVALID" | "HUMANISH_COMPUTER_USE_FAILED" | "HUMANISH_COMPUTER_USE_KEYS_MISSING" | "HUMANISH_COMPUTER_USE_AGENT_MISSING" | "HUMANISH_COMPUTER_USE_AGENT_SIGNIN_REQUIRED" | "HUMANISH_COMPUTER_USE_SUBJECT_ENV_MISSING" | "HUMANISH_COMPUTER_USE_ACTOR_UNSUPPORTED" | "HUMANISH_COMPUTER_USE_SUBJECT_INVALID" | "HUMANISH_COMPUTER_USE_SUBJECT_UNSAFE" | "HUMANISH_COMPUTER_USE_EXECUTOR_NO_PROVIDER" | "HUMANISH_COMPUTER_USE_LOCAL_APP_NO_EXECUTOR" | "HUMANISH_COMPUTER_USE_LOCAL_DESKTOP_MISSING" | "HUMANISH_COMPUTER_USE_FANOUT_INVALID" | "HUMANISH_COMPUTER_USE_RERUN_INVALID" | "HUMANISH_COMPUTER_USE_DEVICE_GEOMETRY" | "HUMANISH_RUN_ID_IN_USE" | "HUMANISH_COMPUTER_USE_UNPRICED_CAP" | "HUMANISH_COMPUTER_USE_COMMS_CATCH_UNREACHABLE" | "HUMANISH_COMPUTER_USE_COMMS_TOKEN_INVALID" | "HUMANISH_WATCH_ALLOW_REQUIRES_OAUTH" | "HUMANISH_WATCH_OAUTH_REQUIRES_TUNNEL" | "HUMANISH_WATCH_OPTION_CONFLICT" | "HUMANISH_WATCH_TUNNEL_REQUIRES_EXPOSE" | "HUMANISH_WATCH_EXPOSE_REQUIRES_EDGE_AUTH" | "HUMANISH_WATCH_EXPOSE_REQUIRES_LIVE_FOLLOW" | "HUMANISH_WATCH_SAFE_NOT_APPLICABLE" | "HUMANISH_SERVE_TUNNEL_NOT_FOUND" | "HUMANISH_SERVE_TUNNEL_START_FAILED";
|
|
186
186
|
/** Subject provenance projection: what the actor actually drove. */
|
|
187
187
|
export interface CuaSubjectProjection {
|
|
188
188
|
source: "app-url" | "clone" | "local-tree";
|
|
@@ -385,7 +385,7 @@ export interface ParticipantModelDeps {
|
|
|
385
385
|
artifactRoot: PreparedOutputRoot;
|
|
386
386
|
redactScreenshots: boolean;
|
|
387
387
|
scrubKnownValues: (text: string) => string;
|
|
388
|
-
/** The study's shared spend ledger, present exactly when
|
|
388
|
+
/** The study's shared spend ledger, present exactly when caps.maxTotalUsd is set on a
|
|
389
389
|
* live run. Preflight already refused the cap on an unpriced model. */
|
|
390
390
|
runBudget?: CuaRunBudget;
|
|
391
391
|
/**
|
package/dist/routes/preview.js
CHANGED
|
@@ -30,6 +30,7 @@ export function admitPreviewPlan(plan, input) {
|
|
|
30
30
|
function runPreviewPlan(plan, input) {
|
|
31
31
|
return runDryRun({
|
|
32
32
|
...(plan.study === undefined ? {} : { study: plan.study }),
|
|
33
|
+
...(plan.warnings === undefined ? {} : { warnings: plan.warnings }),
|
|
33
34
|
cwd: input.cwd,
|
|
34
35
|
dryRun: true,
|
|
35
36
|
participantCount: plan.participantCount,
|
|
@@ -7,7 +7,6 @@ import { browserSurfaces } from "../../actors/scripted-browser/types.js";
|
|
|
7
7
|
import { resolveAutomaticAnalysis } from "../../analysis/automatic-config.js";
|
|
8
8
|
import { isNonEmpty, planBase, provisionedSubject } from "../../study/plan-base.js";
|
|
9
9
|
import { cloneTargetValidationReason, desktopMediaValidationReason, taskProtocolValidationReason, } from "../../study/validation.js";
|
|
10
|
-
import { actorOf, surfaceCount, scenarioRefOf } from "../../study/study-fields.js";
|
|
11
10
|
// Default surface roster is 1 (desktop only): the defaults-table single-participant row governs;
|
|
12
11
|
// `count: 2` is the declared override that adds the mobile surface.
|
|
13
12
|
const DEFAULT_SURFACE_COUNT = 1;
|
|
@@ -43,7 +42,7 @@ export function planScriptedStudy(config, input) {
|
|
|
43
42
|
const cloneTargetReason = cloneTargetValidationReason(config);
|
|
44
43
|
if (cloneTargetReason)
|
|
45
44
|
return refuse("HUMANISH_SCRIPTED_SUBJECT_UNSAFE", cloneTargetReason);
|
|
46
|
-
const actorType =
|
|
45
|
+
const actorType = config.actor?.type ?? "";
|
|
47
46
|
const descriptor = actorRegistry[actorType];
|
|
48
47
|
if (!descriptor || !isScriptedBrowserActorDescriptor(descriptor))
|
|
49
48
|
return refuse("HUMANISH_SCRIPTED_ACTOR_UNSUPPORTED", `actor.type "${actorType}" is not a registered scripted-browser actor.`);
|
|
@@ -64,7 +63,7 @@ export function planScriptedStudy(config, input) {
|
|
|
64
63
|
subject = { kind: "loopback", appUrl };
|
|
65
64
|
}
|
|
66
65
|
// The steps are the actor, so there is no built-in journey to fall back on.
|
|
67
|
-
const scenarioRef =
|
|
66
|
+
const scenarioRef = config.scenario;
|
|
68
67
|
if (!scenarioRef?.trim())
|
|
69
68
|
return refuse("HUMANISH_SCRIPTED_SCENARIO_INVALID", "A scripted-browser study needs `scenario`: the actor runs the browser steps in that scenario file.", { actor, appUrl: evidenceAppUrlOf(subject) });
|
|
70
69
|
const requirements = [];
|
|
@@ -76,7 +75,7 @@ export function planScriptedStudy(config, input) {
|
|
|
76
75
|
// The browser runs on this machine on both subjects; a clone's is pointed at its getHost URL.
|
|
77
76
|
if (!input.dryRun && input.injectedBrowser !== true)
|
|
78
77
|
requirements.push({ kind: "host-browser" });
|
|
79
|
-
const persona =
|
|
78
|
+
const persona = config.actor?.persona;
|
|
80
79
|
const timeoutMs = config.execution?.timeoutMs;
|
|
81
80
|
return {
|
|
82
81
|
ok: true,
|
|
@@ -89,7 +88,7 @@ export function planScriptedStudy(config, input) {
|
|
|
89
88
|
actor,
|
|
90
89
|
subject,
|
|
91
90
|
scenarioRef,
|
|
92
|
-
surfaces: browserSurfaces.slice(0,
|
|
91
|
+
surfaces: browserSurfaces.slice(0, config.surfaces?.length ?? DEFAULT_SURFACE_COUNT),
|
|
93
92
|
...(persona === undefined ? {} : { personaId: persona }),
|
|
94
93
|
...(timeoutMs === undefined ? {} : { sessionTimeoutMs: timeoutMs }),
|
|
95
94
|
requirements,
|
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
//
|
|
7
7
|
// Spend: scripted steps make no model requests, and their traces record zero token usage. A local
|
|
8
8
|
// app-url run spends no sandbox minutes; a live clone run spends E2B minutes to clone and serve the
|
|
9
|
-
// subject. `
|
|
9
|
+
// subject. `mode: live` is required anyway, because a live run actuates a real browser
|
|
10
10
|
// against a real app (it fills forms and clicks buttons). A dry run parses and digest-pins the
|
|
11
11
|
// scenario and writes the contract bundle without touching anything.
|
|
12
12
|
//
|
|
@@ -31,7 +31,6 @@ import { ScriptedSubject } from "./subject.js";
|
|
|
31
31
|
import { runScriptedSessions, writeSurfaceTraces } from "./surface-sessions.js";
|
|
32
32
|
import {} from "./types.js";
|
|
33
33
|
import { studyResultIdentity } from "../../run/study-result.js";
|
|
34
|
-
import { actorOf } from "../../study/study-fields.js";
|
|
35
34
|
/**
|
|
36
35
|
* A refused scripted study's result, at the refusal's stage: a before-scope refusal has its own field
|
|
37
36
|
* order and no analysis record; the others come after the output directory checks.
|
|
@@ -39,7 +38,7 @@ import { actorOf } from "../../study/study-fields.js";
|
|
|
39
38
|
export async function scriptedStudyRefusal(options, refusal) {
|
|
40
39
|
const { config, dryRun } = options;
|
|
41
40
|
const cwd = path.resolve(options.cwd);
|
|
42
|
-
const actorType =
|
|
41
|
+
const actorType = config.actor?.type ?? "";
|
|
43
42
|
if (refusal.beforeScope)
|
|
44
43
|
return {
|
|
45
44
|
...studyResultIdentity("scripted", config.id),
|
|
@@ -95,6 +94,7 @@ async function runScriptedPlanInScope(plan, input, scope) {
|
|
|
95
94
|
mintRunId: makeScriptedRunId,
|
|
96
95
|
mode: dryRun ? "dry-run" : "live",
|
|
97
96
|
study: plan.study,
|
|
97
|
+
warnings: plan.warnings,
|
|
98
98
|
// Only a clone subject is served from a sandbox; an app-url run drives a local browser.
|
|
99
99
|
sandboxes: setup.clone ? undefined : "none",
|
|
100
100
|
renderReview: renderScriptedReviewMarkdown,
|
|
@@ -8,7 +8,7 @@ interface ResolvedScriptedScenario {
|
|
|
8
8
|
sourceDigest: string;
|
|
9
9
|
}
|
|
10
10
|
/**
|
|
11
|
-
* Resolve and consume `scenario
|
|
11
|
+
* Resolve and consume `scenario`. Path-style refs (contain a separator or end .yaml/.yml)
|
|
12
12
|
* resolve against cwd and are clamped inside it: a ../../ escape is rejected, never recorded
|
|
13
13
|
* as repo-relative provenance. Id-style refs must be public-safe tokens and resolve to
|
|
14
14
|
* humanish/scenarios/<ref>.yaml (then .yml). Every failure mode is fail-closed. planScriptedStudy
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
// Resolve a scripted-browser study's `scenario
|
|
1
|
+
// Resolve a scripted-browser study's `scenario` to a committed scenario file and parse its
|
|
2
2
|
// `browser.steps` into the journey the actor runs. The ref is an id or a repo-relative path that must
|
|
3
3
|
// stay inside the project; provenance records it repo-relative with the scenario's digest.
|
|
4
4
|
import { parse as parseYaml } from "yaml";
|
|
@@ -6,11 +6,11 @@ import path from "node:path";
|
|
|
6
6
|
import { parseBrowserPersonaJourneyFromScenario } from "../../actors/scripted-browser/journey.js";
|
|
7
7
|
import { digestText } from "../../evidence/redaction.js";
|
|
8
8
|
import { readContainedRegularFile, } from "../../run/contained-output.js";
|
|
9
|
-
// Same public-safe token shape the study id uses; an id-style scenario
|
|
9
|
+
// Same public-safe token shape the study id uses; an id-style `scenario` must match it before
|
|
10
10
|
// it is interpolated into a repo path.
|
|
11
11
|
const SCENARIO_REF_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_.-]*$/;
|
|
12
12
|
/**
|
|
13
|
-
* Resolve and consume `scenario
|
|
13
|
+
* Resolve and consume `scenario`. Path-style refs (contain a separator or end .yaml/.yml)
|
|
14
14
|
* resolve against cwd and are clamped inside it: a ../../ escape is rejected, never recorded
|
|
15
15
|
* as repo-relative provenance. Id-style refs must be public-safe tokens and resolve to
|
|
16
16
|
* humanish/scenarios/<ref>.yaml (then .yml). Every failure mode is fail-closed. planScriptedStudy
|
|
@@ -47,7 +47,7 @@ export interface ScriptedBrowserStudyResult extends AutomaticAnalysisResult, Stu
|
|
|
47
47
|
killed: boolean;
|
|
48
48
|
};
|
|
49
49
|
hostDigest?: string;
|
|
50
|
-
/** The consumed scenario
|
|
50
|
+
/** The consumed `scenario`: digest-pinned provenance of the executable steps. */
|
|
51
51
|
scenario?: {
|
|
52
52
|
id: string;
|
|
53
53
|
source: string;
|
|
@@ -58,7 +58,7 @@ export interface ScriptedBrowserStudyResult extends AutomaticAnalysisResult, Stu
|
|
|
58
58
|
observer?: ObserverResult;
|
|
59
59
|
warnings: string[];
|
|
60
60
|
error?: {
|
|
61
|
-
code: "HUMANISH_STUDY_ANALYSIS_INVALID" | "HUMANISH_STUDY_TASKS_UNSUPPORTED" | "HUMANISH_STUDY_OPTION_UNSUPPORTED" | "HUMANISH_STUDY_V2_UNSUPPORTED" | "HUMANISH_SCRIPTED_FAILED" | "HUMANISH_SCRIPTED_ACTOR_UNSUPPORTED" | "HUMANISH_SCRIPTED_SCENARIO_INVALID" | "HUMANISH_SCRIPTED_SUBJECT_UNSAFE" | "HUMANISH_SCRIPTED_BROWSER_MISSING" | "HUMANISH_SCRIPTED_KEYS_MISSING" | "HUMANISH_SCRIPTED_SUBJECT_ENV_MISSING" | "HUMANISH_SCRIPTED_GETHOST_UNAVAILABLE" | "HUMANISH_RUN_ID_IN_USE";
|
|
61
|
+
code: "HUMANISH_STUDY_ANALYSIS_INVALID" | "HUMANISH_STUDY_TASKS_UNSUPPORTED" | "HUMANISH_STUDY_OPTION_UNSUPPORTED" | "HUMANISH_STUDY_V2_UNSUPPORTED" | "HUMANISH_STUDY_INVALID" | "HUMANISH_SCRIPTED_FAILED" | "HUMANISH_SCRIPTED_ACTOR_UNSUPPORTED" | "HUMANISH_SCRIPTED_SCENARIO_INVALID" | "HUMANISH_SCRIPTED_SUBJECT_UNSAFE" | "HUMANISH_SCRIPTED_BROWSER_MISSING" | "HUMANISH_SCRIPTED_KEYS_MISSING" | "HUMANISH_SCRIPTED_SUBJECT_ENV_MISSING" | "HUMANISH_SCRIPTED_GETHOST_UNAVAILABLE" | "HUMANISH_RUN_ID_IN_USE";
|
|
62
62
|
message: string;
|
|
63
63
|
};
|
|
64
64
|
}
|
|
@@ -30,7 +30,7 @@ const HANDOFF_DEADLINE_BUDGET_FRACTION = 0.4;
|
|
|
30
30
|
// convergence proof): at most this many single-frame reads before the participant is assumed to be
|
|
31
31
|
// somewhere without a code. Each reader stops the instant it has what it needs, so in practice only
|
|
32
32
|
// a handful fire (a participant reaches its /lobby within a few turns). NOTE: these reads are
|
|
33
|
-
// out-of-band OpenAI calls (external-public route only) and are left out of
|
|
33
|
+
// out-of-band OpenAI calls (external-public route only) and are left out of caps.maxUsd,
|
|
34
34
|
// so this hard cap is what bounds their spend (each read is one cheap single-frame OCR call). If
|
|
35
35
|
// this route ever runs under a strict budget, fold the estimate in.
|
|
36
36
|
const MAX_LOBBY_CODE_VISION_READS = 30;
|
|
@@ -200,7 +200,7 @@ export function participantRunDeps(ctx, live, scrubKnownValues) {
|
|
|
200
200
|
* scrubbed of the run's known secret values.
|
|
201
201
|
*/
|
|
202
202
|
export async function buildParticipantSpecs(participants, cwd, scrubKnownValues) {
|
|
203
|
-
// Only the personas the participants use: an
|
|
203
|
+
// Only the personas the participants use: an actor.persona that every participant overrides
|
|
204
204
|
// is never applied, so it is not read.
|
|
205
205
|
const personaResolution = await resolveCommittedPersonasForCwd(cwd, participants.map((participant) => participant.personaId));
|
|
206
206
|
const actorSpecs = participants.map((participant) => buildActorSpec(participant, personaResolution.personas));
|
|
@@ -10,9 +10,8 @@ import { brainOf, planCaps, desktopRequirements, isNonEmpty, planBase, } from ".
|
|
|
10
10
|
import { sharedWorldParticipants } from "../../study/plan-participants.js";
|
|
11
11
|
import { PUBLIC_TARGET_OWNER_PATTERN } from "../../study/parse/subject.js";
|
|
12
12
|
import { REPO_SLUG_PATTERN } from "../../study/parse/values.js";
|
|
13
|
-
import { concurrentSharedWorldValidationReason, desktopMediaValidationReason, externalPublicSharedWorldValidationReason, outputTokenLimitValidationReason, receivingEmailValidationReason,
|
|
13
|
+
import { concurrentSharedWorldValidationReason, desktopMediaValidationReason, externalPublicSharedWorldValidationReason, outputTokenLimitValidationReason, receivingEmailValidationReason, taskProtocolValidationReason, } from "../../study/validation.js";
|
|
14
14
|
import { MODEL_RATES, unpricedCapMessage } from "../../run/pricing.js";
|
|
15
|
-
import { actorOf, capsOf } from "../../study/study-fields.js";
|
|
16
15
|
/** The registered descriptor for a planned actor; planSharedWorldStudy checked the registry. */
|
|
17
16
|
export function sharedWorldDescriptorOf(actorType) {
|
|
18
17
|
const descriptor = actorRegistry[actorType];
|
|
@@ -39,7 +38,7 @@ export function planSharedWorldStudy(config, input) {
|
|
|
39
38
|
const tasksReason = taskProtocolValidationReason(config, false);
|
|
40
39
|
if (tasksReason)
|
|
41
40
|
return refuse("HUMANISH_STUDY_TASKS_UNSUPPORTED", tasksReason);
|
|
42
|
-
const actorType =
|
|
41
|
+
const actorType = config.actor?.type ?? "";
|
|
43
42
|
const descriptor = actorRegistry[actorType];
|
|
44
43
|
if (!descriptor || !isCuaActorDescriptor(descriptor))
|
|
45
44
|
return refuse("HUMANISH_SHARED_WORLD_ACTOR_UNSUPPORTED", `actor.type "${actorType}" is not a registered computer-use actor.`);
|
|
@@ -54,17 +53,16 @@ export function planSharedWorldStudy(config, input) {
|
|
|
54
53
|
// A library caller's config skips the parser, which refuses these first.
|
|
55
54
|
studyUrlCredentialReason(config) ??
|
|
56
55
|
outputTokenLimitValidationReason(config) ??
|
|
57
|
-
scenarioCapsValidationReason(config) ??
|
|
58
56
|
(externalPublic
|
|
59
57
|
? externalPublicSharedWorldValidationReason(config)
|
|
60
58
|
: concurrentSharedWorldValidationReason(config));
|
|
61
59
|
if (invalidReason)
|
|
62
60
|
return refuse(invalid, invalidReason, actor);
|
|
63
|
-
if (
|
|
61
|
+
if (config.actor?.maxOutputTokens !== undefined && input.hasRunSession === true)
|
|
64
62
|
return refuse(invalid, "maxOutputTokens cannot be enforced by a custom runSession.", actor);
|
|
65
|
-
const caps =
|
|
63
|
+
const caps = config.caps;
|
|
66
64
|
if (!input.dryRun && (caps?.maxUsd !== undefined || caps?.maxTotalUsd !== undefined)) {
|
|
67
|
-
const model = (
|
|
65
|
+
const model = (config.actor?.model ?? DEFAULT_OPENAI_CU_MODEL).trim().toLowerCase();
|
|
68
66
|
if (!MODEL_RATES[model])
|
|
69
67
|
return refuse("HUMANISH_SHARED_WORLD_UNPRICED_CAP", unpricedCapMessage(model), actor);
|
|
70
68
|
}
|
|
@@ -107,7 +105,7 @@ export function planSharedWorldStudy(config, input) {
|
|
|
107
105
|
? {}
|
|
108
106
|
: { sessionTimeoutMs: config.execution.timeoutMs }),
|
|
109
107
|
brain,
|
|
110
|
-
caps: planCaps(config
|
|
108
|
+
caps: planCaps(config),
|
|
111
109
|
requirements: base.dryRun
|
|
112
110
|
? []
|
|
113
111
|
: [
|
|
@@ -48,7 +48,7 @@ import { prepareConcurrentRun } from "./setup.js";
|
|
|
48
48
|
import { browserRouteScorer } from "../../study/adapter-scorer-loader.js";
|
|
49
49
|
import { withLateScorer } from "../../study/route-inputs.js";
|
|
50
50
|
import {} from "./types.js";
|
|
51
|
-
import {
|
|
51
|
+
import { participantList } from "../../study/study-fields.js";
|
|
52
52
|
/** The shared-world code for each local-agent refusal, kind for kind with computer-use. */
|
|
53
53
|
const LOCAL_AGENT_REFUSAL_CODES = {
|
|
54
54
|
"agent-missing": "HUMANISH_SHARED_WORLD_AGENT_MISSING",
|
|
@@ -63,7 +63,7 @@ export function sharedWorldStudyRefusal(options, refusal) {
|
|
|
63
63
|
const fail = concurrentStudyFailure({
|
|
64
64
|
cwd: path.resolve(options.cwd),
|
|
65
65
|
studyId: config.id,
|
|
66
|
-
actor:
|
|
66
|
+
actor: config.actor?.type ?? "",
|
|
67
67
|
participantCount: declared.length,
|
|
68
68
|
// The parser fills concurrency for multi-participant studies, so this fallback serves only
|
|
69
69
|
// library callers: every declared participant runs at once unless the author declared a cap.
|
|
@@ -56,6 +56,7 @@ function startConcurrentRun(study, cwd, scope) {
|
|
|
56
56
|
mintRunId: makeRunId,
|
|
57
57
|
mode: plan.dryRun ? "dry-run" : "live",
|
|
58
58
|
study: plan.study,
|
|
59
|
+
warnings: plan.warnings,
|
|
59
60
|
renderReview: renderConcurrentReviewMarkdown,
|
|
60
61
|
observer: { open: input.open === true, render: deps.renderObserver },
|
|
61
62
|
});
|
|
@@ -54,7 +54,7 @@ export interface SharedWorldRunInput {
|
|
|
54
54
|
* core-stamped onto the bundle as evidence. Absent for library callers. */
|
|
55
55
|
scorerProvenance?: RunScorerProvenance;
|
|
56
56
|
}
|
|
57
|
-
export type ConcurrentSharedWorldStudyErrorCode = "HUMANISH_STUDY_ANALYSIS_INVALID" | "HUMANISH_STUDY_TASKS_UNSUPPORTED" | "HUMANISH_STUDY_OPTION_UNSUPPORTED" | "HUMANISH_STUDY_V2_UNSUPPORTED" | "HUMANISH_SHARED_WORLD_FAILED" | "HUMANISH_SHARED_WORLD_ACTOR_UNSUPPORTED" | "HUMANISH_SHARED_WORLD_INVALID" | "HUMANISH_SHARED_WORLD_KEYS_MISSING"
|
|
57
|
+
export type ConcurrentSharedWorldStudyErrorCode = "HUMANISH_STUDY_ANALYSIS_INVALID" | "HUMANISH_STUDY_TASKS_UNSUPPORTED" | "HUMANISH_STUDY_OPTION_UNSUPPORTED" | "HUMANISH_STUDY_V2_UNSUPPORTED" | "HUMANISH_STUDY_INVALID" | "HUMANISH_SHARED_WORLD_FAILED" | "HUMANISH_SHARED_WORLD_ACTOR_UNSUPPORTED" | "HUMANISH_SHARED_WORLD_INVALID" | "HUMANISH_SHARED_WORLD_KEYS_MISSING"
|
|
58
58
|
/** A local-agent brain's CLI is not on `PATH`. */
|
|
59
59
|
| "HUMANISH_SHARED_WORLD_AGENT_MISSING"
|
|
60
60
|
/** A local-agent brain's CLI is signed out, or could not report its sign-in status. */
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { ActorPersonaRef, ActorTrace } from "../../actors/contract.js";
|
|
2
|
-
import type {
|
|
2
|
+
import type { StudyCaps, StudyRuntimeAuth } from "../../study/types.js";
|
|
3
3
|
import { type RunBundle, type RunCostSummary, type BundleRun } from "../../run/bundle.js";
|
|
4
4
|
import { type Verdict } from "../../run/judge.js";
|
|
5
5
|
import type { TerminalLedgers } from "./types.js";
|
|
@@ -21,7 +21,7 @@ export declare function buildTerminalProductBundle(args: {
|
|
|
21
21
|
persona: ActorPersonaRef;
|
|
22
22
|
productName: string;
|
|
23
23
|
publicSurfaces: string[];
|
|
24
|
-
caps?:
|
|
24
|
+
caps?: StudyCaps;
|
|
25
25
|
runtimeAuth?: string;
|
|
26
26
|
stdin: "disabled" | "planned" | "sent";
|
|
27
27
|
policies: {
|
|
@@ -51,7 +51,7 @@ export declare function buildLiveTerminalProductBundle(args: {
|
|
|
51
51
|
persona: ActorPersonaRef;
|
|
52
52
|
productName: string;
|
|
53
53
|
publicSurfaces: string[];
|
|
54
|
-
caps?:
|
|
54
|
+
caps?: StudyCaps;
|
|
55
55
|
runtimeAuthKeyName: string;
|
|
56
56
|
runtimeAuth?: StudyRuntimeAuth;
|
|
57
57
|
policies: {
|
|
@@ -30,6 +30,7 @@ export async function runDryTerminalStudy(args) {
|
|
|
30
30
|
mintRunId: makeTerminalRunId,
|
|
31
31
|
mode: "dry-run",
|
|
32
32
|
study: plan.study,
|
|
33
|
+
warnings: plan.warnings,
|
|
33
34
|
renderReview: renderTerminalReviewMarkdown,
|
|
34
35
|
observer: { open: input.open === true, render: input.deps?.renderObserver },
|
|
35
36
|
});
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { ActorTokenUsage } from "../../actors/contract.js";
|
|
2
|
-
import type {
|
|
2
|
+
import type { StudyCaps } from "../../study/types.js";
|
|
3
3
|
import { type CostCategory } from "../../run/terminal-contract.js";
|
|
4
4
|
import type { CostLine, NoSpendProof, TerminalCostLedger } from "./types.js";
|
|
5
5
|
/**
|
|
@@ -41,7 +41,7 @@ export declare function buildNoSpendProof(ledger: TerminalCostLedger, maxUsd: nu
|
|
|
41
41
|
* grant a green pass: the no-spend proof reports them as unmeasured. maxMinutes is wall-clock and is
|
|
42
42
|
* enforced separately (runWithWallClock); it is not a ledger-derived cap.
|
|
43
43
|
*/
|
|
44
|
-
export declare function evaluateCapsAgainstLedger(ledger: TerminalCostLedger, caps:
|
|
44
|
+
export declare function evaluateCapsAgainstLedger(ledger: TerminalCostLedger, caps: StudyCaps): {
|
|
45
45
|
ok: true;
|
|
46
46
|
} | {
|
|
47
47
|
ok: false;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// How long a live terminal sandbox may live. Its server-side timeout has to cover everything that
|
|
2
|
-
// runs in it before the codex command, the command's own wall clock (
|
|
2
|
+
// runs in it before the codex command, the command's own wall clock (caps.maxMinutes) and
|
|
3
3
|
// the reclamation buffer after that clock's kill. A timeout shorter than that lets E2B kill the
|
|
4
4
|
// sandbox before the command's deadline and before the evidence is read.
|
|
5
5
|
import { NODE_BOOTSTRAP_TIMEOUT_MS } from "../../subject/node-bootstrap.js";
|
|
@@ -23,11 +23,30 @@ function buildLiveTrace(inputs) {
|
|
|
23
23
|
// chunks before any transcript/trace/event artifact is persisted. Check both each stream and
|
|
24
24
|
// the combined event order that the transcript uses; either view can assemble a split value.
|
|
25
25
|
scrubSplitKnownValues(terminalEvents, knownSecretValues, discardedPrefixes);
|
|
26
|
-
// Build the actor trace first (the cost ledger reads its tokenUsage).
|
|
27
|
-
|
|
26
|
+
// Build the actor trace first (the cost ledger reads its tokenUsage). A session past the cap
|
|
27
|
+
// ends its transcript with what was not stored, and its trace says the same.
|
|
28
|
+
const cut = inputs.recorder.transcriptCut();
|
|
29
|
+
const normalizedTranscript = `${normalizeLocalActorTranscript(terminalEvents.map((e) => e.chunk).join(""))}${cut ? `\n[humanish] ${cut.notice}` : ""}`;
|
|
28
30
|
// Parsed from the full stream: usage records arrive once per turn, and the tail
|
|
29
|
-
// would drop all but the last.
|
|
30
|
-
const terminalTokenUsage = parseTerminalTokenUsage(normalizedTranscript);
|
|
31
|
+
// would drop all but the last. Records past the cap were kept as they arrived.
|
|
32
|
+
const terminalTokenUsage = parseTerminalTokenUsage([normalizedTranscript, ...(cut?.usage ?? [])].join("\n"));
|
|
33
|
+
const read = inputs.recorder.participantText.finish();
|
|
34
|
+
const participant = cut
|
|
35
|
+
? {
|
|
36
|
+
...read,
|
|
37
|
+
items: [
|
|
38
|
+
...read.items,
|
|
39
|
+
{
|
|
40
|
+
id: "notice-transcript",
|
|
41
|
+
kind: "notice",
|
|
42
|
+
lifecycle: "completed",
|
|
43
|
+
status: "truncated",
|
|
44
|
+
title: "terminal output truncated",
|
|
45
|
+
text: cut.notice,
|
|
46
|
+
},
|
|
47
|
+
],
|
|
48
|
+
}
|
|
49
|
+
: read;
|
|
31
50
|
const trace = buildTerminalActorTrace({
|
|
32
51
|
persona,
|
|
33
52
|
productName: product.name,
|
|
@@ -40,7 +59,7 @@ function buildLiveTrace(inputs) {
|
|
|
40
59
|
terminalEvents,
|
|
41
60
|
commandLog,
|
|
42
61
|
transcriptTail: tailOf(normalizedTranscript),
|
|
43
|
-
participant
|
|
62
|
+
participant,
|
|
44
63
|
runtimeAuth: runtimeEnv.mode,
|
|
45
64
|
runtime,
|
|
46
65
|
...(terminalTokenUsage === undefined ? {} : { tokenUsage: terminalTokenUsage }),
|
|
@@ -117,7 +136,7 @@ export async function finishLiveTerminalSession(inputs) {
|
|
|
117
136
|
const judgment = judgeTerminal({ dryRun: false, participant });
|
|
118
137
|
// The run cost summary, as the computer-use route records it: the sandbox's compute time from
|
|
119
138
|
// its span and observed size, and the participant's tokens (unpriced for Codex). It is not part
|
|
120
|
-
// of the cap ledger above, whose lines sum against
|
|
139
|
+
// of the cap ledger above, whose lines sum against caps.maxUsd.
|
|
121
140
|
const desktops = session.runCostDesktops();
|
|
122
141
|
const runCost = buildRunCostSummary({
|
|
123
142
|
participants: [{ trace }],
|
|
@@ -349,9 +349,13 @@ export class LiveTerminalSandbox {
|
|
|
349
349
|
});
|
|
350
350
|
// Score by the verdict-nonce marker over the scrubbed, redacted and normalized transcript, with
|
|
351
351
|
// the exact same logic the local-actor routes use (extractLocalActorVerdict/normalizeLocalActorTranscript).
|
|
352
|
+
// Output past the transcript cap is scanned as it arrives, since it is not stored.
|
|
352
353
|
const rawTranscript = terminalEvents.map((e) => e.chunk).join("");
|
|
353
354
|
const normalizedTranscript = normalizeLocalActorTranscript(rawTranscript);
|
|
354
|
-
const
|
|
355
|
+
const cut = this.inputs.recorder.transcriptCut();
|
|
356
|
+
const markerStatus = extractLocalActorVerdict(normalizedTranscript, verdictNonce) ?? cut?.verdict ?? null;
|
|
357
|
+
if (cut)
|
|
358
|
+
recordLifecycle("terminal-lab.transcript.exceeded", cut.notice);
|
|
355
359
|
if (this.timedOut) {
|
|
356
360
|
this.status = "timed_out";
|
|
357
361
|
this.completionReason = "timed_out";
|
|
@@ -13,7 +13,6 @@ import { NODE_BOOTSTRAP_TIMEOUT_MS } from "../../subject/node-bootstrap.js";
|
|
|
13
13
|
import { terminalSandboxTimeoutMs } from "./lifetime.js";
|
|
14
14
|
import { isExactRuntimeVersion, TERMINAL_RUNTIME_VERSION_TIMEOUT_MS } from "./runtime.js";
|
|
15
15
|
import { PRODUCT_SETUP_TIMEOUT_MS, TERMINAL_SANDBOX_TIMEOUT_BUFFER_MS, } from "./types.js";
|
|
16
|
-
import { actorOf, capsOf } from "../../study/study-fields.js";
|
|
17
16
|
/**
|
|
18
17
|
* Plan a terminal-product study. It is called for any config handed to the terminal runner, not
|
|
19
18
|
* only one routeOf sends here, so a config for another route gets this route's refusal.
|
|
@@ -41,12 +40,12 @@ export function planTerminalStudy(config, input) {
|
|
|
41
40
|
const tasksReason = taskProtocolValidationReason(config, false);
|
|
42
41
|
if (tasksReason)
|
|
43
42
|
return refuse("HUMANISH_STUDY_TASKS_UNSUPPORTED", tasksReason);
|
|
44
|
-
const actorType =
|
|
43
|
+
const actorType = config.actor?.type ?? "";
|
|
45
44
|
const descriptor = actorRegistry[actorType];
|
|
46
45
|
if (!descriptor || !isTerminalActorDescriptor(descriptor))
|
|
47
46
|
return refuse("HUMANISH_TERMINAL_ACTOR_UNSUPPORTED", `actor.type "${actorType}" is not a registered terminal actor.`);
|
|
48
47
|
const runtimeVersion = config.execution?.runtime?.version;
|
|
49
|
-
const actor =
|
|
48
|
+
const actor = config.actor;
|
|
50
49
|
if ((config.execution?.runtime !== undefined && !isExactRuntimeVersion(runtimeVersion)) ||
|
|
51
50
|
(actor?.model !== undefined &&
|
|
52
51
|
(typeof actor.model !== "string" || actor.model.trim().length === 0)) ||
|
|
@@ -84,7 +83,7 @@ export function planTerminalStudy(config, input) {
|
|
|
84
83
|
...(egressAllow === undefined ? {} : { egressAllow }),
|
|
85
84
|
...(stdin === undefined ? {} : { stdin }),
|
|
86
85
|
};
|
|
87
|
-
const caps =
|
|
86
|
+
const caps = config.caps;
|
|
88
87
|
if (input.dryRun)
|
|
89
88
|
return { ok: true, plan: { ...shared, dryRun: true, ...(caps ? { caps } : {}) } };
|
|
90
89
|
// The live route injects the runtime key into one command only, and only a registered actor
|
|
@@ -8,6 +8,7 @@ export declare function createTerminalRecorder(args: {
|
|
|
8
8
|
nowIso: () => string;
|
|
9
9
|
sanitize: (text: string) => string;
|
|
10
10
|
knownSecretValues: readonly string[];
|
|
11
|
+
verdictNonce: string;
|
|
11
12
|
}): {
|
|
12
13
|
lifecycle: LifecycleRecord[];
|
|
13
14
|
commandLog: CommandLogRecord[];
|
|
@@ -22,6 +23,11 @@ export declare function createTerminalRecorder(args: {
|
|
|
22
23
|
append(raw: string): void;
|
|
23
24
|
finish(): import("./participant-text.js").TerminalParticipantText;
|
|
24
25
|
};
|
|
26
|
+
transcriptCut: () => {
|
|
27
|
+
notice: string;
|
|
28
|
+
verdict: ReturnType<() => ("blocked" | "failed" | "passed") | null>;
|
|
29
|
+
usage: string[];
|
|
30
|
+
} | undefined;
|
|
25
31
|
recordLifecycle: (event: string, message: string) => void;
|
|
26
32
|
recordStreamedTerminalChunk: (stream: "stdout" | "stderr", raw: string) => void;
|
|
27
33
|
appendReturnedTerminalOutput: (stream: "stdout" | "stderr", raw: string) => void;
|
|
@@ -1,13 +1,15 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
|
+
import { createLocalActorVerdictScanner } from "../../run/terminal-contract.js";
|
|
2
3
|
import { createTerminalParticipantReader } from "./participant-text.js";
|
|
3
|
-
import {
|
|
4
|
+
import { terminalUsageRecords } from "./token-usage.js";
|
|
5
|
+
import { MAX_TRANSCRIPT_BYTES, PENDING_LINE_CHARS, } from "./types.js";
|
|
4
6
|
/**
|
|
5
7
|
* The terminal run's ledgers and capture buffers, mutated through the live lifecycle. Every chunk
|
|
6
8
|
* is scrubbed and redacted as it is stored. Raw bytes leave the append functions only for the
|
|
7
9
|
* in-memory participant reader, which keeps sanitized decoded text.
|
|
8
10
|
*/
|
|
9
11
|
export function createTerminalRecorder(args) {
|
|
10
|
-
const { nowIso, sanitize, knownSecretValues } = args;
|
|
12
|
+
const { nowIso, sanitize, knownSecretValues, verdictNonce } = args;
|
|
11
13
|
const lifecycle = [];
|
|
12
14
|
const commandLog = [];
|
|
13
15
|
const terminalEvents = [];
|
|
@@ -19,24 +21,76 @@ export function createTerminalRecorder(args) {
|
|
|
19
21
|
// Reads the agent's text from the raw stdout in memory; it keeps only sanitized decoded text.
|
|
20
22
|
const participantText = createTerminalParticipantReader(sanitize);
|
|
21
23
|
let transcriptBytes = 0;
|
|
24
|
+
// Output past the cap is not stored, but the participant reader, the verdict marker and token
|
|
25
|
+
// usage still read all of it, so a long session is judged and priced as a short one is.
|
|
26
|
+
const notStored = { stdout: 0, stderr: 0 };
|
|
27
|
+
const laterVerdict = createLocalActorVerdictScanner(verdictNonce);
|
|
28
|
+
const laterUsage = [];
|
|
29
|
+
// The stdout line in progress past the cap. Undefined while the rest of a line is skipped: the
|
|
30
|
+
// line the cap cut through (its start is stored) or one longer than the pending limit.
|
|
31
|
+
let laterLine;
|
|
32
|
+
let stdoutAtLineStart = true;
|
|
33
|
+
let pastCap = false;
|
|
34
|
+
const readLaterStdout = (raw) => {
|
|
35
|
+
let text = raw;
|
|
36
|
+
if (laterLine === undefined) {
|
|
37
|
+
const end = text.indexOf("\n");
|
|
38
|
+
if (end < 0)
|
|
39
|
+
return;
|
|
40
|
+
text = text.slice(end + 1);
|
|
41
|
+
laterLine = "";
|
|
42
|
+
}
|
|
43
|
+
laterLine += text;
|
|
44
|
+
const end = laterLine.lastIndexOf("\n");
|
|
45
|
+
if (end >= 0) {
|
|
46
|
+
laterUsage.push(...terminalUsageRecords(sanitize(laterLine.slice(0, end))));
|
|
47
|
+
laterLine = laterLine.slice(end + 1);
|
|
48
|
+
}
|
|
49
|
+
if (laterLine.length > PENDING_LINE_CHARS)
|
|
50
|
+
laterLine = undefined;
|
|
51
|
+
};
|
|
22
52
|
const recordLifecycle = (event, message) => {
|
|
23
53
|
lifecycle.push({ at: nowIso(), event, message: sanitize(message) });
|
|
24
54
|
};
|
|
25
55
|
const appendTerminalChunk = (stream, raw) => {
|
|
56
|
+
if (stream === "stdout")
|
|
57
|
+
participantText.append(raw);
|
|
26
58
|
if (transcriptBytes >= MAX_TRANSCRIPT_BYTES) {
|
|
27
59
|
for (const order of [stream, "combined"]) {
|
|
28
60
|
const remaining = maxDiscardedPrefixChars - discardedPrefixes[order].length;
|
|
29
61
|
if (remaining > 0)
|
|
30
62
|
discardedPrefixes[order] += raw.slice(0, remaining);
|
|
31
63
|
}
|
|
64
|
+
if (!pastCap) {
|
|
65
|
+
pastCap = true;
|
|
66
|
+
// The stored stream seeds the scan, so a marker the cap split in two is still found.
|
|
67
|
+
laterVerdict.push(terminalEvents.map((event) => event.chunk).join(""));
|
|
68
|
+
laterLine = stdoutAtLineStart ? "" : undefined;
|
|
69
|
+
}
|
|
70
|
+
notStored[stream] += Buffer.byteLength(raw, "utf8");
|
|
71
|
+
laterVerdict.push(sanitize(raw));
|
|
72
|
+
if (stream === "stdout")
|
|
73
|
+
readLaterStdout(raw);
|
|
32
74
|
return;
|
|
33
75
|
}
|
|
34
76
|
transcriptBytes += Buffer.byteLength(raw, "utf8");
|
|
35
|
-
// Scrub, then redact, at the source. Only the participant reader
|
|
36
|
-
//
|
|
77
|
+
// Scrub, then redact, at the source. Only the participant reader and the readers of output
|
|
78
|
+
// past the cap see raw bytes, and they store none of them.
|
|
37
79
|
terminalEvents.push({ at: nowIso(), stream, chunk: sanitize(raw) });
|
|
38
|
-
if (stream === "stdout")
|
|
39
|
-
|
|
80
|
+
if (stream === "stdout" && raw)
|
|
81
|
+
stdoutAtLineStart = raw.endsWith("\n");
|
|
82
|
+
};
|
|
83
|
+
/** What the cap left out, for the transcript, the trace and the run events; undefined when the
|
|
84
|
+
* whole session was stored. */
|
|
85
|
+
const transcriptCut = () => {
|
|
86
|
+
if (!pastCap)
|
|
87
|
+
return undefined;
|
|
88
|
+
const bytes = notStored.stdout + notStored.stderr;
|
|
89
|
+
return {
|
|
90
|
+
notice: `Output past the ${MAX_TRANSCRIPT_BYTES / 1024} KiB transcript cap was not stored: ${bytes} bytes (${notStored.stdout} stdout, ${notStored.stderr} stderr). The verdict marker, token usage and agent messages were still read from it.`,
|
|
91
|
+
verdict: laterVerdict.verdict(),
|
|
92
|
+
usage: laterUsage,
|
|
93
|
+
};
|
|
40
94
|
};
|
|
41
95
|
// E2B can stream every byte through callbacks and return the same complete output.
|
|
42
96
|
// Track transport delivery, independently per stream, rather than deduplicating participant
|
|
@@ -78,6 +132,7 @@ export function createTerminalRecorder(args) {
|
|
|
78
132
|
interventions,
|
|
79
133
|
discardedPrefixes,
|
|
80
134
|
participantText,
|
|
135
|
+
transcriptCut,
|
|
81
136
|
recordLifecycle,
|
|
82
137
|
recordStreamedTerminalChunk,
|
|
83
138
|
appendReturnedTerminalOutput,
|
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
// spending. Opt-in openai-env injects the raw key command-scoped, never through
|
|
18
18
|
// Sandbox.create({envs}).
|
|
19
19
|
// Enforced in credentials.ts and runtime-auth.ts.
|
|
20
|
-
// 2. Fail-closed cap. The live key is never exercised without
|
|
20
|
+
// 2. Fail-closed cap. The live key is never exercised without `caps` in force: maxUsd
|
|
21
21
|
// (default/require 0 = no-spend) + maxMinutes (wall-clock kill of the codex command).
|
|
22
22
|
// Enforced in plan.ts (caps required), live-sandbox.ts and lifetime.ts (the wall clock).
|
|
23
23
|
// 3. Public surfaces only. The mission references only subject.product.publicSurfaces + the
|
|
@@ -54,7 +54,6 @@ import { terminalRouteScorer } from "../../study/adapter-scorer-loader.js";
|
|
|
54
54
|
import { withLateScorer } from "../../study/route-inputs.js";
|
|
55
55
|
import {} from "./types.js";
|
|
56
56
|
import { studyResultIdentity } from "../../run/study-result.js";
|
|
57
|
-
import { actorOf } from "../../study/study-fields.js";
|
|
58
57
|
/** A refused terminal study's result: the route's envelope, and the analysis record a refusal gets. */
|
|
59
58
|
export function terminalStudyRefusal(options, refusal) {
|
|
60
59
|
const { config, dryRun } = options;
|
|
@@ -62,7 +61,7 @@ export function terminalStudyRefusal(options, refusal) {
|
|
|
62
61
|
...studyResultIdentity("terminal", config.id),
|
|
63
62
|
ok: false,
|
|
64
63
|
cwd: path.resolve(options.cwd),
|
|
65
|
-
actor: refusal.actor ??
|
|
64
|
+
actor: refusal.actor ?? config.actor?.type ?? "",
|
|
66
65
|
product: config.subject.product?.name ?? "",
|
|
67
66
|
dryRun,
|
|
68
67
|
runId: options.runId ?? "not-created",
|