humanish 0.111.0 → 0.111.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/CONTEXT.md +1 -1
- package/dist/actors/codex/restricted-participant-policy.d.ts +2 -0
- package/dist/actors/codex/restricted-participant-policy.js +30 -2
- package/dist/actors/codex/restricted-participant.js +1 -1
- package/dist/actors/computer-use/executor-diagnostic.d.ts +15 -0
- package/dist/actors/computer-use/executor-diagnostic.js +105 -0
- package/dist/actors/computer-use/executor-error.d.ts +18 -3
- package/dist/actors/computer-use/executor-error.js +30 -4
- package/dist/actors/computer-use/loop/actions.d.ts +6 -3
- package/dist/actors/computer-use/loop/actions.js +27 -7
- package/dist/actors/computer-use/loop/ending.js +9 -1
- package/dist/actors/computer-use/loop/types.d.ts +14 -0
- package/dist/actors/computer-use/loop.d.ts +1 -1
- package/dist/actors/computer-use/loop.js +1 -1
- package/dist/actors/computer-use/openai-context.d.ts +8 -2
- package/dist/actors/computer-use/openai-context.js +91 -26
- package/dist/actors/computer-use/openai-provider.js +55 -12
- package/dist/actors/computer-use/openai-requests.d.ts +2 -1
- package/dist/actors/computer-use/openai-requests.js +4 -1
- package/dist/actors/contract.d.ts +7 -1
- package/dist/browser-control/client.js +5 -5
- package/dist/browser-control/protocol.d.ts +30 -1
- package/dist/browser-control/protocol.js +19 -2
- package/dist/browser-control/transport.d.ts +3 -2
- package/dist/browser-control/transport.js +12 -7
- package/dist/cli/commands/study-route-computer-use.js +2 -2
- package/dist/cli/commands/study-route-shared-world.js +2 -2
- package/dist/cli/commands/study.js +1 -2
- package/dist/cli/doctor.js +11 -0
- package/dist/evidence/encoded-text.d.ts +3 -3
- package/dist/evidence/encoded-text.js +10 -5
- package/dist/evidence/redaction.d.ts +2 -2
- package/dist/evidence/redaction.js +18 -7
- package/dist/guest/browser-tools.js +5 -1
- package/dist/guest/chromium-text.js +42 -18
- package/dist/guest/desktop-executor.js +5 -1
- package/dist/keys/key-resolution.d.ts +3 -0
- package/dist/keys/key-resolution.js +15 -2
- package/dist/routes/computer-use/bundle-parts.d.ts +4 -0
- package/dist/routes/computer-use/bundle-parts.js +7 -2
- package/dist/routes/computer-use/fanout-bundle.js +4 -2
- package/dist/routes/computer-use/local-vm.js +1 -2
- package/dist/routes/computer-use/participant-model.js +6 -1
- package/dist/routes/computer-use/plan.js +3 -4
- package/dist/routes/computer-use/route.js +1 -2
- package/dist/routes/computer-use/setup.js +2 -2
- package/dist/routes/preview.js +1 -0
- package/dist/routes/scripted/plan.js +4 -5
- package/dist/routes/scripted/route.js +2 -2
- package/dist/routes/shared-world/plan.js +4 -5
- package/dist/routes/shared-world/route.js +2 -2
- package/dist/routes/shared-world/setup.js +1 -0
- package/dist/routes/terminal/dry-run.js +1 -0
- package/dist/routes/terminal/live-finish.js +24 -5
- package/dist/routes/terminal/live-sandbox.js +5 -1
- package/dist/routes/terminal/plan.js +3 -4
- package/dist/routes/terminal/recorder.d.ts +6 -0
- package/dist/routes/terminal/recorder.js +61 -6
- package/dist/routes/terminal/route.js +1 -2
- package/dist/routes/terminal/session.js +2 -1
- package/dist/routes/terminal/token-usage.d.ts +3 -0
- package/dist/routes/terminal/token-usage.js +5 -0
- package/dist/routes/terminal/types.d.ts +2 -1
- package/dist/routes/terminal/types.js +2 -1
- package/dist/run/dry-run.js +1 -0
- package/dist/run/results.d.ts +2 -0
- package/dist/run/run.d.ts +2 -0
- package/dist/run/run.js +16 -2
- package/dist/run/terminal-contract.d.ts +10 -0
- package/dist/run/terminal-contract.js +27 -9
- package/dist/study/composition-rules.js +17 -18
- package/dist/study/doctor.js +4 -5
- package/dist/study/migrate/v2.d.ts +5 -0
- package/dist/study/migrate/v2.js +23 -0
- package/dist/study/parse/front.js +2 -21
- package/dist/study/parse/url-credentials.d.ts +3 -3
- package/dist/study/parse/url-credentials.js +155 -13
- package/dist/study/persona-resolve.js +2 -2
- package/dist/study/plan-base.js +3 -4
- package/dist/study/plan-participants.js +3 -3
- package/dist/study/plan-types.d.ts +2 -0
- package/dist/study/plan.js +11 -5
- package/dist/study/routing.js +11 -11
- package/dist/study/run-study-options.js +2 -2
- package/dist/study/study-fields.d.ts +1 -13
- package/dist/study/study-fields.js +0 -24
- package/dist/study/summary.js +7 -7
- package/dist/study/types.d.ts +1 -0
- package/dist/study/validation.js +11 -11
- package/dist/study/warnings.d.ts +8 -0
- package/dist/study/warnings.js +22 -7
- package/dist/substrates/local/runtime-config.js +6 -6
- package/docs/architecture/browser-control.md +12 -2
- package/docs/architecture/guest-desktop.md +7 -1
- package/docs/architecture/terminal-product-route.md +3 -2
- package/docs/contracts/schemas.md +16 -5
- package/package.json +1 -1
|
@@ -12,7 +12,6 @@ import { PUBLIC_TARGET_OWNER_PATTERN } from "../../study/parse/subject.js";
|
|
|
12
12
|
import { REPO_SLUG_PATTERN } from "../../study/parse/values.js";
|
|
13
13
|
import { concurrentSharedWorldValidationReason, desktopMediaValidationReason, externalPublicSharedWorldValidationReason, outputTokenLimitValidationReason, receivingEmailValidationReason, taskProtocolValidationReason, } from "../../study/validation.js";
|
|
14
14
|
import { MODEL_RATES, unpricedCapMessage } from "../../run/pricing.js";
|
|
15
|
-
import { actorOf, capsOf } from "../../study/study-fields.js";
|
|
16
15
|
/** The registered descriptor for a planned actor; planSharedWorldStudy checked the registry. */
|
|
17
16
|
export function sharedWorldDescriptorOf(actorType) {
|
|
18
17
|
const descriptor = actorRegistry[actorType];
|
|
@@ -39,7 +38,7 @@ export function planSharedWorldStudy(config, input) {
|
|
|
39
38
|
const tasksReason = taskProtocolValidationReason(config, false);
|
|
40
39
|
if (tasksReason)
|
|
41
40
|
return refuse("HUMANISH_STUDY_TASKS_UNSUPPORTED", tasksReason);
|
|
42
|
-
const actorType =
|
|
41
|
+
const actorType = config.actor?.type ?? "";
|
|
43
42
|
const descriptor = actorRegistry[actorType];
|
|
44
43
|
if (!descriptor || !isCuaActorDescriptor(descriptor))
|
|
45
44
|
return refuse("HUMANISH_SHARED_WORLD_ACTOR_UNSUPPORTED", `actor.type "${actorType}" is not a registered computer-use actor.`);
|
|
@@ -59,11 +58,11 @@ export function planSharedWorldStudy(config, input) {
|
|
|
59
58
|
: concurrentSharedWorldValidationReason(config));
|
|
60
59
|
if (invalidReason)
|
|
61
60
|
return refuse(invalid, invalidReason, actor);
|
|
62
|
-
if (
|
|
61
|
+
if (config.actor?.maxOutputTokens !== undefined && input.hasRunSession === true)
|
|
63
62
|
return refuse(invalid, "maxOutputTokens cannot be enforced by a custom runSession.", actor);
|
|
64
|
-
const caps =
|
|
63
|
+
const caps = config.caps;
|
|
65
64
|
if (!input.dryRun && (caps?.maxUsd !== undefined || caps?.maxTotalUsd !== undefined)) {
|
|
66
|
-
const model = (
|
|
65
|
+
const model = (config.actor?.model ?? DEFAULT_OPENAI_CU_MODEL).trim().toLowerCase();
|
|
67
66
|
if (!MODEL_RATES[model])
|
|
68
67
|
return refuse("HUMANISH_SHARED_WORLD_UNPRICED_CAP", unpricedCapMessage(model), actor);
|
|
69
68
|
}
|
|
@@ -48,7 +48,7 @@ import { prepareConcurrentRun } from "./setup.js";
|
|
|
48
48
|
import { browserRouteScorer } from "../../study/adapter-scorer-loader.js";
|
|
49
49
|
import { withLateScorer } from "../../study/route-inputs.js";
|
|
50
50
|
import {} from "./types.js";
|
|
51
|
-
import {
|
|
51
|
+
import { participantList } from "../../study/study-fields.js";
|
|
52
52
|
/** The shared-world code for each local-agent refusal, kind for kind with computer-use. */
|
|
53
53
|
const LOCAL_AGENT_REFUSAL_CODES = {
|
|
54
54
|
"agent-missing": "HUMANISH_SHARED_WORLD_AGENT_MISSING",
|
|
@@ -63,7 +63,7 @@ export function sharedWorldStudyRefusal(options, refusal) {
|
|
|
63
63
|
const fail = concurrentStudyFailure({
|
|
64
64
|
cwd: path.resolve(options.cwd),
|
|
65
65
|
studyId: config.id,
|
|
66
|
-
actor:
|
|
66
|
+
actor: config.actor?.type ?? "",
|
|
67
67
|
participantCount: declared.length,
|
|
68
68
|
// The parser fills concurrency for multi-participant studies, so this fallback serves only
|
|
69
69
|
// library callers: every declared participant runs at once unless the author declared a cap.
|
|
@@ -56,6 +56,7 @@ function startConcurrentRun(study, cwd, scope) {
|
|
|
56
56
|
mintRunId: makeRunId,
|
|
57
57
|
mode: plan.dryRun ? "dry-run" : "live",
|
|
58
58
|
study: plan.study,
|
|
59
|
+
warnings: plan.warnings,
|
|
59
60
|
renderReview: renderConcurrentReviewMarkdown,
|
|
60
61
|
observer: { open: input.open === true, render: deps.renderObserver },
|
|
61
62
|
});
|
|
@@ -30,6 +30,7 @@ export async function runDryTerminalStudy(args) {
|
|
|
30
30
|
mintRunId: makeTerminalRunId,
|
|
31
31
|
mode: "dry-run",
|
|
32
32
|
study: plan.study,
|
|
33
|
+
warnings: plan.warnings,
|
|
33
34
|
renderReview: renderTerminalReviewMarkdown,
|
|
34
35
|
observer: { open: input.open === true, render: input.deps?.renderObserver },
|
|
35
36
|
});
|
|
@@ -23,11 +23,30 @@ function buildLiveTrace(inputs) {
|
|
|
23
23
|
// chunks before any transcript/trace/event artifact is persisted. Check both each stream and
|
|
24
24
|
// the combined event order that the transcript uses; either view can assemble a split value.
|
|
25
25
|
scrubSplitKnownValues(terminalEvents, knownSecretValues, discardedPrefixes);
|
|
26
|
-
// Build the actor trace first (the cost ledger reads its tokenUsage).
|
|
27
|
-
|
|
26
|
+
// Build the actor trace first (the cost ledger reads its tokenUsage). A session past the cap
|
|
27
|
+
// ends its transcript with what was not stored, and its trace says the same.
|
|
28
|
+
const cut = inputs.recorder.transcriptCut();
|
|
29
|
+
const normalizedTranscript = `${normalizeLocalActorTranscript(terminalEvents.map((e) => e.chunk).join(""))}${cut ? `\n[humanish] ${cut.notice}` : ""}`;
|
|
28
30
|
// Parsed from the full stream: usage records arrive once per turn, and the tail
|
|
29
|
-
// would drop all but the last.
|
|
30
|
-
const terminalTokenUsage = parseTerminalTokenUsage(normalizedTranscript);
|
|
31
|
+
// would drop all but the last. Records past the cap were kept as they arrived.
|
|
32
|
+
const terminalTokenUsage = parseTerminalTokenUsage([normalizedTranscript, ...(cut?.usage ?? [])].join("\n"));
|
|
33
|
+
const read = inputs.recorder.participantText.finish();
|
|
34
|
+
const participant = cut
|
|
35
|
+
? {
|
|
36
|
+
...read,
|
|
37
|
+
items: [
|
|
38
|
+
...read.items,
|
|
39
|
+
{
|
|
40
|
+
id: "notice-transcript",
|
|
41
|
+
kind: "notice",
|
|
42
|
+
lifecycle: "completed",
|
|
43
|
+
status: "truncated",
|
|
44
|
+
title: "terminal output truncated",
|
|
45
|
+
text: cut.notice,
|
|
46
|
+
},
|
|
47
|
+
],
|
|
48
|
+
}
|
|
49
|
+
: read;
|
|
31
50
|
const trace = buildTerminalActorTrace({
|
|
32
51
|
persona,
|
|
33
52
|
productName: product.name,
|
|
@@ -40,7 +59,7 @@ function buildLiveTrace(inputs) {
|
|
|
40
59
|
terminalEvents,
|
|
41
60
|
commandLog,
|
|
42
61
|
transcriptTail: tailOf(normalizedTranscript),
|
|
43
|
-
participant
|
|
62
|
+
participant,
|
|
44
63
|
runtimeAuth: runtimeEnv.mode,
|
|
45
64
|
runtime,
|
|
46
65
|
...(terminalTokenUsage === undefined ? {} : { tokenUsage: terminalTokenUsage }),
|
|
@@ -349,9 +349,13 @@ export class LiveTerminalSandbox {
|
|
|
349
349
|
});
|
|
350
350
|
// Score by the verdict-nonce marker over the scrubbed, redacted and normalized transcript, with
|
|
351
351
|
// the exact same logic the local-actor routes use (extractLocalActorVerdict/normalizeLocalActorTranscript).
|
|
352
|
+
// Output past the transcript cap is scanned as it arrives, since it is not stored.
|
|
352
353
|
const rawTranscript = terminalEvents.map((e) => e.chunk).join("");
|
|
353
354
|
const normalizedTranscript = normalizeLocalActorTranscript(rawTranscript);
|
|
354
|
-
const
|
|
355
|
+
const cut = this.inputs.recorder.transcriptCut();
|
|
356
|
+
const markerStatus = extractLocalActorVerdict(normalizedTranscript, verdictNonce) ?? cut?.verdict ?? null;
|
|
357
|
+
if (cut)
|
|
358
|
+
recordLifecycle("terminal-lab.transcript.exceeded", cut.notice);
|
|
355
359
|
if (this.timedOut) {
|
|
356
360
|
this.status = "timed_out";
|
|
357
361
|
this.completionReason = "timed_out";
|
|
@@ -13,7 +13,6 @@ import { NODE_BOOTSTRAP_TIMEOUT_MS } from "../../subject/node-bootstrap.js";
|
|
|
13
13
|
import { terminalSandboxTimeoutMs } from "./lifetime.js";
|
|
14
14
|
import { isExactRuntimeVersion, TERMINAL_RUNTIME_VERSION_TIMEOUT_MS } from "./runtime.js";
|
|
15
15
|
import { PRODUCT_SETUP_TIMEOUT_MS, TERMINAL_SANDBOX_TIMEOUT_BUFFER_MS, } from "./types.js";
|
|
16
|
-
import { actorOf, capsOf } from "../../study/study-fields.js";
|
|
17
16
|
/**
|
|
18
17
|
* Plan a terminal-product study. It is called for any config handed to the terminal runner, not
|
|
19
18
|
* only one routeOf sends here, so a config for another route gets this route's refusal.
|
|
@@ -41,12 +40,12 @@ export function planTerminalStudy(config, input) {
|
|
|
41
40
|
const tasksReason = taskProtocolValidationReason(config, false);
|
|
42
41
|
if (tasksReason)
|
|
43
42
|
return refuse("HUMANISH_STUDY_TASKS_UNSUPPORTED", tasksReason);
|
|
44
|
-
const actorType =
|
|
43
|
+
const actorType = config.actor?.type ?? "";
|
|
45
44
|
const descriptor = actorRegistry[actorType];
|
|
46
45
|
if (!descriptor || !isTerminalActorDescriptor(descriptor))
|
|
47
46
|
return refuse("HUMANISH_TERMINAL_ACTOR_UNSUPPORTED", `actor.type "${actorType}" is not a registered terminal actor.`);
|
|
48
47
|
const runtimeVersion = config.execution?.runtime?.version;
|
|
49
|
-
const actor =
|
|
48
|
+
const actor = config.actor;
|
|
50
49
|
if ((config.execution?.runtime !== undefined && !isExactRuntimeVersion(runtimeVersion)) ||
|
|
51
50
|
(actor?.model !== undefined &&
|
|
52
51
|
(typeof actor.model !== "string" || actor.model.trim().length === 0)) ||
|
|
@@ -84,7 +83,7 @@ export function planTerminalStudy(config, input) {
|
|
|
84
83
|
...(egressAllow === undefined ? {} : { egressAllow }),
|
|
85
84
|
...(stdin === undefined ? {} : { stdin }),
|
|
86
85
|
};
|
|
87
|
-
const caps =
|
|
86
|
+
const caps = config.caps;
|
|
88
87
|
if (input.dryRun)
|
|
89
88
|
return { ok: true, plan: { ...shared, dryRun: true, ...(caps ? { caps } : {}) } };
|
|
90
89
|
// The live route injects the runtime key into one command only, and only a registered actor
|
|
@@ -8,6 +8,7 @@ export declare function createTerminalRecorder(args: {
|
|
|
8
8
|
nowIso: () => string;
|
|
9
9
|
sanitize: (text: string) => string;
|
|
10
10
|
knownSecretValues: readonly string[];
|
|
11
|
+
verdictNonce: string;
|
|
11
12
|
}): {
|
|
12
13
|
lifecycle: LifecycleRecord[];
|
|
13
14
|
commandLog: CommandLogRecord[];
|
|
@@ -22,6 +23,11 @@ export declare function createTerminalRecorder(args: {
|
|
|
22
23
|
append(raw: string): void;
|
|
23
24
|
finish(): import("./participant-text.js").TerminalParticipantText;
|
|
24
25
|
};
|
|
26
|
+
transcriptCut: () => {
|
|
27
|
+
notice: string;
|
|
28
|
+
verdict: ReturnType<() => ("blocked" | "failed" | "passed") | null>;
|
|
29
|
+
usage: string[];
|
|
30
|
+
} | undefined;
|
|
25
31
|
recordLifecycle: (event: string, message: string) => void;
|
|
26
32
|
recordStreamedTerminalChunk: (stream: "stdout" | "stderr", raw: string) => void;
|
|
27
33
|
appendReturnedTerminalOutput: (stream: "stdout" | "stderr", raw: string) => void;
|
|
@@ -1,13 +1,15 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
|
+
import { createLocalActorVerdictScanner } from "../../run/terminal-contract.js";
|
|
2
3
|
import { createTerminalParticipantReader } from "./participant-text.js";
|
|
3
|
-
import {
|
|
4
|
+
import { terminalUsageRecords } from "./token-usage.js";
|
|
5
|
+
import { MAX_TRANSCRIPT_BYTES, PENDING_LINE_CHARS, } from "./types.js";
|
|
4
6
|
/**
|
|
5
7
|
* The terminal run's ledgers and capture buffers, mutated through the live lifecycle. Every chunk
|
|
6
8
|
* is scrubbed and redacted as it is stored. Raw bytes leave the append functions only for the
|
|
7
9
|
* in-memory participant reader, which keeps sanitized decoded text.
|
|
8
10
|
*/
|
|
9
11
|
export function createTerminalRecorder(args) {
|
|
10
|
-
const { nowIso, sanitize, knownSecretValues } = args;
|
|
12
|
+
const { nowIso, sanitize, knownSecretValues, verdictNonce } = args;
|
|
11
13
|
const lifecycle = [];
|
|
12
14
|
const commandLog = [];
|
|
13
15
|
const terminalEvents = [];
|
|
@@ -19,24 +21,76 @@ export function createTerminalRecorder(args) {
|
|
|
19
21
|
// Reads the agent's text from the raw stdout in memory; it keeps only sanitized decoded text.
|
|
20
22
|
const participantText = createTerminalParticipantReader(sanitize);
|
|
21
23
|
let transcriptBytes = 0;
|
|
24
|
+
// Output past the cap is not stored, but the participant reader, the verdict marker and token
|
|
25
|
+
// usage still read all of it, so a long session is judged and priced as a short one is.
|
|
26
|
+
const notStored = { stdout: 0, stderr: 0 };
|
|
27
|
+
const laterVerdict = createLocalActorVerdictScanner(verdictNonce);
|
|
28
|
+
const laterUsage = [];
|
|
29
|
+
// The stdout line in progress past the cap. Undefined while the rest of a line is skipped: the
|
|
30
|
+
// line the cap cut through (its start is stored) or one longer than the pending limit.
|
|
31
|
+
let laterLine;
|
|
32
|
+
let stdoutAtLineStart = true;
|
|
33
|
+
let pastCap = false;
|
|
34
|
+
const readLaterStdout = (raw) => {
|
|
35
|
+
let text = raw;
|
|
36
|
+
if (laterLine === undefined) {
|
|
37
|
+
const end = text.indexOf("\n");
|
|
38
|
+
if (end < 0)
|
|
39
|
+
return;
|
|
40
|
+
text = text.slice(end + 1);
|
|
41
|
+
laterLine = "";
|
|
42
|
+
}
|
|
43
|
+
laterLine += text;
|
|
44
|
+
const end = laterLine.lastIndexOf("\n");
|
|
45
|
+
if (end >= 0) {
|
|
46
|
+
laterUsage.push(...terminalUsageRecords(sanitize(laterLine.slice(0, end))));
|
|
47
|
+
laterLine = laterLine.slice(end + 1);
|
|
48
|
+
}
|
|
49
|
+
if (laterLine.length > PENDING_LINE_CHARS)
|
|
50
|
+
laterLine = undefined;
|
|
51
|
+
};
|
|
22
52
|
const recordLifecycle = (event, message) => {
|
|
23
53
|
lifecycle.push({ at: nowIso(), event, message: sanitize(message) });
|
|
24
54
|
};
|
|
25
55
|
const appendTerminalChunk = (stream, raw) => {
|
|
56
|
+
if (stream === "stdout")
|
|
57
|
+
participantText.append(raw);
|
|
26
58
|
if (transcriptBytes >= MAX_TRANSCRIPT_BYTES) {
|
|
27
59
|
for (const order of [stream, "combined"]) {
|
|
28
60
|
const remaining = maxDiscardedPrefixChars - discardedPrefixes[order].length;
|
|
29
61
|
if (remaining > 0)
|
|
30
62
|
discardedPrefixes[order] += raw.slice(0, remaining);
|
|
31
63
|
}
|
|
64
|
+
if (!pastCap) {
|
|
65
|
+
pastCap = true;
|
|
66
|
+
// The stored stream seeds the scan, so a marker the cap split in two is still found.
|
|
67
|
+
laterVerdict.push(terminalEvents.map((event) => event.chunk).join(""));
|
|
68
|
+
laterLine = stdoutAtLineStart ? "" : undefined;
|
|
69
|
+
}
|
|
70
|
+
notStored[stream] += Buffer.byteLength(raw, "utf8");
|
|
71
|
+
laterVerdict.push(sanitize(raw));
|
|
72
|
+
if (stream === "stdout")
|
|
73
|
+
readLaterStdout(raw);
|
|
32
74
|
return;
|
|
33
75
|
}
|
|
34
76
|
transcriptBytes += Buffer.byteLength(raw, "utf8");
|
|
35
|
-
// Scrub, then redact, at the source. Only the participant reader
|
|
36
|
-
//
|
|
77
|
+
// Scrub, then redact, at the source. Only the participant reader and the readers of output
|
|
78
|
+
// past the cap see raw bytes, and they store none of them.
|
|
37
79
|
terminalEvents.push({ at: nowIso(), stream, chunk: sanitize(raw) });
|
|
38
|
-
if (stream === "stdout")
|
|
39
|
-
|
|
80
|
+
if (stream === "stdout" && raw)
|
|
81
|
+
stdoutAtLineStart = raw.endsWith("\n");
|
|
82
|
+
};
|
|
83
|
+
/** What the cap left out, for the transcript, the trace and the run events; undefined when the
|
|
84
|
+
* whole session was stored. */
|
|
85
|
+
const transcriptCut = () => {
|
|
86
|
+
if (!pastCap)
|
|
87
|
+
return undefined;
|
|
88
|
+
const bytes = notStored.stdout + notStored.stderr;
|
|
89
|
+
return {
|
|
90
|
+
notice: `Output past the ${MAX_TRANSCRIPT_BYTES / 1024} KiB transcript cap was not stored: ${bytes} bytes (${notStored.stdout} stdout, ${notStored.stderr} stderr). The verdict marker, token usage and agent messages were still read from it.`,
|
|
91
|
+
verdict: laterVerdict.verdict(),
|
|
92
|
+
usage: laterUsage,
|
|
93
|
+
};
|
|
40
94
|
};
|
|
41
95
|
// E2B can stream every byte through callbacks and return the same complete output.
|
|
42
96
|
// Track transport delivery, independently per stream, rather than deduplicating participant
|
|
@@ -78,6 +132,7 @@ export function createTerminalRecorder(args) {
|
|
|
78
132
|
interventions,
|
|
79
133
|
discardedPrefixes,
|
|
80
134
|
participantText,
|
|
135
|
+
transcriptCut,
|
|
81
136
|
recordLifecycle,
|
|
82
137
|
recordStreamedTerminalChunk,
|
|
83
138
|
appendReturnedTerminalOutput,
|
|
@@ -54,7 +54,6 @@ import { terminalRouteScorer } from "../../study/adapter-scorer-loader.js";
|
|
|
54
54
|
import { withLateScorer } from "../../study/route-inputs.js";
|
|
55
55
|
import {} from "./types.js";
|
|
56
56
|
import { studyResultIdentity } from "../../run/study-result.js";
|
|
57
|
-
import { actorOf } from "../../study/study-fields.js";
|
|
58
57
|
/** A refused terminal study's result: the route's envelope, and the analysis record a refusal gets. */
|
|
59
58
|
export function terminalStudyRefusal(options, refusal) {
|
|
60
59
|
const { config, dryRun } = options;
|
|
@@ -62,7 +61,7 @@ export function terminalStudyRefusal(options, refusal) {
|
|
|
62
61
|
...studyResultIdentity("terminal", config.id),
|
|
63
62
|
ok: false,
|
|
64
63
|
cwd: path.resolve(options.cwd),
|
|
65
|
-
actor: refusal.actor ??
|
|
64
|
+
actor: refusal.actor ?? config.actor?.type ?? "",
|
|
66
65
|
product: config.subject.product?.name ?? "",
|
|
67
66
|
dryRun,
|
|
68
67
|
runId: options.runId ?? "not-created",
|
|
@@ -61,6 +61,7 @@ export async function runLiveTerminalSession(args) {
|
|
|
61
61
|
// This entry point is the live terminal route; its dry-run sibling is a separate function.
|
|
62
62
|
mode: "live",
|
|
63
63
|
study: plan.study,
|
|
64
|
+
warnings: plan.warnings,
|
|
64
65
|
renderReview: renderTerminalReviewMarkdown,
|
|
65
66
|
observer: { open: input.open === true, render: deps.renderObserver },
|
|
66
67
|
now,
|
|
@@ -77,7 +78,7 @@ export async function runLiveTerminalSession(args) {
|
|
|
77
78
|
});
|
|
78
79
|
const e2bApiKey = env.E2B_API_KEY?.trim() ?? "";
|
|
79
80
|
// The ledgers + capture buffers, mutated through the live lifecycle.
|
|
80
|
-
const recorder = createTerminalRecorder({ nowIso, sanitize, knownSecretValues });
|
|
81
|
+
const recorder = createTerminalRecorder({ nowIso, sanitize, knownSecretValues, verdictNonce });
|
|
81
82
|
const { version, model, modelSource, reasoningEffort } = plan.runtime;
|
|
82
83
|
const runtime = declaredRuntimeProvenance({
|
|
83
84
|
...(version === undefined ? {} : { version }),
|
|
@@ -1,4 +1,7 @@
|
|
|
1
1
|
import type { ActorTokenUsage } from "../../actors/contract.js";
|
|
2
|
+
/** The usage records in complete stdout lines, kept from output past the transcript cap so
|
|
3
|
+
* parseTerminalTokenUsage can count them with the stored ones. */
|
|
4
|
+
export declare function terminalUsageRecords(lines: string): string[];
|
|
2
5
|
/**
|
|
3
6
|
* Accumulate runtime-turn usage from a captured `codex exec --json` stream. A Codex turn may
|
|
4
7
|
* include multiple provider requests; these records cannot establish per-request pricing tiers.
|
|
@@ -19,6 +19,11 @@ const USAGE_RE = /"type"\s*:\s*"turn\.completed"\s*,\s*"usage"\s*:\s*(\{[^}\n]*\
|
|
|
19
19
|
function num(value) {
|
|
20
20
|
return typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : undefined;
|
|
21
21
|
}
|
|
22
|
+
/** The usage records in complete stdout lines, kept from output past the transcript cap so
|
|
23
|
+
* parseTerminalTokenUsage can count them with the stored ones. */
|
|
24
|
+
export function terminalUsageRecords(lines) {
|
|
25
|
+
return Array.from(lines.matchAll(USAGE_RE), (match) => match[0]);
|
|
26
|
+
}
|
|
22
27
|
/**
|
|
23
28
|
* Accumulate runtime-turn usage from a captured `codex exec --json` stream. A Codex turn may
|
|
24
29
|
* include multiple provider requests; these records cannot establish per-request pricing tiers.
|
|
@@ -53,7 +53,8 @@ export interface TerminalProductScoringContext {
|
|
|
53
53
|
/**
|
|
54
54
|
* The full normalized transcript of the in-sandbox agent session, scrubbed (literal known
|
|
55
55
|
* values) then redacted (shape patterns) at the source, capped at MAX_TRANSCRIPT_BYTES, and
|
|
56
|
-
* byte-identical to the persisted terminal-transcript.txt artifact.
|
|
56
|
+
* byte-identical to the persisted terminal-transcript.txt artifact. A capped transcript ends
|
|
57
|
+
* with a `[humanish]` line giving the byte count that was not stored. The trace's transcriptTail
|
|
57
58
|
* is a ~2KB projection of this; a scorer needs the whole session so a rubric can find
|
|
58
59
|
* command-tier evidence anywhere in it.
|
|
59
60
|
*/
|
|
@@ -32,5 +32,6 @@ export const TEXT_ITEMS_BYTES = 128 * 1024;
|
|
|
32
32
|
// The longest stdout line the participant reader holds while waiting for its newline.
|
|
33
33
|
export const PENDING_LINE_CHARS = 1024 * 1024;
|
|
34
34
|
// Hard cap on the retained event-stream + transcript size, so a runaway agent cannot balloon the
|
|
35
|
-
// bundle. Redaction runs pre-truncation so a cut can never split a secret past the scrubber.
|
|
35
|
+
// bundle. Redaction runs pre-truncation so a cut can never split a secret past the scrubber. Output
|
|
36
|
+
// past the cap is still read for the verdict marker, token usage and the trace's agent text.
|
|
36
37
|
export const MAX_TRANSCRIPT_BYTES = 512 * 1024;
|
package/dist/run/dry-run.js
CHANGED
|
@@ -67,6 +67,7 @@ async function runDryRunInScope(options, scope) {
|
|
|
67
67
|
mintRunId: () => `dryrun-${createdAt.replace(/[:.]/g, "-")}-${randomUUID().slice(0, 8)}`,
|
|
68
68
|
mode: options.dryRun ? "dry-run" : "live",
|
|
69
69
|
study: options.study,
|
|
70
|
+
warnings: options.warnings,
|
|
70
71
|
renderReview: renderReviewMarkdown,
|
|
71
72
|
// The run starts at the time its id and source were stamped with.
|
|
72
73
|
now: () => Date.parse(createdAt),
|
package/dist/run/results.d.ts
CHANGED
|
@@ -5,6 +5,8 @@ export declare const CLEANUP_SCHEMA = "humanish.cleanup-result.v1";
|
|
|
5
5
|
export interface RunOptions {
|
|
6
6
|
/** Which manifest produced this run. */
|
|
7
7
|
study?: RunStudyProvenance;
|
|
8
|
+
/** Warnings about the study's own fields, recorded in the bundle as warn events. */
|
|
9
|
+
warnings?: readonly string[];
|
|
8
10
|
cwd: string;
|
|
9
11
|
dryRun?: boolean;
|
|
10
12
|
runId?: string;
|
package/dist/run/run.d.ts
CHANGED
|
@@ -22,6 +22,8 @@ interface StartRunOptions {
|
|
|
22
22
|
};
|
|
23
23
|
/** Clock for `createdAt` and the latest pointer. */
|
|
24
24
|
now?: (() => number) | undefined;
|
|
25
|
+
/** Warnings about the study's own fields. Every bundle write records each as a warn event. */
|
|
26
|
+
warnings?: readonly string[] | undefined;
|
|
25
27
|
}
|
|
26
28
|
/** How a run ended as an execution, as run.json records it: the result's ok and execution outcome. */
|
|
27
29
|
interface RecordedOutcome {
|
package/dist/run/run.js
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
import { redactText } from "../evidence/redaction.js";
|
|
6
6
|
import { buildObserverData } from "../observer/data.js";
|
|
7
7
|
import { renderObserver } from "../observer/render.js";
|
|
8
|
-
import { RUN_BUNDLE_FILE, PUBLIC_TARGET_CWD } from "./bundle.js";
|
|
8
|
+
import { RUN_BUNDLE_FILE, PUBLIC_TARGET_CWD, } from "./bundle.js";
|
|
9
9
|
import { judgeExecution, } from "./judge.js";
|
|
10
10
|
import {} from "./results.js";
|
|
11
11
|
import { createRunArtifactPaths, validatePreparedRunArtifactPaths, } from "./paths.js";
|
|
@@ -112,8 +112,20 @@ function observerTarget(options) {
|
|
|
112
112
|
};
|
|
113
113
|
}
|
|
114
114
|
function runPublisher(args) {
|
|
115
|
-
const { options, runId, paths, runStatus, now, admit, participantsRan } = args;
|
|
115
|
+
const { options, runId, createdAt, paths, runStatus, now, admit, participantsRan } = args;
|
|
116
116
|
const observer = observerTarget(options);
|
|
117
|
+
// The study's warnings join whatever events the route wrote, once, in every write.
|
|
118
|
+
const studyWarnings = (options.warnings ?? []).map((message, index) => ({
|
|
119
|
+
id: `event-study-warning-${String(index + 1).padStart(3, "0")}`,
|
|
120
|
+
at: createdAt,
|
|
121
|
+
level: "warn",
|
|
122
|
+
type: "study.warning",
|
|
123
|
+
message,
|
|
124
|
+
}));
|
|
125
|
+
const withStudyWarnings = (events) => [
|
|
126
|
+
...events.filter((event) => !studyWarnings.some((warning) => warning.id === event.id)),
|
|
127
|
+
...studyWarnings,
|
|
128
|
+
];
|
|
117
129
|
let pointerWritten = false;
|
|
118
130
|
// The bundle of the last write asked for, and the outcome every later write carries once a
|
|
119
131
|
// signal stopped the run, so no write after the interrupt can drop it.
|
|
@@ -145,6 +157,7 @@ function runPublisher(args) {
|
|
|
145
157
|
const recorded = interrupted ?? outcome;
|
|
146
158
|
const publicBundle = await withPublicSandboxIds(paths, {
|
|
147
159
|
...evidence,
|
|
160
|
+
events: withStudyWarnings(evidence.events),
|
|
148
161
|
cwd: PUBLIC_TARGET_CWD,
|
|
149
162
|
...(recorded === undefined ? {} : { outcome: recorded }),
|
|
150
163
|
});
|
|
@@ -271,6 +284,7 @@ export async function runScope(fn) {
|
|
|
271
284
|
const publisher = runPublisher({
|
|
272
285
|
options,
|
|
273
286
|
runId,
|
|
287
|
+
createdAt,
|
|
274
288
|
paths,
|
|
275
289
|
runStatus,
|
|
276
290
|
now,
|
|
@@ -22,4 +22,14 @@ type ActorVerdict = "passed" | "blocked" | "failed";
|
|
|
22
22
|
* divergent verdict logic would let the two routes disagree about what "passed" means.
|
|
23
23
|
*/
|
|
24
24
|
export declare function extractLocalActorVerdict(transcript: string, verdictNonce: string): ActorVerdict | null;
|
|
25
|
+
/**
|
|
26
|
+
* The same search over text that arrives in pieces and is not kept: the terminal route reads
|
|
27
|
+
* output past its transcript cap through this. Each piece is normalized and loses its whitespace
|
|
28
|
+
* as the whole transcript does above, and the end of the previous piece is carried so a marker
|
|
29
|
+
* split between pieces is found. The first marker wins, as above.
|
|
30
|
+
*/
|
|
31
|
+
export declare function createLocalActorVerdictScanner(verdictNonce: string): {
|
|
32
|
+
push(text: string): void;
|
|
33
|
+
verdict(): ActorVerdict | null;
|
|
34
|
+
};
|
|
25
35
|
export {};
|
|
@@ -27,6 +27,9 @@ export function normalizeLocalActorTranscript(transcript) {
|
|
|
27
27
|
.replace(/\r/g, "\n")
|
|
28
28
|
.replace(/[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]/g, "");
|
|
29
29
|
}
|
|
30
|
+
// The nonce is mandatory: a bare HUMANISH_ACTOR_VERDICT=<status> marker echoed by an actor (or
|
|
31
|
+
// replayed from untrusted text) must never satisfy verdict extraction.
|
|
32
|
+
const verdictPattern = (verdictNonce) => new RegExp(`HUMANISH_ACTOR_VERDICT=(passed|blocked|failed)HUMANISH_ACTOR_NONCE=${escapeRegExp(verdictNonce)}`, "i");
|
|
30
33
|
/**
|
|
31
34
|
* Extract the per-run verdict from a normalized transcript: the agent must print exactly
|
|
32
35
|
* `HUMANISH_ACTOR_VERDICT=<status> HUMANISH_ACTOR_NONCE=<nonce>`, and the nonce is mandatory so a
|
|
@@ -35,13 +38,28 @@ export function normalizeLocalActorTranscript(transcript) {
|
|
|
35
38
|
* divergent verdict logic would let the two routes disagree about what "passed" means.
|
|
36
39
|
*/
|
|
37
40
|
export function extractLocalActorVerdict(transcript, verdictNonce) {
|
|
38
|
-
const
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
41
|
+
const match = verdictPattern(verdictNonce).exec(transcript.replace(/\s+/g, ""));
|
|
42
|
+
return match?.[1]?.toLowerCase() ?? null;
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* The same search over text that arrives in pieces and is not kept: the terminal route reads
|
|
46
|
+
* output past its transcript cap through this. Each piece is normalized and loses its whitespace
|
|
47
|
+
* as the whole transcript does above, and the end of the previous piece is carried so a marker
|
|
48
|
+
* split between pieces is found. The first marker wins, as above.
|
|
49
|
+
*/
|
|
50
|
+
export function createLocalActorVerdictScanner(verdictNonce) {
|
|
51
|
+
const pattern = verdictPattern(verdictNonce);
|
|
52
|
+
const carried = `HUMANISH_ACTOR_VERDICT=blockedHUMANISH_ACTOR_NONCE=${verdictNonce}`.length - 1;
|
|
53
|
+
let tail = "";
|
|
54
|
+
let found = null;
|
|
55
|
+
return {
|
|
56
|
+
push(text) {
|
|
57
|
+
if (found)
|
|
58
|
+
return;
|
|
59
|
+
const window = tail + normalizeLocalActorTranscript(text).replace(/\s+/g, "");
|
|
60
|
+
found = pattern.exec(window)?.[1]?.toLowerCase() ?? null;
|
|
61
|
+
tail = window.slice(-carried);
|
|
62
|
+
},
|
|
63
|
+
verdict: () => found,
|
|
64
|
+
};
|
|
47
65
|
}
|