humanish 0.111.0 → 0.111.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/CONTEXT.md +1 -1
- package/dist/actors/codex/restricted-participant-policy.d.ts +2 -0
- package/dist/actors/codex/restricted-participant-policy.js +30 -2
- package/dist/actors/codex/restricted-participant.js +1 -1
- package/dist/actors/computer-use/executor-diagnostic.d.ts +15 -0
- package/dist/actors/computer-use/executor-diagnostic.js +105 -0
- package/dist/actors/computer-use/executor-error.d.ts +18 -3
- package/dist/actors/computer-use/executor-error.js +30 -4
- package/dist/actors/computer-use/loop/actions.d.ts +6 -3
- package/dist/actors/computer-use/loop/actions.js +27 -7
- package/dist/actors/computer-use/loop/ending.js +9 -1
- package/dist/actors/computer-use/loop/types.d.ts +14 -0
- package/dist/actors/computer-use/loop.d.ts +1 -1
- package/dist/actors/computer-use/loop.js +1 -1
- package/dist/actors/computer-use/openai-context.d.ts +8 -2
- package/dist/actors/computer-use/openai-context.js +91 -26
- package/dist/actors/computer-use/openai-provider.js +55 -12
- package/dist/actors/computer-use/openai-requests.d.ts +2 -1
- package/dist/actors/computer-use/openai-requests.js +4 -1
- package/dist/actors/contract.d.ts +7 -1
- package/dist/browser-control/client.js +5 -5
- package/dist/browser-control/protocol.d.ts +30 -1
- package/dist/browser-control/protocol.js +19 -2
- package/dist/browser-control/transport.d.ts +3 -2
- package/dist/browser-control/transport.js +12 -7
- package/dist/cli/commands/study-route-computer-use.js +2 -2
- package/dist/cli/commands/study-route-shared-world.js +2 -2
- package/dist/cli/commands/study.js +1 -2
- package/dist/cli/doctor.js +11 -0
- package/dist/evidence/encoded-text.d.ts +3 -3
- package/dist/evidence/encoded-text.js +10 -5
- package/dist/evidence/redaction.d.ts +2 -2
- package/dist/evidence/redaction.js +18 -7
- package/dist/guest/browser-tools.js +5 -1
- package/dist/guest/chromium-text.js +42 -18
- package/dist/guest/desktop-executor.js +5 -1
- package/dist/keys/key-resolution.d.ts +3 -0
- package/dist/keys/key-resolution.js +15 -2
- package/dist/observer-app.html +6 -6
- package/dist/routes/computer-use/bundle-parts.d.ts +4 -0
- package/dist/routes/computer-use/bundle-parts.js +7 -2
- package/dist/routes/computer-use/fanout-bundle.js +4 -2
- package/dist/routes/computer-use/local-vm.js +1 -2
- package/dist/routes/computer-use/participant-model.js +6 -1
- package/dist/routes/computer-use/plan.js +3 -4
- package/dist/routes/computer-use/route.js +1 -2
- package/dist/routes/computer-use/setup.js +2 -2
- package/dist/routes/preview.js +1 -0
- package/dist/routes/scripted/plan.js +4 -5
- package/dist/routes/scripted/route.js +2 -2
- package/dist/routes/shared-world/plan.js +4 -5
- package/dist/routes/shared-world/route.js +2 -2
- package/dist/routes/shared-world/setup.js +1 -0
- package/dist/routes/terminal/dry-run.js +1 -0
- package/dist/routes/terminal/live-finish.js +24 -5
- package/dist/routes/terminal/live-sandbox.js +5 -1
- package/dist/routes/terminal/plan.js +3 -4
- package/dist/routes/terminal/recorder.d.ts +6 -0
- package/dist/routes/terminal/recorder.js +61 -6
- package/dist/routes/terminal/route.js +1 -2
- package/dist/routes/terminal/session.js +2 -1
- package/dist/routes/terminal/token-usage.d.ts +3 -0
- package/dist/routes/terminal/token-usage.js +5 -0
- package/dist/routes/terminal/types.d.ts +2 -1
- package/dist/routes/terminal/types.js +2 -1
- package/dist/run/dry-run.js +1 -0
- package/dist/run/results.d.ts +2 -0
- package/dist/run/run.d.ts +2 -0
- package/dist/run/run.js +16 -2
- package/dist/run/terminal-contract.d.ts +10 -0
- package/dist/run/terminal-contract.js +27 -9
- package/dist/study/composition-rules.js +17 -18
- package/dist/study/doctor.js +4 -5
- package/dist/study/migrate/v2.d.ts +5 -0
- package/dist/study/migrate/v2.js +23 -0
- package/dist/study/parse/front.js +2 -21
- package/dist/study/parse/url-credentials.d.ts +3 -3
- package/dist/study/parse/url-credentials.js +155 -13
- package/dist/study/persona-resolve.js +2 -2
- package/dist/study/plan-base.js +3 -4
- package/dist/study/plan-participants.js +3 -3
- package/dist/study/plan-types.d.ts +2 -0
- package/dist/study/plan.js +11 -5
- package/dist/study/routing.js +11 -11
- package/dist/study/run-study-options.js +2 -2
- package/dist/study/study-fields.d.ts +1 -13
- package/dist/study/study-fields.js +0 -24
- package/dist/study/summary.js +7 -7
- package/dist/study/types.d.ts +1 -0
- package/dist/study/validation.js +11 -11
- package/dist/study/warnings.d.ts +8 -0
- package/dist/study/warnings.js +22 -7
- package/dist/substrates/local/runtime-config.js +6 -6
- package/docs/architecture/browser-control.md +12 -2
- package/docs/architecture/guest-desktop.md +7 -1
- package/docs/architecture/terminal-product-route.md +3 -2
- package/docs/contracts/schemas.md +16 -5
- package/package.json +1 -1
|
@@ -43,6 +43,10 @@ export declare function providerResourcesForOutcome(args: {
|
|
|
43
43
|
ids: ParticipantIds;
|
|
44
44
|
participantId: string;
|
|
45
45
|
}): RunProviderResource[];
|
|
46
|
+
/**
|
|
47
|
+
* The app URL as a run records it: the URL, or its digest when verify would flag it. verify reads the
|
|
48
|
+
* URL decoded, so an E2B URL percent-encoded in a parameter gets the digest too.
|
|
49
|
+
*/
|
|
46
50
|
export declare function publicSafeAppUrlLabel(url: string): string;
|
|
47
51
|
/** Short id-safe suffix for a subject-phase RunEvent: drops the shared prefix/suffix so each
|
|
48
52
|
* phase gets a distinct bundle event id (e.g. "clone", "state-before-build"). */
|
|
@@ -3,7 +3,8 @@
|
|
|
3
3
|
// public-safe app URL label, the URL digest and the subject-phase event id suffix. The builders are bundle.ts (the dispatcher and the judge),
|
|
4
4
|
// single-bundle.ts and fanout-bundle.ts.
|
|
5
5
|
import { feedbackProofCommands } from "../../feedback/proof.js";
|
|
6
|
-
import {
|
|
6
|
+
import { scanEncodedText } from "../../evidence/encoded-text.js";
|
|
7
|
+
import { digestText, redactText } from "../../evidence/redaction.js";
|
|
7
8
|
import {} from "../../run/bundle.js";
|
|
8
9
|
import { participantResourceIds } from "../../run/participant-records.js";
|
|
9
10
|
import { resolveSelfReportedFriction } from "./self-report.js";
|
|
@@ -169,8 +170,12 @@ export function providerResourcesForOutcome(args) {
|
|
|
169
170
|
},
|
|
170
171
|
];
|
|
171
172
|
}
|
|
173
|
+
/**
|
|
174
|
+
* The app URL as a run records it: the URL, or its digest when verify would flag it. verify reads the
|
|
175
|
+
* URL decoded, so an E2B URL percent-encoded in a parameter gets the digest too.
|
|
176
|
+
*/
|
|
172
177
|
export function publicSafeAppUrlLabel(url) {
|
|
173
|
-
return
|
|
178
|
+
return scanEncodedText(url).sensitive ? `[target-url:${digestUrl(url)}]` : url;
|
|
174
179
|
}
|
|
175
180
|
/** Short id-safe suffix for a subject-phase RunEvent: drops the shared prefix/suffix so each
|
|
176
181
|
* phase gets a distinct bundle event id (e.g. "clone", "state-before-build"). */
|
|
@@ -4,7 +4,7 @@ import { actorEnding } from "../../actors/stop-cause.js";
|
|
|
4
4
|
import { redactText } from "../../evidence/redaction.js";
|
|
5
5
|
import { REVIEW_SCHEMA, bundleArtifacts, bundleHead, } from "../../run/bundle.js";
|
|
6
6
|
import { aggregateTaskFunnels, formatParticipantOutcomes, formatRunTaskFunnel, tallyParticipantOutcomes, withCuaReviewProvenance, } from "../../run/outcomes.js";
|
|
7
|
-
import { participantFeedbackCandidates, providerResourcesForOutcome } from "./bundle-parts.js";
|
|
7
|
+
import { participantFeedbackCandidates, providerResourcesForOutcome, publicSafeAppUrlLabel, } from "./bundle-parts.js";
|
|
8
8
|
import { participantFactsOf } from "./participant-facts.js";
|
|
9
9
|
import { participantPassed, participantStatus } from "../../run/judge.js";
|
|
10
10
|
import { buildRunCostSummary, desktopSpanToMinutes } from "../../run/cost-summary.js";
|
|
@@ -72,10 +72,12 @@ function fanoutReview(args, streams) {
|
|
|
72
72
|
verdict,
|
|
73
73
|
...(participants === undefined ? {} : { participants }),
|
|
74
74
|
...(runTasks === undefined ? {} : { tasks: runTasks }),
|
|
75
|
+
// The app URL is named as the participant records name it: one on an E2B host names a
|
|
76
|
+
// sandbox, so the summary carries its digest, which verify accepts in a shared run.
|
|
75
77
|
summary: args.inProgress === true
|
|
76
78
|
? `Live computer-use fan-out is running (${specs.length} participants, one world each); terminal participant evidence has not been written yet.`
|
|
77
79
|
: args.dryRun
|
|
78
|
-
? `${args.rerun ? `Rerun from ${args.rerun.sourceRunId}: ` : ""}Dry-run fan-out: ${specs.length} participants composed for ${args.descriptor.id} against ${args.appUrl}, one world each; no desktops launched, $0 spend.`
|
|
80
|
+
? `${args.rerun ? `Rerun from ${args.rerun.sourceRunId}: ` : ""}Dry-run fan-out: ${specs.length} participants composed for ${args.descriptor.id} against ${publicSafeAppUrlLabel(args.appUrl)}, one world each; no desktops launched, $0 spend.`
|
|
79
81
|
: `${args.rerun ? `Rerun from ${args.rerun.sourceRunId}: ` : ""}Computer-use fan-out (${specs.length} participants, one world each): ${passedParticipants}/${plural(specs.length, "participant")} reached a terminal, engaged verdict${participants ? `: ${formatParticipantOutcomes(participants, participantEndings)}` : ""}${runTasks ? `; tasks: ${formatRunTaskFunnel(runTasks)}` : ""}.`,
|
|
80
82
|
gaps: args.inProgress === true
|
|
81
83
|
? ["Live fan-out session is still running."]
|
|
@@ -10,7 +10,6 @@ import { createRestrictedCodexParticipant } from "../../actors/codex/restricted-
|
|
|
10
10
|
import { withCloseReport } from "./participant-model.js";
|
|
11
11
|
import { guestMediaConfigSchema } from "../../guest/media-config.js";
|
|
12
12
|
import { startLocalCapturedInbox } from "../../substrates/local/captured-inbox.js";
|
|
13
|
-
import { actorOf } from "../../study/study-fields.js";
|
|
14
13
|
function guestMedia(config) {
|
|
15
14
|
const declaredMedia = config.execution?.desktop?.media;
|
|
16
15
|
return declaredMedia === undefined
|
|
@@ -165,7 +164,7 @@ export function prepareLocalVmRun(options) {
|
|
|
165
164
|
const media = guestMedia(config);
|
|
166
165
|
// A caller-supplied provider replaces the Codex account participant, so the account is
|
|
167
166
|
// neither checked nor used.
|
|
168
|
-
const account =
|
|
167
|
+
const account = config.actor?.type === "local-agent" && options.createProvider === undefined;
|
|
169
168
|
const state = { sessions: [], participants: [], cleanupUnconfirmed: false };
|
|
170
169
|
const context = {
|
|
171
170
|
config,
|
|
@@ -228,10 +228,15 @@ export async function recordParticipantTrace(spec, deps, session, warnings) {
|
|
|
228
228
|
* carried it on every request. It says why, and how many early turns became a text summary.
|
|
229
229
|
*/
|
|
230
230
|
export function explicitContextWarning(conversation) {
|
|
231
|
+
const answer = conversation.rejection === "stored_item"
|
|
232
|
+
? " (a stored item the request referenced was not found)"
|
|
233
|
+
: conversation.rejection === "previous_response"
|
|
234
|
+
? " (the previous response was refused or not found)"
|
|
235
|
+
: "";
|
|
231
236
|
const why = conversation.explicitReason === "zdr_rejection"
|
|
232
237
|
? `The OpenAI organization rejected server-side conversation state (zero data retention)${conversation.switchedAtRequest === undefined
|
|
233
238
|
? ""
|
|
234
|
-
: ` on request ${conversation.switchedAtRequest}`}`
|
|
239
|
+
: ` on request ${conversation.switchedAtRequest}`}${answer}`
|
|
235
240
|
: "zeroDataRetention is set";
|
|
236
241
|
const summarized = conversation.summarizedTurns === 0
|
|
237
242
|
? ""
|
|
@@ -16,7 +16,6 @@ import { isLocalBrowserStudy } from "../../substrates/local/runtime-config.js";
|
|
|
16
16
|
import { boundedConcurrency, defaultSessionTimeoutMs, resolveParticipantSandboxMs, } from "./participant-runs.js";
|
|
17
17
|
import { MAX_SANDBOX_MS } from "../../substrates/e2b/lifetime.js";
|
|
18
18
|
import {} from "./types.js";
|
|
19
|
-
import { actorOf, declaresSharedWorld } from "../../study/study-fields.js";
|
|
20
19
|
/** The registered descriptor for a planned actor id; planComputerUseStudy checked the registry. */
|
|
21
20
|
export function cuaDescriptorOf(actor) {
|
|
22
21
|
const descriptor = actorRegistry[actor];
|
|
@@ -87,7 +86,7 @@ function unsupportedDeclarationReason(config, hasRunSession, driving, { inProces
|
|
|
87
86
|
const reason = desktopMediaValidationReason(config) || outputTokenLimitValidationReason(config);
|
|
88
87
|
if (reason)
|
|
89
88
|
return invalid(reason);
|
|
90
|
-
if (
|
|
89
|
+
if (config.actor?.maxOutputTokens !== undefined &&
|
|
91
90
|
(hasRunSession || driving.createProvider || driving.inProcess))
|
|
92
91
|
return invalid("maxOutputTokens cannot be enforced by a custom runSession/provider/executor route.");
|
|
93
92
|
const receivingReason = receivingEmailValidationReason(config);
|
|
@@ -219,7 +218,7 @@ export function planComputerUseStudy(config, input) {
|
|
|
219
218
|
if (tasksReason)
|
|
220
219
|
return refuse("before-scope", "HUMANISH_STUDY_TASKS_UNSUPPORTED", tasksReason);
|
|
221
220
|
// The parser checks the actor too; a library caller skips the parser.
|
|
222
|
-
const actorType =
|
|
221
|
+
const actorType = config.actor?.type ?? "";
|
|
223
222
|
const descriptor = actorRegistry[actorType];
|
|
224
223
|
if (!descriptor || !isCuaActorDescriptor(descriptor))
|
|
225
224
|
return refuse("in-scope", "HUMANISH_COMPUTER_USE_ACTOR_UNSUPPORTED", `actor.type "${actorType}" is not a registered computer-use actor.`);
|
|
@@ -230,7 +229,7 @@ export function planComputerUseStudy(config, input) {
|
|
|
230
229
|
// A shared world runs every participant against one app; this route would run them as separate
|
|
231
230
|
// participants. It comes after the rules above, so a shared-world config that breaks one of them,
|
|
232
231
|
// which runStudyWith sends here, still gets that rule's message.
|
|
233
|
-
if (
|
|
232
|
+
if (config.route === "shared-world")
|
|
234
233
|
return refuse("in-scope", "HUMANISH_COMPUTER_USE_SUBJECT_INVALID", "route: shared-world studies run every participant against one shared app on the shared-world route; this route runs independent participants. The shared-world route takes subject.source clone or local-tree, or app-url with policies.allowPublicTargets: true, on execution.target e2b-desktop.", actor);
|
|
235
234
|
// The in-process route drives subject.appUrl on this machine and creates no desktop, so it would
|
|
236
235
|
// skip the subject a clone, local-tree or desktop-cli study declares.
|
|
@@ -32,7 +32,6 @@ import { runStudyParticipants } from "./live-phase.js";
|
|
|
32
32
|
import { admitCuaRun, refuseCuaStudy, startCuaRun } from "./setup.js";
|
|
33
33
|
import {} from "./types.js";
|
|
34
34
|
import { studyResultIdentity } from "../../run/study-result.js";
|
|
35
|
-
import { actorOf } from "../../study/study-fields.js";
|
|
36
35
|
/**
|
|
37
36
|
* A refused computer-use study's result, at the refusal's stage: a before-scope refusal has its own
|
|
38
37
|
* envelope and no analysis record; the others come after the cwd checks, and the participant cap after
|
|
@@ -45,7 +44,7 @@ export async function computerUseStudyRefusal(options, refusal) {
|
|
|
45
44
|
...studyResultIdentity("computer-use", config.id),
|
|
46
45
|
ok: false,
|
|
47
46
|
cwd: path.resolve(options.cwd),
|
|
48
|
-
actor:
|
|
47
|
+
actor: config.actor?.type ?? "",
|
|
49
48
|
dryRun,
|
|
50
49
|
runId: options.runId ?? "not-created",
|
|
51
50
|
appUrl: config.subject.appUrl ?? config.subject.serve?.url ?? "",
|
|
@@ -25,7 +25,6 @@ import { projectParticipantSubjects, subjectProvenanceArg } from "./subject-proj
|
|
|
25
25
|
import { participantSubjectEnv, } from "./types.js";
|
|
26
26
|
import { studyPersonaIds } from "../../study/persona-resolve.js";
|
|
27
27
|
import { studyResultIdentity } from "../../run/study-result.js";
|
|
28
|
-
import { actorOf } from "../../study/study-fields.js";
|
|
29
28
|
/** The physical project, bound before any caller hook runs. */
|
|
30
29
|
async function bindProject(cwd) {
|
|
31
30
|
// Capture the physical project before reading or invoking any caller hook. A supported
|
|
@@ -48,7 +47,7 @@ export async function refuseCuaStudy(options, refusal) {
|
|
|
48
47
|
...studyResultIdentity("computer-use", config.id),
|
|
49
48
|
ok: false,
|
|
50
49
|
cwd: projectRoot.physicalPath,
|
|
51
|
-
actor: refusal.actor ??
|
|
50
|
+
actor: refusal.actor ?? config.actor?.type ?? "",
|
|
52
51
|
appUrl: declaredAppUrl(config),
|
|
53
52
|
dryRun,
|
|
54
53
|
runId: options.runId ?? "not-created",
|
|
@@ -214,6 +213,7 @@ export async function startCuaRun(plan, input, admitted, scope) {
|
|
|
214
213
|
mintRunId: makeCuaRunId,
|
|
215
214
|
mode: dryRun ? "dry-run" : "live",
|
|
216
215
|
study: plan.study,
|
|
216
|
+
warnings: plan.warnings,
|
|
217
217
|
renderReview: renderCuaReviewMarkdown,
|
|
218
218
|
observer: { open: input.open === true, render: seams.renderObserver },
|
|
219
219
|
});
|
package/dist/routes/preview.js
CHANGED
|
@@ -30,6 +30,7 @@ export function admitPreviewPlan(plan, input) {
|
|
|
30
30
|
function runPreviewPlan(plan, input) {
|
|
31
31
|
return runDryRun({
|
|
32
32
|
...(plan.study === undefined ? {} : { study: plan.study }),
|
|
33
|
+
...(plan.warnings === undefined ? {} : { warnings: plan.warnings }),
|
|
33
34
|
cwd: input.cwd,
|
|
34
35
|
dryRun: true,
|
|
35
36
|
participantCount: plan.participantCount,
|
|
@@ -7,7 +7,6 @@ import { browserSurfaces } from "../../actors/scripted-browser/types.js";
|
|
|
7
7
|
import { resolveAutomaticAnalysis } from "../../analysis/automatic-config.js";
|
|
8
8
|
import { isNonEmpty, planBase, provisionedSubject } from "../../study/plan-base.js";
|
|
9
9
|
import { cloneTargetValidationReason, desktopMediaValidationReason, taskProtocolValidationReason, } from "../../study/validation.js";
|
|
10
|
-
import { actorOf, surfaceCount, scenarioRefOf } from "../../study/study-fields.js";
|
|
11
10
|
// Default surface roster is 1 (desktop only): the defaults-table single-participant row governs;
|
|
12
11
|
// `count: 2` is the declared override that adds the mobile surface.
|
|
13
12
|
const DEFAULT_SURFACE_COUNT = 1;
|
|
@@ -43,7 +42,7 @@ export function planScriptedStudy(config, input) {
|
|
|
43
42
|
const cloneTargetReason = cloneTargetValidationReason(config);
|
|
44
43
|
if (cloneTargetReason)
|
|
45
44
|
return refuse("HUMANISH_SCRIPTED_SUBJECT_UNSAFE", cloneTargetReason);
|
|
46
|
-
const actorType =
|
|
45
|
+
const actorType = config.actor?.type ?? "";
|
|
47
46
|
const descriptor = actorRegistry[actorType];
|
|
48
47
|
if (!descriptor || !isScriptedBrowserActorDescriptor(descriptor))
|
|
49
48
|
return refuse("HUMANISH_SCRIPTED_ACTOR_UNSUPPORTED", `actor.type "${actorType}" is not a registered scripted-browser actor.`);
|
|
@@ -64,7 +63,7 @@ export function planScriptedStudy(config, input) {
|
|
|
64
63
|
subject = { kind: "loopback", appUrl };
|
|
65
64
|
}
|
|
66
65
|
// The steps are the actor, so there is no built-in journey to fall back on.
|
|
67
|
-
const scenarioRef =
|
|
66
|
+
const scenarioRef = config.scenario;
|
|
68
67
|
if (!scenarioRef?.trim())
|
|
69
68
|
return refuse("HUMANISH_SCRIPTED_SCENARIO_INVALID", "A scripted-browser study needs `scenario`: the actor runs the browser steps in that scenario file.", { actor, appUrl: evidenceAppUrlOf(subject) });
|
|
70
69
|
const requirements = [];
|
|
@@ -76,7 +75,7 @@ export function planScriptedStudy(config, input) {
|
|
|
76
75
|
// The browser runs on this machine on both subjects; a clone's is pointed at its getHost URL.
|
|
77
76
|
if (!input.dryRun && input.injectedBrowser !== true)
|
|
78
77
|
requirements.push({ kind: "host-browser" });
|
|
79
|
-
const persona =
|
|
78
|
+
const persona = config.actor?.persona;
|
|
80
79
|
const timeoutMs = config.execution?.timeoutMs;
|
|
81
80
|
return {
|
|
82
81
|
ok: true,
|
|
@@ -89,7 +88,7 @@ export function planScriptedStudy(config, input) {
|
|
|
89
88
|
actor,
|
|
90
89
|
subject,
|
|
91
90
|
scenarioRef,
|
|
92
|
-
surfaces: browserSurfaces.slice(0,
|
|
91
|
+
surfaces: browserSurfaces.slice(0, config.surfaces?.length ?? DEFAULT_SURFACE_COUNT),
|
|
93
92
|
...(persona === undefined ? {} : { personaId: persona }),
|
|
94
93
|
...(timeoutMs === undefined ? {} : { sessionTimeoutMs: timeoutMs }),
|
|
95
94
|
requirements,
|
|
@@ -31,7 +31,6 @@ import { ScriptedSubject } from "./subject.js";
|
|
|
31
31
|
import { runScriptedSessions, writeSurfaceTraces } from "./surface-sessions.js";
|
|
32
32
|
import {} from "./types.js";
|
|
33
33
|
import { studyResultIdentity } from "../../run/study-result.js";
|
|
34
|
-
import { actorOf } from "../../study/study-fields.js";
|
|
35
34
|
/**
|
|
36
35
|
* A refused scripted study's result, at the refusal's stage: a before-scope refusal has its own field
|
|
37
36
|
* order and no analysis record; the others come after the output directory checks.
|
|
@@ -39,7 +38,7 @@ import { actorOf } from "../../study/study-fields.js";
|
|
|
39
38
|
export async function scriptedStudyRefusal(options, refusal) {
|
|
40
39
|
const { config, dryRun } = options;
|
|
41
40
|
const cwd = path.resolve(options.cwd);
|
|
42
|
-
const actorType =
|
|
41
|
+
const actorType = config.actor?.type ?? "";
|
|
43
42
|
if (refusal.beforeScope)
|
|
44
43
|
return {
|
|
45
44
|
...studyResultIdentity("scripted", config.id),
|
|
@@ -95,6 +94,7 @@ async function runScriptedPlanInScope(plan, input, scope) {
|
|
|
95
94
|
mintRunId: makeScriptedRunId,
|
|
96
95
|
mode: dryRun ? "dry-run" : "live",
|
|
97
96
|
study: plan.study,
|
|
97
|
+
warnings: plan.warnings,
|
|
98
98
|
// Only a clone subject is served from a sandbox; an app-url run drives a local browser.
|
|
99
99
|
sandboxes: setup.clone ? undefined : "none",
|
|
100
100
|
renderReview: renderScriptedReviewMarkdown,
|
|
@@ -12,7 +12,6 @@ import { PUBLIC_TARGET_OWNER_PATTERN } from "../../study/parse/subject.js";
|
|
|
12
12
|
import { REPO_SLUG_PATTERN } from "../../study/parse/values.js";
|
|
13
13
|
import { concurrentSharedWorldValidationReason, desktopMediaValidationReason, externalPublicSharedWorldValidationReason, outputTokenLimitValidationReason, receivingEmailValidationReason, taskProtocolValidationReason, } from "../../study/validation.js";
|
|
14
14
|
import { MODEL_RATES, unpricedCapMessage } from "../../run/pricing.js";
|
|
15
|
-
import { actorOf, capsOf } from "../../study/study-fields.js";
|
|
16
15
|
/** The registered descriptor for a planned actor; planSharedWorldStudy checked the registry. */
|
|
17
16
|
export function sharedWorldDescriptorOf(actorType) {
|
|
18
17
|
const descriptor = actorRegistry[actorType];
|
|
@@ -39,7 +38,7 @@ export function planSharedWorldStudy(config, input) {
|
|
|
39
38
|
const tasksReason = taskProtocolValidationReason(config, false);
|
|
40
39
|
if (tasksReason)
|
|
41
40
|
return refuse("HUMANISH_STUDY_TASKS_UNSUPPORTED", tasksReason);
|
|
42
|
-
const actorType =
|
|
41
|
+
const actorType = config.actor?.type ?? "";
|
|
43
42
|
const descriptor = actorRegistry[actorType];
|
|
44
43
|
if (!descriptor || !isCuaActorDescriptor(descriptor))
|
|
45
44
|
return refuse("HUMANISH_SHARED_WORLD_ACTOR_UNSUPPORTED", `actor.type "${actorType}" is not a registered computer-use actor.`);
|
|
@@ -59,11 +58,11 @@ export function planSharedWorldStudy(config, input) {
|
|
|
59
58
|
: concurrentSharedWorldValidationReason(config));
|
|
60
59
|
if (invalidReason)
|
|
61
60
|
return refuse(invalid, invalidReason, actor);
|
|
62
|
-
if (
|
|
61
|
+
if (config.actor?.maxOutputTokens !== undefined && input.hasRunSession === true)
|
|
63
62
|
return refuse(invalid, "maxOutputTokens cannot be enforced by a custom runSession.", actor);
|
|
64
|
-
const caps =
|
|
63
|
+
const caps = config.caps;
|
|
65
64
|
if (!input.dryRun && (caps?.maxUsd !== undefined || caps?.maxTotalUsd !== undefined)) {
|
|
66
|
-
const model = (
|
|
65
|
+
const model = (config.actor?.model ?? DEFAULT_OPENAI_CU_MODEL).trim().toLowerCase();
|
|
67
66
|
if (!MODEL_RATES[model])
|
|
68
67
|
return refuse("HUMANISH_SHARED_WORLD_UNPRICED_CAP", unpricedCapMessage(model), actor);
|
|
69
68
|
}
|
|
@@ -48,7 +48,7 @@ import { prepareConcurrentRun } from "./setup.js";
|
|
|
48
48
|
import { browserRouteScorer } from "../../study/adapter-scorer-loader.js";
|
|
49
49
|
import { withLateScorer } from "../../study/route-inputs.js";
|
|
50
50
|
import {} from "./types.js";
|
|
51
|
-
import {
|
|
51
|
+
import { participantList } from "../../study/study-fields.js";
|
|
52
52
|
/** The shared-world code for each local-agent refusal, kind for kind with computer-use. */
|
|
53
53
|
const LOCAL_AGENT_REFUSAL_CODES = {
|
|
54
54
|
"agent-missing": "HUMANISH_SHARED_WORLD_AGENT_MISSING",
|
|
@@ -63,7 +63,7 @@ export function sharedWorldStudyRefusal(options, refusal) {
|
|
|
63
63
|
const fail = concurrentStudyFailure({
|
|
64
64
|
cwd: path.resolve(options.cwd),
|
|
65
65
|
studyId: config.id,
|
|
66
|
-
actor:
|
|
66
|
+
actor: config.actor?.type ?? "",
|
|
67
67
|
participantCount: declared.length,
|
|
68
68
|
// The parser fills concurrency for multi-participant studies, so this fallback serves only
|
|
69
69
|
// library callers: every declared participant runs at once unless the author declared a cap.
|
|
@@ -56,6 +56,7 @@ function startConcurrentRun(study, cwd, scope) {
|
|
|
56
56
|
mintRunId: makeRunId,
|
|
57
57
|
mode: plan.dryRun ? "dry-run" : "live",
|
|
58
58
|
study: plan.study,
|
|
59
|
+
warnings: plan.warnings,
|
|
59
60
|
renderReview: renderConcurrentReviewMarkdown,
|
|
60
61
|
observer: { open: input.open === true, render: deps.renderObserver },
|
|
61
62
|
});
|
|
@@ -30,6 +30,7 @@ export async function runDryTerminalStudy(args) {
|
|
|
30
30
|
mintRunId: makeTerminalRunId,
|
|
31
31
|
mode: "dry-run",
|
|
32
32
|
study: plan.study,
|
|
33
|
+
warnings: plan.warnings,
|
|
33
34
|
renderReview: renderTerminalReviewMarkdown,
|
|
34
35
|
observer: { open: input.open === true, render: input.deps?.renderObserver },
|
|
35
36
|
});
|
|
@@ -23,11 +23,30 @@ function buildLiveTrace(inputs) {
|
|
|
23
23
|
// chunks before any transcript/trace/event artifact is persisted. Check both each stream and
|
|
24
24
|
// the combined event order that the transcript uses; either view can assemble a split value.
|
|
25
25
|
scrubSplitKnownValues(terminalEvents, knownSecretValues, discardedPrefixes);
|
|
26
|
-
// Build the actor trace first (the cost ledger reads its tokenUsage).
|
|
27
|
-
|
|
26
|
+
// Build the actor trace first (the cost ledger reads its tokenUsage). A session past the cap
|
|
27
|
+
// ends its transcript with what was not stored, and its trace says the same.
|
|
28
|
+
const cut = inputs.recorder.transcriptCut();
|
|
29
|
+
const normalizedTranscript = `${normalizeLocalActorTranscript(terminalEvents.map((e) => e.chunk).join(""))}${cut ? `\n[humanish] ${cut.notice}` : ""}`;
|
|
28
30
|
// Parsed from the full stream: usage records arrive once per turn, and the tail
|
|
29
|
-
// would drop all but the last.
|
|
30
|
-
const terminalTokenUsage = parseTerminalTokenUsage(normalizedTranscript);
|
|
31
|
+
// would drop all but the last. Records past the cap were kept as they arrived.
|
|
32
|
+
const terminalTokenUsage = parseTerminalTokenUsage([normalizedTranscript, ...(cut?.usage ?? [])].join("\n"));
|
|
33
|
+
const read = inputs.recorder.participantText.finish();
|
|
34
|
+
const participant = cut
|
|
35
|
+
? {
|
|
36
|
+
...read,
|
|
37
|
+
items: [
|
|
38
|
+
...read.items,
|
|
39
|
+
{
|
|
40
|
+
id: "notice-transcript",
|
|
41
|
+
kind: "notice",
|
|
42
|
+
lifecycle: "completed",
|
|
43
|
+
status: "truncated",
|
|
44
|
+
title: "terminal output truncated",
|
|
45
|
+
text: cut.notice,
|
|
46
|
+
},
|
|
47
|
+
],
|
|
48
|
+
}
|
|
49
|
+
: read;
|
|
31
50
|
const trace = buildTerminalActorTrace({
|
|
32
51
|
persona,
|
|
33
52
|
productName: product.name,
|
|
@@ -40,7 +59,7 @@ function buildLiveTrace(inputs) {
|
|
|
40
59
|
terminalEvents,
|
|
41
60
|
commandLog,
|
|
42
61
|
transcriptTail: tailOf(normalizedTranscript),
|
|
43
|
-
participant
|
|
62
|
+
participant,
|
|
44
63
|
runtimeAuth: runtimeEnv.mode,
|
|
45
64
|
runtime,
|
|
46
65
|
...(terminalTokenUsage === undefined ? {} : { tokenUsage: terminalTokenUsage }),
|
|
@@ -349,9 +349,13 @@ export class LiveTerminalSandbox {
|
|
|
349
349
|
});
|
|
350
350
|
// Score by the verdict-nonce marker over the scrubbed, redacted and normalized transcript, with
|
|
351
351
|
// the exact same logic the local-actor routes use (extractLocalActorVerdict/normalizeLocalActorTranscript).
|
|
352
|
+
// Output past the transcript cap is scanned as it arrives, since it is not stored.
|
|
352
353
|
const rawTranscript = terminalEvents.map((e) => e.chunk).join("");
|
|
353
354
|
const normalizedTranscript = normalizeLocalActorTranscript(rawTranscript);
|
|
354
|
-
const
|
|
355
|
+
const cut = this.inputs.recorder.transcriptCut();
|
|
356
|
+
const markerStatus = extractLocalActorVerdict(normalizedTranscript, verdictNonce) ?? cut?.verdict ?? null;
|
|
357
|
+
if (cut)
|
|
358
|
+
recordLifecycle("terminal-lab.transcript.exceeded", cut.notice);
|
|
355
359
|
if (this.timedOut) {
|
|
356
360
|
this.status = "timed_out";
|
|
357
361
|
this.completionReason = "timed_out";
|
|
@@ -13,7 +13,6 @@ import { NODE_BOOTSTRAP_TIMEOUT_MS } from "../../subject/node-bootstrap.js";
|
|
|
13
13
|
import { terminalSandboxTimeoutMs } from "./lifetime.js";
|
|
14
14
|
import { isExactRuntimeVersion, TERMINAL_RUNTIME_VERSION_TIMEOUT_MS } from "./runtime.js";
|
|
15
15
|
import { PRODUCT_SETUP_TIMEOUT_MS, TERMINAL_SANDBOX_TIMEOUT_BUFFER_MS, } from "./types.js";
|
|
16
|
-
import { actorOf, capsOf } from "../../study/study-fields.js";
|
|
17
16
|
/**
|
|
18
17
|
* Plan a terminal-product study. It is called for any config handed to the terminal runner, not
|
|
19
18
|
* only one routeOf sends here, so a config for another route gets this route's refusal.
|
|
@@ -41,12 +40,12 @@ export function planTerminalStudy(config, input) {
|
|
|
41
40
|
const tasksReason = taskProtocolValidationReason(config, false);
|
|
42
41
|
if (tasksReason)
|
|
43
42
|
return refuse("HUMANISH_STUDY_TASKS_UNSUPPORTED", tasksReason);
|
|
44
|
-
const actorType =
|
|
43
|
+
const actorType = config.actor?.type ?? "";
|
|
45
44
|
const descriptor = actorRegistry[actorType];
|
|
46
45
|
if (!descriptor || !isTerminalActorDescriptor(descriptor))
|
|
47
46
|
return refuse("HUMANISH_TERMINAL_ACTOR_UNSUPPORTED", `actor.type "${actorType}" is not a registered terminal actor.`);
|
|
48
47
|
const runtimeVersion = config.execution?.runtime?.version;
|
|
49
|
-
const actor =
|
|
48
|
+
const actor = config.actor;
|
|
50
49
|
if ((config.execution?.runtime !== undefined && !isExactRuntimeVersion(runtimeVersion)) ||
|
|
51
50
|
(actor?.model !== undefined &&
|
|
52
51
|
(typeof actor.model !== "string" || actor.model.trim().length === 0)) ||
|
|
@@ -84,7 +83,7 @@ export function planTerminalStudy(config, input) {
|
|
|
84
83
|
...(egressAllow === undefined ? {} : { egressAllow }),
|
|
85
84
|
...(stdin === undefined ? {} : { stdin }),
|
|
86
85
|
};
|
|
87
|
-
const caps =
|
|
86
|
+
const caps = config.caps;
|
|
88
87
|
if (input.dryRun)
|
|
89
88
|
return { ok: true, plan: { ...shared, dryRun: true, ...(caps ? { caps } : {}) } };
|
|
90
89
|
// The live route injects the runtime key into one command only, and only a registered actor
|
|
@@ -8,6 +8,7 @@ export declare function createTerminalRecorder(args: {
|
|
|
8
8
|
nowIso: () => string;
|
|
9
9
|
sanitize: (text: string) => string;
|
|
10
10
|
knownSecretValues: readonly string[];
|
|
11
|
+
verdictNonce: string;
|
|
11
12
|
}): {
|
|
12
13
|
lifecycle: LifecycleRecord[];
|
|
13
14
|
commandLog: CommandLogRecord[];
|
|
@@ -22,6 +23,11 @@ export declare function createTerminalRecorder(args: {
|
|
|
22
23
|
append(raw: string): void;
|
|
23
24
|
finish(): import("./participant-text.js").TerminalParticipantText;
|
|
24
25
|
};
|
|
26
|
+
transcriptCut: () => {
|
|
27
|
+
notice: string;
|
|
28
|
+
verdict: ReturnType<() => ("blocked" | "failed" | "passed") | null>;
|
|
29
|
+
usage: string[];
|
|
30
|
+
} | undefined;
|
|
25
31
|
recordLifecycle: (event: string, message: string) => void;
|
|
26
32
|
recordStreamedTerminalChunk: (stream: "stdout" | "stderr", raw: string) => void;
|
|
27
33
|
appendReturnedTerminalOutput: (stream: "stdout" | "stderr", raw: string) => void;
|
|
@@ -1,13 +1,15 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
|
+
import { createLocalActorVerdictScanner } from "../../run/terminal-contract.js";
|
|
2
3
|
import { createTerminalParticipantReader } from "./participant-text.js";
|
|
3
|
-
import {
|
|
4
|
+
import { terminalUsageRecords } from "./token-usage.js";
|
|
5
|
+
import { MAX_TRANSCRIPT_BYTES, PENDING_LINE_CHARS, } from "./types.js";
|
|
4
6
|
/**
|
|
5
7
|
* The terminal run's ledgers and capture buffers, mutated through the live lifecycle. Every chunk
|
|
6
8
|
* is scrubbed and redacted as it is stored. Raw bytes leave the append functions only for the
|
|
7
9
|
* in-memory participant reader, which keeps sanitized decoded text.
|
|
8
10
|
*/
|
|
9
11
|
export function createTerminalRecorder(args) {
|
|
10
|
-
const { nowIso, sanitize, knownSecretValues } = args;
|
|
12
|
+
const { nowIso, sanitize, knownSecretValues, verdictNonce } = args;
|
|
11
13
|
const lifecycle = [];
|
|
12
14
|
const commandLog = [];
|
|
13
15
|
const terminalEvents = [];
|
|
@@ -19,24 +21,76 @@ export function createTerminalRecorder(args) {
|
|
|
19
21
|
// Reads the agent's text from the raw stdout in memory; it keeps only sanitized decoded text.
|
|
20
22
|
const participantText = createTerminalParticipantReader(sanitize);
|
|
21
23
|
let transcriptBytes = 0;
|
|
24
|
+
// Output past the cap is not stored, but the participant reader, the verdict marker and token
|
|
25
|
+
// usage still read all of it, so a long session is judged and priced as a short one is.
|
|
26
|
+
const notStored = { stdout: 0, stderr: 0 };
|
|
27
|
+
const laterVerdict = createLocalActorVerdictScanner(verdictNonce);
|
|
28
|
+
const laterUsage = [];
|
|
29
|
+
// The stdout line in progress past the cap. Undefined while the rest of a line is skipped: the
|
|
30
|
+
// line the cap cut through (its start is stored) or one longer than the pending limit.
|
|
31
|
+
let laterLine;
|
|
32
|
+
let stdoutAtLineStart = true;
|
|
33
|
+
let pastCap = false;
|
|
34
|
+
const readLaterStdout = (raw) => {
|
|
35
|
+
let text = raw;
|
|
36
|
+
if (laterLine === undefined) {
|
|
37
|
+
const end = text.indexOf("\n");
|
|
38
|
+
if (end < 0)
|
|
39
|
+
return;
|
|
40
|
+
text = text.slice(end + 1);
|
|
41
|
+
laterLine = "";
|
|
42
|
+
}
|
|
43
|
+
laterLine += text;
|
|
44
|
+
const end = laterLine.lastIndexOf("\n");
|
|
45
|
+
if (end >= 0) {
|
|
46
|
+
laterUsage.push(...terminalUsageRecords(sanitize(laterLine.slice(0, end))));
|
|
47
|
+
laterLine = laterLine.slice(end + 1);
|
|
48
|
+
}
|
|
49
|
+
if (laterLine.length > PENDING_LINE_CHARS)
|
|
50
|
+
laterLine = undefined;
|
|
51
|
+
};
|
|
22
52
|
const recordLifecycle = (event, message) => {
|
|
23
53
|
lifecycle.push({ at: nowIso(), event, message: sanitize(message) });
|
|
24
54
|
};
|
|
25
55
|
const appendTerminalChunk = (stream, raw) => {
|
|
56
|
+
if (stream === "stdout")
|
|
57
|
+
participantText.append(raw);
|
|
26
58
|
if (transcriptBytes >= MAX_TRANSCRIPT_BYTES) {
|
|
27
59
|
for (const order of [stream, "combined"]) {
|
|
28
60
|
const remaining = maxDiscardedPrefixChars - discardedPrefixes[order].length;
|
|
29
61
|
if (remaining > 0)
|
|
30
62
|
discardedPrefixes[order] += raw.slice(0, remaining);
|
|
31
63
|
}
|
|
64
|
+
if (!pastCap) {
|
|
65
|
+
pastCap = true;
|
|
66
|
+
// The stored stream seeds the scan, so a marker the cap split in two is still found.
|
|
67
|
+
laterVerdict.push(terminalEvents.map((event) => event.chunk).join(""));
|
|
68
|
+
laterLine = stdoutAtLineStart ? "" : undefined;
|
|
69
|
+
}
|
|
70
|
+
notStored[stream] += Buffer.byteLength(raw, "utf8");
|
|
71
|
+
laterVerdict.push(sanitize(raw));
|
|
72
|
+
if (stream === "stdout")
|
|
73
|
+
readLaterStdout(raw);
|
|
32
74
|
return;
|
|
33
75
|
}
|
|
34
76
|
transcriptBytes += Buffer.byteLength(raw, "utf8");
|
|
35
|
-
// Scrub, then redact, at the source. Only the participant reader
|
|
36
|
-
//
|
|
77
|
+
// Scrub, then redact, at the source. Only the participant reader and the readers of output
|
|
78
|
+
// past the cap see raw bytes, and they store none of them.
|
|
37
79
|
terminalEvents.push({ at: nowIso(), stream, chunk: sanitize(raw) });
|
|
38
|
-
if (stream === "stdout")
|
|
39
|
-
|
|
80
|
+
if (stream === "stdout" && raw)
|
|
81
|
+
stdoutAtLineStart = raw.endsWith("\n");
|
|
82
|
+
};
|
|
83
|
+
/** What the cap left out, for the transcript, the trace and the run events; undefined when the
|
|
84
|
+
* whole session was stored. */
|
|
85
|
+
const transcriptCut = () => {
|
|
86
|
+
if (!pastCap)
|
|
87
|
+
return undefined;
|
|
88
|
+
const bytes = notStored.stdout + notStored.stderr;
|
|
89
|
+
return {
|
|
90
|
+
notice: `Output past the ${MAX_TRANSCRIPT_BYTES / 1024} KiB transcript cap was not stored: ${bytes} bytes (${notStored.stdout} stdout, ${notStored.stderr} stderr). The verdict marker, token usage and agent messages were still read from it.`,
|
|
91
|
+
verdict: laterVerdict.verdict(),
|
|
92
|
+
usage: laterUsage,
|
|
93
|
+
};
|
|
40
94
|
};
|
|
41
95
|
// E2B can stream every byte through callbacks and return the same complete output.
|
|
42
96
|
// Track transport delivery, independently per stream, rather than deduplicating participant
|
|
@@ -78,6 +132,7 @@ export function createTerminalRecorder(args) {
|
|
|
78
132
|
interventions,
|
|
79
133
|
discardedPrefixes,
|
|
80
134
|
participantText,
|
|
135
|
+
transcriptCut,
|
|
81
136
|
recordLifecycle,
|
|
82
137
|
recordStreamedTerminalChunk,
|
|
83
138
|
appendReturnedTerminalOutput,
|
|
@@ -54,7 +54,6 @@ import { terminalRouteScorer } from "../../study/adapter-scorer-loader.js";
|
|
|
54
54
|
import { withLateScorer } from "../../study/route-inputs.js";
|
|
55
55
|
import {} from "./types.js";
|
|
56
56
|
import { studyResultIdentity } from "../../run/study-result.js";
|
|
57
|
-
import { actorOf } from "../../study/study-fields.js";
|
|
58
57
|
/** A refused terminal study's result: the route's envelope, and the analysis record a refusal gets. */
|
|
59
58
|
export function terminalStudyRefusal(options, refusal) {
|
|
60
59
|
const { config, dryRun } = options;
|
|
@@ -62,7 +61,7 @@ export function terminalStudyRefusal(options, refusal) {
|
|
|
62
61
|
...studyResultIdentity("terminal", config.id),
|
|
63
62
|
ok: false,
|
|
64
63
|
cwd: path.resolve(options.cwd),
|
|
65
|
-
actor: refusal.actor ??
|
|
64
|
+
actor: refusal.actor ?? config.actor?.type ?? "",
|
|
66
65
|
product: config.subject.product?.name ?? "",
|
|
67
66
|
dryRun,
|
|
68
67
|
runId: options.runId ?? "not-created",
|
|
@@ -61,6 +61,7 @@ export async function runLiveTerminalSession(args) {
|
|
|
61
61
|
// This entry point is the live terminal route; its dry-run sibling is a separate function.
|
|
62
62
|
mode: "live",
|
|
63
63
|
study: plan.study,
|
|
64
|
+
warnings: plan.warnings,
|
|
64
65
|
renderReview: renderTerminalReviewMarkdown,
|
|
65
66
|
observer: { open: input.open === true, render: deps.renderObserver },
|
|
66
67
|
now,
|
|
@@ -77,7 +78,7 @@ export async function runLiveTerminalSession(args) {
|
|
|
77
78
|
});
|
|
78
79
|
const e2bApiKey = env.E2B_API_KEY?.trim() ?? "";
|
|
79
80
|
// The ledgers + capture buffers, mutated through the live lifecycle.
|
|
80
|
-
const recorder = createTerminalRecorder({ nowIso, sanitize, knownSecretValues });
|
|
81
|
+
const recorder = createTerminalRecorder({ nowIso, sanitize, knownSecretValues, verdictNonce });
|
|
81
82
|
const { version, model, modelSource, reasoningEffort } = plan.runtime;
|
|
82
83
|
const runtime = declaredRuntimeProvenance({
|
|
83
84
|
...(version === undefined ? {} : { version }),
|
|
@@ -1,4 +1,7 @@
|
|
|
1
1
|
import type { ActorTokenUsage } from "../../actors/contract.js";
|
|
2
|
+
/** The usage records in complete stdout lines, kept from output past the transcript cap so
|
|
3
|
+
* parseTerminalTokenUsage can count them with the stored ones. */
|
|
4
|
+
export declare function terminalUsageRecords(lines: string): string[];
|
|
2
5
|
/**
|
|
3
6
|
* Accumulate runtime-turn usage from a captured `codex exec --json` stream. A Codex turn may
|
|
4
7
|
* include multiple provider requests; these records cannot establish per-request pricing tiers.
|
|
@@ -19,6 +19,11 @@ const USAGE_RE = /"type"\s*:\s*"turn\.completed"\s*,\s*"usage"\s*:\s*(\{[^}\n]*\
|
|
|
19
19
|
function num(value) {
|
|
20
20
|
return typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : undefined;
|
|
21
21
|
}
|
|
22
|
+
/** The usage records in complete stdout lines, kept from output past the transcript cap so
|
|
23
|
+
* parseTerminalTokenUsage can count them with the stored ones. */
|
|
24
|
+
export function terminalUsageRecords(lines) {
|
|
25
|
+
return Array.from(lines.matchAll(USAGE_RE), (match) => match[0]);
|
|
26
|
+
}
|
|
22
27
|
/**
|
|
23
28
|
* Accumulate runtime-turn usage from a captured `codex exec --json` stream. A Codex turn may
|
|
24
29
|
* include multiple provider requests; these records cannot establish per-request pricing tiers.
|
|
@@ -53,7 +53,8 @@ export interface TerminalProductScoringContext {
|
|
|
53
53
|
/**
|
|
54
54
|
* The full normalized transcript of the in-sandbox agent session, scrubbed (literal known
|
|
55
55
|
* values) then redacted (shape patterns) at the source, capped at MAX_TRANSCRIPT_BYTES, and
|
|
56
|
-
* byte-identical to the persisted terminal-transcript.txt artifact.
|
|
56
|
+
* byte-identical to the persisted terminal-transcript.txt artifact. A capped transcript ends
|
|
57
|
+
* with a `[humanish]` line giving the byte count that was not stored. The trace's transcriptTail
|
|
57
58
|
* is a ~2KB projection of this; a scorer needs the whole session so a rubric can find
|
|
58
59
|
* command-tier evidence anywhere in it.
|
|
59
60
|
*/
|
|
@@ -32,5 +32,6 @@ export const TEXT_ITEMS_BYTES = 128 * 1024;
|
|
|
32
32
|
// The longest stdout line the participant reader holds while waiting for its newline.
|
|
33
33
|
export const PENDING_LINE_CHARS = 1024 * 1024;
|
|
34
34
|
// Hard cap on the retained event-stream + transcript size, so a runaway agent cannot balloon the
|
|
35
|
-
// bundle. Redaction runs pre-truncation so a cut can never split a secret past the scrubber.
|
|
35
|
+
// bundle. Redaction runs pre-truncation so a cut can never split a secret past the scrubber. Output
|
|
36
|
+
// past the cap is still read for the verdict marker, token usage and the trace's agent text.
|
|
36
37
|
export const MAX_TRANSCRIPT_BYTES = 512 * 1024;
|