@sema-agent/core 5.6.0 → 5.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +55 -0
- package/dist/agents/cascade.d.ts +1 -1
- package/dist/agents/subagent.d.ts +4 -7
- package/dist/agents/subagent.js +10 -19
- package/dist/agents/teacher.js +3 -3
- package/dist/agents/team.d.ts +1 -1
- package/dist/agents/team.js +1 -1
- package/dist/agents/verify.js +1 -1
- package/dist/brain/anthropic.js +24 -35
- package/dist/brain/openai.js +22 -33
- package/dist/brain/stream-engine.d.ts +0 -2
- package/dist/brain/stream-engine.js +5 -47
- package/dist/brain/stream-shared.d.ts +0 -10
- package/dist/brain/stream-shared.js +0 -23
- package/dist/brain/terminal-cause.d.ts +0 -1
- package/dist/brain/terminal-cause.js +0 -3
- package/dist/brain/timeout.d.ts +4 -2
- package/dist/brain/timeout.js +4 -14
- package/dist/config/catalog.d.ts +3 -11
- package/dist/config/catalog.js +30 -69
- package/dist/config/defaults.d.ts +0 -3
- package/dist/config/defaults.js +0 -3
- package/dist/core/auto-compaction.d.ts +2 -11
- package/dist/core/auto-compaction.js +3 -67
- package/dist/core/checkpoint-store.d.ts +15 -10
- package/dist/core/checkpoint-store.js +13 -10
- package/dist/core/memory-engine/file-backend.js +20 -17
- package/dist/core/memory-engine/layout.d.ts +5 -0
- package/dist/core/memory-engine/layout.js +72 -47
- package/dist/core/runner/assemble-result.d.ts +2 -8
- package/dist/core/runner/assemble-result.js +15 -16
- package/dist/core/runner/prepare-task.d.ts +26 -6
- package/dist/core/runner/prepare-task.js +203 -141
- package/dist/core/runner/runtask.d.ts +0 -2
- package/dist/core/runner/runtask.js +267 -253
- package/dist/core/secret-env.d.ts +2 -0
- package/dist/core/secret-env.js +16 -4
- package/dist/core/session-policy-store.d.ts +5 -0
- package/dist/core/session-policy-store.js +3 -1
- package/dist/core/store-contracts/checkpoint-store-contract.js +2 -2
- package/dist/core/task-notification.d.ts +1 -0
- package/dist/core/task-notification.js +3 -0
- package/dist/core/task-registry-agent.d.ts +2 -0
- package/dist/core/task-registry-agent.js +33 -1
- package/dist/core/task-registry.d.ts +2 -0
- package/dist/core/task-registry.js +9 -1
- package/dist/core/tool-errors.js +10 -2
- package/dist/core/trace.d.ts +1 -8
- package/dist/core/types.d.ts +24 -30
- package/dist/core/usage-window-store.d.ts +39 -0
- package/dist/core/usage-window-store.js +115 -0
- package/dist/engine/execution-env/node-execution-env.js +0 -22
- package/dist/engine/harness/agent-harness.d.ts +0 -7
- package/dist/engine/harness/agent-harness.js +13 -53
- package/dist/engine/harness/types.d.ts +2 -15
- package/dist/engine/llm/types.d.ts +1 -2
- package/dist/engine/loop/agent-loop.d.ts +1 -1
- package/dist/engine/loop/agent-loop.js +5 -171
- package/dist/engine/loop/types.d.ts +0 -18
- package/dist/index.d.ts +10 -5
- package/dist/index.js +8 -4
- package/dist/orchestration/goal.js +1 -5
- package/dist/orchestration/run-workflow-tool.d.ts +1 -1
- package/dist/orchestration/run-workflow-tool.js +1 -2
- package/dist/orchestration/workflow-governance.d.ts +6 -6
- package/dist/orchestration/workflow-governance.js +33 -47
- package/dist/prompt-assembly/event-registry.js +2 -3
- package/dist/stores/cc/lockfile.js +9 -11
- package/dist/stores/cc/mailbox-store.js +49 -38
- package/dist/stores/file/index.d.ts +3 -0
- package/dist/stores/file/index.js +4 -0
- package/dist/stores/file/usage-window-store.d.ts +9 -0
- package/dist/stores/file/usage-window-store.js +86 -0
- package/dist/tools/fs/fs-bash.d.ts +1 -10
- package/dist/tools/fs/fs-bash.js +20 -60
- package/dist/tools/fs/index.d.ts +0 -2
- package/dist/tools/fs/index.js +1 -2
- package/package.json +1 -4
- package/dist/bin/sema-tb.d.ts +0 -31
- package/dist/bin/sema-tb.js +0 -448
- package/dist/bin/tb-env.d.ts +0 -2
- package/dist/bin/tb-env.js +0 -17
- package/dist/brain/walltime.d.ts +0 -1
- package/dist/brain/walltime.js +0 -1
- package/dist/core/runner/call-cap.d.ts +0 -67
- package/dist/core/runner/call-cap.js +0 -145
- package/dist/core/runner/cut-kill.d.ts +0 -10
- package/dist/core/runner/cut-kill.js +0 -37
|
@@ -1,15 +1,14 @@
|
|
|
1
1
|
import { AgentHarness, DEFAULT_COMPACTION_SETTINGS, uuidv7 } from "../../internal/harness.js";
|
|
2
|
-
import { CheckpointError, BINDING_CHECKPOINT_VERSION, checkpointVersionOf, MAX_SUPPORTED_CHECKPOINT_VERSION, remainingBudgetMicroUsd,
|
|
3
|
-
import { effectiveCushionMs, estimateCycleMs, hasRealContent, minLiveWriteoutWindowMs, recordCallSample, recordToolCycleSample, recordToolSample, recordTtftSample, softExecDeadlineMs, writeoutCushionMs } from "./call-cap.js";
|
|
2
|
+
import { CheckpointError, BINDING_CHECKPOINT_VERSION, checkpointVersionOf, MAX_SUPPORTED_CHECKPOINT_VERSION, remainingBudgetMicroUsd, remainingTokens, validatePendingSteer, winnerFromOutcome, } from "../checkpoint-store.js";
|
|
4
3
|
import { engineVersion } from "../version.js";
|
|
5
4
|
import { CONFIG_CATALOG_VERSION, declarationReasons, resolveEffectiveConfig } from "../../config/catalog.js";
|
|
6
5
|
import { eventDefaultOn } from "../../prompt-assembly/event-registry.js";
|
|
7
|
-
import { DEFAULT_COMPACTION_INSTRUCTIONS, createRapidRefillState, isCompactionManualCancel,
|
|
8
|
-
import { ASK_USER_QUESTION_TOOL_NAME } from "../ask-question.js";
|
|
6
|
+
import { DEFAULT_COMPACTION_INSTRUCTIONS, createRapidRefillState, isCompactionManualCancel, maybeCompact, nextTrimForceBackoff, recordCompactionAndCheckRapidRefill } from "../auto-compaction.js";
|
|
7
|
+
import { ASK_USER_QUESTION_TOOL_NAME, QUESTION_AWAITS_RESUME } from "../ask-question.js";
|
|
9
8
|
import { computeCostMicroUsd, modelCostToPricing } from "../pricing.js";
|
|
10
9
|
import { emitTrace } from "../trace.js";
|
|
11
10
|
import { emitTaskOutcome } from "../task-outcome.js";
|
|
12
|
-
import { isDegenerateCutMessage
|
|
11
|
+
import { isDegenerateCutMessage } from "../../brain/terminal-cause.js";
|
|
13
12
|
import { primaryActivityArg } from "../arg-summary.js";
|
|
14
13
|
import { resolveReasoning } from "../../brain/reasoning.js";
|
|
15
14
|
import { readDegradation } from "../../brain/degrading.js";
|
|
@@ -38,20 +37,18 @@ import { RunnerSharedToolResultStore } from "../tool-result-store.js";
|
|
|
38
37
|
import { formatDiagnosticsBlock } from "../lsp-diagnostics.js";
|
|
39
38
|
import { refuseOutOfContractDecision, toolPolicyNameSets } from "../tool-policy.js";
|
|
40
39
|
import { defaultTaskRegistry } from "../task-registry.js";
|
|
41
|
-
import { discloseDroppedPending, PendingSessionNotifications, renderTaskNotificationXml, SystemInjectionQueue, taskNotificationDedupKey } from "../task-notification.js";
|
|
40
|
+
import { discloseDroppedPending, isDelegatedAgentTerminal, PendingSessionNotifications, renderTaskNotificationXml, SystemInjectionQueue, taskNotificationDedupKey } from "../task-notification.js";
|
|
42
41
|
import { ToolDetachHub } from "../tool-detach.js";
|
|
43
42
|
import { workflowSizeGuidelineChangeNotice } from "../../orchestration/workflow-size-guideline.js";
|
|
44
|
-
import { DEFAULT_MAX_TURNS } from "../../config/defaults.js";
|
|
45
43
|
import { structuredFrom, toolOutputFrom } from "./tool-output-projection.js";
|
|
46
|
-
export { DEFAULT_MAX_TURNS };
|
|
47
44
|
function createRunState() {
|
|
48
45
|
return {
|
|
49
46
|
telemetry: { cacheFamily: "input-excludes-cached", pricing: { inputPer1M: 0, outputPer1M: 0 }, pricingConfigured: true, unpricedSpend: false, tracer: undefined, taskId: "", taskStart: 0, taskStartMonotonic: 0, cacheBreakReported: false },
|
|
50
47
|
degrade: { degradeToModel: undefined, degraded: undefined, outputErrorStreak: 0, outputInvalid: false, recordDegraded: () => { } },
|
|
51
|
-
limits: { turnsExceeded: false, budgetHit: undefined, outputRetryCap: 0, effectiveMaxTurns: undefined },
|
|
52
|
-
budget: { remainingMicroUsd: undefined, maxCostMicroUsd: undefined, overBudget: () =>
|
|
48
|
+
limits: { turnsExceeded: false, budgetHit: undefined, budgetAxis: undefined, platformTerminal: undefined, outputRetryCap: 0, effectiveMaxTurns: undefined },
|
|
49
|
+
budget: { remainingMicroUsd: undefined, maxCostMicroUsd: undefined, remainingTokens: undefined, maxTokensWindow: undefined, overBudget: () => undefined, streamCancel: false, callOutputChars: 0, lastStreamBudgetCheck: 0, projectedOverBudget: () => undefined },
|
|
53
50
|
turn: { callStartAt: undefined, firstTokenAt: undefined, turnUsage: undefined, turnUsageMissing: false, turnStopReason: undefined, lastTurnHadToolCalls: false, toolBatch: [] },
|
|
54
|
-
counters: {
|
|
51
|
+
counters: { approachNoticesSent: 0, walltimeSyncBackstopFired: false, compactionFloor: 0, trimForceBackoff: false, repetitionCuts: 0, repetitionSpared: 0, repetitionEvents: [], REPETITION_EVENTS_CAP: 0, preemptIgnoredReported: false, wroteThisRun: false, finalVerifyInjections: 0, groundingSignalPreR9: false, groundingSignalPostR9: false, cadenceTurns: 0 },
|
|
55
52
|
attach: { attachmentsCfg: undefined, agentListingOn: false, skillsListingOn: false, attachState: undefined, dateState: undefined, instrProbe: undefined, instrState: undefined, sizeGuidelineState: undefined, attachmentsInjected: 0 },
|
|
56
53
|
};
|
|
57
54
|
}
|
|
@@ -96,14 +93,19 @@ const SUGGESTIONS_TIMEOUT_MS = 30_000;
|
|
|
96
93
|
const SUGGESTIONS_TRANSCRIPT_MESSAGES = 12;
|
|
97
94
|
const SNAPSHOT_TIMEOUT_MS = 30_000;
|
|
98
95
|
const SNAPSHOT_TOO_LARGE_TTL_MS = 30 * 60_000;
|
|
96
|
+
const LIMIT_APPROACH_DEFAULT_THRESHOLDS = [0.8, 0.95];
|
|
97
|
+
const DEFAULT_PRECALL_OUTPUT_TOKENS = 4096;
|
|
98
|
+
function limitApproachFrames(index, axis, percent) {
|
|
99
|
+
return index === 0
|
|
100
|
+
? `<system-reminder>[limit approach] This task has used about ${percent}% of its ${axis}. ` +
|
|
101
|
+
`Start converging: finish the step you are on, drop work that is not required for the deliverable, and ` +
|
|
102
|
+
`avoid starting anything you cannot complete within what is left.</system-reminder>`
|
|
103
|
+
: `<system-reminder>[limit approach] About ${percent}% of the ${axis} is gone. Deliver now: ` +
|
|
104
|
+
`stop exploring, write up what you have — including what is incomplete and what you would do next — ` +
|
|
105
|
+
`and end the task. Work that is not reported before the allowance runs out is lost.</system-reminder>`;
|
|
106
|
+
}
|
|
99
107
|
function resolveMaxTurns(limits) {
|
|
100
|
-
|
|
101
|
-
return limits.maxTurns;
|
|
102
|
-
}
|
|
103
|
-
if (limits?.timeoutSec !== undefined && limits.timeoutSec > 0) {
|
|
104
|
-
return undefined;
|
|
105
|
-
}
|
|
106
|
-
return DEFAULT_MAX_TURNS;
|
|
108
|
+
return limits?.maxTurns;
|
|
107
109
|
}
|
|
108
110
|
function resumeDecisionWasNegative(resume) {
|
|
109
111
|
const o = resume.outcome;
|
|
@@ -135,7 +137,24 @@ function deepJsonEqual(a, b) {
|
|
|
135
137
|
return aKeys.every((k) => Object.prototype.hasOwnProperty.call(bo, k) && deepJsonEqual(ao[k], bo[k]));
|
|
136
138
|
}
|
|
137
139
|
function sameWinner(a, b) {
|
|
138
|
-
return a.boundCallId === b.boundCallId &&
|
|
140
|
+
return (a.boundCallId === b.boundCallId &&
|
|
141
|
+
a.decision === b.decision &&
|
|
142
|
+
deepJsonEqual(a.updatedInput, b.updatedInput) &&
|
|
143
|
+
deepJsonEqual(a.answer, b.answer));
|
|
144
|
+
}
|
|
145
|
+
function pendingContentAskCallId(cp) {
|
|
146
|
+
return cp.pendingAction.kind === "tool_approval" && cp.pendingAction.toolName === ASK_USER_QUESTION_TOOL_NAME
|
|
147
|
+
? cp.pendingAction.toolCallId
|
|
148
|
+
: undefined;
|
|
149
|
+
}
|
|
150
|
+
function answerFaceForRedeemedCall(answer, redeemedCallId, base) {
|
|
151
|
+
return async (req, signal) => {
|
|
152
|
+
if (req.toolCallId === redeemedCallId)
|
|
153
|
+
return answer;
|
|
154
|
+
if (base !== undefined)
|
|
155
|
+
return base(req, signal);
|
|
156
|
+
throw new Error("this resumed leg's answer was bound to the decided question only — a new question has no answer on this leg");
|
|
157
|
+
};
|
|
139
158
|
}
|
|
140
159
|
function writeFamilyOfCanonical(name) {
|
|
141
160
|
if (name === "TaskCreate" || name === "TaskUpdate")
|
|
@@ -211,62 +230,85 @@ function resumeContinuation(resume) {
|
|
|
211
230
|
`conversation above. Do NOT restart the task or re-run any tool you already ran; continue from this ` +
|
|
212
231
|
`exact point, building on the existing results, and finish the remaining work.`);
|
|
213
232
|
}
|
|
233
|
+
function platformLimitTerminal(reason, retryAfterMs, moment = "turn_boundary") {
|
|
234
|
+
const message = moment === "entry"
|
|
235
|
+
? `a deployment usage window is exhausted (RunnerDeps.usageWindows), so the task was refused before its first model call — nothing ran and nothing was spent. There is no checkpoint to suspend into at this point, so the caller re-submits after the window frees, in ${String(retryAfterMs ?? 0)}ms.`
|
|
236
|
+
: reason === "env_lifetime"
|
|
237
|
+
? "the execution environment's declared lifetime is expiring and the run could not be suspended durably (not eligible, or the suspend was refused — see remoteEnvFailures / onError). It was stopped at a clean turn boundary instead of being killed mid-turn when the platform reclaims the environment."
|
|
238
|
+
: `a deployment usage window is exhausted (RunnerDeps.usageWindows) and the run could not be suspended durably, so it was stopped at a clean turn boundary. The window frees in ${String(retryAfterMs ?? 0)}ms.`;
|
|
239
|
+
const e = new Error(message);
|
|
240
|
+
e.code = reason === "env_lifetime" ? "env.lifetime_expired" : "usage.window_exhausted";
|
|
241
|
+
if (retryAfterMs !== undefined)
|
|
242
|
+
e.retryAfterMs = retryAfterMs;
|
|
243
|
+
return e;
|
|
244
|
+
}
|
|
214
245
|
function makeTurnBoundary(prepared, stats, rs, deps) {
|
|
215
|
-
const { spec, queue, manualCompactRef, todoToolMounted, taskToolsMounted,
|
|
246
|
+
const { spec, queue, manualCompactRef, todoToolMounted, taskToolsMounted, walltimeMonotonicDeadline, timeout, ident, postToolBatchHook, batchArgs, compactionBrain, withinTaskCompaction, compactionBreaker, windowSafetyOptions, rapidRefill, drainManualCompact, runnerHooks } = deps;
|
|
216
247
|
let finalVerifySeenAtLastBoundary = 0;
|
|
217
248
|
const onTurnBoundary = async (event) => {
|
|
218
|
-
if (prepared.callCapRef && turnToolSpan.startMin !== undefined && turnToolSpan.endMax !== undefined) {
|
|
219
|
-
recordToolCycleSample(prepared.callCapRef.state, turnToolSpan.endMax - turnToolSpan.startMin);
|
|
220
|
-
}
|
|
221
|
-
turnToolSpan.startMin = undefined;
|
|
222
|
-
turnToolSpan.endMax = undefined;
|
|
223
249
|
let boundarySteered = false;
|
|
224
|
-
if (prepared.suspendForResource === undefined) {
|
|
225
|
-
const nowMs = Date.now();
|
|
226
|
-
if (!rs.counters.finalizeInjected &&
|
|
227
|
-
spec.limits?.gracefulFinalize !== false &&
|
|
228
|
-
walltimeDeadlineMs !== undefined &&
|
|
229
|
-
prepared.callCapRef !== undefined) {
|
|
230
|
-
const remainingMs = walltimeDeadlineMs - nowMs;
|
|
231
|
-
if (remainingMs > 0 && remainingMs < estimateCycleMs(prepared.callCapRef.state) + (effectiveCushionMs(prepared.callCapRef) ?? 0)) {
|
|
232
|
-
rs.counters.finalizeInjected = true;
|
|
233
|
-
prepared.callCapRef.finalizeMode = true;
|
|
234
|
-
void prepared.harness.setThinkingLevel("off").catch(() => { });
|
|
235
|
-
prepared.turnSnapshot?.refreshRequestPolicy({ thinkingLevel: "off" });
|
|
236
|
-
rs.counters.nudgeIdx = nudgeSchedule.length;
|
|
237
|
-
const finalizeText = buildFinalizeText();
|
|
238
|
-
void prepared.harness.steer(finalizeText, { engineMinted: true }).catch(() => { });
|
|
239
|
-
boundarySteered = true;
|
|
240
|
-
queue.push({ type: "steering_injected", source: "finalize", preview: finalizeText.slice(0, 220), ...ident() });
|
|
241
|
-
}
|
|
242
|
-
}
|
|
243
|
-
let due;
|
|
244
|
-
while (rs.counters.nudgeIdx < nudgeSchedule.length && nowMs >= nudgeSchedule[rs.counters.nudgeIdx].atMs) {
|
|
245
|
-
due = nudgeSchedule[rs.counters.nudgeIdx++];
|
|
246
|
-
}
|
|
247
|
-
if (due) {
|
|
248
|
-
void prepared.harness.steer(due.text, { engineMinted: true }).catch(() => { });
|
|
249
|
-
rs.counters.nudgesSent += 1;
|
|
250
|
-
boundarySteered = true;
|
|
251
|
-
queue.push({ type: "steering_injected", source: "deadline_nudge", preview: due.text.slice(0, 220), ...ident() });
|
|
252
|
-
}
|
|
253
|
-
}
|
|
254
250
|
const walltimeHit = walltimeMonotonicDeadline !== undefined && performance.now() >= walltimeMonotonicDeadline;
|
|
255
251
|
const preemptHit = spec.preemptSignal?.aborted === true;
|
|
256
252
|
if (preemptHit && !prepared.suspendForResource && !rs.counters.preemptIgnoredReported) {
|
|
257
253
|
rs.counters.preemptIgnoredReported = true;
|
|
258
254
|
emitTrace(rs.telemetry.tracer, () => ({ kind: "preempt.ignored", version: 1, taskId: rs.telemetry.taskId, ts: Date.now() }));
|
|
259
255
|
}
|
|
256
|
+
let platformCause;
|
|
257
|
+
let platformRetryAfterMs;
|
|
258
|
+
const preemptWillSuspend = preemptHit && prepared.suspendForResource !== undefined;
|
|
259
|
+
if (!preemptWillSuspend && prepared.suspendRef.token === undefined && !prepared.abortController.signal.aborted) {
|
|
260
|
+
if (prepared.envLifetimeSuspendAt !== undefined && Date.now() >= prepared.envLifetimeSuspendAt) {
|
|
261
|
+
platformCause = "env_lifetime";
|
|
262
|
+
}
|
|
263
|
+
else if (prepared.usageGovernance !== undefined) {
|
|
264
|
+
const governance = prepared.usageGovernance;
|
|
265
|
+
try {
|
|
266
|
+
const at = Date.now();
|
|
267
|
+
await governance.commit(stats.tokens, at);
|
|
268
|
+
platformRetryAfterMs = await governance.check(at);
|
|
269
|
+
if (platformRetryAfterMs !== undefined)
|
|
270
|
+
platformCause = "usage_window";
|
|
271
|
+
}
|
|
272
|
+
catch (govErr) {
|
|
273
|
+
rs.limits.platformTerminal = govErr instanceof Error ? govErr : new Error(String(govErr));
|
|
274
|
+
void prepared.harness.abort();
|
|
275
|
+
prepared.abortController.abort();
|
|
276
|
+
return undefined;
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
if (platformCause !== undefined) {
|
|
281
|
+
const platformSpend = { costMicroUsd: stats.costMicroUsd, tokens: stats.tokens, turns: stats.turns, walltimeMs: Math.round(performance.now() - rs.telemetry.taskStartMonotonic) };
|
|
282
|
+
const committed = prepared.suspendForPlatformLimit !== undefined &&
|
|
283
|
+
(await prepared.suspendForPlatformLimit(platformCause, platformSpend, platformRetryAfterMs !== undefined ? { resumeAfterMs: platformRetryAfterMs } : undefined));
|
|
284
|
+
if (committed) {
|
|
285
|
+
rs.limits.turnsExceeded = false;
|
|
286
|
+
rs.limits.budgetHit = undefined;
|
|
287
|
+
return undefined;
|
|
288
|
+
}
|
|
289
|
+
const terminal = platformLimitTerminal(platformCause, platformRetryAfterMs);
|
|
290
|
+
rs.limits.platformTerminal = terminal;
|
|
291
|
+
try {
|
|
292
|
+
runnerHooks.onError?.(terminal, { phase: "config", sessionId: prepared.sessionId });
|
|
293
|
+
}
|
|
294
|
+
catch {
|
|
295
|
+
}
|
|
296
|
+
void prepared.harness.abort();
|
|
297
|
+
prepared.abortController.abort();
|
|
298
|
+
return undefined;
|
|
299
|
+
}
|
|
260
300
|
if (prepared.suspendForResource &&
|
|
261
301
|
(rs.limits.turnsExceeded || rs.limits.budgetHit !== undefined || walltimeHit || preemptHit) &&
|
|
262
302
|
prepared.suspendRef.token === undefined) {
|
|
263
303
|
const reason = preemptHit
|
|
264
304
|
? "preempt"
|
|
265
305
|
: rs.limits.budgetHit !== undefined
|
|
266
|
-
? "
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
306
|
+
? rs.limits.budgetAxis === "tokens"
|
|
307
|
+
? "budget_tokens"
|
|
308
|
+
: "budget_cost"
|
|
309
|
+
: rs.limits.turnsExceeded
|
|
310
|
+
? "turns"
|
|
311
|
+
: "walltime";
|
|
270
312
|
const sliceSpend = { costMicroUsd: stats.costMicroUsd, tokens: stats.tokens, turns: stats.turns, walltimeMs: Math.round(performance.now() - rs.telemetry.taskStartMonotonic) };
|
|
271
313
|
if (await prepared.suspendForResource(reason, sliceSpend)) {
|
|
272
314
|
rs.limits.turnsExceeded = false;
|
|
@@ -291,6 +333,39 @@ function makeTurnBoundary(prepared, stats, rs, deps) {
|
|
|
291
333
|
}
|
|
292
334
|
return undefined;
|
|
293
335
|
}
|
|
336
|
+
const approachCfg = spec.limits?.approachNotice;
|
|
337
|
+
if (approachCfg !== false &&
|
|
338
|
+
rs.counters.approachNoticesSent < LIMIT_APPROACH_DEFAULT_THRESHOLDS.length &&
|
|
339
|
+
prepared.suspendForResource === undefined &&
|
|
340
|
+
rs.turn.lastTurnHadToolCalls &&
|
|
341
|
+
!boundarySteered &&
|
|
342
|
+
!prepared.abortController.signal.aborted) {
|
|
343
|
+
const thresholds = approachCfg?.at ?? LIMIT_APPROACH_DEFAULT_THRESHOLDS;
|
|
344
|
+
const ratios = [];
|
|
345
|
+
if (rs.budget.maxTokensWindow !== undefined && rs.budget.maxTokensWindow > 0) {
|
|
346
|
+
ratios.push({ axis: "token budget", ratio: stats.tokens / rs.budget.maxTokensWindow });
|
|
347
|
+
}
|
|
348
|
+
if (rs.budget.maxCostMicroUsd !== undefined && rs.budget.maxCostMicroUsd > 0) {
|
|
349
|
+
ratios.push({ axis: "cost budget", ratio: stats.costMicroUsd / rs.budget.maxCostMicroUsd });
|
|
350
|
+
}
|
|
351
|
+
const declaredMaxTurns = spec.limits?.maxTurns;
|
|
352
|
+
if (declaredMaxTurns !== undefined && declaredMaxTurns > 0) {
|
|
353
|
+
ratios.push({ axis: "turn budget", ratio: (stats.turns + 1) / declaredMaxTurns });
|
|
354
|
+
}
|
|
355
|
+
let tightest;
|
|
356
|
+
for (const r of ratios)
|
|
357
|
+
if (tightest === undefined || r.ratio > tightest.ratio)
|
|
358
|
+
tightest = r;
|
|
359
|
+
const index = rs.counters.approachNoticesSent;
|
|
360
|
+
if (tightest !== undefined && tightest.ratio >= thresholds[index]) {
|
|
361
|
+
rs.counters.approachNoticesSent += 1;
|
|
362
|
+
const percent = Math.min(100, Math.round(tightest.ratio * 100));
|
|
363
|
+
const body = limitApproachFrames(index === 0 ? 0 : 1, tightest.axis, percent);
|
|
364
|
+
void prepared.harness.steer(body, { engineMinted: true }).catch(() => undefined);
|
|
365
|
+
queue.push({ type: "steering_injected", source: "limit_approach", preview: body.slice(0, 220), ...ident() });
|
|
366
|
+
boundarySteered = true;
|
|
367
|
+
}
|
|
368
|
+
}
|
|
294
369
|
if (prepared.reviewRequestRef.pending !== undefined) {
|
|
295
370
|
const { reason } = prepared.reviewRequestRef.pending;
|
|
296
371
|
prepared.reviewRequestRef.pending = undefined;
|
|
@@ -320,7 +395,6 @@ function makeTurnBoundary(prepared, stats, rs, deps) {
|
|
|
320
395
|
rs.attach.sizeGuidelineState !== undefined ||
|
|
321
396
|
rs.attach.attachState !== undefined) &&
|
|
322
397
|
!boundarySteered &&
|
|
323
|
-
!rs.counters.finalizeInjected &&
|
|
324
398
|
rs.counters.finalVerifyInjections === 0 &&
|
|
325
399
|
!prepared.abortController.signal.aborted) {
|
|
326
400
|
const due = [];
|
|
@@ -508,7 +582,7 @@ function makeTurnBoundary(prepared, stats, rs, deps) {
|
|
|
508
582
|
const batch = rs.turn.toolBatch;
|
|
509
583
|
rs.turn.toolBatch = [];
|
|
510
584
|
batchArgs?.clear();
|
|
511
|
-
const injectedThisTurn = finalVerifyInjectedThisTurn ? "final_verification" :
|
|
585
|
+
const injectedThisTurn = finalVerifyInjectedThisTurn ? "final_verification" : undefined;
|
|
512
586
|
if (!boundarySteered && !prepared.abortController.signal.aborted) {
|
|
513
587
|
try {
|
|
514
588
|
const r = await postToolBatchHook(batch, injectedThisTurn !== undefined ? { injectedThisTurn } : undefined);
|
|
@@ -538,7 +612,6 @@ function makeTurnBoundary(prepared, stats, rs, deps) {
|
|
|
538
612
|
}
|
|
539
613
|
const forceManual = manualCompactRef.requested;
|
|
540
614
|
if ((!withinTaskCompaction && !forceManual) ||
|
|
541
|
-
(rs.counters.finalizeInjected && !forceManual) ||
|
|
542
615
|
prepared.abortController.signal.aborted ||
|
|
543
616
|
prepared.suspendRef.token !== undefined ||
|
|
544
617
|
prepared.reviewRef.token !== undefined) {
|
|
@@ -600,9 +673,6 @@ function makeTurnBoundary(prepared, stats, rs, deps) {
|
|
|
600
673
|
const trimPressure = trimPressureArmed && !rs.counters.trimForceBackoff;
|
|
601
674
|
const passTrigger = trimPressure ? "forced" : forceManual ? "manual" : "auto";
|
|
602
675
|
let manualOutcome = "failed";
|
|
603
|
-
if (rs.counters.finalizeInjected && forceManual) {
|
|
604
|
-
emitTrace(rs.telemetry.tracer, () => ({ kind: "compaction.manual_in_finalize", version: 1, taskId: rs.telemetry.taskId, ts: Date.now() }));
|
|
605
|
-
}
|
|
606
676
|
const manualInstructions = claimedManual?.instructions;
|
|
607
677
|
try {
|
|
608
678
|
const comp = await maybeCompact({
|
|
@@ -727,16 +797,10 @@ function makeTurnBoundary(prepared, stats, rs, deps) {
|
|
|
727
797
|
manualOutcome = "mooted";
|
|
728
798
|
}
|
|
729
799
|
else if (!prepared.abortController.signal.aborted) {
|
|
730
|
-
|
|
731
|
-
if (!walltimeAbort)
|
|
732
|
-
compactionBreaker.failures += 1;
|
|
800
|
+
compactionBreaker.failures += 1;
|
|
733
801
|
runnerHooks.onError?.(err, { phase: "compaction", sessionId: prepared.sessionId });
|
|
734
802
|
const msg = String(err instanceof Error ? err.message : err);
|
|
735
|
-
const failReason =
|
|
736
|
-
? "walltime soft-deadline abort (engine-initiated write-out protection; not counted toward the breaker)"
|
|
737
|
-
: msg.length > 512
|
|
738
|
-
? `${msg.slice(0, 512)}…`
|
|
739
|
-
: msg;
|
|
803
|
+
const failReason = msg.length > 512 ? `${msg.slice(0, 512)}…` : msg;
|
|
740
804
|
emitTrace(rs.telemetry.tracer, () => ({
|
|
741
805
|
kind: "compaction.failed",
|
|
742
806
|
version: 1,
|
|
@@ -759,7 +823,7 @@ function makeTurnBoundary(prepared, stats, rs, deps) {
|
|
|
759
823
|
return onTurnBoundary;
|
|
760
824
|
}
|
|
761
825
|
function makeHarnessHandlers(prepared, stats, rs, deps) {
|
|
762
|
-
const { spec, queue, internals,
|
|
826
|
+
const { spec, queue, internals, ident, parentToolCallId, subagentName, pushContent, emitCommitted, startedToolCallIds, toolStartAt, writeFamilyOf, toolLabels, postToolBatchHook, batchArgs } = deps;
|
|
763
827
|
let lastWorkspaceCwd = prepared.cwdRef?.current;
|
|
764
828
|
const announceWorkspaceMove = () => {
|
|
765
829
|
const cwdNow = prepared.cwdRef?.current;
|
|
@@ -784,8 +848,10 @@ function makeHarnessHandlers(prepared, stats, rs, deps) {
|
|
|
784
848
|
rs.budget.callOutputChars += ev.delta.length;
|
|
785
849
|
if (rs.budget.callOutputChars - rs.budget.lastStreamBudgetCheck >= 256) {
|
|
786
850
|
rs.budget.lastStreamBudgetCheck = rs.budget.callOutputChars;
|
|
787
|
-
|
|
851
|
+
const projectedAxis = rs.budget.projectedOverBudget();
|
|
852
|
+
if (projectedAxis !== undefined) {
|
|
788
853
|
rs.limits.budgetHit = "exceeded";
|
|
854
|
+
rs.limits.budgetAxis = projectedAxis;
|
|
789
855
|
void prepared.harness.abort();
|
|
790
856
|
prepared.abortController.abort();
|
|
791
857
|
}
|
|
@@ -824,19 +890,8 @@ function makeHarnessHandlers(prepared, stats, rs, deps) {
|
|
|
824
890
|
rs.turn.turnUsage.cacheWriteTokens += u.cacheWrite || 0;
|
|
825
891
|
rs.turn.turnUsage.costMicroUsd += turnCostMicroUsd;
|
|
826
892
|
const callEndAt = Date.now();
|
|
827
|
-
const callIssuedAt = prepared.callIssuedAtRef
|
|
828
|
-
|
|
829
|
-
prepared.callIssuedAtRef.current = undefined;
|
|
830
|
-
const capLast = prepared.callCapRef?.state.last;
|
|
831
|
-
if (prepared.callCapRef) {
|
|
832
|
-
recordCallSample(prepared.callCapRef.state, u.output || 0, rs.turn.callStartAt !== undefined ? callEndAt - rs.turn.callStartAt : 0, {
|
|
833
|
-
contentValid: m.usageMissing === true && hasRealContent(m.content),
|
|
834
|
-
});
|
|
835
|
-
if (rs.turn.callStartAt !== undefined && rs.turn.firstTokenAt !== undefined) {
|
|
836
|
-
recordTtftSample(prepared.callCapRef.state, rs.turn.firstTokenAt - rs.turn.callStartAt);
|
|
837
|
-
}
|
|
838
|
-
prepared.callCapRef.state.last = undefined;
|
|
839
|
-
}
|
|
893
|
+
const callIssuedAt = prepared.callIssuedAtRef.current;
|
|
894
|
+
prepared.callIssuedAtRef.current = undefined;
|
|
840
895
|
emitTrace(rs.telemetry.tracer, () => ({
|
|
841
896
|
kind: "brain.call",
|
|
842
897
|
version: 1,
|
|
@@ -853,25 +908,11 @@ function makeHarnessHandlers(prepared, stats, rs, deps) {
|
|
|
853
908
|
firstTokenMs: rs.turn.callStartAt !== undefined && rs.turn.firstTokenAt !== undefined ? rs.turn.firstTokenAt - rs.turn.callStartAt : undefined,
|
|
854
909
|
...(callIssuedAt !== undefined ? { callStartedAt: callIssuedAt } : {}),
|
|
855
910
|
...(rs.telemetry.pricingConfigured ? { costMicroUsd: turnCostMicroUsd } : {}),
|
|
856
|
-
...(capLast !== undefined ? { callCap: capLast.cap, capThinkingSkipped: capLast.thinkingSkipped } : {}),
|
|
857
911
|
...(m.role === "assistant" && typeof m.stopReason === "string"
|
|
858
912
|
? { stopReason: m.stopReason }
|
|
859
913
|
: {}),
|
|
860
914
|
ts: callEndAt,
|
|
861
915
|
}));
|
|
862
|
-
if (prepared.callCapRef &&
|
|
863
|
-
prepared.callCapRef.reasoningObserved !== true &&
|
|
864
|
-
capLast !== undefined &&
|
|
865
|
-
m.role === "assistant" &&
|
|
866
|
-
m.stopReason === "length") {
|
|
867
|
-
const blocks = Array.isArray(m.content)
|
|
868
|
-
? (m.content)
|
|
869
|
-
: [];
|
|
870
|
-
const productive = blocks.some((c) => c?.type === "toolCall") ||
|
|
871
|
-
blocks.some((c) => c?.type === "text" && typeof c.text === "string" && c.text.trim() !== "");
|
|
872
|
-
if (!productive)
|
|
873
|
-
prepared.callCapRef.reasoningObserved = true;
|
|
874
|
-
}
|
|
875
916
|
if (prepared.cacheBreakDetector && prepared.cacheFingerprint) {
|
|
876
917
|
const finding = prepared.cacheBreakDetector.observe({
|
|
877
918
|
turn: stats.turns + 1,
|
|
@@ -889,20 +930,6 @@ function makeHarnessHandlers(prepared, stats, rs, deps) {
|
|
|
889
930
|
if (m.role === "assistant" && typeof m.stopReason === "string") {
|
|
890
931
|
rs.turn.turnStopReason = m.stopReason;
|
|
891
932
|
}
|
|
892
|
-
if (m.role === "assistant" && isWalltimeCutMessage(m)) {
|
|
893
|
-
rs.counters.callCutoffs += 1;
|
|
894
|
-
if (prepared.callCapRef)
|
|
895
|
-
prepared.callCapRef.finalizeMode = true;
|
|
896
|
-
if (!rs.counters.finalizeInjected) {
|
|
897
|
-
const finalizeText = buildFinalizeText();
|
|
898
|
-
void prepared.harness.steer(finalizeText, { engineMinted: true }).catch(() => { });
|
|
899
|
-
void prepared.harness.setThinkingLevel("off").catch(() => { });
|
|
900
|
-
prepared.turnSnapshot?.refreshRequestPolicy({ thinkingLevel: "off" });
|
|
901
|
-
rs.counters.finalizeInjected = true;
|
|
902
|
-
rs.counters.nudgeIdx = nudgeSchedule.length;
|
|
903
|
-
queue.push({ type: "steering_injected", source: "finalize", preview: finalizeText.slice(0, 220), ...ident() });
|
|
904
|
-
}
|
|
905
|
-
}
|
|
906
933
|
if (m.role === "assistant") {
|
|
907
934
|
const rep = m.repetition;
|
|
908
935
|
if (rep !== undefined) {
|
|
@@ -933,13 +960,6 @@ function makeHarnessHandlers(prepared, stats, rs, deps) {
|
|
|
933
960
|
}
|
|
934
961
|
}
|
|
935
962
|
}
|
|
936
|
-
if (prepared.callCapRef &&
|
|
937
|
-
prepared.callCapRef.reasoningObserved !== true &&
|
|
938
|
-
m.role === "assistant" &&
|
|
939
|
-
Array.isArray(m.content) &&
|
|
940
|
-
m.content.some((c) => c?.type === "thinking")) {
|
|
941
|
-
prepared.callCapRef.reasoningObserved = true;
|
|
942
|
-
}
|
|
943
963
|
if (m.role === "assistant") {
|
|
944
964
|
const content = m.content;
|
|
945
965
|
if (rs.attach.attachState !== undefined)
|
|
@@ -978,11 +998,6 @@ function makeHarnessHandlers(prepared, stats, rs, deps) {
|
|
|
978
998
|
const toolStarted = toolStartAt.get(event.toolCallId);
|
|
979
999
|
toolStartAt.delete(event.toolCallId);
|
|
980
1000
|
const toolNow = Date.now();
|
|
981
|
-
if (prepared.callCapRef && toolStarted !== undefined) {
|
|
982
|
-
recordToolSample(prepared.callCapRef.state, toolNow - toolStarted);
|
|
983
|
-
turnToolSpan.startMin = turnToolSpan.startMin === undefined ? toolStarted : Math.min(turnToolSpan.startMin, toolStarted);
|
|
984
|
-
turnToolSpan.endMax = turnToolSpan.endMax === undefined ? toolNow : Math.max(turnToolSpan.endMax, toolNow);
|
|
985
|
-
}
|
|
986
1001
|
if (!rs.counters.wroteThisRun && (prepared.toolEffects.get(event.toolName) ?? "write") !== "read") {
|
|
987
1002
|
rs.counters.wroteThisRun = true;
|
|
988
1003
|
}
|
|
@@ -1102,21 +1117,17 @@ function makeHarnessHandlers(prepared, stats, rs, deps) {
|
|
|
1102
1117
|
}
|
|
1103
1118
|
else {
|
|
1104
1119
|
if (rs.degrade.degraded === undefined &&
|
|
1105
|
-
spec.degrade &&
|
|
1120
|
+
spec.limits?.degrade &&
|
|
1106
1121
|
rs.degrade.degradeToModel &&
|
|
1107
1122
|
rs.budget.maxCostMicroUsd !== undefined &&
|
|
1108
|
-
stats.costMicroUsd >= spec.degrade.atCostFraction * rs.budget.maxCostMicroUsd) {
|
|
1123
|
+
stats.costMicroUsd >= spec.limits.degrade.atCostFraction * rs.budget.maxCostMicroUsd) {
|
|
1109
1124
|
void prepared.harness.setModel(rs.degrade.degradeToModel);
|
|
1110
1125
|
rs.degrade.recordDegraded({ from: prepared.model.id, to: rs.degrade.degradeToModel.id, reason: "budget", atTurn: stats.turns }, rs.degrade.degradeToModel);
|
|
1111
|
-
if (prepared.callCapRef &&
|
|
1112
|
-
spec.limits?.maxOutputTokens === undefined &&
|
|
1113
|
-
rs.degrade.degradeToModel.maxTokens !== undefined &&
|
|
1114
|
-
(prepared.callCapRef.staticCapTokens === undefined || rs.degrade.degradeToModel.maxTokens < prepared.callCapRef.staticCapTokens)) {
|
|
1115
|
-
prepared.callCapRef.staticCapTokens = rs.degrade.degradeToModel.maxTokens;
|
|
1116
|
-
}
|
|
1117
1126
|
}
|
|
1118
|
-
|
|
1127
|
+
const overAxis = rs.limits.budgetHit ? undefined : rs.budget.overBudget();
|
|
1128
|
+
if (overAxis !== undefined) {
|
|
1119
1129
|
rs.limits.budgetHit = "exceeded";
|
|
1130
|
+
rs.limits.budgetAxis = overAxis;
|
|
1120
1131
|
if (prepared.suspendForResource === undefined) {
|
|
1121
1132
|
void prepared.harness.abort();
|
|
1122
1133
|
prepared.abortController.abort();
|
|
@@ -1559,6 +1570,32 @@ export class Runner {
|
|
|
1559
1570
|
let notificationIdent = () => ({});
|
|
1560
1571
|
let notificationLaneLive = true;
|
|
1561
1572
|
let notificationSessionId;
|
|
1573
|
+
let descendantAnchorDisclosed = false;
|
|
1574
|
+
const parkTaskNotification = (payload, priority) => {
|
|
1575
|
+
if (notificationSessionId !== undefined)
|
|
1576
|
+
this.pendingSessionNotifications.pend(notificationSessionId, payload);
|
|
1577
|
+
if (!isDelegatedAgentTerminal(payload))
|
|
1578
|
+
return;
|
|
1579
|
+
const uplink = internals?.parentNotify;
|
|
1580
|
+
if (uplink !== undefined) {
|
|
1581
|
+
try {
|
|
1582
|
+
uplink(payload, { priority });
|
|
1583
|
+
}
|
|
1584
|
+
catch (e) {
|
|
1585
|
+
this.deps.onError?.(e, { phase: "degraded", sessionId: notificationSessionId ?? "", classification: "descendant-terminal-uplink" });
|
|
1586
|
+
}
|
|
1587
|
+
return;
|
|
1588
|
+
}
|
|
1589
|
+
const rootAnchor = internals?.rootSessionId;
|
|
1590
|
+
if (rootAnchor !== undefined && rootAnchor !== notificationSessionId) {
|
|
1591
|
+
this.pendingSessionNotifications.pend(rootAnchor, payload);
|
|
1592
|
+
return;
|
|
1593
|
+
}
|
|
1594
|
+
if (internals?.parentSessionId !== undefined && !descendantAnchorDisclosed) {
|
|
1595
|
+
descendantAnchorDisclosed = true;
|
|
1596
|
+
this.deps.onError?.(new Error(`background-agent terminal ${payload.task_id} parked on this run's own session — the delegation tree's root session is not reachable from this run (no uplink, no root anchor)`), { phase: "degraded", sessionId: notificationSessionId ?? "", classification: "descendant-terminal-unrouted" });
|
|
1597
|
+
}
|
|
1598
|
+
};
|
|
1562
1599
|
const unsubscribeTaskNotifications = taskNotificationQueue.subscribe((item) => {
|
|
1563
1600
|
queue.push({ type: "task_notification", notification: item.payload, ...notificationIdent() });
|
|
1564
1601
|
if (notificationHarness) {
|
|
@@ -1567,14 +1604,12 @@ export class Runner {
|
|
|
1567
1604
|
? notificationHarness.followUp(xml, { provenance: "engine-note", enginePayload: item.payload })
|
|
1568
1605
|
: notificationHarness.steer(xml, { provenance: "engine-note", enginePayload: item.payload });
|
|
1569
1606
|
void deliver.then(() => item.onDisposition?.("queued"), () => {
|
|
1570
|
-
|
|
1571
|
-
this.pendingSessionNotifications.pend(notificationSessionId, item.payload);
|
|
1607
|
+
parkTaskNotification(item.payload, item.priority);
|
|
1572
1608
|
item.onDisposition?.("parked");
|
|
1573
1609
|
});
|
|
1574
1610
|
}
|
|
1575
1611
|
else {
|
|
1576
|
-
|
|
1577
|
-
this.pendingSessionNotifications.pend(notificationSessionId, item.payload);
|
|
1612
|
+
parkTaskNotification(item.payload, item.priority);
|
|
1578
1613
|
item.onDisposition?.("parked");
|
|
1579
1614
|
}
|
|
1580
1615
|
});
|
|
@@ -1584,8 +1619,7 @@ export class Runner {
|
|
|
1584
1619
|
if (deliveredAtTurnOpen.has(taskNotificationDedupKey(notification)))
|
|
1585
1620
|
return Promise.resolve("dropped_duplicate");
|
|
1586
1621
|
if (!notificationLaneLive) {
|
|
1587
|
-
|
|
1588
|
-
this.pendingSessionNotifications.pend(notificationSessionId, notification);
|
|
1622
|
+
parkTaskNotification(notification, opts?.priority ?? "later");
|
|
1589
1623
|
return Promise.resolve("parked");
|
|
1590
1624
|
}
|
|
1591
1625
|
return new Promise((resolve) => {
|
|
@@ -1662,9 +1696,9 @@ export class Runner {
|
|
|
1662
1696
|
rs.telemetry.cacheFamily = cacheFamilyOf(prepared.model);
|
|
1663
1697
|
rs.telemetry.pricing = this.deps.pricing?.[prepared.model.id] ?? modelCostToPricing(prepared.model.cost);
|
|
1664
1698
|
rs.telemetry.pricingConfigured = this.deps.pricing?.[prepared.model.id] !== undefined || prepared.model.cost !== undefined;
|
|
1665
|
-
if (spec.degrade) {
|
|
1699
|
+
if (spec.limits?.degrade) {
|
|
1666
1700
|
try {
|
|
1667
|
-
rs.degrade.degradeToModel = resolveModel(spec.degrade.to, this.deps.models);
|
|
1701
|
+
rs.degrade.degradeToModel = resolveModel(spec.limits.degrade.to, this.deps.models);
|
|
1668
1702
|
}
|
|
1669
1703
|
catch (e) {
|
|
1670
1704
|
this.deps.onError?.(e, { phase: "degraded", sessionId: prepared.sessionId });
|
|
@@ -1736,7 +1770,7 @@ export class Runner {
|
|
|
1736
1770
|
catch {
|
|
1737
1771
|
}
|
|
1738
1772
|
};
|
|
1739
|
-
const sliceWindowMicroUsd = spec.maxCostUsd !== undefined ? Math.round(spec.maxCostUsd * 1e6) : undefined;
|
|
1773
|
+
const sliceWindowMicroUsd = spec.limits?.maxCostUsd !== undefined ? Math.round(spec.limits.maxCostUsd * 1e6) : undefined;
|
|
1740
1774
|
rs.budget.remainingMicroUsd = prepared.resourceLedger
|
|
1741
1775
|
? remainingBudgetMicroUsd(prepared.resourceLedger)
|
|
1742
1776
|
: spec.resourceSuspend?.totalBudgetUsd !== undefined
|
|
@@ -1746,24 +1780,36 @@ export class Runner {
|
|
|
1746
1780
|
sliceWindowMicroUsd !== undefined && rs.budget.remainingMicroUsd !== undefined
|
|
1747
1781
|
? Math.min(sliceWindowMicroUsd, rs.budget.remainingMicroUsd)
|
|
1748
1782
|
: (sliceWindowMicroUsd ?? rs.budget.remainingMicroUsd);
|
|
1749
|
-
rs.budget.
|
|
1750
|
-
(
|
|
1783
|
+
rs.budget.remainingTokens = prepared.resourceLedger
|
|
1784
|
+
? remainingTokens(prepared.resourceLedger)
|
|
1785
|
+
: spec.resourceSuspend?.totalTokens;
|
|
1786
|
+
rs.budget.maxTokensWindow =
|
|
1787
|
+
spec.limits?.maxTokens !== undefined && rs.budget.remainingTokens !== undefined
|
|
1788
|
+
? Math.min(spec.limits.maxTokens, rs.budget.remainingTokens)
|
|
1789
|
+
: (spec.limits?.maxTokens ?? rs.budget.remainingTokens);
|
|
1790
|
+
rs.budget.overBudget = () => {
|
|
1791
|
+
if (rs.budget.maxTokensWindow !== undefined && stats.tokens > rs.budget.maxTokensWindow)
|
|
1792
|
+
return "tokens";
|
|
1793
|
+
if (rs.budget.maxCostMicroUsd !== undefined && stats.costMicroUsd > rs.budget.maxCostMicroUsd)
|
|
1794
|
+
return "cost";
|
|
1795
|
+
return undefined;
|
|
1796
|
+
};
|
|
1751
1797
|
rs.budget.streamCancel =
|
|
1752
|
-
(spec.budgetStreamCancel ?? rs.budget.maxCostMicroUsd !== undefined) && prepared.suspendForResource === undefined;
|
|
1798
|
+
(spec.limits?.budgetStreamCancel ?? rs.budget.maxCostMicroUsd !== undefined) && prepared.suspendForResource === undefined;
|
|
1753
1799
|
rs.budget.callOutputChars = 0;
|
|
1754
1800
|
rs.budget.lastStreamBudgetCheck = 0;
|
|
1755
1801
|
rs.budget.projectedOverBudget = () => {
|
|
1756
1802
|
const estOut = Math.ceil(rs.budget.callOutputChars / 4);
|
|
1803
|
+
if (rs.budget.maxTokensWindow !== undefined && stats.tokens + estOut > rs.budget.maxTokensWindow)
|
|
1804
|
+
return "tokens";
|
|
1757
1805
|
if (rs.budget.maxCostMicroUsd !== undefined) {
|
|
1758
1806
|
const estOutMicro = computeCostMicroUsd({ totalInputTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0, cacheWriteTokensLong: 0, outputTokens: estOut }, rs.telemetry.pricing);
|
|
1759
1807
|
if (stats.costMicroUsd + estOutMicro > rs.budget.maxCostMicroUsd)
|
|
1760
|
-
return
|
|
1808
|
+
return "cost";
|
|
1761
1809
|
}
|
|
1762
|
-
return
|
|
1810
|
+
return undefined;
|
|
1763
1811
|
};
|
|
1764
1812
|
rs.turn.turnUsageMissing = false;
|
|
1765
|
-
rs.counters.nudgesSent = 0;
|
|
1766
|
-
rs.counters.callCutoffs = 0;
|
|
1767
1813
|
rs.counters.repetitionCuts = 0;
|
|
1768
1814
|
rs.counters.repetitionSpared = 0;
|
|
1769
1815
|
rs.counters.repetitionEvents = [];
|
|
@@ -1959,7 +2005,6 @@ export class Runner {
|
|
|
1959
2005
|
}
|
|
1960
2006
|
const writeFamilyOf = (name) => writeFamilyByName.get(name) ?? writeFamilyOfCanonical(name);
|
|
1961
2007
|
const toolStartAt = new Map();
|
|
1962
|
-
const turnToolSpan = {};
|
|
1963
2008
|
const startedToolCallIds = new Set();
|
|
1964
2009
|
emitTrace(rs.telemetry.tracer, () => ({ kind: "task.start", version: 1, taskId: rs.telemetry.taskId, model: prepared.model.id, engineVersion: engineVersion(), ts: rs.telemetry.taskStart }));
|
|
1965
2010
|
emitTrace(rs.telemetry.tracer, () => ({
|
|
@@ -2014,61 +2059,9 @@ export class Runner {
|
|
|
2014
2059
|
};
|
|
2015
2060
|
});
|
|
2016
2061
|
}
|
|
2017
|
-
const
|
|
2018
|
-
const remainingWalltimeBudgetMs = prepared.resourceLedger?.totalWalltimeSec !== undefined
|
|
2019
|
-
? remainingWalltimeMs(prepared.resourceLedger)
|
|
2020
|
-
: spec.resourceSuspend?.totalWalltimeSec !== undefined
|
|
2021
|
-
? Math.max(0, spec.resourceSuspend.totalWalltimeSec * 1000 - (prepared.resourceLedger?.spentWalltimeMs ?? 0))
|
|
2022
|
-
: undefined;
|
|
2023
|
-
const effectiveTimeoutMs = sliceWindowWalltimeMs !== undefined && remainingWalltimeBudgetMs !== undefined
|
|
2024
|
-
? Math.min(sliceWindowWalltimeMs, remainingWalltimeBudgetMs)
|
|
2025
|
-
: (sliceWindowWalltimeMs ?? remainingWalltimeBudgetMs);
|
|
2026
|
-
const walltimeDeadlineMs = effectiveTimeoutMs !== undefined ? rs.telemetry.taskStart + effectiveTimeoutMs : undefined;
|
|
2062
|
+
const effectiveTimeoutMs = spec.limits?.maxWalltimeMs;
|
|
2027
2063
|
const walltimeMonotonicDeadline = effectiveTimeoutMs !== undefined ? rs.telemetry.taskStartMonotonic + effectiveTimeoutMs : undefined;
|
|
2028
|
-
if (prepared.callCapRef && prepared.suspendForResource === undefined && walltimeDeadlineMs !== undefined) {
|
|
2029
|
-
prepared.callCapRef.deadlineMs = walltimeDeadlineMs;
|
|
2030
|
-
if (spec.limits?.gracefulFinalize !== false) {
|
|
2031
|
-
prepared.callCapRef.cushionMs = writeoutCushionMs(effectiveTimeoutMs);
|
|
2032
|
-
prepared.callCapRef.budgetMs = effectiveTimeoutMs;
|
|
2033
|
-
}
|
|
2034
|
-
}
|
|
2035
|
-
rs.counters.finalizeInjected = false;
|
|
2036
2064
|
rs.counters.walltimeSyncBackstopFired = false;
|
|
2037
|
-
const nudgeSchedule = [];
|
|
2038
|
-
{
|
|
2039
|
-
const conf = spec.limits?.deadlineNudge;
|
|
2040
|
-
const budgetSec = effectiveTimeoutMs !== undefined ? effectiveTimeoutMs / 1000 : undefined;
|
|
2041
|
-
if (conf !== false && budgetSec && budgetSec > 0) {
|
|
2042
|
-
const [a, b] = conf?.at ?? [0.8, 0.95];
|
|
2043
|
-
if (a > 0 && a < 1) {
|
|
2044
|
-
const t1 = rs.telemetry.taskStart + a * budgetSec * 1000;
|
|
2045
|
-
const remainMin = Math.max(1, Math.round((budgetSec * 1000 * (1 - a)) / 60_000));
|
|
2046
|
-
const budgetMin = Math.max(1, Math.round(budgetSec / 60));
|
|
2047
|
-
nudgeSchedule.push({
|
|
2048
|
-
atMs: t1,
|
|
2049
|
-
text: `<system-reminder>[deadline] ~${remainMin} min of the ~${budgetMin}-min wall-clock budget remain. ` +
|
|
2050
|
-
`Start converging: persist finished work (write files / commit), prefer completing in-flight items over opening new ones.</system-reminder>`,
|
|
2051
|
-
});
|
|
2052
|
-
const t2 = rs.telemetry.taskStart + Math.max(Math.min(b * budgetSec * 1000, budgetSec * 1000 - 120_000), a * budgetSec * 1000 + 1);
|
|
2053
|
-
if (b > a && b < 1) {
|
|
2054
|
-
nudgeSchedule.push({
|
|
2055
|
-
atMs: t2,
|
|
2056
|
-
text: `<system-reminder>[deadline] the wall-clock budget is nearly exhausted. STOP new work — ` +
|
|
2057
|
-
`write out deliverables and give your final answer NOW; an unfinished answer beats a hard timeout.</system-reminder>`,
|
|
2058
|
-
});
|
|
2059
|
-
}
|
|
2060
|
-
}
|
|
2061
|
-
}
|
|
2062
|
-
}
|
|
2063
|
-
rs.counters.nudgeIdx = 0;
|
|
2064
|
-
const buildFinalizeText = () => `<system-reminder>[deadline] the wall-clock budget is exhausted — this is the FINAL turn. ` +
|
|
2065
|
-
`Write your best current work to the deliverable path NOW, even if incomplete — persist to disk FIRST; everything else comes after. ` +
|
|
2066
|
-
`If a better version exists anywhere else (a draft, scratch or temp file), copy it to the deliverable path now. ` +
|
|
2067
|
-
`Do NOT open new work, new investigations, or long commands; write out and stop.` +
|
|
2068
|
-
(finalVerificationOn
|
|
2069
|
-
? ` After writing, if seconds permit, confirm the deliverable file actually EXISTS at the required path with your latest content.`
|
|
2070
|
-
: "") +
|
|
2071
|
-
`</system-reminder>`;
|
|
2072
2065
|
const timeout = startTimeout(prepared.harness, prepared.abortController, walltimeMonotonicDeadline !== undefined ? walltimeMonotonicDeadline - performance.now() : undefined, prepared.suspendForResource !== undefined);
|
|
2073
2066
|
const parentToolCallId = internals?.parentToolCallId;
|
|
2074
2067
|
const ident = () => parentToolCallId !== undefined ? { eventId: uuidv7(), parentToolCallId, sourceTaskId: rs.telemetry.taskId } : { eventId: uuidv7() };
|
|
@@ -2124,8 +2117,8 @@ export class Runner {
|
|
|
2124
2117
|
const batchArgs = postToolBatchHook ? new Map() : undefined;
|
|
2125
2118
|
rs.turn.toolBatch = [];
|
|
2126
2119
|
const { onMessageUpdate, onMessageEnd, onToolStart, onToolEnd, onTurnEnd, announceWorkspaceMove } = makeHarnessHandlers(prepared, stats, rs, {
|
|
2127
|
-
spec, queue, internals,
|
|
2128
|
-
pushContent, emitCommitted, startedToolCallIds, toolStartAt,
|
|
2120
|
+
spec, queue, internals, ident, parentToolCallId, subagentName,
|
|
2121
|
+
pushContent, emitCommitted, startedToolCallIds, toolStartAt, writeFamilyOf,
|
|
2129
2122
|
toolLabels, postToolBatchHook, batchArgs,
|
|
2130
2123
|
runnerHooks: { onError: (e, info) => this.deps.onError?.(e, info) },
|
|
2131
2124
|
});
|
|
@@ -2166,7 +2159,6 @@ export class Runner {
|
|
|
2166
2159
|
};
|
|
2167
2160
|
const withinTaskCompaction = (spec.compaction?.enabled ?? true) && (spec.compaction?.withinTask ?? true);
|
|
2168
2161
|
const compactionBreaker = { failures: 0 };
|
|
2169
|
-
const compactionThroughputRef = {};
|
|
2170
2162
|
const windowSafetyOptions = (mainModel) => ({
|
|
2171
2163
|
...(rs.budget.maxCostMicroUsd !== undefined
|
|
2172
2164
|
? {
|
|
@@ -2177,15 +2169,6 @@ export class Runner {
|
|
|
2177
2169
|
},
|
|
2178
2170
|
}
|
|
2179
2171
|
: {}),
|
|
2180
|
-
...(walltimeDeadlineMs !== undefined && prepared.callCapRef !== undefined
|
|
2181
|
-
? {
|
|
2182
|
-
walltime: {
|
|
2183
|
-
deadlineMs: walltimeDeadlineMs,
|
|
2184
|
-
cushionMs: () => effectiveCushionMs(prepared.callCapRef) ?? 0,
|
|
2185
|
-
throughputRef: compactionThroughputRef,
|
|
2186
|
-
},
|
|
2187
|
-
}
|
|
2188
|
-
: {}),
|
|
2189
2172
|
onWindowSafety: (info) => {
|
|
2190
2173
|
if (info.kind === "fallback") {
|
|
2191
2174
|
emitTrace(rs.telemetry.tracer, () => ({
|
|
@@ -2223,7 +2206,6 @@ export class Runner {
|
|
|
2223
2206
|
(rs.counters.finalVerifyInjections === 0 || (rs.counters.finalVerifyInjections === 1 && rs.counters.groundingSignalPreR9 && !rs.counters.groundingSignalPostR9)) &&
|
|
2224
2207
|
rs.counters.wroteThisRun &&
|
|
2225
2208
|
prepared.outputRef.set !== true &&
|
|
2226
|
-
!rs.counters.finalizeInjected &&
|
|
2227
2209
|
!(rs.limits.effectiveMaxTurns !== undefined && rs.limits.effectiveMaxTurns > 0 && stats.turns >= rs.limits.effectiveMaxTurns - 1)) {
|
|
2228
2210
|
rs.counters.finalVerifyInjections += 1;
|
|
2229
2211
|
if (rs.counters.finalVerifyInjections === 2) {
|
|
@@ -2331,18 +2313,6 @@ export class Runner {
|
|
|
2331
2313
|
malformedToolUse: {},
|
|
2332
2314
|
thinkingOnly: {},
|
|
2333
2315
|
degenerateOutput: { detect: (m) => isDegenerateCutMessage(m) },
|
|
2334
|
-
walltimeCutoff: {
|
|
2335
|
-
detect: (m) => isWalltimeCutMessage(m),
|
|
2336
|
-
hasLiveWriteoutWindow: () => {
|
|
2337
|
-
const ref = prepared.callCapRef;
|
|
2338
|
-
if (ref === undefined)
|
|
2339
|
-
return true;
|
|
2340
|
-
const soft = softExecDeadlineMs(ref);
|
|
2341
|
-
if (soft === undefined)
|
|
2342
|
-
return true;
|
|
2343
|
-
return soft - Date.now() >= minLiveWriteoutWindowMs(ref.state);
|
|
2344
|
-
},
|
|
2345
|
-
},
|
|
2346
2316
|
promptTooLong: {
|
|
2347
2317
|
recover: async () => {
|
|
2348
2318
|
if (!(spec.compaction?.enabled ?? true))
|
|
@@ -2416,8 +2386,7 @@ export class Runner {
|
|
|
2416
2386
|
return comp.compacted === true;
|
|
2417
2387
|
}
|
|
2418
2388
|
catch (err) {
|
|
2419
|
-
|
|
2420
|
-
compactionBreaker.failures += 1;
|
|
2389
|
+
compactionBreaker.failures += 1;
|
|
2421
2390
|
const msg = String(err instanceof Error ? err.message : err);
|
|
2422
2391
|
queue.push({ type: "compaction_outcome", outcome: "failed", trigger: "forced", reason: msg.length > 512 ? `${msg.slice(0, 512)}…` : msg, ...ident() });
|
|
2423
2392
|
return false;
|
|
@@ -2432,8 +2401,8 @@ export class Runner {
|
|
|
2432
2401
|
w.resolve(outcome);
|
|
2433
2402
|
};
|
|
2434
2403
|
const onTurnBoundary = makeTurnBoundary(prepared, stats, rs, {
|
|
2435
|
-
spec, queue, manualCompactRef, todoToolMounted, taskToolsMounted,
|
|
2436
|
-
|
|
2404
|
+
spec, queue, manualCompactRef, todoToolMounted, taskToolsMounted,
|
|
2405
|
+
walltimeMonotonicDeadline,
|
|
2437
2406
|
timeout, ident, postToolBatchHook, batchArgs, compactionBrain, withinTaskCompaction,
|
|
2438
2407
|
compactionBreaker, windowSafetyOptions, rapidRefill, drainManualCompact,
|
|
2439
2408
|
runnerHooks: {
|
|
@@ -2501,8 +2470,13 @@ export class Runner {
|
|
|
2501
2470
|
void prepared.harness.abort();
|
|
2502
2471
|
prepared.abortController.abort();
|
|
2503
2472
|
}
|
|
2473
|
+
else if (rs.budget.maxTokensWindow !== undefined && rs.budget.maxTokensWindow <= 0) {
|
|
2474
|
+
rs.limits.budgetHit = "exceeded";
|
|
2475
|
+
rs.limits.budgetAxis = "tokens";
|
|
2476
|
+
}
|
|
2504
2477
|
else if (rs.budget.maxCostMicroUsd !== undefined && rs.budget.maxCostMicroUsd <= 0) {
|
|
2505
2478
|
rs.limits.budgetHit = "exceeded";
|
|
2479
|
+
rs.limits.budgetAxis = "cost";
|
|
2506
2480
|
}
|
|
2507
2481
|
else {
|
|
2508
2482
|
let continuation = resumeContinuation(resume);
|
|
@@ -2601,13 +2575,30 @@ export class Runner {
|
|
|
2601
2575
|
cacheReadTokens: 0,
|
|
2602
2576
|
cacheWriteTokens: 0,
|
|
2603
2577
|
cacheWriteTokensLong: 0,
|
|
2604
|
-
outputTokens: prepared.model.maxTokens ??
|
|
2578
|
+
outputTokens: prepared.model.maxTokens ?? DEFAULT_PRECALL_OUTPUT_TOKENS,
|
|
2605
2579
|
}, rs.telemetry.pricing);
|
|
2606
2580
|
const precallCeilingMicroUsd = prepared.suspendForResource !== undefined ? rs.budget.remainingMicroUsd : rs.budget.maxCostMicroUsd;
|
|
2581
|
+
const precallTokens = Math.ceil(effectiveObjective.length / 4) + (prepared.model.maxTokens ?? DEFAULT_PRECALL_OUTPUT_TOKENS);
|
|
2582
|
+
const precallCeilingTokens = prepared.suspendForResource !== undefined ? rs.budget.remainingTokens : rs.budget.maxTokensWindow;
|
|
2583
|
+
const entryUsageRetryAfterMs = prepared.usageGovernance !== undefined ? await prepared.usageGovernance.check(Date.now()) : undefined;
|
|
2607
2584
|
if (promptBlocked) {
|
|
2608
2585
|
}
|
|
2586
|
+
else if (entryUsageRetryAfterMs !== undefined) {
|
|
2587
|
+
const terminal = platformLimitTerminal("usage_window", entryUsageRetryAfterMs, "entry");
|
|
2588
|
+
rs.limits.platformTerminal = terminal;
|
|
2589
|
+
try {
|
|
2590
|
+
this.deps.onError?.(terminal, { phase: "config", sessionId: prepared.sessionId });
|
|
2591
|
+
}
|
|
2592
|
+
catch {
|
|
2593
|
+
}
|
|
2594
|
+
}
|
|
2595
|
+
else if (precallCeilingTokens !== undefined && precallTokens > precallCeilingTokens) {
|
|
2596
|
+
rs.limits.budgetHit = "precall";
|
|
2597
|
+
rs.limits.budgetAxis = "tokens";
|
|
2598
|
+
}
|
|
2609
2599
|
else if (precallCeilingMicroUsd !== undefined && precallMicroUsd > precallCeilingMicroUsd) {
|
|
2610
2600
|
rs.limits.budgetHit = "precall";
|
|
2601
|
+
rs.limits.budgetAxis = "cost";
|
|
2611
2602
|
}
|
|
2612
2603
|
else if (effectiveTimeoutMs !== undefined && effectiveTimeoutMs <= 0) {
|
|
2613
2604
|
timeout.fired = true;
|
|
@@ -2662,6 +2653,17 @@ export class Runner {
|
|
|
2662
2653
|
if (threw === undefined && prepared.brainCallGuardrailRef.timedOut !== undefined) {
|
|
2663
2654
|
threw = prepared.brainCallGuardrailRef.timedOut;
|
|
2664
2655
|
}
|
|
2656
|
+
if (rs.limits.platformTerminal !== undefined) {
|
|
2657
|
+
threw = rs.limits.platformTerminal;
|
|
2658
|
+
}
|
|
2659
|
+
if (prepared.usageGovernance !== undefined) {
|
|
2660
|
+
try {
|
|
2661
|
+
await prepared.usageGovernance.commit(stats.tokens, Date.now());
|
|
2662
|
+
}
|
|
2663
|
+
catch (flushErr) {
|
|
2664
|
+
this.deps.onError?.(flushErr, { phase: "config", sessionId: prepared.sessionId });
|
|
2665
|
+
}
|
|
2666
|
+
}
|
|
2665
2667
|
const suspended = prepared.suspendRef.token !== undefined && prepared.suspendRef.gate !== undefined;
|
|
2666
2668
|
const reviewPaused = prepared.reviewRef.token !== undefined && prepared.reviewRef.gate !== undefined;
|
|
2667
2669
|
const durablyPaused = suspended || reviewPaused;
|
|
@@ -2760,24 +2762,12 @@ export class Runner {
|
|
|
2760
2762
|
gates: [...prepared.humanReviewRef.gates],
|
|
2761
2763
|
};
|
|
2762
2764
|
}
|
|
2763
|
-
|
|
2764
|
-
const toolClamps = prepared.callCapRef?.toolClamps ?? 0;
|
|
2765
|
-
const toolCuts = prepared.callCapRef?.toolCuts ?? 0;
|
|
2766
|
-
const pendingCutTools = prepared.callCapRef?.pendingCutTools ?? 0;
|
|
2767
|
-
const toolCutKills = prepared.cutKills?.kills ?? 0;
|
|
2768
|
-
if (rs.counters.finalizeInjected || rs.counters.nudgesSent > 0 || capShrinks > 0 || rs.counters.callCutoffs > 0 || toolClamps > 0 || toolCuts > 0 || toolCutKills > 0 || pendingCutTools > 0 || rs.counters.finalVerifyInjections > 0 || rs.attach.attachmentsInjected > 0 || rs.counters.repetitionCuts > 0 || rs.counters.repetitionSpared > 0) {
|
|
2765
|
+
if (rs.counters.finalVerifyInjections > 0 || rs.attach.attachmentsInjected > 0 || rs.counters.repetitionCuts > 0 || rs.counters.repetitionSpared > 0 || rs.counters.approachNoticesSent > 0) {
|
|
2769
2766
|
stats.mechanisms = {
|
|
2770
|
-
...(rs.counters.finalizeInjected ? { finalizeInjected: true } : {}),
|
|
2771
|
-
...(rs.counters.nudgesSent > 0 ? { nudgesSent: rs.counters.nudgesSent } : {}),
|
|
2772
|
-
...(capShrinks > 0 ? { capShrinks } : {}),
|
|
2773
|
-
...(rs.counters.callCutoffs > 0 ? { callCutoffs: rs.counters.callCutoffs } : {}),
|
|
2774
|
-
...(toolClamps > 0 ? { toolClamps } : {}),
|
|
2775
|
-
...(toolCuts > 0 ? { toolCuts } : {}),
|
|
2776
|
-
...(toolCutKills > 0 ? { toolCutKills } : {}),
|
|
2777
|
-
...(pendingCutTools > 0 ? { pendingCutTools } : {}),
|
|
2778
2767
|
...(rs.counters.finalVerifyInjections > 0 ? { finalVerifyInjected: true } : {}),
|
|
2779
2768
|
...(rs.counters.finalVerifyInjections > 0 ? { finalVerifyInjections: rs.counters.finalVerifyInjections } : {}),
|
|
2780
2769
|
...(rs.attach.attachmentsInjected > 0 ? { attachmentsInjected: rs.attach.attachmentsInjected } : {}),
|
|
2770
|
+
...(rs.counters.approachNoticesSent > 0 ? { approachNoticesSent: rs.counters.approachNoticesSent } : {}),
|
|
2781
2771
|
...(rs.counters.repetitionCuts > 0 ? { repetitionCuts: rs.counters.repetitionCuts } : {}),
|
|
2782
2772
|
...(rs.counters.repetitionSpared > 0 ? { repetitionSpared: rs.counters.repetitionSpared } : {}),
|
|
2783
2773
|
...(rs.counters.repetitionEvents.length > 0 ? { repetitionEvents: rs.counters.repetitionEvents } : {}),
|
|
@@ -2793,6 +2783,7 @@ export class Runner {
|
|
|
2793
2783
|
abortedForTurns: rs.limits.turnsExceeded,
|
|
2794
2784
|
abortedLive,
|
|
2795
2785
|
budgetHit: rs.limits.budgetHit,
|
|
2786
|
+
budgetAxis: rs.limits.budgetAxis,
|
|
2796
2787
|
blockedReason: prepared.blockedRef.reason,
|
|
2797
2788
|
conflict: prepared.conflictRef.hit,
|
|
2798
2789
|
outputInvalid: rs.degrade.outputInvalid,
|
|
@@ -2851,7 +2842,6 @@ export class Runner {
|
|
|
2851
2842
|
result.status === "failed" &&
|
|
2852
2843
|
final?.stopReason === "error" &&
|
|
2853
2844
|
!isDegenerateCutMessage(final) &&
|
|
2854
|
-
!isWalltimeCutMessage(final) &&
|
|
2855
2845
|
threw === undefined &&
|
|
2856
2846
|
rs.limits.budgetHit === undefined &&
|
|
2857
2847
|
!rs.degrade.outputInvalid &&
|
|
@@ -2919,7 +2909,7 @@ export class Runner {
|
|
|
2919
2909
|
ts: Date.now(),
|
|
2920
2910
|
}));
|
|
2921
2911
|
setResult(result);
|
|
2922
|
-
if (parentToolCallId !== undefined && (result.status === "completed" || result.status === "failed" || result.status === "
|
|
2912
|
+
if (parentToolCallId !== undefined && (result.status === "completed" || result.status === "failed" || result.status === "blocked")) {
|
|
2923
2913
|
const terminalTick = {
|
|
2924
2914
|
type: "task_progress",
|
|
2925
2915
|
taskId: rs.telemetry.taskId,
|
|
@@ -3240,10 +3230,30 @@ export class Runner {
|
|
|
3240
3230
|
throw new CheckpointError("checkpoint.reopen_revote", `this checkpoint was reopened after an env-restore failure (env_failed) — a re-resume is a system retry of the already-approved action, not a re-vote; the supplied decision must replay the persisted winner (boundCallId "${cp.resolvedOutcome.boundCallId}", decision "${cp.resolvedOutcome.decision}"), refusing a different decision`);
|
|
3241
3231
|
}
|
|
3242
3232
|
}
|
|
3233
|
+
const contentAskCallId = pendingContentAskCallId(cp);
|
|
3234
|
+
if (outcome.answer !== undefined && (contentAskCallId === undefined || outcome.decision !== "allow")) {
|
|
3235
|
+
throw new CheckpointError("checkpoint.invalid_outcome", contentAskCallId === undefined
|
|
3236
|
+
? "resume carried a content-ask `answer` but the checkpoint's pending call is not the reserved question tool — an answer has no consumer on a side-effecting tool's approval; refusing rather than dropping it silently"
|
|
3237
|
+
: "resume carried a content-ask `answer` on a `deny` — a denial injects a refusal, never an answer; refusing rather than dropping it silently", { field: "answer" });
|
|
3238
|
+
}
|
|
3239
|
+
if (contentAskCallId !== undefined && outcome.decision === "allow" && outcome.answer === undefined) {
|
|
3240
|
+
const liveFace = taskConfig.onQuestion ?? this.deps.onQuestion;
|
|
3241
|
+
if (liveFace === undefined || liveFace === QUESTION_AWAITS_RESUME) {
|
|
3242
|
+
throw new CheckpointError("checkpoint.invalid_outcome", "resume approved a content-ask (the reserved question tool) without an `answer`, and this resume has no live answering face — " +
|
|
3243
|
+
'executing the question against nothing would hand the model a fabricated "no human is available" default while consuming the approval; ' +
|
|
3244
|
+
"re-resume with the operator's answer on the outcome (or deny it), the checkpoint stays pending", { field: "answer" });
|
|
3245
|
+
}
|
|
3246
|
+
}
|
|
3243
3247
|
}
|
|
3244
3248
|
if (checkpointVersionOf(cp) > MAX_SUPPORTED_CHECKPOINT_VERSION) {
|
|
3245
3249
|
throw new CheckpointError("checkpoint.unsupported_version", `checkpoint version ${checkpointVersionOf(cp)} is newer than this worker supports (max ${MAX_SUPPORTED_CHECKPOINT_VERSION})`);
|
|
3246
3250
|
}
|
|
3251
|
+
const retiredWalltimeTotal = cp.resourceLedger?.totalWalltimeSec;
|
|
3252
|
+
if (retiredWalltimeTotal !== undefined) {
|
|
3253
|
+
throw new CheckpointError("checkpoint.walltime_axis_retired", "this checkpoint's ledger carries a cross-slice WALL-CLOCK allocation (resourceLedger.totalWalltimeSec), an axis this engine version retired — " +
|
|
3254
|
+
"resuming here would run the leg with that ceiling unenforced. Rejected pre-CAS (the checkpoint stays pending): finish it on a worker of the " +
|
|
3255
|
+
"previous release, or start a fresh task bounded by resourceSuspend.totalTokens / totalBudgetUsd.");
|
|
3256
|
+
}
|
|
3247
3257
|
if (taskConfig.signal?.aborted === true && cp.status === "pending") {
|
|
3248
3258
|
throw new CheckpointError("checkpoint.resume_aborted", "the resume was handed an ALREADY-ABORTED signal — refusing to consume the approval on a leg that cannot run it (the checkpoint stays pending and is resumable with a live signal)");
|
|
3249
3259
|
}
|
|
@@ -3279,12 +3289,16 @@ export class Runner {
|
|
|
3279
3289
|
await internals.afterCheckpointResolve();
|
|
3280
3290
|
consumeFlipDone = true;
|
|
3281
3291
|
}
|
|
3292
|
+
const answerFace = outcome.gate === "policy_ask" && outcome.answer !== undefined
|
|
3293
|
+
? answerFaceForRedeemedCall(outcome.answer, outcome.boundCallId, taskConfig.onQuestion ?? this.deps.onQuestion)
|
|
3294
|
+
: undefined;
|
|
3282
3295
|
const spec = {
|
|
3283
3296
|
...taskConfig,
|
|
3284
3297
|
objective: "",
|
|
3285
3298
|
sessionId: cp.sessionId,
|
|
3286
3299
|
requireExistingSession: true,
|
|
3287
3300
|
preemptSignal: taskConfig.preemptSignal?.aborted ? undefined : taskConfig.preemptSignal,
|
|
3301
|
+
...(answerFace !== undefined ? { onQuestion: answerFace } : {}),
|
|
3288
3302
|
};
|
|
3289
3303
|
const reopenFn = store.reopen?.bind(store);
|
|
3290
3304
|
const onEnvRestoreFailed = reopenFn
|