@sema-agent/core 5.7.0 → 5.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +37 -0
- package/dist/agents/cascade.d.ts +1 -1
- package/dist/agents/subagent.d.ts +4 -7
- package/dist/agents/subagent.js +10 -19
- package/dist/agents/teacher.js +3 -3
- package/dist/agents/team.d.ts +1 -1
- package/dist/agents/team.js +1 -1
- package/dist/agents/verify.js +1 -1
- package/dist/brain/anthropic.js +24 -35
- package/dist/brain/openai.js +22 -33
- package/dist/brain/stream-engine.d.ts +0 -2
- package/dist/brain/stream-engine.js +5 -47
- package/dist/brain/stream-shared.d.ts +0 -10
- package/dist/brain/stream-shared.js +0 -23
- package/dist/brain/terminal-cause.d.ts +0 -1
- package/dist/brain/terminal-cause.js +0 -3
- package/dist/brain/timeout.d.ts +4 -2
- package/dist/brain/timeout.js +4 -14
- package/dist/config/catalog.d.ts +3 -11
- package/dist/config/catalog.js +30 -69
- package/dist/config/defaults.d.ts +0 -3
- package/dist/config/defaults.js +0 -3
- package/dist/core/auto-compaction.d.ts +2 -11
- package/dist/core/auto-compaction.js +3 -67
- package/dist/core/checkpoint-store.d.ts +10 -8
- package/dist/core/checkpoint-store.js +7 -7
- package/dist/core/memory-engine/file-backend.js +20 -17
- package/dist/core/memory-engine/layout.d.ts +5 -0
- package/dist/core/memory-engine/layout.js +72 -47
- package/dist/core/runner/assemble-result.d.ts +2 -8
- package/dist/core/runner/assemble-result.js +15 -16
- package/dist/core/runner/prepare-task.d.ts +26 -6
- package/dist/core/runner/prepare-task.js +203 -141
- package/dist/core/runner/runtask.d.ts +0 -2
- package/dist/core/runner/runtask.js +200 -244
- package/dist/core/session-policy-store.d.ts +5 -0
- package/dist/core/session-policy-store.js +3 -1
- package/dist/core/store-contracts/checkpoint-store-contract.js +2 -2
- package/dist/core/task-registry-agent.d.ts +2 -0
- package/dist/core/task-registry-agent.js +33 -1
- package/dist/core/task-registry.d.ts +2 -0
- package/dist/core/task-registry.js +9 -1
- package/dist/core/tool-errors.js +10 -2
- package/dist/core/trace.d.ts +1 -8
- package/dist/core/types.d.ts +24 -30
- package/dist/core/usage-window-store.d.ts +39 -0
- package/dist/core/usage-window-store.js +115 -0
- package/dist/engine/execution-env/node-execution-env.js +0 -22
- package/dist/engine/harness/agent-harness.d.ts +0 -7
- package/dist/engine/harness/agent-harness.js +13 -53
- package/dist/engine/harness/types.d.ts +2 -15
- package/dist/engine/llm/types.d.ts +1 -2
- package/dist/engine/loop/agent-loop.d.ts +1 -1
- package/dist/engine/loop/agent-loop.js +5 -171
- package/dist/engine/loop/types.d.ts +0 -18
- package/dist/index.d.ts +10 -5
- package/dist/index.js +8 -4
- package/dist/orchestration/goal.js +1 -5
- package/dist/orchestration/run-workflow-tool.d.ts +1 -1
- package/dist/orchestration/run-workflow-tool.js +1 -2
- package/dist/orchestration/workflow-governance.d.ts +6 -6
- package/dist/orchestration/workflow-governance.js +33 -47
- package/dist/prompt-assembly/event-registry.js +2 -3
- package/dist/stores/cc/lockfile.js +9 -11
- package/dist/stores/cc/mailbox-store.js +49 -38
- package/dist/stores/file/index.d.ts +3 -0
- package/dist/stores/file/index.js +4 -0
- package/dist/stores/file/usage-window-store.d.ts +9 -0
- package/dist/stores/file/usage-window-store.js +86 -0
- package/dist/tools/fs/fs-bash.d.ts +1 -10
- package/dist/tools/fs/fs-bash.js +20 -60
- package/dist/tools/fs/index.d.ts +0 -2
- package/dist/tools/fs/index.js +1 -2
- package/package.json +1 -4
- package/dist/bin/sema-tb.d.ts +0 -31
- package/dist/bin/sema-tb.js +0 -448
- package/dist/bin/tb-env.d.ts +0 -2
- package/dist/bin/tb-env.js +0 -17
- package/dist/brain/walltime.d.ts +0 -1
- package/dist/brain/walltime.js +0 -1
- package/dist/core/runner/call-cap.d.ts +0 -67
- package/dist/core/runner/call-cap.js +0 -145
- package/dist/core/runner/cut-kill.d.ts +0 -10
- package/dist/core/runner/cut-kill.js +0 -37
|
@@ -1,15 +1,14 @@
|
|
|
1
1
|
import { AgentHarness, DEFAULT_COMPACTION_SETTINGS, uuidv7 } from "../../internal/harness.js";
|
|
2
|
-
import { CheckpointError, BINDING_CHECKPOINT_VERSION, checkpointVersionOf, MAX_SUPPORTED_CHECKPOINT_VERSION, remainingBudgetMicroUsd,
|
|
3
|
-
import { effectiveCushionMs, estimateCycleMs, hasRealContent, minLiveWriteoutWindowMs, recordCallSample, recordToolCycleSample, recordToolSample, recordTtftSample, softExecDeadlineMs, writeoutCushionMs } from "./call-cap.js";
|
|
2
|
+
import { CheckpointError, BINDING_CHECKPOINT_VERSION, checkpointVersionOf, MAX_SUPPORTED_CHECKPOINT_VERSION, remainingBudgetMicroUsd, remainingTokens, validatePendingSteer, winnerFromOutcome, } from "../checkpoint-store.js";
|
|
4
3
|
import { engineVersion } from "../version.js";
|
|
5
4
|
import { CONFIG_CATALOG_VERSION, declarationReasons, resolveEffectiveConfig } from "../../config/catalog.js";
|
|
6
5
|
import { eventDefaultOn } from "../../prompt-assembly/event-registry.js";
|
|
7
|
-
import { DEFAULT_COMPACTION_INSTRUCTIONS, createRapidRefillState, isCompactionManualCancel,
|
|
6
|
+
import { DEFAULT_COMPACTION_INSTRUCTIONS, createRapidRefillState, isCompactionManualCancel, maybeCompact, nextTrimForceBackoff, recordCompactionAndCheckRapidRefill } from "../auto-compaction.js";
|
|
8
7
|
import { ASK_USER_QUESTION_TOOL_NAME, QUESTION_AWAITS_RESUME } from "../ask-question.js";
|
|
9
8
|
import { computeCostMicroUsd, modelCostToPricing } from "../pricing.js";
|
|
10
9
|
import { emitTrace } from "../trace.js";
|
|
11
10
|
import { emitTaskOutcome } from "../task-outcome.js";
|
|
12
|
-
import { isDegenerateCutMessage
|
|
11
|
+
import { isDegenerateCutMessage } from "../../brain/terminal-cause.js";
|
|
13
12
|
import { primaryActivityArg } from "../arg-summary.js";
|
|
14
13
|
import { resolveReasoning } from "../../brain/reasoning.js";
|
|
15
14
|
import { readDegradation } from "../../brain/degrading.js";
|
|
@@ -41,17 +40,15 @@ import { defaultTaskRegistry } from "../task-registry.js";
|
|
|
41
40
|
import { discloseDroppedPending, isDelegatedAgentTerminal, PendingSessionNotifications, renderTaskNotificationXml, SystemInjectionQueue, taskNotificationDedupKey } from "../task-notification.js";
|
|
42
41
|
import { ToolDetachHub } from "../tool-detach.js";
|
|
43
42
|
import { workflowSizeGuidelineChangeNotice } from "../../orchestration/workflow-size-guideline.js";
|
|
44
|
-
import { DEFAULT_MAX_TURNS } from "../../config/defaults.js";
|
|
45
43
|
import { structuredFrom, toolOutputFrom } from "./tool-output-projection.js";
|
|
46
|
-
export { DEFAULT_MAX_TURNS };
|
|
47
44
|
function createRunState() {
|
|
48
45
|
return {
|
|
49
46
|
telemetry: { cacheFamily: "input-excludes-cached", pricing: { inputPer1M: 0, outputPer1M: 0 }, pricingConfigured: true, unpricedSpend: false, tracer: undefined, taskId: "", taskStart: 0, taskStartMonotonic: 0, cacheBreakReported: false },
|
|
50
47
|
degrade: { degradeToModel: undefined, degraded: undefined, outputErrorStreak: 0, outputInvalid: false, recordDegraded: () => { } },
|
|
51
|
-
limits: { turnsExceeded: false, budgetHit: undefined, outputRetryCap: 0, effectiveMaxTurns: undefined },
|
|
52
|
-
budget: { remainingMicroUsd: undefined, maxCostMicroUsd: undefined, overBudget: () =>
|
|
48
|
+
limits: { turnsExceeded: false, budgetHit: undefined, budgetAxis: undefined, platformTerminal: undefined, outputRetryCap: 0, effectiveMaxTurns: undefined },
|
|
49
|
+
budget: { remainingMicroUsd: undefined, maxCostMicroUsd: undefined, remainingTokens: undefined, maxTokensWindow: undefined, overBudget: () => undefined, streamCancel: false, callOutputChars: 0, lastStreamBudgetCheck: 0, projectedOverBudget: () => undefined },
|
|
53
50
|
turn: { callStartAt: undefined, firstTokenAt: undefined, turnUsage: undefined, turnUsageMissing: false, turnStopReason: undefined, lastTurnHadToolCalls: false, toolBatch: [] },
|
|
54
|
-
counters: {
|
|
51
|
+
counters: { approachNoticesSent: 0, walltimeSyncBackstopFired: false, compactionFloor: 0, trimForceBackoff: false, repetitionCuts: 0, repetitionSpared: 0, repetitionEvents: [], REPETITION_EVENTS_CAP: 0, preemptIgnoredReported: false, wroteThisRun: false, finalVerifyInjections: 0, groundingSignalPreR9: false, groundingSignalPostR9: false, cadenceTurns: 0 },
|
|
55
52
|
attach: { attachmentsCfg: undefined, agentListingOn: false, skillsListingOn: false, attachState: undefined, dateState: undefined, instrProbe: undefined, instrState: undefined, sizeGuidelineState: undefined, attachmentsInjected: 0 },
|
|
56
53
|
};
|
|
57
54
|
}
|
|
@@ -96,14 +93,19 @@ const SUGGESTIONS_TIMEOUT_MS = 30_000;
|
|
|
96
93
|
const SUGGESTIONS_TRANSCRIPT_MESSAGES = 12;
|
|
97
94
|
const SNAPSHOT_TIMEOUT_MS = 30_000;
|
|
98
95
|
const SNAPSHOT_TOO_LARGE_TTL_MS = 30 * 60_000;
|
|
96
|
+
const LIMIT_APPROACH_DEFAULT_THRESHOLDS = [0.8, 0.95];
|
|
97
|
+
const DEFAULT_PRECALL_OUTPUT_TOKENS = 4096;
|
|
98
|
+
function limitApproachFrames(index, axis, percent) {
|
|
99
|
+
return index === 0
|
|
100
|
+
? `<system-reminder>[limit approach] This task has used about ${percent}% of its ${axis}. ` +
|
|
101
|
+
`Start converging: finish the step you are on, drop work that is not required for the deliverable, and ` +
|
|
102
|
+
`avoid starting anything you cannot complete within what is left.</system-reminder>`
|
|
103
|
+
: `<system-reminder>[limit approach] About ${percent}% of the ${axis} is gone. Deliver now: ` +
|
|
104
|
+
`stop exploring, write up what you have — including what is incomplete and what you would do next — ` +
|
|
105
|
+
`and end the task. Work that is not reported before the allowance runs out is lost.</system-reminder>`;
|
|
106
|
+
}
|
|
99
107
|
function resolveMaxTurns(limits) {
|
|
100
|
-
|
|
101
|
-
return limits.maxTurns;
|
|
102
|
-
}
|
|
103
|
-
if (limits?.timeoutSec !== undefined && limits.timeoutSec > 0) {
|
|
104
|
-
return undefined;
|
|
105
|
-
}
|
|
106
|
-
return DEFAULT_MAX_TURNS;
|
|
108
|
+
return limits?.maxTurns;
|
|
107
109
|
}
|
|
108
110
|
function resumeDecisionWasNegative(resume) {
|
|
109
111
|
const o = resume.outcome;
|
|
@@ -228,62 +230,85 @@ function resumeContinuation(resume) {
|
|
|
228
230
|
`conversation above. Do NOT restart the task or re-run any tool you already ran; continue from this ` +
|
|
229
231
|
`exact point, building on the existing results, and finish the remaining work.`);
|
|
230
232
|
}
|
|
233
|
+
function platformLimitTerminal(reason, retryAfterMs, moment = "turn_boundary") {
|
|
234
|
+
const message = moment === "entry"
|
|
235
|
+
? `a deployment usage window is exhausted (RunnerDeps.usageWindows), so the task was refused before its first model call — nothing ran and nothing was spent. There is no checkpoint to suspend into at this point, so the caller re-submits after the window frees, in ${String(retryAfterMs ?? 0)}ms.`
|
|
236
|
+
: reason === "env_lifetime"
|
|
237
|
+
? "the execution environment's declared lifetime is expiring and the run could not be suspended durably (not eligible, or the suspend was refused — see remoteEnvFailures / onError). It was stopped at a clean turn boundary instead of being killed mid-turn when the platform reclaims the environment."
|
|
238
|
+
: `a deployment usage window is exhausted (RunnerDeps.usageWindows) and the run could not be suspended durably, so it was stopped at a clean turn boundary. The window frees in ${String(retryAfterMs ?? 0)}ms.`;
|
|
239
|
+
const e = new Error(message);
|
|
240
|
+
e.code = reason === "env_lifetime" ? "env.lifetime_expired" : "usage.window_exhausted";
|
|
241
|
+
if (retryAfterMs !== undefined)
|
|
242
|
+
e.retryAfterMs = retryAfterMs;
|
|
243
|
+
return e;
|
|
244
|
+
}
|
|
231
245
|
function makeTurnBoundary(prepared, stats, rs, deps) {
|
|
232
|
-
const { spec, queue, manualCompactRef, todoToolMounted, taskToolsMounted,
|
|
246
|
+
const { spec, queue, manualCompactRef, todoToolMounted, taskToolsMounted, walltimeMonotonicDeadline, timeout, ident, postToolBatchHook, batchArgs, compactionBrain, withinTaskCompaction, compactionBreaker, windowSafetyOptions, rapidRefill, drainManualCompact, runnerHooks } = deps;
|
|
233
247
|
let finalVerifySeenAtLastBoundary = 0;
|
|
234
248
|
const onTurnBoundary = async (event) => {
|
|
235
|
-
if (prepared.callCapRef && turnToolSpan.startMin !== undefined && turnToolSpan.endMax !== undefined) {
|
|
236
|
-
recordToolCycleSample(prepared.callCapRef.state, turnToolSpan.endMax - turnToolSpan.startMin);
|
|
237
|
-
}
|
|
238
|
-
turnToolSpan.startMin = undefined;
|
|
239
|
-
turnToolSpan.endMax = undefined;
|
|
240
249
|
let boundarySteered = false;
|
|
241
|
-
if (prepared.suspendForResource === undefined) {
|
|
242
|
-
const nowMs = Date.now();
|
|
243
|
-
if (!rs.counters.finalizeInjected &&
|
|
244
|
-
spec.limits?.gracefulFinalize !== false &&
|
|
245
|
-
walltimeDeadlineMs !== undefined &&
|
|
246
|
-
prepared.callCapRef !== undefined) {
|
|
247
|
-
const remainingMs = walltimeDeadlineMs - nowMs;
|
|
248
|
-
if (remainingMs > 0 && remainingMs < estimateCycleMs(prepared.callCapRef.state) + (effectiveCushionMs(prepared.callCapRef) ?? 0)) {
|
|
249
|
-
rs.counters.finalizeInjected = true;
|
|
250
|
-
prepared.callCapRef.finalizeMode = true;
|
|
251
|
-
void prepared.harness.setThinkingLevel("off").catch(() => { });
|
|
252
|
-
prepared.turnSnapshot?.refreshRequestPolicy({ thinkingLevel: "off" });
|
|
253
|
-
rs.counters.nudgeIdx = nudgeSchedule.length;
|
|
254
|
-
const finalizeText = buildFinalizeText();
|
|
255
|
-
void prepared.harness.steer(finalizeText, { engineMinted: true }).catch(() => { });
|
|
256
|
-
boundarySteered = true;
|
|
257
|
-
queue.push({ type: "steering_injected", source: "finalize", preview: finalizeText.slice(0, 220), ...ident() });
|
|
258
|
-
}
|
|
259
|
-
}
|
|
260
|
-
let due;
|
|
261
|
-
while (rs.counters.nudgeIdx < nudgeSchedule.length && nowMs >= nudgeSchedule[rs.counters.nudgeIdx].atMs) {
|
|
262
|
-
due = nudgeSchedule[rs.counters.nudgeIdx++];
|
|
263
|
-
}
|
|
264
|
-
if (due) {
|
|
265
|
-
void prepared.harness.steer(due.text, { engineMinted: true }).catch(() => { });
|
|
266
|
-
rs.counters.nudgesSent += 1;
|
|
267
|
-
boundarySteered = true;
|
|
268
|
-
queue.push({ type: "steering_injected", source: "deadline_nudge", preview: due.text.slice(0, 220), ...ident() });
|
|
269
|
-
}
|
|
270
|
-
}
|
|
271
250
|
const walltimeHit = walltimeMonotonicDeadline !== undefined && performance.now() >= walltimeMonotonicDeadline;
|
|
272
251
|
const preemptHit = spec.preemptSignal?.aborted === true;
|
|
273
252
|
if (preemptHit && !prepared.suspendForResource && !rs.counters.preemptIgnoredReported) {
|
|
274
253
|
rs.counters.preemptIgnoredReported = true;
|
|
275
254
|
emitTrace(rs.telemetry.tracer, () => ({ kind: "preempt.ignored", version: 1, taskId: rs.telemetry.taskId, ts: Date.now() }));
|
|
276
255
|
}
|
|
256
|
+
let platformCause;
|
|
257
|
+
let platformRetryAfterMs;
|
|
258
|
+
const preemptWillSuspend = preemptHit && prepared.suspendForResource !== undefined;
|
|
259
|
+
if (!preemptWillSuspend && prepared.suspendRef.token === undefined && !prepared.abortController.signal.aborted) {
|
|
260
|
+
if (prepared.envLifetimeSuspendAt !== undefined && Date.now() >= prepared.envLifetimeSuspendAt) {
|
|
261
|
+
platformCause = "env_lifetime";
|
|
262
|
+
}
|
|
263
|
+
else if (prepared.usageGovernance !== undefined) {
|
|
264
|
+
const governance = prepared.usageGovernance;
|
|
265
|
+
try {
|
|
266
|
+
const at = Date.now();
|
|
267
|
+
await governance.commit(stats.tokens, at);
|
|
268
|
+
platformRetryAfterMs = await governance.check(at);
|
|
269
|
+
if (platformRetryAfterMs !== undefined)
|
|
270
|
+
platformCause = "usage_window";
|
|
271
|
+
}
|
|
272
|
+
catch (govErr) {
|
|
273
|
+
rs.limits.platformTerminal = govErr instanceof Error ? govErr : new Error(String(govErr));
|
|
274
|
+
void prepared.harness.abort();
|
|
275
|
+
prepared.abortController.abort();
|
|
276
|
+
return undefined;
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
if (platformCause !== undefined) {
|
|
281
|
+
const platformSpend = { costMicroUsd: stats.costMicroUsd, tokens: stats.tokens, turns: stats.turns, walltimeMs: Math.round(performance.now() - rs.telemetry.taskStartMonotonic) };
|
|
282
|
+
const committed = prepared.suspendForPlatformLimit !== undefined &&
|
|
283
|
+
(await prepared.suspendForPlatformLimit(platformCause, platformSpend, platformRetryAfterMs !== undefined ? { resumeAfterMs: platformRetryAfterMs } : undefined));
|
|
284
|
+
if (committed) {
|
|
285
|
+
rs.limits.turnsExceeded = false;
|
|
286
|
+
rs.limits.budgetHit = undefined;
|
|
287
|
+
return undefined;
|
|
288
|
+
}
|
|
289
|
+
const terminal = platformLimitTerminal(platformCause, platformRetryAfterMs);
|
|
290
|
+
rs.limits.platformTerminal = terminal;
|
|
291
|
+
try {
|
|
292
|
+
runnerHooks.onError?.(terminal, { phase: "config", sessionId: prepared.sessionId });
|
|
293
|
+
}
|
|
294
|
+
catch {
|
|
295
|
+
}
|
|
296
|
+
void prepared.harness.abort();
|
|
297
|
+
prepared.abortController.abort();
|
|
298
|
+
return undefined;
|
|
299
|
+
}
|
|
277
300
|
if (prepared.suspendForResource &&
|
|
278
301
|
(rs.limits.turnsExceeded || rs.limits.budgetHit !== undefined || walltimeHit || preemptHit) &&
|
|
279
302
|
prepared.suspendRef.token === undefined) {
|
|
280
303
|
const reason = preemptHit
|
|
281
304
|
? "preempt"
|
|
282
305
|
: rs.limits.budgetHit !== undefined
|
|
283
|
-
? "
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
306
|
+
? rs.limits.budgetAxis === "tokens"
|
|
307
|
+
? "budget_tokens"
|
|
308
|
+
: "budget_cost"
|
|
309
|
+
: rs.limits.turnsExceeded
|
|
310
|
+
? "turns"
|
|
311
|
+
: "walltime";
|
|
287
312
|
const sliceSpend = { costMicroUsd: stats.costMicroUsd, tokens: stats.tokens, turns: stats.turns, walltimeMs: Math.round(performance.now() - rs.telemetry.taskStartMonotonic) };
|
|
288
313
|
if (await prepared.suspendForResource(reason, sliceSpend)) {
|
|
289
314
|
rs.limits.turnsExceeded = false;
|
|
@@ -308,6 +333,39 @@ function makeTurnBoundary(prepared, stats, rs, deps) {
|
|
|
308
333
|
}
|
|
309
334
|
return undefined;
|
|
310
335
|
}
|
|
336
|
+
const approachCfg = spec.limits?.approachNotice;
|
|
337
|
+
if (approachCfg !== false &&
|
|
338
|
+
rs.counters.approachNoticesSent < LIMIT_APPROACH_DEFAULT_THRESHOLDS.length &&
|
|
339
|
+
prepared.suspendForResource === undefined &&
|
|
340
|
+
rs.turn.lastTurnHadToolCalls &&
|
|
341
|
+
!boundarySteered &&
|
|
342
|
+
!prepared.abortController.signal.aborted) {
|
|
343
|
+
const thresholds = approachCfg?.at ?? LIMIT_APPROACH_DEFAULT_THRESHOLDS;
|
|
344
|
+
const ratios = [];
|
|
345
|
+
if (rs.budget.maxTokensWindow !== undefined && rs.budget.maxTokensWindow > 0) {
|
|
346
|
+
ratios.push({ axis: "token budget", ratio: stats.tokens / rs.budget.maxTokensWindow });
|
|
347
|
+
}
|
|
348
|
+
if (rs.budget.maxCostMicroUsd !== undefined && rs.budget.maxCostMicroUsd > 0) {
|
|
349
|
+
ratios.push({ axis: "cost budget", ratio: stats.costMicroUsd / rs.budget.maxCostMicroUsd });
|
|
350
|
+
}
|
|
351
|
+
const declaredMaxTurns = spec.limits?.maxTurns;
|
|
352
|
+
if (declaredMaxTurns !== undefined && declaredMaxTurns > 0) {
|
|
353
|
+
ratios.push({ axis: "turn budget", ratio: (stats.turns + 1) / declaredMaxTurns });
|
|
354
|
+
}
|
|
355
|
+
let tightest;
|
|
356
|
+
for (const r of ratios)
|
|
357
|
+
if (tightest === undefined || r.ratio > tightest.ratio)
|
|
358
|
+
tightest = r;
|
|
359
|
+
const index = rs.counters.approachNoticesSent;
|
|
360
|
+
if (tightest !== undefined && tightest.ratio >= thresholds[index]) {
|
|
361
|
+
rs.counters.approachNoticesSent += 1;
|
|
362
|
+
const percent = Math.min(100, Math.round(tightest.ratio * 100));
|
|
363
|
+
const body = limitApproachFrames(index === 0 ? 0 : 1, tightest.axis, percent);
|
|
364
|
+
void prepared.harness.steer(body, { engineMinted: true }).catch(() => undefined);
|
|
365
|
+
queue.push({ type: "steering_injected", source: "limit_approach", preview: body.slice(0, 220), ...ident() });
|
|
366
|
+
boundarySteered = true;
|
|
367
|
+
}
|
|
368
|
+
}
|
|
311
369
|
if (prepared.reviewRequestRef.pending !== undefined) {
|
|
312
370
|
const { reason } = prepared.reviewRequestRef.pending;
|
|
313
371
|
prepared.reviewRequestRef.pending = undefined;
|
|
@@ -337,7 +395,6 @@ function makeTurnBoundary(prepared, stats, rs, deps) {
|
|
|
337
395
|
rs.attach.sizeGuidelineState !== undefined ||
|
|
338
396
|
rs.attach.attachState !== undefined) &&
|
|
339
397
|
!boundarySteered &&
|
|
340
|
-
!rs.counters.finalizeInjected &&
|
|
341
398
|
rs.counters.finalVerifyInjections === 0 &&
|
|
342
399
|
!prepared.abortController.signal.aborted) {
|
|
343
400
|
const due = [];
|
|
@@ -525,7 +582,7 @@ function makeTurnBoundary(prepared, stats, rs, deps) {
|
|
|
525
582
|
const batch = rs.turn.toolBatch;
|
|
526
583
|
rs.turn.toolBatch = [];
|
|
527
584
|
batchArgs?.clear();
|
|
528
|
-
const injectedThisTurn = finalVerifyInjectedThisTurn ? "final_verification" :
|
|
585
|
+
const injectedThisTurn = finalVerifyInjectedThisTurn ? "final_verification" : undefined;
|
|
529
586
|
if (!boundarySteered && !prepared.abortController.signal.aborted) {
|
|
530
587
|
try {
|
|
531
588
|
const r = await postToolBatchHook(batch, injectedThisTurn !== undefined ? { injectedThisTurn } : undefined);
|
|
@@ -555,7 +612,6 @@ function makeTurnBoundary(prepared, stats, rs, deps) {
|
|
|
555
612
|
}
|
|
556
613
|
const forceManual = manualCompactRef.requested;
|
|
557
614
|
if ((!withinTaskCompaction && !forceManual) ||
|
|
558
|
-
(rs.counters.finalizeInjected && !forceManual) ||
|
|
559
615
|
prepared.abortController.signal.aborted ||
|
|
560
616
|
prepared.suspendRef.token !== undefined ||
|
|
561
617
|
prepared.reviewRef.token !== undefined) {
|
|
@@ -617,9 +673,6 @@ function makeTurnBoundary(prepared, stats, rs, deps) {
|
|
|
617
673
|
const trimPressure = trimPressureArmed && !rs.counters.trimForceBackoff;
|
|
618
674
|
const passTrigger = trimPressure ? "forced" : forceManual ? "manual" : "auto";
|
|
619
675
|
let manualOutcome = "failed";
|
|
620
|
-
if (rs.counters.finalizeInjected && forceManual) {
|
|
621
|
-
emitTrace(rs.telemetry.tracer, () => ({ kind: "compaction.manual_in_finalize", version: 1, taskId: rs.telemetry.taskId, ts: Date.now() }));
|
|
622
|
-
}
|
|
623
676
|
const manualInstructions = claimedManual?.instructions;
|
|
624
677
|
try {
|
|
625
678
|
const comp = await maybeCompact({
|
|
@@ -744,16 +797,10 @@ function makeTurnBoundary(prepared, stats, rs, deps) {
|
|
|
744
797
|
manualOutcome = "mooted";
|
|
745
798
|
}
|
|
746
799
|
else if (!prepared.abortController.signal.aborted) {
|
|
747
|
-
|
|
748
|
-
if (!walltimeAbort)
|
|
749
|
-
compactionBreaker.failures += 1;
|
|
800
|
+
compactionBreaker.failures += 1;
|
|
750
801
|
runnerHooks.onError?.(err, { phase: "compaction", sessionId: prepared.sessionId });
|
|
751
802
|
const msg = String(err instanceof Error ? err.message : err);
|
|
752
|
-
const failReason =
|
|
753
|
-
? "walltime soft-deadline abort (engine-initiated write-out protection; not counted toward the breaker)"
|
|
754
|
-
: msg.length > 512
|
|
755
|
-
? `${msg.slice(0, 512)}…`
|
|
756
|
-
: msg;
|
|
803
|
+
const failReason = msg.length > 512 ? `${msg.slice(0, 512)}…` : msg;
|
|
757
804
|
emitTrace(rs.telemetry.tracer, () => ({
|
|
758
805
|
kind: "compaction.failed",
|
|
759
806
|
version: 1,
|
|
@@ -776,7 +823,7 @@ function makeTurnBoundary(prepared, stats, rs, deps) {
|
|
|
776
823
|
return onTurnBoundary;
|
|
777
824
|
}
|
|
778
825
|
function makeHarnessHandlers(prepared, stats, rs, deps) {
|
|
779
|
-
const { spec, queue, internals,
|
|
826
|
+
const { spec, queue, internals, ident, parentToolCallId, subagentName, pushContent, emitCommitted, startedToolCallIds, toolStartAt, writeFamilyOf, toolLabels, postToolBatchHook, batchArgs } = deps;
|
|
780
827
|
let lastWorkspaceCwd = prepared.cwdRef?.current;
|
|
781
828
|
const announceWorkspaceMove = () => {
|
|
782
829
|
const cwdNow = prepared.cwdRef?.current;
|
|
@@ -801,8 +848,10 @@ function makeHarnessHandlers(prepared, stats, rs, deps) {
|
|
|
801
848
|
rs.budget.callOutputChars += ev.delta.length;
|
|
802
849
|
if (rs.budget.callOutputChars - rs.budget.lastStreamBudgetCheck >= 256) {
|
|
803
850
|
rs.budget.lastStreamBudgetCheck = rs.budget.callOutputChars;
|
|
804
|
-
|
|
851
|
+
const projectedAxis = rs.budget.projectedOverBudget();
|
|
852
|
+
if (projectedAxis !== undefined) {
|
|
805
853
|
rs.limits.budgetHit = "exceeded";
|
|
854
|
+
rs.limits.budgetAxis = projectedAxis;
|
|
806
855
|
void prepared.harness.abort();
|
|
807
856
|
prepared.abortController.abort();
|
|
808
857
|
}
|
|
@@ -841,19 +890,8 @@ function makeHarnessHandlers(prepared, stats, rs, deps) {
|
|
|
841
890
|
rs.turn.turnUsage.cacheWriteTokens += u.cacheWrite || 0;
|
|
842
891
|
rs.turn.turnUsage.costMicroUsd += turnCostMicroUsd;
|
|
843
892
|
const callEndAt = Date.now();
|
|
844
|
-
const callIssuedAt = prepared.callIssuedAtRef
|
|
845
|
-
|
|
846
|
-
prepared.callIssuedAtRef.current = undefined;
|
|
847
|
-
const capLast = prepared.callCapRef?.state.last;
|
|
848
|
-
if (prepared.callCapRef) {
|
|
849
|
-
recordCallSample(prepared.callCapRef.state, u.output || 0, rs.turn.callStartAt !== undefined ? callEndAt - rs.turn.callStartAt : 0, {
|
|
850
|
-
contentValid: m.usageMissing === true && hasRealContent(m.content),
|
|
851
|
-
});
|
|
852
|
-
if (rs.turn.callStartAt !== undefined && rs.turn.firstTokenAt !== undefined) {
|
|
853
|
-
recordTtftSample(prepared.callCapRef.state, rs.turn.firstTokenAt - rs.turn.callStartAt);
|
|
854
|
-
}
|
|
855
|
-
prepared.callCapRef.state.last = undefined;
|
|
856
|
-
}
|
|
893
|
+
const callIssuedAt = prepared.callIssuedAtRef.current;
|
|
894
|
+
prepared.callIssuedAtRef.current = undefined;
|
|
857
895
|
emitTrace(rs.telemetry.tracer, () => ({
|
|
858
896
|
kind: "brain.call",
|
|
859
897
|
version: 1,
|
|
@@ -870,25 +908,11 @@ function makeHarnessHandlers(prepared, stats, rs, deps) {
|
|
|
870
908
|
firstTokenMs: rs.turn.callStartAt !== undefined && rs.turn.firstTokenAt !== undefined ? rs.turn.firstTokenAt - rs.turn.callStartAt : undefined,
|
|
871
909
|
...(callIssuedAt !== undefined ? { callStartedAt: callIssuedAt } : {}),
|
|
872
910
|
...(rs.telemetry.pricingConfigured ? { costMicroUsd: turnCostMicroUsd } : {}),
|
|
873
|
-
...(capLast !== undefined ? { callCap: capLast.cap, capThinkingSkipped: capLast.thinkingSkipped } : {}),
|
|
874
911
|
...(m.role === "assistant" && typeof m.stopReason === "string"
|
|
875
912
|
? { stopReason: m.stopReason }
|
|
876
913
|
: {}),
|
|
877
914
|
ts: callEndAt,
|
|
878
915
|
}));
|
|
879
|
-
if (prepared.callCapRef &&
|
|
880
|
-
prepared.callCapRef.reasoningObserved !== true &&
|
|
881
|
-
capLast !== undefined &&
|
|
882
|
-
m.role === "assistant" &&
|
|
883
|
-
m.stopReason === "length") {
|
|
884
|
-
const blocks = Array.isArray(m.content)
|
|
885
|
-
? (m.content)
|
|
886
|
-
: [];
|
|
887
|
-
const productive = blocks.some((c) => c?.type === "toolCall") ||
|
|
888
|
-
blocks.some((c) => c?.type === "text" && typeof c.text === "string" && c.text.trim() !== "");
|
|
889
|
-
if (!productive)
|
|
890
|
-
prepared.callCapRef.reasoningObserved = true;
|
|
891
|
-
}
|
|
892
916
|
if (prepared.cacheBreakDetector && prepared.cacheFingerprint) {
|
|
893
917
|
const finding = prepared.cacheBreakDetector.observe({
|
|
894
918
|
turn: stats.turns + 1,
|
|
@@ -906,20 +930,6 @@ function makeHarnessHandlers(prepared, stats, rs, deps) {
|
|
|
906
930
|
if (m.role === "assistant" && typeof m.stopReason === "string") {
|
|
907
931
|
rs.turn.turnStopReason = m.stopReason;
|
|
908
932
|
}
|
|
909
|
-
if (m.role === "assistant" && isWalltimeCutMessage(m)) {
|
|
910
|
-
rs.counters.callCutoffs += 1;
|
|
911
|
-
if (prepared.callCapRef)
|
|
912
|
-
prepared.callCapRef.finalizeMode = true;
|
|
913
|
-
if (!rs.counters.finalizeInjected) {
|
|
914
|
-
const finalizeText = buildFinalizeText();
|
|
915
|
-
void prepared.harness.steer(finalizeText, { engineMinted: true }).catch(() => { });
|
|
916
|
-
void prepared.harness.setThinkingLevel("off").catch(() => { });
|
|
917
|
-
prepared.turnSnapshot?.refreshRequestPolicy({ thinkingLevel: "off" });
|
|
918
|
-
rs.counters.finalizeInjected = true;
|
|
919
|
-
rs.counters.nudgeIdx = nudgeSchedule.length;
|
|
920
|
-
queue.push({ type: "steering_injected", source: "finalize", preview: finalizeText.slice(0, 220), ...ident() });
|
|
921
|
-
}
|
|
922
|
-
}
|
|
923
933
|
if (m.role === "assistant") {
|
|
924
934
|
const rep = m.repetition;
|
|
925
935
|
if (rep !== undefined) {
|
|
@@ -950,13 +960,6 @@ function makeHarnessHandlers(prepared, stats, rs, deps) {
|
|
|
950
960
|
}
|
|
951
961
|
}
|
|
952
962
|
}
|
|
953
|
-
if (prepared.callCapRef &&
|
|
954
|
-
prepared.callCapRef.reasoningObserved !== true &&
|
|
955
|
-
m.role === "assistant" &&
|
|
956
|
-
Array.isArray(m.content) &&
|
|
957
|
-
m.content.some((c) => c?.type === "thinking")) {
|
|
958
|
-
prepared.callCapRef.reasoningObserved = true;
|
|
959
|
-
}
|
|
960
963
|
if (m.role === "assistant") {
|
|
961
964
|
const content = m.content;
|
|
962
965
|
if (rs.attach.attachState !== undefined)
|
|
@@ -995,11 +998,6 @@ function makeHarnessHandlers(prepared, stats, rs, deps) {
|
|
|
995
998
|
const toolStarted = toolStartAt.get(event.toolCallId);
|
|
996
999
|
toolStartAt.delete(event.toolCallId);
|
|
997
1000
|
const toolNow = Date.now();
|
|
998
|
-
if (prepared.callCapRef && toolStarted !== undefined) {
|
|
999
|
-
recordToolSample(prepared.callCapRef.state, toolNow - toolStarted);
|
|
1000
|
-
turnToolSpan.startMin = turnToolSpan.startMin === undefined ? toolStarted : Math.min(turnToolSpan.startMin, toolStarted);
|
|
1001
|
-
turnToolSpan.endMax = turnToolSpan.endMax === undefined ? toolNow : Math.max(turnToolSpan.endMax, toolNow);
|
|
1002
|
-
}
|
|
1003
1001
|
if (!rs.counters.wroteThisRun && (prepared.toolEffects.get(event.toolName) ?? "write") !== "read") {
|
|
1004
1002
|
rs.counters.wroteThisRun = true;
|
|
1005
1003
|
}
|
|
@@ -1119,21 +1117,17 @@ function makeHarnessHandlers(prepared, stats, rs, deps) {
|
|
|
1119
1117
|
}
|
|
1120
1118
|
else {
|
|
1121
1119
|
if (rs.degrade.degraded === undefined &&
|
|
1122
|
-
spec.degrade &&
|
|
1120
|
+
spec.limits?.degrade &&
|
|
1123
1121
|
rs.degrade.degradeToModel &&
|
|
1124
1122
|
rs.budget.maxCostMicroUsd !== undefined &&
|
|
1125
|
-
stats.costMicroUsd >= spec.degrade.atCostFraction * rs.budget.maxCostMicroUsd) {
|
|
1123
|
+
stats.costMicroUsd >= spec.limits.degrade.atCostFraction * rs.budget.maxCostMicroUsd) {
|
|
1126
1124
|
void prepared.harness.setModel(rs.degrade.degradeToModel);
|
|
1127
1125
|
rs.degrade.recordDegraded({ from: prepared.model.id, to: rs.degrade.degradeToModel.id, reason: "budget", atTurn: stats.turns }, rs.degrade.degradeToModel);
|
|
1128
|
-
if (prepared.callCapRef &&
|
|
1129
|
-
spec.limits?.maxOutputTokens === undefined &&
|
|
1130
|
-
rs.degrade.degradeToModel.maxTokens !== undefined &&
|
|
1131
|
-
(prepared.callCapRef.staticCapTokens === undefined || rs.degrade.degradeToModel.maxTokens < prepared.callCapRef.staticCapTokens)) {
|
|
1132
|
-
prepared.callCapRef.staticCapTokens = rs.degrade.degradeToModel.maxTokens;
|
|
1133
|
-
}
|
|
1134
1126
|
}
|
|
1135
|
-
|
|
1127
|
+
const overAxis = rs.limits.budgetHit ? undefined : rs.budget.overBudget();
|
|
1128
|
+
if (overAxis !== undefined) {
|
|
1136
1129
|
rs.limits.budgetHit = "exceeded";
|
|
1130
|
+
rs.limits.budgetAxis = overAxis;
|
|
1137
1131
|
if (prepared.suspendForResource === undefined) {
|
|
1138
1132
|
void prepared.harness.abort();
|
|
1139
1133
|
prepared.abortController.abort();
|
|
@@ -1702,9 +1696,9 @@ export class Runner {
|
|
|
1702
1696
|
rs.telemetry.cacheFamily = cacheFamilyOf(prepared.model);
|
|
1703
1697
|
rs.telemetry.pricing = this.deps.pricing?.[prepared.model.id] ?? modelCostToPricing(prepared.model.cost);
|
|
1704
1698
|
rs.telemetry.pricingConfigured = this.deps.pricing?.[prepared.model.id] !== undefined || prepared.model.cost !== undefined;
|
|
1705
|
-
if (spec.degrade) {
|
|
1699
|
+
if (spec.limits?.degrade) {
|
|
1706
1700
|
try {
|
|
1707
|
-
rs.degrade.degradeToModel = resolveModel(spec.degrade.to, this.deps.models);
|
|
1701
|
+
rs.degrade.degradeToModel = resolveModel(spec.limits.degrade.to, this.deps.models);
|
|
1708
1702
|
}
|
|
1709
1703
|
catch (e) {
|
|
1710
1704
|
this.deps.onError?.(e, { phase: "degraded", sessionId: prepared.sessionId });
|
|
@@ -1776,7 +1770,7 @@ export class Runner {
|
|
|
1776
1770
|
catch {
|
|
1777
1771
|
}
|
|
1778
1772
|
};
|
|
1779
|
-
const sliceWindowMicroUsd = spec.maxCostUsd !== undefined ? Math.round(spec.maxCostUsd * 1e6) : undefined;
|
|
1773
|
+
const sliceWindowMicroUsd = spec.limits?.maxCostUsd !== undefined ? Math.round(spec.limits.maxCostUsd * 1e6) : undefined;
|
|
1780
1774
|
rs.budget.remainingMicroUsd = prepared.resourceLedger
|
|
1781
1775
|
? remainingBudgetMicroUsd(prepared.resourceLedger)
|
|
1782
1776
|
: spec.resourceSuspend?.totalBudgetUsd !== undefined
|
|
@@ -1786,24 +1780,36 @@ export class Runner {
|
|
|
1786
1780
|
sliceWindowMicroUsd !== undefined && rs.budget.remainingMicroUsd !== undefined
|
|
1787
1781
|
? Math.min(sliceWindowMicroUsd, rs.budget.remainingMicroUsd)
|
|
1788
1782
|
: (sliceWindowMicroUsd ?? rs.budget.remainingMicroUsd);
|
|
1789
|
-
rs.budget.
|
|
1790
|
-
(
|
|
1783
|
+
rs.budget.remainingTokens = prepared.resourceLedger
|
|
1784
|
+
? remainingTokens(prepared.resourceLedger)
|
|
1785
|
+
: spec.resourceSuspend?.totalTokens;
|
|
1786
|
+
rs.budget.maxTokensWindow =
|
|
1787
|
+
spec.limits?.maxTokens !== undefined && rs.budget.remainingTokens !== undefined
|
|
1788
|
+
? Math.min(spec.limits.maxTokens, rs.budget.remainingTokens)
|
|
1789
|
+
: (spec.limits?.maxTokens ?? rs.budget.remainingTokens);
|
|
1790
|
+
rs.budget.overBudget = () => {
|
|
1791
|
+
if (rs.budget.maxTokensWindow !== undefined && stats.tokens > rs.budget.maxTokensWindow)
|
|
1792
|
+
return "tokens";
|
|
1793
|
+
if (rs.budget.maxCostMicroUsd !== undefined && stats.costMicroUsd > rs.budget.maxCostMicroUsd)
|
|
1794
|
+
return "cost";
|
|
1795
|
+
return undefined;
|
|
1796
|
+
};
|
|
1791
1797
|
rs.budget.streamCancel =
|
|
1792
|
-
(spec.budgetStreamCancel ?? rs.budget.maxCostMicroUsd !== undefined) && prepared.suspendForResource === undefined;
|
|
1798
|
+
(spec.limits?.budgetStreamCancel ?? rs.budget.maxCostMicroUsd !== undefined) && prepared.suspendForResource === undefined;
|
|
1793
1799
|
rs.budget.callOutputChars = 0;
|
|
1794
1800
|
rs.budget.lastStreamBudgetCheck = 0;
|
|
1795
1801
|
rs.budget.projectedOverBudget = () => {
|
|
1796
1802
|
const estOut = Math.ceil(rs.budget.callOutputChars / 4);
|
|
1803
|
+
if (rs.budget.maxTokensWindow !== undefined && stats.tokens + estOut > rs.budget.maxTokensWindow)
|
|
1804
|
+
return "tokens";
|
|
1797
1805
|
if (rs.budget.maxCostMicroUsd !== undefined) {
|
|
1798
1806
|
const estOutMicro = computeCostMicroUsd({ totalInputTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0, cacheWriteTokensLong: 0, outputTokens: estOut }, rs.telemetry.pricing);
|
|
1799
1807
|
if (stats.costMicroUsd + estOutMicro > rs.budget.maxCostMicroUsd)
|
|
1800
|
-
return
|
|
1808
|
+
return "cost";
|
|
1801
1809
|
}
|
|
1802
|
-
return
|
|
1810
|
+
return undefined;
|
|
1803
1811
|
};
|
|
1804
1812
|
rs.turn.turnUsageMissing = false;
|
|
1805
|
-
rs.counters.nudgesSent = 0;
|
|
1806
|
-
rs.counters.callCutoffs = 0;
|
|
1807
1813
|
rs.counters.repetitionCuts = 0;
|
|
1808
1814
|
rs.counters.repetitionSpared = 0;
|
|
1809
1815
|
rs.counters.repetitionEvents = [];
|
|
@@ -1999,7 +2005,6 @@ export class Runner {
|
|
|
1999
2005
|
}
|
|
2000
2006
|
const writeFamilyOf = (name) => writeFamilyByName.get(name) ?? writeFamilyOfCanonical(name);
|
|
2001
2007
|
const toolStartAt = new Map();
|
|
2002
|
-
const turnToolSpan = {};
|
|
2003
2008
|
const startedToolCallIds = new Set();
|
|
2004
2009
|
emitTrace(rs.telemetry.tracer, () => ({ kind: "task.start", version: 1, taskId: rs.telemetry.taskId, model: prepared.model.id, engineVersion: engineVersion(), ts: rs.telemetry.taskStart }));
|
|
2005
2010
|
emitTrace(rs.telemetry.tracer, () => ({
|
|
@@ -2054,61 +2059,9 @@ export class Runner {
|
|
|
2054
2059
|
};
|
|
2055
2060
|
});
|
|
2056
2061
|
}
|
|
2057
|
-
const
|
|
2058
|
-
const remainingWalltimeBudgetMs = prepared.resourceLedger?.totalWalltimeSec !== undefined
|
|
2059
|
-
? remainingWalltimeMs(prepared.resourceLedger)
|
|
2060
|
-
: spec.resourceSuspend?.totalWalltimeSec !== undefined
|
|
2061
|
-
? Math.max(0, spec.resourceSuspend.totalWalltimeSec * 1000 - (prepared.resourceLedger?.spentWalltimeMs ?? 0))
|
|
2062
|
-
: undefined;
|
|
2063
|
-
const effectiveTimeoutMs = sliceWindowWalltimeMs !== undefined && remainingWalltimeBudgetMs !== undefined
|
|
2064
|
-
? Math.min(sliceWindowWalltimeMs, remainingWalltimeBudgetMs)
|
|
2065
|
-
: (sliceWindowWalltimeMs ?? remainingWalltimeBudgetMs);
|
|
2066
|
-
const walltimeDeadlineMs = effectiveTimeoutMs !== undefined ? rs.telemetry.taskStart + effectiveTimeoutMs : undefined;
|
|
2062
|
+
const effectiveTimeoutMs = spec.limits?.maxWalltimeMs;
|
|
2067
2063
|
const walltimeMonotonicDeadline = effectiveTimeoutMs !== undefined ? rs.telemetry.taskStartMonotonic + effectiveTimeoutMs : undefined;
|
|
2068
|
-
if (prepared.callCapRef && prepared.suspendForResource === undefined && walltimeDeadlineMs !== undefined) {
|
|
2069
|
-
prepared.callCapRef.deadlineMs = walltimeDeadlineMs;
|
|
2070
|
-
if (spec.limits?.gracefulFinalize !== false) {
|
|
2071
|
-
prepared.callCapRef.cushionMs = writeoutCushionMs(effectiveTimeoutMs);
|
|
2072
|
-
prepared.callCapRef.budgetMs = effectiveTimeoutMs;
|
|
2073
|
-
}
|
|
2074
|
-
}
|
|
2075
|
-
rs.counters.finalizeInjected = false;
|
|
2076
2064
|
rs.counters.walltimeSyncBackstopFired = false;
|
|
2077
|
-
const nudgeSchedule = [];
|
|
2078
|
-
{
|
|
2079
|
-
const conf = spec.limits?.deadlineNudge;
|
|
2080
|
-
const budgetSec = effectiveTimeoutMs !== undefined ? effectiveTimeoutMs / 1000 : undefined;
|
|
2081
|
-
if (conf !== false && budgetSec && budgetSec > 0) {
|
|
2082
|
-
const [a, b] = conf?.at ?? [0.8, 0.95];
|
|
2083
|
-
if (a > 0 && a < 1) {
|
|
2084
|
-
const t1 = rs.telemetry.taskStart + a * budgetSec * 1000;
|
|
2085
|
-
const remainMin = Math.max(1, Math.round((budgetSec * 1000 * (1 - a)) / 60_000));
|
|
2086
|
-
const budgetMin = Math.max(1, Math.round(budgetSec / 60));
|
|
2087
|
-
nudgeSchedule.push({
|
|
2088
|
-
atMs: t1,
|
|
2089
|
-
text: `<system-reminder>[deadline] ~${remainMin} min of the ~${budgetMin}-min wall-clock budget remain. ` +
|
|
2090
|
-
`Start converging: persist finished work (write files / commit), prefer completing in-flight items over opening new ones.</system-reminder>`,
|
|
2091
|
-
});
|
|
2092
|
-
const t2 = rs.telemetry.taskStart + Math.max(Math.min(b * budgetSec * 1000, budgetSec * 1000 - 120_000), a * budgetSec * 1000 + 1);
|
|
2093
|
-
if (b > a && b < 1) {
|
|
2094
|
-
nudgeSchedule.push({
|
|
2095
|
-
atMs: t2,
|
|
2096
|
-
text: `<system-reminder>[deadline] the wall-clock budget is nearly exhausted. STOP new work — ` +
|
|
2097
|
-
`write out deliverables and give your final answer NOW; an unfinished answer beats a hard timeout.</system-reminder>`,
|
|
2098
|
-
});
|
|
2099
|
-
}
|
|
2100
|
-
}
|
|
2101
|
-
}
|
|
2102
|
-
}
|
|
2103
|
-
rs.counters.nudgeIdx = 0;
|
|
2104
|
-
const buildFinalizeText = () => `<system-reminder>[deadline] the wall-clock budget is exhausted — this is the FINAL turn. ` +
|
|
2105
|
-
`Write your best current work to the deliverable path NOW, even if incomplete — persist to disk FIRST; everything else comes after. ` +
|
|
2106
|
-
`If a better version exists anywhere else (a draft, scratch or temp file), copy it to the deliverable path now. ` +
|
|
2107
|
-
`Do NOT open new work, new investigations, or long commands; write out and stop.` +
|
|
2108
|
-
(finalVerificationOn
|
|
2109
|
-
? ` After writing, if seconds permit, confirm the deliverable file actually EXISTS at the required path with your latest content.`
|
|
2110
|
-
: "") +
|
|
2111
|
-
`</system-reminder>`;
|
|
2112
2065
|
const timeout = startTimeout(prepared.harness, prepared.abortController, walltimeMonotonicDeadline !== undefined ? walltimeMonotonicDeadline - performance.now() : undefined, prepared.suspendForResource !== undefined);
|
|
2113
2066
|
const parentToolCallId = internals?.parentToolCallId;
|
|
2114
2067
|
const ident = () => parentToolCallId !== undefined ? { eventId: uuidv7(), parentToolCallId, sourceTaskId: rs.telemetry.taskId } : { eventId: uuidv7() };
|
|
@@ -2164,8 +2117,8 @@ export class Runner {
|
|
|
2164
2117
|
const batchArgs = postToolBatchHook ? new Map() : undefined;
|
|
2165
2118
|
rs.turn.toolBatch = [];
|
|
2166
2119
|
const { onMessageUpdate, onMessageEnd, onToolStart, onToolEnd, onTurnEnd, announceWorkspaceMove } = makeHarnessHandlers(prepared, stats, rs, {
|
|
2167
|
-
spec, queue, internals,
|
|
2168
|
-
pushContent, emitCommitted, startedToolCallIds, toolStartAt,
|
|
2120
|
+
spec, queue, internals, ident, parentToolCallId, subagentName,
|
|
2121
|
+
pushContent, emitCommitted, startedToolCallIds, toolStartAt, writeFamilyOf,
|
|
2169
2122
|
toolLabels, postToolBatchHook, batchArgs,
|
|
2170
2123
|
runnerHooks: { onError: (e, info) => this.deps.onError?.(e, info) },
|
|
2171
2124
|
});
|
|
@@ -2206,7 +2159,6 @@ export class Runner {
|
|
|
2206
2159
|
};
|
|
2207
2160
|
const withinTaskCompaction = (spec.compaction?.enabled ?? true) && (spec.compaction?.withinTask ?? true);
|
|
2208
2161
|
const compactionBreaker = { failures: 0 };
|
|
2209
|
-
const compactionThroughputRef = {};
|
|
2210
2162
|
const windowSafetyOptions = (mainModel) => ({
|
|
2211
2163
|
...(rs.budget.maxCostMicroUsd !== undefined
|
|
2212
2164
|
? {
|
|
@@ -2217,15 +2169,6 @@ export class Runner {
|
|
|
2217
2169
|
},
|
|
2218
2170
|
}
|
|
2219
2171
|
: {}),
|
|
2220
|
-
...(walltimeDeadlineMs !== undefined && prepared.callCapRef !== undefined
|
|
2221
|
-
? {
|
|
2222
|
-
walltime: {
|
|
2223
|
-
deadlineMs: walltimeDeadlineMs,
|
|
2224
|
-
cushionMs: () => effectiveCushionMs(prepared.callCapRef) ?? 0,
|
|
2225
|
-
throughputRef: compactionThroughputRef,
|
|
2226
|
-
},
|
|
2227
|
-
}
|
|
2228
|
-
: {}),
|
|
2229
2172
|
onWindowSafety: (info) => {
|
|
2230
2173
|
if (info.kind === "fallback") {
|
|
2231
2174
|
emitTrace(rs.telemetry.tracer, () => ({
|
|
@@ -2263,7 +2206,6 @@ export class Runner {
|
|
|
2263
2206
|
(rs.counters.finalVerifyInjections === 0 || (rs.counters.finalVerifyInjections === 1 && rs.counters.groundingSignalPreR9 && !rs.counters.groundingSignalPostR9)) &&
|
|
2264
2207
|
rs.counters.wroteThisRun &&
|
|
2265
2208
|
prepared.outputRef.set !== true &&
|
|
2266
|
-
!rs.counters.finalizeInjected &&
|
|
2267
2209
|
!(rs.limits.effectiveMaxTurns !== undefined && rs.limits.effectiveMaxTurns > 0 && stats.turns >= rs.limits.effectiveMaxTurns - 1)) {
|
|
2268
2210
|
rs.counters.finalVerifyInjections += 1;
|
|
2269
2211
|
if (rs.counters.finalVerifyInjections === 2) {
|
|
@@ -2371,18 +2313,6 @@ export class Runner {
|
|
|
2371
2313
|
malformedToolUse: {},
|
|
2372
2314
|
thinkingOnly: {},
|
|
2373
2315
|
degenerateOutput: { detect: (m) => isDegenerateCutMessage(m) },
|
|
2374
|
-
walltimeCutoff: {
|
|
2375
|
-
detect: (m) => isWalltimeCutMessage(m),
|
|
2376
|
-
hasLiveWriteoutWindow: () => {
|
|
2377
|
-
const ref = prepared.callCapRef;
|
|
2378
|
-
if (ref === undefined)
|
|
2379
|
-
return true;
|
|
2380
|
-
const soft = softExecDeadlineMs(ref);
|
|
2381
|
-
if (soft === undefined)
|
|
2382
|
-
return true;
|
|
2383
|
-
return soft - Date.now() >= minLiveWriteoutWindowMs(ref.state);
|
|
2384
|
-
},
|
|
2385
|
-
},
|
|
2386
2316
|
promptTooLong: {
|
|
2387
2317
|
recover: async () => {
|
|
2388
2318
|
if (!(spec.compaction?.enabled ?? true))
|
|
@@ -2456,8 +2386,7 @@ export class Runner {
|
|
|
2456
2386
|
return comp.compacted === true;
|
|
2457
2387
|
}
|
|
2458
2388
|
catch (err) {
|
|
2459
|
-
|
|
2460
|
-
compactionBreaker.failures += 1;
|
|
2389
|
+
compactionBreaker.failures += 1;
|
|
2461
2390
|
const msg = String(err instanceof Error ? err.message : err);
|
|
2462
2391
|
queue.push({ type: "compaction_outcome", outcome: "failed", trigger: "forced", reason: msg.length > 512 ? `${msg.slice(0, 512)}…` : msg, ...ident() });
|
|
2463
2392
|
return false;
|
|
@@ -2472,8 +2401,8 @@ export class Runner {
|
|
|
2472
2401
|
w.resolve(outcome);
|
|
2473
2402
|
};
|
|
2474
2403
|
const onTurnBoundary = makeTurnBoundary(prepared, stats, rs, {
|
|
2475
|
-
spec, queue, manualCompactRef, todoToolMounted, taskToolsMounted,
|
|
2476
|
-
|
|
2404
|
+
spec, queue, manualCompactRef, todoToolMounted, taskToolsMounted,
|
|
2405
|
+
walltimeMonotonicDeadline,
|
|
2477
2406
|
timeout, ident, postToolBatchHook, batchArgs, compactionBrain, withinTaskCompaction,
|
|
2478
2407
|
compactionBreaker, windowSafetyOptions, rapidRefill, drainManualCompact,
|
|
2479
2408
|
runnerHooks: {
|
|
@@ -2541,8 +2470,13 @@ export class Runner {
|
|
|
2541
2470
|
void prepared.harness.abort();
|
|
2542
2471
|
prepared.abortController.abort();
|
|
2543
2472
|
}
|
|
2473
|
+
else if (rs.budget.maxTokensWindow !== undefined && rs.budget.maxTokensWindow <= 0) {
|
|
2474
|
+
rs.limits.budgetHit = "exceeded";
|
|
2475
|
+
rs.limits.budgetAxis = "tokens";
|
|
2476
|
+
}
|
|
2544
2477
|
else if (rs.budget.maxCostMicroUsd !== undefined && rs.budget.maxCostMicroUsd <= 0) {
|
|
2545
2478
|
rs.limits.budgetHit = "exceeded";
|
|
2479
|
+
rs.limits.budgetAxis = "cost";
|
|
2546
2480
|
}
|
|
2547
2481
|
else {
|
|
2548
2482
|
let continuation = resumeContinuation(resume);
|
|
@@ -2641,13 +2575,30 @@ export class Runner {
|
|
|
2641
2575
|
cacheReadTokens: 0,
|
|
2642
2576
|
cacheWriteTokens: 0,
|
|
2643
2577
|
cacheWriteTokensLong: 0,
|
|
2644
|
-
outputTokens: prepared.model.maxTokens ??
|
|
2578
|
+
outputTokens: prepared.model.maxTokens ?? DEFAULT_PRECALL_OUTPUT_TOKENS,
|
|
2645
2579
|
}, rs.telemetry.pricing);
|
|
2646
2580
|
const precallCeilingMicroUsd = prepared.suspendForResource !== undefined ? rs.budget.remainingMicroUsd : rs.budget.maxCostMicroUsd;
|
|
2581
|
+
const precallTokens = Math.ceil(effectiveObjective.length / 4) + (prepared.model.maxTokens ?? DEFAULT_PRECALL_OUTPUT_TOKENS);
|
|
2582
|
+
const precallCeilingTokens = prepared.suspendForResource !== undefined ? rs.budget.remainingTokens : rs.budget.maxTokensWindow;
|
|
2583
|
+
const entryUsageRetryAfterMs = prepared.usageGovernance !== undefined ? await prepared.usageGovernance.check(Date.now()) : undefined;
|
|
2647
2584
|
if (promptBlocked) {
|
|
2648
2585
|
}
|
|
2586
|
+
else if (entryUsageRetryAfterMs !== undefined) {
|
|
2587
|
+
const terminal = platformLimitTerminal("usage_window", entryUsageRetryAfterMs, "entry");
|
|
2588
|
+
rs.limits.platformTerminal = terminal;
|
|
2589
|
+
try {
|
|
2590
|
+
this.deps.onError?.(terminal, { phase: "config", sessionId: prepared.sessionId });
|
|
2591
|
+
}
|
|
2592
|
+
catch {
|
|
2593
|
+
}
|
|
2594
|
+
}
|
|
2595
|
+
else if (precallCeilingTokens !== undefined && precallTokens > precallCeilingTokens) {
|
|
2596
|
+
rs.limits.budgetHit = "precall";
|
|
2597
|
+
rs.limits.budgetAxis = "tokens";
|
|
2598
|
+
}
|
|
2649
2599
|
else if (precallCeilingMicroUsd !== undefined && precallMicroUsd > precallCeilingMicroUsd) {
|
|
2650
2600
|
rs.limits.budgetHit = "precall";
|
|
2601
|
+
rs.limits.budgetAxis = "cost";
|
|
2651
2602
|
}
|
|
2652
2603
|
else if (effectiveTimeoutMs !== undefined && effectiveTimeoutMs <= 0) {
|
|
2653
2604
|
timeout.fired = true;
|
|
@@ -2702,6 +2653,17 @@ export class Runner {
|
|
|
2702
2653
|
if (threw === undefined && prepared.brainCallGuardrailRef.timedOut !== undefined) {
|
|
2703
2654
|
threw = prepared.brainCallGuardrailRef.timedOut;
|
|
2704
2655
|
}
|
|
2656
|
+
if (rs.limits.platformTerminal !== undefined) {
|
|
2657
|
+
threw = rs.limits.platformTerminal;
|
|
2658
|
+
}
|
|
2659
|
+
if (prepared.usageGovernance !== undefined) {
|
|
2660
|
+
try {
|
|
2661
|
+
await prepared.usageGovernance.commit(stats.tokens, Date.now());
|
|
2662
|
+
}
|
|
2663
|
+
catch (flushErr) {
|
|
2664
|
+
this.deps.onError?.(flushErr, { phase: "config", sessionId: prepared.sessionId });
|
|
2665
|
+
}
|
|
2666
|
+
}
|
|
2705
2667
|
const suspended = prepared.suspendRef.token !== undefined && prepared.suspendRef.gate !== undefined;
|
|
2706
2668
|
const reviewPaused = prepared.reviewRef.token !== undefined && prepared.reviewRef.gate !== undefined;
|
|
2707
2669
|
const durablyPaused = suspended || reviewPaused;
|
|
@@ -2800,24 +2762,12 @@ export class Runner {
|
|
|
2800
2762
|
gates: [...prepared.humanReviewRef.gates],
|
|
2801
2763
|
};
|
|
2802
2764
|
}
|
|
2803
|
-
|
|
2804
|
-
const toolClamps = prepared.callCapRef?.toolClamps ?? 0;
|
|
2805
|
-
const toolCuts = prepared.callCapRef?.toolCuts ?? 0;
|
|
2806
|
-
const pendingCutTools = prepared.callCapRef?.pendingCutTools ?? 0;
|
|
2807
|
-
const toolCutKills = prepared.cutKills?.kills ?? 0;
|
|
2808
|
-
if (rs.counters.finalizeInjected || rs.counters.nudgesSent > 0 || capShrinks > 0 || rs.counters.callCutoffs > 0 || toolClamps > 0 || toolCuts > 0 || toolCutKills > 0 || pendingCutTools > 0 || rs.counters.finalVerifyInjections > 0 || rs.attach.attachmentsInjected > 0 || rs.counters.repetitionCuts > 0 || rs.counters.repetitionSpared > 0) {
|
|
2765
|
+
if (rs.counters.finalVerifyInjections > 0 || rs.attach.attachmentsInjected > 0 || rs.counters.repetitionCuts > 0 || rs.counters.repetitionSpared > 0 || rs.counters.approachNoticesSent > 0) {
|
|
2809
2766
|
stats.mechanisms = {
|
|
2810
|
-
...(rs.counters.finalizeInjected ? { finalizeInjected: true } : {}),
|
|
2811
|
-
...(rs.counters.nudgesSent > 0 ? { nudgesSent: rs.counters.nudgesSent } : {}),
|
|
2812
|
-
...(capShrinks > 0 ? { capShrinks } : {}),
|
|
2813
|
-
...(rs.counters.callCutoffs > 0 ? { callCutoffs: rs.counters.callCutoffs } : {}),
|
|
2814
|
-
...(toolClamps > 0 ? { toolClamps } : {}),
|
|
2815
|
-
...(toolCuts > 0 ? { toolCuts } : {}),
|
|
2816
|
-
...(toolCutKills > 0 ? { toolCutKills } : {}),
|
|
2817
|
-
...(pendingCutTools > 0 ? { pendingCutTools } : {}),
|
|
2818
2767
|
...(rs.counters.finalVerifyInjections > 0 ? { finalVerifyInjected: true } : {}),
|
|
2819
2768
|
...(rs.counters.finalVerifyInjections > 0 ? { finalVerifyInjections: rs.counters.finalVerifyInjections } : {}),
|
|
2820
2769
|
...(rs.attach.attachmentsInjected > 0 ? { attachmentsInjected: rs.attach.attachmentsInjected } : {}),
|
|
2770
|
+
...(rs.counters.approachNoticesSent > 0 ? { approachNoticesSent: rs.counters.approachNoticesSent } : {}),
|
|
2821
2771
|
...(rs.counters.repetitionCuts > 0 ? { repetitionCuts: rs.counters.repetitionCuts } : {}),
|
|
2822
2772
|
...(rs.counters.repetitionSpared > 0 ? { repetitionSpared: rs.counters.repetitionSpared } : {}),
|
|
2823
2773
|
...(rs.counters.repetitionEvents.length > 0 ? { repetitionEvents: rs.counters.repetitionEvents } : {}),
|
|
@@ -2833,6 +2783,7 @@ export class Runner {
|
|
|
2833
2783
|
abortedForTurns: rs.limits.turnsExceeded,
|
|
2834
2784
|
abortedLive,
|
|
2835
2785
|
budgetHit: rs.limits.budgetHit,
|
|
2786
|
+
budgetAxis: rs.limits.budgetAxis,
|
|
2836
2787
|
blockedReason: prepared.blockedRef.reason,
|
|
2837
2788
|
conflict: prepared.conflictRef.hit,
|
|
2838
2789
|
outputInvalid: rs.degrade.outputInvalid,
|
|
@@ -2891,7 +2842,6 @@ export class Runner {
|
|
|
2891
2842
|
result.status === "failed" &&
|
|
2892
2843
|
final?.stopReason === "error" &&
|
|
2893
2844
|
!isDegenerateCutMessage(final) &&
|
|
2894
|
-
!isWalltimeCutMessage(final) &&
|
|
2895
2845
|
threw === undefined &&
|
|
2896
2846
|
rs.limits.budgetHit === undefined &&
|
|
2897
2847
|
!rs.degrade.outputInvalid &&
|
|
@@ -2959,7 +2909,7 @@ export class Runner {
|
|
|
2959
2909
|
ts: Date.now(),
|
|
2960
2910
|
}));
|
|
2961
2911
|
setResult(result);
|
|
2962
|
-
if (parentToolCallId !== undefined && (result.status === "completed" || result.status === "failed" || result.status === "
|
|
2912
|
+
if (parentToolCallId !== undefined && (result.status === "completed" || result.status === "failed" || result.status === "blocked")) {
|
|
2963
2913
|
const terminalTick = {
|
|
2964
2914
|
type: "task_progress",
|
|
2965
2915
|
taskId: rs.telemetry.taskId,
|
|
@@ -3298,6 +3248,12 @@ export class Runner {
|
|
|
3298
3248
|
if (checkpointVersionOf(cp) > MAX_SUPPORTED_CHECKPOINT_VERSION) {
|
|
3299
3249
|
throw new CheckpointError("checkpoint.unsupported_version", `checkpoint version ${checkpointVersionOf(cp)} is newer than this worker supports (max ${MAX_SUPPORTED_CHECKPOINT_VERSION})`);
|
|
3300
3250
|
}
|
|
3251
|
+
const retiredWalltimeTotal = cp.resourceLedger?.totalWalltimeSec;
|
|
3252
|
+
if (retiredWalltimeTotal !== undefined) {
|
|
3253
|
+
throw new CheckpointError("checkpoint.walltime_axis_retired", "this checkpoint's ledger carries a cross-slice WALL-CLOCK allocation (resourceLedger.totalWalltimeSec), an axis this engine version retired — " +
|
|
3254
|
+
"resuming here would run the leg with that ceiling unenforced. Rejected pre-CAS (the checkpoint stays pending): finish it on a worker of the " +
|
|
3255
|
+
"previous release, or start a fresh task bounded by resourceSuspend.totalTokens / totalBudgetUsd.");
|
|
3256
|
+
}
|
|
3301
3257
|
if (taskConfig.signal?.aborted === true && cp.status === "pending") {
|
|
3302
3258
|
throw new CheckpointError("checkpoint.resume_aborted", "the resume was handed an ALREADY-ABORTED signal — refusing to consume the approval on a leg that cannot run it (the checkpoint stays pending and is resumable with a live signal)");
|
|
3303
3259
|
}
|