@sema-agent/core 1.284.0 → 1.286.0

package/dist/core/runner/runtask.js

@@ -1,7 +1,8 @@
 import { AgentHarness, DEFAULT_COMPACTION_SETTINGS, uuidv7 } from "../../internal/harness.js";
 import { CheckpointError, BINDING_CHECKPOINT_VERSION, checkpointVersionOf, MAX_SUPPORTED_CHECKPOINT_VERSION, remainingBudgetMicroUsd, validatePendingSteer, winnerFromOutcome, } from "../checkpoint-store.js";
-import { effectiveCushionMs, estimateCycleMs, recordCallSample, recordToolSample, writeoutCushionMs } from "./call-cap.js";
-import { DEFAULT_COMPACTION_INSTRUCTIONS, createRapidRefillState, maybeCompact, nextTrimForceBackoff, recordCompactionAndCheckRapidRefill } from "../auto-compaction.js";
+import { effectiveCushionMs, estimateCycleMs, minLiveWriteoutWindowMs, recordCallSample, recordToolSample, recordTtftSample, softExecDeadlineMs, writeoutCushionMs } from "./call-cap.js";
+import { engineVersion } from "../version.js";
+import { DEFAULT_COMPACTION_INSTRUCTIONS, createRapidRefillState, isCompactionWalltimeAbort, maybeCompact, nextTrimForceBackoff, recordCompactionAndCheckRapidRefill } from "../auto-compaction.js";
 import { ASK_USER_QUESTION_TOOL_NAME } from "../ask-question.js";
 import { computeCostMicroUsd, modelCostToPricing } from "../pricing.js";
 import { emitTrace } from "../trace.js";
@@ -303,6 +304,7 @@ export class Runner {
         const manualCompactRef = { requested: false, waiters: [] };
         const drainManualCompactWaiters = (outcome) => {
             manualCompactRef.requested = false;
+            manualCompactRef.instructions = undefined;
             for (const w of manualCompactRef.waiters.splice(0))
                 w(outcome);
         };
@@ -496,7 +498,7 @@ export class Runner {
                 }
                 notifyRef.inject(payload, opts);
             },
-            compact: async () => {
+            compact: async (opts) => {
                 if (resultValue)
                     throw steeringError("the task has already finished");
                 const h = handle ?? (await orTimeout(ready));
@@ -504,6 +506,9 @@ export class Runner {
                     throw steeringError("the task is not running");
                 return new Promise((resolve) => {
                     manualCompactRef.requested = true;
+                    if (opts?.instructions !== undefined) {
+                        manualCompactRef.instructions = opts.instructions;
+                    }
                     manualCompactRef.waiters.push(resolve);
                 });
             },
@@ -796,7 +801,7 @@ export class Runner {
         const writeFamilyOf = (name) => writeFamilyByName.get(name) ?? writeFamilyOfCanonical(canonicalToolName(name));
         const toolStartAt = new Map();
         const startedToolCallIds = new Set();
-        emitTrace(tracer, () => ({ kind: "task.start", version: 1, taskId, model: prepared.model.id, ts: taskStart }));
+        emitTrace(tracer, () => ({ kind: "task.start", version: 1, taskId, model: prepared.model.id, engineVersion: engineVersion(), ts: taskStart }));
         emitTrace(tracer, () => ({
             kind: "prompt.assembled",
             version: 1,
@@ -1007,6 +1012,9 @@ export class Runner {
                 const capLast = prepared.callCapRef?.state.last;
                 if (prepared.callCapRef) {
                     recordCallSample(prepared.callCapRef.state, u.output || 0, callStartAt !== undefined ? callEndAt - callStartAt : 0);
+                    if (callStartAt !== undefined && firstTokenAt !== undefined) {
+                        recordTtftSample(prepared.callCapRef.state, firstTokenAt - callStartAt);
+                    }
                     prepared.callCapRef.state.last = undefined;
                 }
                 emitTrace(tracer, () => ({
@@ -1330,6 +1338,52 @@ export class Runner {
         };
         const withinTaskCompaction = (spec.compaction?.enabled ?? true) && (spec.compaction?.withinTask ?? true);
         const compactionBreaker = { failures: 0 };
+        const compactionThroughputRef = {};
+        const windowSafetyOptions = (mainModel) => ({
+            ...(maxCostMicroUsd !== undefined
+                ? {
+                    fallbackBudget: {
+                        spentMicroUsd: () => stats.costMicroUsd,
+                        capMicroUsd: maxCostMicroUsd,
+                        mainInputPer1M: (this.deps.pricing?.[mainModel.id] ?? modelCostToPricing(mainModel.cost)).inputPer1M,
+                    },
+                }
+                : {}),
+            ...(walltimeDeadlineMs !== undefined && prepared.callCapRef !== undefined
+                ? {
+                    walltime: {
+                        deadlineMs: walltimeDeadlineMs,
+                        cushionMs: () => effectiveCushionMs(prepared.callCapRef) ?? 0,
+                        throughputRef: compactionThroughputRef,
+                    },
+                }
+                : {}),
+            onWindowSafety: (info) => {
+                if (info.kind === "fallback") {
+                    emitTrace(tracer, () => ({
+                        kind: "compaction.model_fallback",
+                        version: 1,
+                        taskId,
+                        contentTokens: info.contentTokens,
+                        headroomTokens: info.headroomTokens ?? 0,
+                        truncationRatio: info.truncationRatio,
+                        ...(info.estCostMicroUsd !== undefined ? { estCostMicroUsd: info.estCostMicroUsd } : {}),
+                        ts: Date.now(),
+                    }));
+                }
+                else {
+                    emitTrace(tracer, () => ({
+                        kind: "compaction.clamp_disclosure",
+                        version: 1,
+                        taskId,
+                        truncationRatio: info.truncationRatio,
+                        reason: info.reason ?? "tolerance",
+                        ...(info.estCostMicroUsd !== undefined ? { estCostMicroUsd: info.estCostMicroUsd } : {}),
+                        ts: Date.now(),
+                    }));
+                }
+            },
+        });
         const stopHook = (spec.hooks ?? this.deps.hooks)?.stop;
         const finalVerificationOn = spec.finalVerification === true;
         if (stopHook || finalVerificationOn) {
@@ -1434,7 +1488,18 @@ export class Runner {
             malformedToolUse: {},
             thinkingOnly: {},
             degenerateOutput: { detect: (m) => m.errorMessage === DEGENERATE_MESSAGE },
-            walltimeCutoff: { detect: (m) => m.errorMessage === WALLTIME_CUTOFF_MESSAGE },
+            walltimeCutoff: {
+                detect: (m) => m.errorMessage === WALLTIME_CUTOFF_MESSAGE,
+                hasLiveWriteoutWindow: () => {
+                    const ref = prepared.callCapRef;
+                    if (ref === undefined)
+                        return true;
+                    const soft = softExecDeadlineMs(ref);
+                    if (soft === undefined)
+                        return true;
+                    return soft - Date.now() >= minLiveWriteoutWindowMs(ref.state);
+                },
+            },
             promptTooLong: {
                 recover: async () => {
                     if (!(spec.compaction?.enabled ?? true))
@@ -1466,6 +1531,7 @@ export class Runner {
                                 }
                                 : undefined,
                             ...this.seamCCompactionOptions(prepared),
+                            ...windowSafetyOptions(prepared.harness.getModel()),
                             ...this.compactionHookOptions(spec, prepared.sessionId, "forced"),
                         });
                         if (comp.compacted) {
@@ -1484,6 +1550,10 @@ export class Runner {
                                 ...(comp.durationMs !== undefined ? { durationMs: comp.durationMs } : {}),
                                 ...(comp.firstKeptEntryId !== undefined ? { preserved_segment: { firstKeptEntryId: comp.firstKeptEntryId } } : {}),
                                 ...(comp.attachedFiles !== undefined ? { attachedFiles: comp.attachedFiles } : {}),
+                                ...(comp.modelFallback ? { modelFallback: true } : {}),
+                                ...(comp.fallbackReason !== undefined ? { fallbackReason: comp.fallbackReason } : {}),
+                                ...(comp.clampedRatio !== undefined ? { clampedRatio: comp.clampedRatio } : {}),
+                                ...(comp.clampReason !== undefined ? { clampReason: comp.clampReason } : {}),
                             });
                             prepared.cacheBreakDetector?.notifyCompaction();
                             if (attachState !== undefined) {
@@ -1496,8 +1566,9 @@ export class Runner {
                         }
                         return comp.compacted === true;
                     }
-                    catch {
-                        compactionBreaker.failures += 1;
+                    catch (err) {
+                        if (!isCompactionWalltimeAbort(err))
+                            compactionBreaker.failures += 1;
                         return false;
                     }
                 },
@@ -1508,6 +1579,7 @@ export class Runner {
         let trimForceBackoff = false;
         const drainManualCompact = (outcome) => {
             manualCompactRef.requested = false;
+            manualCompactRef.instructions = undefined;
             for (const w of manualCompactRef.waiters.splice(0))
                 w(outcome);
         };
@@ -1743,11 +1815,27 @@ export class Runner {
                 }
                 return undefined;
             }
+            const claimedManual = forceManual
+                ? { instructions: manualCompactRef.instructions, waiters: manualCompactRef.waiters.splice(0) }
+                : undefined;
+            if (forceManual) {
+                manualCompactRef.requested = false;
+                manualCompactRef.instructions = undefined;
+            }
+            const resolveClaimedWaiters = (outcome) => {
+                if (claimedManual !== undefined)
+                    for (const w of claimedManual.waiters.splice(0))
+                        w(outcome);
+            };
             const trimPressureArmed = prepared.trimPressureRef.droppedMessages;
             if (trimPressureArmed)
                 prepared.trimPressureRef.droppedMessages = false;
             const trimPressure = trimPressureArmed && !trimForceBackoff;
             let manualOutcome = "failed";
+            if (finalizeInjected && forceManual) {
+                emitTrace(tracer, () => ({ kind: "compaction.manual_in_finalize", version: 1, taskId, ts: Date.now() }));
+            }
+            const manualInstructions = claimedManual?.instructions;
             try {
                 const comp = await maybeCompact({
                     session: prepared.session,
@@ -1757,7 +1845,7 @@ export class Runner {
                     getApiKeyAndHeaders: spec.getApiKeyAndHeaders,
                     thinking: prepared.thinking,
                     settings: { ...DEFAULT_COMPACTION_SETTINGS, ...spec.compaction },
-                    customInstructions: spec.compaction?.instructions ?? DEFAULT_COMPACTION_INSTRUCTIONS,
+                    customInstructions: manualInstructions ?? spec.compaction?.instructions ?? DEFAULT_COMPACTION_INSTRUCTIONS,
                     signal: prepared.abortController.signal,
                     minTokens: forceManual ? 0 : compactionFloor,
                     force: forceManual,
@@ -1772,6 +1860,7 @@ export class Runner {
                         }
                         : undefined,
                     ...this.seamCCompactionOptions(prepared),
+                    ...windowSafetyOptions(event.model),
                     ...this.compactionHookOptions(spec, prepared.sessionId, trimPressure ? "forced" : forceManual ? "manual" : "auto"),
                 });
                 if (comp.blocked) {
@@ -1824,6 +1913,10 @@ export class Runner {
                         ...(comp.durationMs !== undefined ? { durationMs: comp.durationMs } : {}),
                         ...(comp.firstKeptEntryId !== undefined ? { preserved_segment: { firstKeptEntryId: comp.firstKeptEntryId } } : {}),
                         ...(comp.attachedFiles !== undefined ? { attachedFiles: comp.attachedFiles } : {}),
+                        ...(comp.modelFallback ? { modelFallback: true } : {}),
+                        ...(comp.fallbackReason !== undefined ? { fallbackReason: comp.fallbackReason } : {}),
+                        ...(comp.clampedRatio !== undefined ? { clampedRatio: comp.clampedRatio } : {}),
+                        ...(comp.clampReason !== undefined ? { clampReason: comp.clampReason } : {}),
                     });
                     prepared.cacheBreakDetector?.notifyCompaction();
                     if (attachState !== undefined) {
@@ -1839,7 +1932,9 @@ export class Runner {
             }
             catch (err) {
                 if (!prepared.abortController.signal.aborted) {
-                    compactionBreaker.failures += 1;
+                    const walltimeAbort = isCompactionWalltimeAbort(err);
+                    if (!walltimeAbort)
+                        compactionBreaker.failures += 1;
                     this.deps.onError?.(err, { phase: "compaction", sessionId: prepared.sessionId });
                     const msg = String(err instanceof Error ? err.message : err);
                     emitTrace(tracer, () => ({
@@ -1847,7 +1942,11 @@ export class Runner {
                         version: 1,
                         taskId,
                         trigger: trimPressure ? "forced" : forceManual ? "manual" : "auto",
-                        reason: msg.length > 512 ? `${msg.slice(0, 512)}…` : msg,
+                        reason: walltimeAbort
+                            ? "walltime soft-deadline abort (engine-initiated write-out protection; not counted toward the breaker)"
+                            : msg.length > 512
+                                ? `${msg.slice(0, 512)}…`
+                                : msg,
                         ts: Date.now(),
                     }));
                     manualOutcome = "failed";
@@ -1857,7 +1956,7 @@ export class Runner {
                 }
             }
             if (forceManual)
-                drainManualCompact(manualOutcome);
+                resolveClaimedWaiters(manualOutcome);
             return undefined;
         };
         const unsubBoundary = prepared.harness.on("turn_boundary", onTurnBoundary);
@@ -2027,6 +2126,7 @@ export class Runner {
             skipCompaction: durablyPaused || compactionBreaker.failures >= MAX_CONSECUTIVE_COMPACTION_FAILURES,
             minTokens: compactionFloor,
             brain: compactionBrain,
+            windowSafety: windowSafetyOptions(prepared.model),
         });
         try {
             if (comp?.compacted) {
@@ -2039,6 +2139,10 @@ export class Runner {
                     ...(comp.durationMs !== undefined ? { durationMs: comp.durationMs } : {}),
                     ...(comp.firstKeptEntryId !== undefined ? { preserved_segment: { firstKeptEntryId: comp.firstKeptEntryId } } : {}),
                     ...(comp.attachedFiles !== undefined ? { attachedFiles: comp.attachedFiles } : {}),
+                    ...(comp.modelFallback ? { modelFallback: true } : {}),
+                    ...(comp.fallbackReason !== undefined ? { fallbackReason: comp.fallbackReason } : {}),
+                    ...(comp.clampedRatio !== undefined ? { clampedRatio: comp.clampedRatio } : {}),
+                    ...(comp.clampReason !== undefined ? { clampReason: comp.clampReason } : {}),
                 });
                 prepared.cacheBreakDetector?.notifyCompaction();
             }
@@ -2187,6 +2291,9 @@ export class Runner {
                 costMicroUsd: stats.costMicroUsd,
                 durationMs: Date.now() - taskStart,
                 ...(prepared.outputRef.set ? { hasStructuredOutput: true } : {}),
+                ...(stats.mechanisms !== undefined
+                    ? { mechanisms: (({ repetitionEvents: _events, ...scalars }) => scalars)(stats.mechanisms) }
+                    : {}),
                 ts: Date.now(),
             }));
             setResult(result);
@@ -2782,6 +2889,7 @@ export class Runner {
                         keptChars: info.keptChars,
                         ts: Date.now(),
                     })),
+                    ...(opts?.windowSafety ?? {}),
                     workingFileAttachments: spec.compaction?.attachWorkingFiles !== false && prepared.readTaskFile
                         ? {
                             readFile: prepared.readTaskFile,