@sema-agent/core 1.279.0 → 1.281.0

package/dist/core/runner/runtask.js

@@ -1,6 +1,6 @@
 import { AgentHarness, DEFAULT_COMPACTION_SETTINGS, uuidv7 } from "../../internal/harness.js";
 import { CheckpointError, BINDING_CHECKPOINT_VERSION, checkpointVersionOf, MAX_SUPPORTED_CHECKPOINT_VERSION, remainingBudgetMicroUsd, winnerFromOutcome, } from "../checkpoint-store.js";
-import { estimateCycleMs, recordCallSample, recordToolSample, writeoutCushionMs } from "./call-cap.js";
+import { effectiveCushionMs, estimateCycleMs, recordCallSample, recordToolSample, writeoutCushionMs } from "./call-cap.js";
 import { DEFAULT_COMPACTION_INSTRUCTIONS, createRapidRefillState, maybeCompact, nextTrimForceBackoff, recordCompactionAndCheckRapidRefill } from "../auto-compaction.js";
 import { ASK_USER_QUESTION_TOOL_NAME } from "../ask-question.js";
 import { computeCostMicroUsd, modelCostToPricing } from "../pricing.js";
@@ -300,6 +300,8 @@ export class Runner {
             for (const w of manualCompactRef.waiters.splice(0))
                 w(outcome);
         };
+        const runStartedAt = Date.now();
+        const taskIdRef = { current: spec.taskId };
         const run = (async () => {
             const releaseLock = spec.sessionId
                 ? await this.acquireSessionLock(spec.sessionId)
@@ -316,7 +318,7 @@ export class Runner {
                     publishReady(h);
                 }, (s) => {
                     reapHandle = s;
-                }, manualCompactRef, resume, { ...(internals ?? {}), detachHub });
+                }, manualCompactRef, taskIdRef, resume, { ...(internals ?? {}), detachHub });
             }
             finally {
                 releaseLock?.();
@@ -346,8 +348,17 @@ export class Runner {
                 catch {
                 }
             }
+            if (resultValue !== undefined) {
+                try {
+                    this.deps.onError?.(err instanceof Error ? err : new Error(String(err)), { phase: "config", sessionId: resultValue.sessionId });
+                }
+                catch {
+                }
+                queue.close();
+                return;
+            }
             resultValue = {
-                taskId: spec.taskId ?? "unknown",
+                taskId: taskIdRef.current ?? "unknown",
                 sessionId: spec.sessionId ?? "unknown",
                 status: "failed",
                 result: "",
@@ -355,6 +366,17 @@ export class Runner {
                 errorCode: code,
                 stats: { turns: 0, tokens: 0, toolCalls: 0, cachedTokens: 0, costUsd: 0 },
             };
+            emitTrace(spec.tracer ?? this.deps.tracer, () => ({
+                kind: "task.end",
+                version: 1,
+                taskId: taskIdRef.current ?? "unknown",
+                status: "failed",
+                errorCode: code,
+                turns: 0,
+                tokens: 0,
+                durationMs: Date.now() - runStartedAt,
+                ts: Date.now(),
+            }));
             queue.push({ type: "done", result: resultValue });
             queue.close();
         });
@@ -460,7 +482,7 @@ export class Runner {
             })()),
         };
     }
-    async runLocked(spec, queue, setResult, onConsolidation, onSuggestions, onReady, onSuspend, manualCompactRef, resume, internals) {
+    async runLocked(spec, queue, setResult, onConsolidation, onSuggestions, onReady, onSuspend, manualCompactRef, taskIdRef, resume, internals) {
         const prepareResume = resume
             ? {
                 leafId: resume.cp.leafId,
@@ -569,6 +591,8 @@ export class Runner {
         const effectiveMaxTurns = resolveMaxTurns(spec.limits);
         const tracer = spec.tracer ?? this.deps.tracer;
         const taskId = spec.taskId ?? prepared.sessionId;
+        if (taskIdRef)
+            taskIdRef.current = taskId;
         const taskStart = Date.now();
         void Promise.resolve(this.sessions.noteTaskRun?.(prepared.sessionId, taskId)).catch(() => { });
         const recordDegraded = (info, toModel) => {
@@ -718,6 +742,7 @@ export class Runner {
         }
         const writeFamilyOf = (name) => writeFamilyByName.get(name) ?? writeFamilyOfCanonical(canonicalToolName(name));
         const toolStartAt = new Map();
+        const startedToolCallIds = new Set();
         emitTrace(tracer, () => ({ kind: "task.start", version: 1, taskId, model: prepared.model.id, ts: taskStart }));
         emitTrace(tracer, () => ({
             kind: "prompt.assembled",
@@ -751,6 +776,7 @@ export class Runner {
             prepared.callCapRef.deadlineMs = walltimeDeadlineMs;
             if (spec.limits?.gracefulFinalize !== false) {
                 prepared.callCapRef.cushionMs = writeoutCushionMs(spec.limits.timeoutSec * 1000);
+                prepared.callCapRef.budgetMs = spec.limits.timeoutSec * 1000;
             }
         }
         let finalizeInjected = false;
@@ -796,6 +822,8 @@ export class Runner {
             }
         };
         const emitCommitted = (entryId, role, toolCallId) => {
+            if (role === "toolResult" && toolCallId !== undefined)
+                startedToolCallIds.delete(toolCallId);
             queue.push({ type: "message_committed", entryId, role, ...(toolCallId !== undefined ? { toolCallId } : {}), ...ident() });
         };
         const subagentName = parentToolCallId !== undefined && internals?.agentName !== undefined ? inlineUntrusted(internals.agentName.slice(0, 320), 80) : undefined;
@@ -971,6 +999,12 @@ export class Runner {
             }
             if (m.role === "assistant" && m.errorMessage === WALLTIME_CUTOFF_MESSAGE) {
                 callCutoffs += 1;
+                if (prepared.callCapRef)
+                    prepared.callCapRef.finalizeMode = true;
+                if (!finalizeInjected) {
+                    finalizeInjected = true;
+                    nudgeIdx = nudgeSchedule.length;
+                }
             }
             if (m.role === "assistant") {
                 const rep = m.repetition;
@@ -1030,6 +1064,7 @@ export class Runner {
         let toolBatch = [];
         const onToolStart = (event) => {
             toolStartAt.set(event.toolCallId, Date.now());
+            startedToolCallIds.add(event.toolCallId);
             batchArgs?.set(event.toolCallId, event.args);
             stats.toolCalls += 1;
             if (spec.finalVerification === true && !(finalVerifyInjections === 0 ? groundingSignalPreR9 : groundingSignalPostR9) && hasVerifiableStructureSignal(event.args)) {
@@ -1421,7 +1456,7 @@ export class Runner {
                     walltimeDeadlineMs !== undefined &&
                     prepared.callCapRef !== undefined) {
                     const remainingMs = walltimeDeadlineMs - nowMs;
-                    if (remainingMs > 0 && remainingMs < estimateCycleMs(prepared.callCapRef.state) + (prepared.callCapRef.cushionMs ?? 0)) {
+                    if (remainingMs > 0 && remainingMs < estimateCycleMs(prepared.callCapRef.state) + (effectiveCushionMs(prepared.callCapRef) ?? 0)) {
                         finalizeInjected = true;
                         prepared.callCapRef.finalizeMode = true;
                         nudgeIdx = nudgeSchedule.length;
@@ -1808,7 +1843,7 @@ export class Runner {
                         stampWriteAnchor(attachState, family, cadenceTurns);
                     if (details !== undefined)
                         reduceToolEnd(attachState, details);
-                });
+                }, (toolCallId) => startedToolCallIds.add(toolCallId));
                 if (maxCostMicroUsd !== undefined && maxCostMicroUsd <= 0) {
                     budgetHit = "exceeded";
                 }
@@ -1903,7 +1938,7 @@ export class Runner {
             (threw !== undefined || timeout.fired || turnsExceeded || budgetHit !== undefined || abortedLive || final?.stopReason === "aborted");
         if (interrupted) {
             try {
-                const report = await reconcileInterruptedSession(prepared.session, prepared.toolEffects);
+                const report = await reconcileInterruptedSession(prepared.session, prepared.toolEffects, undefined, startedToolCallIds);
                 for (const orphan of report.recovered) {
                     queue.push({ type: "tool_end", toolCallId: orphan.toolCallId, toolName: orphan.toolName, isError: true, ...ident() });
                     emitCommitted(orphan.entryId, "toolResult", orphan.toolCallId);
@@ -1985,13 +2020,17 @@ export class Runner {
             }
             const capShrinks = prepared.callCapRef?.shrinks ?? 0;
             const toolClamps = prepared.callCapRef?.toolClamps ?? 0;
-            if (finalizeInjected || nudgesSent > 0 || capShrinks > 0 || callCutoffs > 0 || toolClamps > 0 || finalVerifyInjections > 0 || attachmentsInjected > 0 || repetitionCuts > 0 || repetitionSpared > 0) {
+            const toolCuts = prepared.callCapRef?.toolCuts ?? 0;
+            const pendingCutTools = prepared.callCapRef?.pendingCutTools ?? 0;
+            if (finalizeInjected || nudgesSent > 0 || capShrinks > 0 || callCutoffs > 0 || toolClamps > 0 || toolCuts > 0 || pendingCutTools > 0 || finalVerifyInjections > 0 || attachmentsInjected > 0 || repetitionCuts > 0 || repetitionSpared > 0) {
                 stats.mechanisms = {
                     ...(finalizeInjected ? { finalizeInjected: true } : {}),
                     ...(nudgesSent > 0 ? { nudgesSent } : {}),
                     ...(capShrinks > 0 ? { capShrinks } : {}),
                     ...(callCutoffs > 0 ? { callCutoffs } : {}),
                     ...(toolClamps > 0 ? { toolClamps } : {}),
+                    ...(toolCuts > 0 ? { toolCuts } : {}),
+                    ...(pendingCutTools > 0 ? { pendingCutTools } : {}),
                     ...(finalVerifyInjections > 0 ? { finalVerifyInjected: true } : {}),
                     ...(finalVerifyInjections > 0 ? { finalVerifyInjections } : {}),
                     ...(attachmentsInjected > 0 ? { attachmentsInjected } : {}),
@@ -2423,7 +2462,7 @@ export class Runner {
             : undefined;
         return this.runTaskStream(spec, { cp, outcome, onEnvRestoreFailed }, internals);
     }
-    async applyResumeDecision(prepared, resume, emit, emitCommitted, onResolvedToolSuccess) {
+    async applyResumeDecision(prepared, resume, emit, emitCommitted, onResolvedToolSuccess, onExecuteStart) {
         const { pendingAction } = resume.cp;
         if (resume.outcome.gate === "dry_run_review")
             return;
@@ -2454,7 +2493,7 @@ export class Runner {
         }
         for (const id of pendingAction.batchToolCallIds) {
             if (id === pendingAction.toolCallId) {
-                await this.resolvePendingCall(prepared, resume, emit, emitCommitted, onResolvedToolSuccess);
+                await this.resolvePendingCall(prepared, resume, emit, emitCommitted, onResolvedToolSuccess, onExecuteStart);
             }
             else if (!completed.has(id)) {
                 const name = names.get(id) ?? "unknown";
@@ -2464,7 +2503,7 @@ export class Runner {
             }
         }
     }
-    async resolvePendingCall(prepared, resume, emit, emitCommitted, onResolvedToolSuccess) {
+    async resolvePendingCall(prepared, resume, emit, emitCommitted, onResolvedToolSuccess, onExecuteStart) {
         const { pendingAction } = resume.cp;
         const { outcome } = resume;
         if (pendingAction.kind !== "tool_approval" || outcome.gate !== "policy_ask")
@@ -2495,26 +2534,29 @@ export class Runner {
             throw e;
         }
         const args = resolvedArgs;
+        onExecuteStart?.(pendingAction.toolCallId);
+        let res;
         try {
-            const res = await tool.execute(pendingAction.toolCallId, args, prepared.abortController.signal);
-            emitEnd(false);
-            onResolvedToolSuccess?.(canonicalToolName(pendingAction.toolName), res.details);
-            const eid = await prepared.session.appendMessage({
-                role: "toolResult",
-                toolCallId: pendingAction.toolCallId,
-                toolName: pendingAction.toolName,
-                content: res.content,
-                details: res.details,
-                isError: false,
-                timestamp: Date.now(),
-            });
-            emitCommitted(eid, "toolResult", pendingAction.toolCallId);
+            res = await tool.execute(pendingAction.toolCallId, args, prepared.abortController.signal);
         }
         catch (err) {
             emitEnd(true);
             const eid = await prepared.session.appendMessage(toolResultMsg(pendingAction.toolCallId, pendingAction.toolName, `Error: ${err instanceof Error ? err.message : String(err)}`, true));
             emitCommitted(eid, "toolResult", pendingAction.toolCallId);
+            return;
         }
+        emitEnd(false);
+        onResolvedToolSuccess?.(canonicalToolName(pendingAction.toolName), res.details);
+        const eid = await prepared.session.appendMessage({
+            role: "toolResult",
+            toolCallId: pendingAction.toolCallId,
+            toolName: pendingAction.toolName,
+            content: res.content,
+            details: res.details,
+            isError: false,
+            timestamp: Date.now(),
+        });
+        emitCommitted(eid, "toolResult", pendingAction.toolCallId);
     }
     seamCCompactionOptions(prepared) {
         if (!this.deps.summaryProvider)