@sema-agent/core 1.451.0 → 1.452.0

package/dist/orchestration/workflow.d.ts

@@ -93,6 +93,15 @@ export interface WorkflowRun {
     error?: string;
     result?: string;
     resultFull?: string;
+    completionId?: string;
+    resume?: {
+        fromRunId: string;
+        journalEntries: number;
+        replayed: number;
+        divergedAtOrdinal?: number;
+        divergedReason?: string;
+    };
+    journalSkips?: number;
 }
 export type WorkflowEvent = {
     type: "run_start";

package/dist/orchestration/workflow.js

@@ -2,10 +2,11 @@ import { resolveModelDisplayLabel } from "../core/roles.js";
 import { AsyncLocalStorage } from "node:async_hooks";
 import { createHash, randomUUID } from "node:crypto";
 import { availableParallelism } from "node:os";
+import { uuidv7 } from "../internal/harness.js";
 import { builtinAgentDefinitions } from "../agents/builtin-agents.js";
 import { GENERAL_PURPOSE_SUBAGENT_TYPE } from "../agents/subagent.js";
 import { combinePolicies, createAllowDenyPolicy } from "../core/tool-policy.js";
-import { callKeyOrdinal } from "../core/workflow-journal-store.js";
+import { callKeyOrdinal, oversizeJournalResult, journalOversizeTombstone, JOURNAL_OVERSIZE_ERROR_CODE, MAX_JOURNAL_RESULT_BYTES } from "../core/workflow-journal-store.js";
 import { isWorkflowRunActive, closeWorkflowChannel, markWorkflowActive, publishWorkflowEvent } from "./workflow-observe.js";
 import { isDurablePause, mapNestedSuspend } from "../agents/suspend-guard.js";
 import { boundInputHashOf } from "../core/canonical-json.js";
@@ -551,13 +552,39 @@ export function startWorkflow(runner, fn, opts = {}, internals) {
         });
     };
     let journalTail = Promise.resolve();
-    const journalAppend = async (callKey, result) => {
+    const journalAppend = async (callKey, result, label) => {
         const dbg = typeof process !== "undefined" && process.env?.SEMA_DEBUG_WORKFLOW_JOURNAL === "1";
         if (!journalStore) {
             if (dbg)
                 console.error(`[sema:wf-journal] runId=${runId} callKey=${callKey} SKIP (no journalStore on this run)`);
             return;
         }
+        let serialized;
+        try {
+            serialized = JSON.stringify(result);
+        }
+        catch {
+            serialized = undefined;
+        }
+        if (serialized !== undefined && oversizeJournalResult(serialized)) {
+            const bytes = Buffer.byteLength(serialized, "utf8");
+            if (!finalized)
+                run.journalSkips = (run.journalSkips ?? 0) + 1;
+            emitRunLog(`resume-journal: agent #${callKeyOrdinal(callKey)}${label !== undefined ? ` "${label.slice(0, 80)}"` : ""} result is ${bytes} bytes, ` +
+                `over the ${MAX_JOURNAL_RESULT_BYTES}-byte per-entry cap — NOT cached. A resume from this run re-runs this agent and everything after it live.`);
+            const tombstone = journalStore.append(runId, scope, { callKey, result: journalOversizeTombstone(result, bytes) });
+            journalTail = journalTail.then(() => tombstone).catch(() => undefined);
+            try {
+                await tombstone;
+                if (dbg)
+                    console.error(`[sema:wf-journal] runId=${runId} callKey=${callKey} OVERSIZE-TOMBSTONE (${bytes} bytes)`);
+            }
+            catch (err) {
+                if (dbg)
+                    console.error(`[sema:wf-journal] runId=${runId} callKey=${callKey} TOMBSTONE-ERROR ${err instanceof Error ? err.message : String(err)}`);
+            }
+            return;
+        }
         const p = journalStore.append(runId, scope, { callKey, result });
         journalTail = journalTail.then(() => p).catch(() => undefined);
         try {
@@ -623,6 +650,17 @@ export function startWorkflow(runner, fn, opts = {}, internals) {
             : message;
         emit({ type: "log", runId, message: capped, ts: now() });
     };
+    let divergenceNoted = false;
+    const noteDivergence = (ordinal, reason) => {
+        if (opts.resumeFromRunId === undefined || divergenceNoted)
+            return;
+        divergenceNoted = true;
+        if (run.resume) {
+            run.resume.divergedAtOrdinal = ordinal;
+            run.resume.divergedReason = reason;
+        }
+        emitRunLog(`resume: replay stopped at agent #${ordinal} (${reason}) — this call and every later call run live.`);
+    };
     const settleAgentError = (rec, result, label, rawOutput) => {
         if (result.status !== "completed") {
             if (result.errorCode !== undefined)
@@ -695,6 +733,8 @@ export function startWorkflow(runner, fn, opts = {}, internals) {
                         ...(rs.toolCalls !== undefined ? { toolCalls: rs.toolCalls } : {}),
                         output: cachedOutput,
                     };
+                    if (run.resume)
+                        run.resume.replayed += 1;
                     run.agents.push(replayRec);
                     if (phaseInstance)
                         agentPhaseOf.set(replayRec, phaseInstance);
@@ -710,6 +750,17 @@ export function startWorkflow(runner, fn, opts = {}, internals) {
                     }
                     return r;
                 }
+                noteDivergence(run.agents.length, cached === undefined
+                    ? run.resume?.journalEntries === 0
+                        ? "the journal loaded no entries"
+                        : run.agents.length >= replayByOrdinal.length
+                            ? "a new call past the journaled prefix — the prior run ended before this point (the intended extend-the-script resume shape)"
+                            : "no journal entry at this ordinal (never recorded, e.g. an oversize result under a pre-tombstone engine, or dropped)"
+                    : cached.callKey !== callKey
+                        ? "the call key changed — the script or its args differ here"
+                        : cached.result.errorCode === JOURNAL_OVERSIZE_ERROR_CODE
+                            ? "the prior result exceeded the journal size cap and was never cached"
+                            : `the journaled result was ${cached.result.status}, not completed`);
                 diverged = true;
             }
             if (budgetTotal !== null && spent() >= budgetTotal) {
@@ -963,7 +1014,7 @@ export function startWorkflow(runner, fn, opts = {}, internals) {
                         status: "failed",
                         result: boundedRedactedSummary(err instanceof Error ? err.message : String(err), 500),
                         stats: { turns: 0, tokens: 0, costMicroUsd: 0 },
-                    }).catch(() => undefined);
+                    }, label).catch(() => undefined);
                     bceTerminal(callKey, "failed", rec.output ?? (err instanceof Error ? err.message : String(err)), rec.sessionId, rec.stats);
                 }
                 throw err;
@@ -992,7 +1043,7 @@ export function startWorkflow(runner, fn, opts = {}, internals) {
             emit({ type: "agent_end", runId, label, phase, ...(groupId !== undefined ? { groupId } : {}), status: rec.status, output, ...(rec.errorCode !== undefined ? { errorCode: rec.errorCode } : {}), ...(result.stats.toolCalls !== undefined ? { toolCalls: result.stats.toolCalls } : {}), ts: rec.endedAt });
             void persist("update");
             bceTerminal(callKey, rec.status === "completed" ? "completed" : "failed", output, result.sessionId || undefined, rec.stats);
-            await journalAppend(callKey, result);
+            await journalAppend(callKey, result, label);
             if (agentOpts.schema && result.status === "completed" && result.structuredOutput === undefined) {
                 throw new WorkflowAgentSchemaError(label, result);
             }
@@ -1026,6 +1077,7 @@ export function startWorkflow(runner, fn, opts = {}, internals) {
             const callKey = workflowAgentCallKey(run.agents.length, specForIdentity, agentOpts);
             const prompt = boundedRedactedSummary(spec.systemPrompt ? `${spec.systemPrompt}\n\n${spec.objective}` : spec.objective, MAX_TRANSCRIPT_CHARS);
             const model = workflowModelLabel(specForIdentity);
+            noteDivergence(run.agents.length, "ctx.agentStream results are never replayed");
             diverged = true;
             const { tail: activityTail, onActivity } = makeActivityCapture(callKey, label, groupId);
             const rec = { label, callKey, ...(groupId !== undefined ? { groupId } : {}), phase, prompt, ...(model !== undefined ? { model } : {}), status: "running", queuedAt: now() };
@@ -1189,7 +1241,7 @@ export function startWorkflow(runner, fn, opts = {}, internals) {
                 emit({ type: "agent_end", runId, label, phase, ...(groupId !== undefined ? { groupId } : {}), status: rec.status, output, ...(rec.errorCode !== undefined ? { errorCode: rec.errorCode } : {}), ...(result.stats.toolCalls !== undefined ? { toolCalls: result.stats.toolCalls } : {}), ts: rec.endedAt });
                 void persist("update");
                 bceTerminal(callKey, rec.status === "completed" ? "completed" : "failed", output, result.sessionId || undefined, rec.stats);
-                await journalAppend(callKey, result).catch(() => undefined);
+                await journalAppend(callKey, result, label).catch(() => undefined);
                 if (agentOpts.schema && result.status === "completed" && result.structuredOutput === undefined) {
                     throw new WorkflowAgentSchemaError(label, result);
                 }
@@ -1239,6 +1291,7 @@ export function startWorkflow(runner, fn, opts = {}, internals) {
             const fanOutOpts = trailing !== undefined && typeof trailing !== "function" ? trailing : undefined;
             const stages = (fanOutOpts !== undefined ? stagesAndOpts.slice(0, -1) : stagesAndOpts);
             const errorsOut = fanOutOpts?.errors;
+            noteDivergence(run.agents.length, "ctx.pipeline ordinals are latency-dependent, so its calls always run live on a resume");
             diverged = true;
             const settled = await Promise.all(items.map((item, index) => stages
                 .reduce((acc, stage) => acc.then((prev) => stage(prev, item, index)), Promise.resolve(item))
@@ -1370,6 +1423,19 @@ export function startWorkflow(runner, fn, opts = {}, internals) {
                     const entries = await journalStore.load(opts.resumeFromRunId, scope);
                     for (const e of entries)
                         replayByOrdinal[callKeyOrdinal(e.callKey)] = e;
+                    run.resume = {
+                        fromRunId: opts.resumeFromRunId.replace(/[^A-Za-z0-9_.:-]/g, "").slice(0, 64),
+                        journalEntries: entries.length,
+                        replayed: 0,
+                    };
+                    if (entries.length === 0) {
+                        emitRunLog("resume: the prior run's journal yielded NO entries under this run's scope — nothing will replay; the entire script runs live. " +
+                            "The prior run id may be unknown or pruned, recorded under a different scope, or its journal unreadable.");
+                    }
+                    else {
+                        emitRunLog(`resume: loaded ${entries.length} journaled result${entries.length === 1 ? "" : "s"} — the longest unchanged prefix replays; the first changed/new call and everything after run live.`);
+                    }
+                    void persist("update");
                 }
                 return fn(ctx);
             };
@@ -1416,6 +1482,7 @@ export function startWorkflow(runner, fn, opts = {}, internals) {
             if (run.result !== fullResult)
                 run.resultFull = fullResult;
             run.status = "completed";
+            run.completionId ??= uuidv7();
             run.endedAt = now();
             closeAbandonedAgents(run.endedAt);
             restampPhaseFailures();
@@ -1431,6 +1498,7 @@ export function startWorkflow(runner, fn, opts = {}, internals) {
             settleActiveUsageBeats();
             closeOpenMarker("failed");
             run.status = "failed";
+            run.completionId ??= uuidv7();
             run.endedAt = now();
             run.error = err instanceof Error ? err.message : String(err);
             closeAbandonedAgents(run.endedAt);
@@ -1440,6 +1508,13 @@ export function startWorkflow(runner, fn, opts = {}, internals) {
                 run.agentFailures = failedRunFailures;
             emit({ type: "run_end", runId, status: "failed", ...(failedRunFailures > 0 ? { agentFailures: failedRunFailures } : {}), ts: run.endedAt });
             await persist("update");
+            if (err !== null && (typeof err === "object" || typeof err === "function")) {
+                try {
+                    Object.defineProperty(err, "workflowRun", { value: run, enumerable: false, configurable: true });
+                }
+                catch {
+                }
+            }
             throw err;
         }
         finally {

package/dist/stores/cc/mailbox-store.js

@@ -89,9 +89,12 @@ export function createCcFileMailboxStore(opts) {
             leases.set(key, { owner, expiresAt: t + ttlMs, maxSeq });
             return Promise.resolve({ maxSeq, messages: pending });
         },
-        ack: (scope, handle, upToSeq) => {
+        ack: (scope, handle, owner, upToSeq) => {
             requireDefaultScope(scope);
             const path = inboxPath(handle);
+            const held = leases.get(path);
+            if (held === undefined || held.owner !== owner)
+                return Promise.resolve();
             if (!existsSync(path))
                 return Promise.resolve();
             return withCcLock(path, () => {
@@ -105,6 +108,8 @@ export function createCcFileMailboxStore(opts) {
                 }
                 if (changed)
                     saveBox(path, box);
+                if (held.maxSeq <= upToSeq)
+                    leases.delete(path);
             });
         },
         releaseLease: (scope, handle, owner) => {

package/dist/stores/file/background-agent-store.js

@@ -1,5 +1,5 @@
 import { join } from "node:path";
-import { BackgroundAgentStoreError, } from "../../core/background-agent-store.js";
+import { BackgroundAgentStoreError, STALE_RUNNING_REAP_ATTRIBUTION, } from "../../core/background-agent-store.js";
 import { canonicalStoreKey, AppendLog, atomicWriteFile, ensureDir, readJsonlRecords } from "./fs-atomic.js";
 const sharedAgentDirs = new Map();
 export class FileBackgroundAgentStore {
@@ -205,7 +205,8 @@ export class FileBackgroundAgentStore {
                     const next = structuredClone(live);
                     next.status = "failed";
                     next.stoppedBy = "system";
-                    next.summary = next.summary ?? "host process interrupted (stale running row reaped)";
+                    next.summary = next.summary ?? STALE_RUNNING_REAP_ATTRIBUTION;
+                    next.error = next.error ?? STALE_RUNNING_REAP_ATTRIBUTION;
                     next.settledAt = now;
                     next.updatedAt = now;
                     next.rev = live.rev + 1;

package/dist/stores/file/mailbox-store.d.ts

@@ -21,7 +21,7 @@ export declare class FileMailboxStore implements MailboxStore {
         sentAt: number;
     }): Promise<number>;
     claimLease(scope: string, handle: string, owner: string, ttlMs: number, now?: number): Promise<MailboxLease | null>;
-    ack(scope: string, handle: string, upToSeq: number): Promise<void>;
+    ack(scope: string, handle: string, owner: string, upToSeq: number): Promise<void>;
     releaseLease(scope: string, handle: string, owner: string): Promise<void>;
     peekCount(scope: string, handle: string): Promise<number>;
     drop(scope: string, handle: string): Promise<void>;

package/dist/stores/file/mailbox-store.js

@@ -1,6 +1,6 @@
 import { join, resolve, sep } from "node:path";
 import { existsSync, realpathSync } from "node:fs";
-import {} from "../../core/mailbox-store.js";
+import { newestSentAt, } from "../../core/mailbox-store.js";
 import { AppendLog, atomicWriteFile, canonicalStoreKey, ensureDir, readJsonlRecords, sanitizeScope, sanitizePathComponent } from "./fs-atomic.js";
 function realpathSyncSafe(p) {
     try {
@@ -86,7 +86,7 @@ export class FileMailboxStore {
             lines.push(JSON.stringify({ t: "lease", ...b.lease }));
         b.log.closeForSwap();
         atomicWriteFile(this.tmpDir, path, lines.length ? `${lines.join("\n")}\n` : "");
-        b.events = lines.length;
+        b.events = 0;
     }
     async append(scope, handle, msg) {
         if (scope === undefined || scope === "")
@@ -111,9 +111,11 @@ export class FileMailboxStore {
             return { messages: b.messages.map((m) => ({ ...m })), maxSeq };
         });
     }
-    async ack(scope, handle, upToSeq) {
+    async ack(scope, handle, owner, upToSeq) {
         return withPathLock(this.lockKey(scope, handle), () => {
             const b = this.load(scope, handle);
+            if (b.lease === undefined || b.lease.owner !== owner)
+                return;
             this.commit(b, this.boxPath(scope, handle), { t: "ack", upToSeq });
         });
     }
@@ -160,15 +162,15 @@ export class FileMailboxStore {
         for (const [key, b] of [...sharedBoxes]) {
             if (!b.path.startsWith(prefix))
                 continue;
-            const seen = b.messages[b.messages.length - 1];
-            if (seen === undefined || seen.sentAt >= now - maxAgeMs)
+            const seen = newestSentAt(b.messages);
+            if (seen === undefined || seen >= now - maxAgeMs)
                 continue;
             const swept = await withPathLock(key, () => {
                 const live = sharedBoxes.get(key);
                 if (live === undefined || live !== b)
                     return false;
-                const newest = b.messages[b.messages.length - 1];
-                if (newest === undefined || newest.sentAt >= now - maxAgeMs)
+                const newest = newestSentAt(b.messages);
+                if (newest === undefined || newest >= now - maxAgeMs)
                     return false;
                 b.messages = [];
                 delete b.lease;

package/dist/tools/fs/encoding.d.ts

@@ -13,3 +13,8 @@ export declare function detectFileEncoding(bytes: Uint8Array): DetectedFileEncod
 export declare function decodeTextBytes(bytes: Uint8Array): DecodedTextFile;
 export declare function encodeTextForFile(text: string, encoding: DetectedFileEncoding, endings: DetectedLineEndings | "preserve"): string | Uint8Array;
 export declare function normalizeEditText(s: string): string;
+export declare function splitLeadingBom(text: string): {
+    hadBom: boolean;
+    text: string;
+};
+export declare function normalizeFileText(s: string): string;

package/dist/tools/fs/encoding.js

@@ -41,3 +41,9 @@ export function encodeTextForFile(text, encoding, endings) {
 export function normalizeEditText(s) {
     return s.replaceAll("\r\n", "\n");
 }
+export function splitLeadingBom(text) {
+    return text.charCodeAt(0) === 0xfeff ? { hadBom: true, text: text.slice(1) } : { hadBom: false, text };
+}
+export function normalizeFileText(s) {
+    return normalizeEditText(splitLeadingBom(s).text);
+}