larkin 0.5.16 → 0.5.18

package/dist/runtime/runtime-host.cjs

@@ -308,6 +308,16 @@ function createRuntimeHost(options) {
         maxAttempts: options.retryPolicy?.maxAttempts ?? 6,
         stableWindowMs: options.retryPolicy?.stableWindowMs ?? 30_000,
     };
+    const turnEndRetryPolicy = {
+        baseDelayMs: options.turnEndRetryPolicy?.baseDelayMs ?? 1_000,
+        maxDelayMs: options.turnEndRetryPolicy?.maxDelayMs ?? 30_000,
+        maxAttempts: options.turnEndRetryPolicy?.maxAttempts ?? 3,
+    };
+    const turnEndRetryScheduler = options.turnEndRetryScheduler ?? {
+        now: Date.now,
+        setTimeout,
+        clearTimeout,
+    };
     const emit = (event) => {
         if (event.type === "delivery")
             telemetry?.delivery(event.agentId, event.messageId, event.status);
@@ -316,6 +326,7 @@ function createRuntimeHost(options) {
     };
     const runtimeEnv = (config, generation) => {
         const base = {
+            LARKIN_RUNTIME: "1",
             LARKIN_AGENT_ID: config.agentId,
             ...(generation ? {
                 LARKIN_RUNTIME_OBSERVATION_GENERATION: generation,
@@ -361,6 +372,8 @@ function createRuntimeHost(options) {
         record.updatedAt = now();
         agent.records.set(record.deliveryId, record);
         agent.byMessage.set(record.messageId, record.deliveryId);
+        if (status === "consumed" || status === "error")
+            clearPendingRetryTimerWhenIdle(agent);
         emitConsumed(agent, persist(agent));
         return agent.records.get(record.deliveryId) ?? record;
     };
@@ -466,8 +479,42 @@ function createRuntimeHost(options) {
                 emit({ type: "delivery", agentId: agent.config.agentId, deliveryId: next.deliveryId, messageId: next.messageId, status: "consumed" });
             }
         }
+        clearPendingRetryTimerWhenIdle(agent);
     };
     const TURN_END_RETRY_REASON = "runtime turn ended before Inbox consumption was observed";
+    const turnEndRetryDelayMs = (attempt) => Math.min(turnEndRetryPolicy.maxDelayMs, turnEndRetryPolicy.baseDelayMs * 2 ** (attempt - 1));
+    const isRetryDue = (record) => !record.retryNotBefore || Number.isNaN(Date.parse(record.retryNotBefore))
+        || Date.parse(record.retryNotBefore) <= turnEndRetryScheduler.now();
+    const clearDeliveryRetryTimer = (agent) => {
+        if (!agent.deliveryRetryTimer)
+            return;
+        turnEndRetryScheduler.clearTimeout(agent.deliveryRetryTimer);
+        agent.deliveryRetryTimer = null;
+    };
+    function clearPendingRetryTimerWhenIdle(agent) {
+        if (!agent.deliveryRetryTimer)
+            return;
+        if ([...agent.records.values()].some((record) => record.status === "pending" && record.retryNotBefore))
+            return;
+        clearDeliveryRetryTimer(agent);
+    }
+    const schedulePendingRetry = (agent) => {
+        if (agent.stopped || agent.deliveryRetryTimer)
+            return;
+        const next = [...agent.records.values()]
+            .filter((record) => record.status === "pending" && record.retryNotBefore)
+            .map((record) => Date.parse(record.retryNotBefore))
+            .filter((at) => Number.isFinite(at))
+            .sort((left, right) => left - right)[0];
+        if (next === undefined)
+            return;
+        const delay = Math.max(0, next - turnEndRetryScheduler.now());
+        agent.deliveryRetryTimer = turnEndRetryScheduler.setTimeout(() => {
+            agent.deliveryRetryTimer = null;
+            void retryPending(agent);
+        }, delay);
+        agent.deliveryRetryTimer.unref?.();
+    };
     const INBOX_UPDATE_PROMOTE_REASON = "accepted inbox_update promoted after Agent became idle";
     const isInboxUpdateKind = (kind) => kind === "inbox_update" || kind === "inbox-update";
     const agentIsIdleForInboxScan = (agent) => !agent.stopped && !agent.busy && !agent.submitting && !agent.turnInProgress && !agent.starting && Boolean(agent.session);
@@ -495,13 +542,32 @@ function createRuntimeHost(options) {
             for (const record of agent.records.values()) {
                 if (record.status !== "accepted")
                     continue;
-                emit({ type: "delivery", agentId: agent.config.agentId, deliveryId: record.deliveryId,
-                    messageId: record.messageId, status: "deferred", reason: TURN_END_RETRY_REASON });
-            }
+                const attempts = record.turnEndRetryAttempts ?? 0;
+                if (attempts >= turnEndRetryPolicy.maxAttempts) {
+                    record.reason = `turn-end redelivery exhausted after ${turnEndRetryPolicy.maxAttempts} attempts`;
+                    record.retryable = false;
+                    const terminal = setRecord(agent, record, "error");
+                    emit({ type: "delivery", agentId: agent.config.agentId, deliveryId: terminal.deliveryId,
+                        messageId: terminal.messageId, status: "error", reason: terminal.reason });
+                    continue;
+                }
+                const attempt = attempts + 1;
+                const delay = turnEndRetryDelayMs(attempt);
+                record.turnEndRetryAttempts = attempt;
+                record.retryNotBefore = new Date(turnEndRetryScheduler.now() + delay).toISOString();
+                record.reason = TURN_END_RETRY_REASON;
+                record.retryable = true;
+                record.input = { ...record.input, attempt: (record.input.attempt ?? 0) + 1 };
+                const deferred = setRecord(agent, record, "pending");
+                emit({ type: "delivery", agentId: agent.config.agentId, deliveryId: deferred.deliveryId,
+                    messageId: deferred.messageId, status: "deferred", reason: TURN_END_RETRY_REASON });
+            }
+            schedulePendingRetry(agent);
             return;
         }
         const consumed = [];
         const deferred = [];
+        const exhausted = [];
         try {
             store.withInboxTransaction(() => {
                 const inboxIds = new Set(store.readNdjson("inbox").flatMap((row) => typeof row?.message_id === "string" ? [row.message_id] : []));
@@ -522,8 +588,31 @@ function createRuntimeHost(options) {
                     if (current.status !== "accepted" || candidate.status !== "accepted" || !inboxIds.has(candidate.messageId)) {
                         return candidate;
                     }
-                    const next = { ...candidate, status: "pending", updatedAt: now(),
-                        reason: TURN_END_RETRY_REASON, retryable: true };
+                    const attempts = candidate.turnEndRetryAttempts ?? 0;
+                    if (attempts >= turnEndRetryPolicy.maxAttempts) {
+                        const terminal = { ...candidate, status: "error", updatedAt: now(),
+                            reason: `turn-end redelivery exhausted after ${turnEndRetryPolicy.maxAttempts} attempts`,
+                            retryable: false };
+                        changed = true;
+                        pendingUpdates.push(terminal);
+                        return terminal;
+                    }
+                    const attempt = attempts + 1;
+                    const delay = turnEndRetryDelayMs(attempt);
+                    const staleInput = candidate.input && typeof candidate.input === "object" ? candidate.input : null;
+                    const next = {
+                        ...candidate,
+                        status: "pending",
+                        updatedAt: now(),
+                        reason: TURN_END_RETRY_REASON,
+                        retryable: true,
+                        turnEndRetryAttempts: attempt,
+                        retryNotBefore: new Date(turnEndRetryScheduler.now() + delay).toISOString(),
+                        input: {
+                            ...(staleInput ?? { inputId: candidate.deliveryId, deliveryId: candidate.deliveryId, kind: "wake", text: "" }),
+                            attempt: (Number.isSafeInteger(staleInput?.attempt) && Number(staleInput?.attempt) >= 0 ? Number(staleInput.attempt) : 0) + 1,
+                        },
+                    };
                     changed = true;
                     pendingUpdates.push(next);
                     return next;
@@ -532,7 +621,10 @@ function createRuntimeHost(options) {
                     store.writeJson("runtimeDeliveries", { ...disk, records });
                     for (const next of pendingUpdates) {
                         agent.records.set(next.deliveryId, next);
-                        deferred.push(next);
+                        if (next.status === "error")
+                            exhausted.push(next);
+                        else
+                            deferred.push(next);
                     }
                 }
             });
@@ -545,6 +637,11 @@ function createRuntimeHost(options) {
         for (const record of deferred)
             emit({ type: "delivery", agentId: agent.config.agentId,
                 deliveryId: record.deliveryId, messageId: record.messageId, status: "deferred", reason: TURN_END_RETRY_REASON });
+        for (const record of exhausted)
+            emit({ type: "delivery", agentId: agent.config.agentId,
+                deliveryId: record.deliveryId, messageId: record.messageId, status: "error", reason: record.reason });
+        if (deferred.length)
+            schedulePendingRetry(agent);
     };
     // Production callers persist the canonical Inbox before calling deliver(). If a
     // concurrent drain wins between that append and ledger creation, close only the
@@ -743,9 +840,11 @@ function createRuntimeHost(options) {
                 if (idleGate)
                     await idleGate;
             }
-            const record = [...agent.records.values()].find((candidate) => candidate.status === "pending" || candidate.status === "submitting");
+            const record = [...agent.records.values()].find((candidate) => (candidate.status === "pending" && isRetryDue(candidate)) || candidate.status === "submitting");
             if (record)
                 await submit(agent, record, false);
+            else
+                schedulePendingRetry(agent);
         })();
         agent.retryPendingInFlight = run;
         try {
@@ -1519,6 +1618,8 @@ function createRuntimeHost(options) {
                         clearInterval(previous.poller);
                     if (previous.retryTimer)
                         clearTimeout(previous.retryTimer);
+                    if (previous.deliveryRetryTimer)
+                        turnEndRetryScheduler.clearTimeout(previous.deliveryRetryTimer);
                     if (previous.stabilityTimer)
                         clearTimeout(previous.stabilityTimer);
                     const previousFailure = currentAuthFailure(previous);
@@ -1530,7 +1631,7 @@ function createRuntimeHost(options) {
                         }, previousFailure));
                     const candidate = {
                         ...previous, config, adapter, session, launchId: node_crypto_1.default.randomUUID(), busy: false, submitting: false,
-                        starting: null, retryAfterSubmit: false, generation: 0, poller: null, retryTimer: null,
+                        starting: null, retryAfterSubmit: false, generation: 0, poller: null, retryTimer: null, deliveryRetryTimer: null,
                         recreateAttempts: 0, stabilityTimer: null, recreateReason: null, stopped: false,
                         disabledReason: null, configurationRecovery: null,
                         authFailureActive: scoped && previous.authFailureActive,
@@ -1623,6 +1724,7 @@ function createRuntimeHost(options) {
                 if (pendingAfterCreate > 0) {
                     throw new RuntimeSessionResetError("inbox_backlog", `Agent ${agentId} received Inbox backlog during reset`, pendingAfterCreate);
                 }
+                clearDeliveryRetryTimer(agent);
                 agent.generation += 1;
                 agent.launchId = node_crypto_1.default.randomUUID();
                 agent.config.sessionId = null;
@@ -1893,7 +1995,7 @@ function createRuntimeHost(options) {
                     retryPendingInFlight: null,
                     records: new Map(persisted.records.map((record) => [record.deliveryId, record])),
                     byMessage: new Map(persisted.records.map((record) => [record.messageId, record.deliveryId])),
-                    generation: 0, poller: null, retryTimer: null, recreateAttempts: 0,
+                    generation: 0, poller: null, retryTimer: null, deliveryRetryTimer: null, recreateAttempts: 0,
                     stabilityTimer: null, recreateReason: null, stopped: false, disabledReason: null,
                     configurationRecovery: null, stateStore,
                     compactionBreaker: config.runtime === "pi" && stateStore
@@ -2065,6 +2167,7 @@ function createRuntimeHost(options) {
                 clearInterval(agent.poller);
             if (agent.retryTimer)
                 clearTimeout(agent.retryTimer);
+            clearDeliveryRetryTimer(agent);
             if (agent.stabilityTimer)
                 clearTimeout(agent.stabilityTimer);
             if (agent.busy)

package/dist/runtime/runtime-host.mjs

@@ -299,6 +299,16 @@ export function createRuntimeHost(options) {
         maxAttempts: options.retryPolicy?.maxAttempts ?? 6,
         stableWindowMs: options.retryPolicy?.stableWindowMs ?? 30_000,
     };
+    const turnEndRetryPolicy = {
+        baseDelayMs: options.turnEndRetryPolicy?.baseDelayMs ?? 1_000,
+        maxDelayMs: options.turnEndRetryPolicy?.maxDelayMs ?? 30_000,
+        maxAttempts: options.turnEndRetryPolicy?.maxAttempts ?? 3,
+    };
+    const turnEndRetryScheduler = options.turnEndRetryScheduler ?? {
+        now: Date.now,
+        setTimeout,
+        clearTimeout,
+    };
     const emit = (event) => {
         if (event.type === "delivery")
             telemetry?.delivery(event.agentId, event.messageId, event.status);
@@ -307,6 +317,7 @@ export function createRuntimeHost(options) {
     };
     const runtimeEnv = (config, generation) => {
         const base = {
+            LARKIN_RUNTIME: "1",
             LARKIN_AGENT_ID: config.agentId,
             ...(generation ? {
                 LARKIN_RUNTIME_OBSERVATION_GENERATION: generation,
@@ -352,6 +363,8 @@ export function createRuntimeHost(options) {
         record.updatedAt = now();
         agent.records.set(record.deliveryId, record);
         agent.byMessage.set(record.messageId, record.deliveryId);
+        if (status === "consumed" || status === "error")
+            clearPendingRetryTimerWhenIdle(agent);
         emitConsumed(agent, persist(agent));
         return agent.records.get(record.deliveryId) ?? record;
     };
@@ -457,8 +470,42 @@ export function createRuntimeHost(options) {
                 emit({ type: "delivery", agentId: agent.config.agentId, deliveryId: next.deliveryId, messageId: next.messageId, status: "consumed" });
             }
         }
+        clearPendingRetryTimerWhenIdle(agent);
     };
     const TURN_END_RETRY_REASON = "runtime turn ended before Inbox consumption was observed";
+    const turnEndRetryDelayMs = (attempt) => Math.min(turnEndRetryPolicy.maxDelayMs, turnEndRetryPolicy.baseDelayMs * 2 ** (attempt - 1));
+    const isRetryDue = (record) => !record.retryNotBefore || Number.isNaN(Date.parse(record.retryNotBefore))
+        || Date.parse(record.retryNotBefore) <= turnEndRetryScheduler.now();
+    const clearDeliveryRetryTimer = (agent) => {
+        if (!agent.deliveryRetryTimer)
+            return;
+        turnEndRetryScheduler.clearTimeout(agent.deliveryRetryTimer);
+        agent.deliveryRetryTimer = null;
+    };
+    function clearPendingRetryTimerWhenIdle(agent) {
+        if (!agent.deliveryRetryTimer)
+            return;
+        if ([...agent.records.values()].some((record) => record.status === "pending" && record.retryNotBefore))
+            return;
+        clearDeliveryRetryTimer(agent);
+    }
+    const schedulePendingRetry = (agent) => {
+        if (agent.stopped || agent.deliveryRetryTimer)
+            return;
+        const next = [...agent.records.values()]
+            .filter((record) => record.status === "pending" && record.retryNotBefore)
+            .map((record) => Date.parse(record.retryNotBefore))
+            .filter((at) => Number.isFinite(at))
+            .sort((left, right) => left - right)[0];
+        if (next === undefined)
+            return;
+        const delay = Math.max(0, next - turnEndRetryScheduler.now());
+        agent.deliveryRetryTimer = turnEndRetryScheduler.setTimeout(() => {
+            agent.deliveryRetryTimer = null;
+            void retryPending(agent);
+        }, delay);
+        agent.deliveryRetryTimer.unref?.();
+    };
     const INBOX_UPDATE_PROMOTE_REASON = "accepted inbox_update promoted after Agent became idle";
     const isInboxUpdateKind = (kind) => kind === "inbox_update" || kind === "inbox-update";
     const agentIsIdleForInboxScan = (agent) => !agent.stopped && !agent.busy && !agent.submitting && !agent.turnInProgress && !agent.starting && Boolean(agent.session);
@@ -486,13 +533,32 @@ export function createRuntimeHost(options) {
             for (const record of agent.records.values()) {
                 if (record.status !== "accepted")
                     continue;
-                emit({ type: "delivery", agentId: agent.config.agentId, deliveryId: record.deliveryId,
-                    messageId: record.messageId, status: "deferred", reason: TURN_END_RETRY_REASON });
-            }
+                const attempts = record.turnEndRetryAttempts ?? 0;
+                if (attempts >= turnEndRetryPolicy.maxAttempts) {
+                    record.reason = `turn-end redelivery exhausted after ${turnEndRetryPolicy.maxAttempts} attempts`;
+                    record.retryable = false;
+                    const terminal = setRecord(agent, record, "error");
+                    emit({ type: "delivery", agentId: agent.config.agentId, deliveryId: terminal.deliveryId,
+                        messageId: terminal.messageId, status: "error", reason: terminal.reason });
+                    continue;
+                }
+                const attempt = attempts + 1;
+                const delay = turnEndRetryDelayMs(attempt);
+                record.turnEndRetryAttempts = attempt;
+                record.retryNotBefore = new Date(turnEndRetryScheduler.now() + delay).toISOString();
+                record.reason = TURN_END_RETRY_REASON;
+                record.retryable = true;
+                record.input = { ...record.input, attempt: (record.input.attempt ?? 0) + 1 };
+                const deferred = setRecord(agent, record, "pending");
+                emit({ type: "delivery", agentId: agent.config.agentId, deliveryId: deferred.deliveryId,
+                    messageId: deferred.messageId, status: "deferred", reason: TURN_END_RETRY_REASON });
+            }
+            schedulePendingRetry(agent);
             return;
         }
         const consumed = [];
         const deferred = [];
+        const exhausted = [];
         try {
             store.withInboxTransaction(() => {
                 const inboxIds = new Set(store.readNdjson("inbox").flatMap((row) => typeof row?.message_id === "string" ? [row.message_id] : []));
@@ -513,8 +579,31 @@ export function createRuntimeHost(options) {
                     if (current.status !== "accepted" || candidate.status !== "accepted" || !inboxIds.has(candidate.messageId)) {
                         return candidate;
                     }
-                    const next = { ...candidate, status: "pending", updatedAt: now(),
-                        reason: TURN_END_RETRY_REASON, retryable: true };
+                    const attempts = candidate.turnEndRetryAttempts ?? 0;
+                    if (attempts >= turnEndRetryPolicy.maxAttempts) {
+                        const terminal = { ...candidate, status: "error", updatedAt: now(),
+                            reason: `turn-end redelivery exhausted after ${turnEndRetryPolicy.maxAttempts} attempts`,
+                            retryable: false };
+                        changed = true;
+                        pendingUpdates.push(terminal);
+                        return terminal;
+                    }
+                    const attempt = attempts + 1;
+                    const delay = turnEndRetryDelayMs(attempt);
+                    const staleInput = candidate.input && typeof candidate.input === "object" ? candidate.input : null;
+                    const next = {
+                        ...candidate,
+                        status: "pending",
+                        updatedAt: now(),
+                        reason: TURN_END_RETRY_REASON,
+                        retryable: true,
+                        turnEndRetryAttempts: attempt,
+                        retryNotBefore: new Date(turnEndRetryScheduler.now() + delay).toISOString(),
+                        input: {
+                            ...(staleInput ?? { inputId: candidate.deliveryId, deliveryId: candidate.deliveryId, kind: "wake", text: "" }),
+                            attempt: (Number.isSafeInteger(staleInput?.attempt) && Number(staleInput?.attempt) >= 0 ? Number(staleInput.attempt) : 0) + 1,
+                        },
+                    };
                     changed = true;
                     pendingUpdates.push(next);
                     return next;
@@ -523,7 +612,10 @@ export function createRuntimeHost(options) {
                     store.writeJson("runtimeDeliveries", { ...disk, records });
                     for (const next of pendingUpdates) {
                         agent.records.set(next.deliveryId, next);
-                        deferred.push(next);
+                        if (next.status === "error")
+                            exhausted.push(next);
+                        else
+                            deferred.push(next);
                     }
                 }
             });
@@ -536,6 +628,11 @@ export function createRuntimeHost(options) {
         for (const record of deferred)
             emit({ type: "delivery", agentId: agent.config.agentId,
                 deliveryId: record.deliveryId, messageId: record.messageId, status: "deferred", reason: TURN_END_RETRY_REASON });
+        for (const record of exhausted)
+            emit({ type: "delivery", agentId: agent.config.agentId,
+                deliveryId: record.deliveryId, messageId: record.messageId, status: "error", reason: record.reason });
+        if (deferred.length)
+            schedulePendingRetry(agent);
     };
     // Production callers persist the canonical Inbox before calling deliver(). If a
     // concurrent drain wins between that append and ledger creation, close only the
@@ -734,9 +831,11 @@ export function createRuntimeHost(options) {
                 if (idleGate)
                     await idleGate;
             }
-            const record = [...agent.records.values()].find((candidate) => candidate.status === "pending" || candidate.status === "submitting");
+            const record = [...agent.records.values()].find((candidate) => (candidate.status === "pending" && isRetryDue(candidate)) || candidate.status === "submitting");
             if (record)
                 await submit(agent, record, false);
+            else
+                schedulePendingRetry(agent);
         })();
         agent.retryPendingInFlight = run;
         try {
@@ -1510,6 +1609,8 @@ export function createRuntimeHost(options) {
                         clearInterval(previous.poller);
                     if (previous.retryTimer)
                         clearTimeout(previous.retryTimer);
+                    if (previous.deliveryRetryTimer)
+                        turnEndRetryScheduler.clearTimeout(previous.deliveryRetryTimer);
                     if (previous.stabilityTimer)
                         clearTimeout(previous.stabilityTimer);
                     const previousFailure = currentAuthFailure(previous);
@@ -1521,7 +1622,7 @@ export function createRuntimeHost(options) {
                         }, previousFailure));
                     const candidate = {
                         ...previous, config, adapter, session, launchId: crypto.randomUUID(), busy: false, submitting: false,
-                        starting: null, retryAfterSubmit: false, generation: 0, poller: null, retryTimer: null,
+                        starting: null, retryAfterSubmit: false, generation: 0, poller: null, retryTimer: null, deliveryRetryTimer: null,
                         recreateAttempts: 0, stabilityTimer: null, recreateReason: null, stopped: false,
                         disabledReason: null, configurationRecovery: null,
                         authFailureActive: scoped && previous.authFailureActive,
@@ -1614,6 +1715,7 @@ export function createRuntimeHost(options) {
                 if (pendingAfterCreate > 0) {
                     throw new RuntimeSessionResetError("inbox_backlog", `Agent ${agentId} received Inbox backlog during reset`, pendingAfterCreate);
                 }
+                clearDeliveryRetryTimer(agent);
                 agent.generation += 1;
                 agent.launchId = crypto.randomUUID();
                 agent.config.sessionId = null;
@@ -1884,7 +1986,7 @@ export function createRuntimeHost(options) {
                     retryPendingInFlight: null,
                     records: new Map(persisted.records.map((record) => [record.deliveryId, record])),
                     byMessage: new Map(persisted.records.map((record) => [record.messageId, record.deliveryId])),
-                    generation: 0, poller: null, retryTimer: null, recreateAttempts: 0,
+                    generation: 0, poller: null, retryTimer: null, deliveryRetryTimer: null, recreateAttempts: 0,
                     stabilityTimer: null, recreateReason: null, stopped: false, disabledReason: null,
                     configurationRecovery: null, stateStore,
                     compactionBreaker: config.runtime === "pi" && stateStore
@@ -2056,6 +2158,7 @@ export function createRuntimeHost(options) {
                 clearInterval(agent.poller);
             if (agent.retryTimer)
                 clearTimeout(agent.retryTimer);
+            clearDeliveryRetryTimer(agent);
             if (agent.stabilityTimer)
                 clearTimeout(agent.stabilityTimer);
             if (agent.busy)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "larkin",
-  "version": "0.5.16",
+  "version": "0.5.18",
   "private": false,
   "larkinPackageRole": "npm-published",
   "files": [