@teamvibe/poller 0.8.2-rc.1 → 0.8.3-rc.2

package/dist/poller.js

@@ -1,9 +1,15 @@
+import { poolRunTelemetry } from './pool-telemetry.js';
 import { config, POLLER_VERSION } from './config.js';
 import { logger, setLogContext } from './logger.js';
 import { pollMessages, deleteMessage, } from './sqs-poller.js';
-import { spawnClaudeCode, isAtCapacity, getActiveProcessCount, killAllActiveProcesses, killZombieForThread } from './claude-spawner.js';
+import { spawnClaudeCode, spawnPoolClaudeCode, isAtCapacity, getActiveProcessCount, killAllActiveProcesses, killZombieForThread } from './claude-spawner.js';
 import { sendSlackError, sendSlackMessage, addReaction, getUserInfo, startTypingIndicator } from './slack-client.js';
 import { classifyFailedIteration } from './classify-failed-iteration.js';
+import { fetchSecretsForSpawn } from './secrets-client.js';
+import { PoolDeferredError, fetchRuntimePool, subscriptionUsageStore } from './subscription-runtime.js';
+import { POOL_RECHECK_MS } from './subscription-selection.js';
+import { isPoolDeferred, readPoolDefer, deferPool, setPoolNotice, clearPoolDefer } from './pool-defer.js';
+import { isQuotaPaused, pauseForQuota, clearQuotaPauseAfterSuccess, QuotaPausedError } from './quota-pause.js';
 import { acquireSessionLock, releaseSessionLock, updateSessionId } from './session-store.js';
 import { getBrainPath, ensureDirectories, ensureBaseBrain, pushBrainChanges } from './brain-manager.js';
 import { cleanupOrphanWorktreesForAllBrains, isWorktreeIsolationEnabled } from './worktree-manager.js';
@@ -14,7 +20,7 @@ import { registerBrain, startMaintenanceLoop, stopMaintenanceLoop } from './main
 import { bootstrapTrackedBrains } from './brains-bootstrap.js';
 import { startEventQueue, stopEventQueue, flush as flushEventQueue, emitEvent, emitError } from './event-queue.js';
 import { startMetricsCron, stopMetricsCron } from './metrics/scheduler.js';
-import { writeMessage, pickNextMessage, markProcessed, markFailed, hasUnprocessed, requeueMessage } from './inbox-manager.js';
+import { writeMessage, pickNextMessage, markProcessed, markFailed, hasUnprocessed, requeueMessage, saveInboxContext, saveInboxRuntime, readInboxRuntime, clearInboxRuntime } from './inbox-manager.js';
 import { startInboxWatcher, stopInboxWatcher } from './inbox-watcher.js';
 import { Cron } from 'croner';
 import { isControlMessage, handleControlMessage } from './control-handler.js';
@@ -239,6 +245,15 @@ export function endOfTurnFallbackText(hasSlackContext, result) {
     return text.split(END_OF_TURN_NO_FURTHER_RESPONSE_SENTINEL).join(END_OF_TURN_SENTINEL_DISPLAY);
 }
 export async function runInboxLoop(threadId, kbPath, queueMessage, sessionLog) {
+    if (isQuotaPaused() && !getPollerId())
+        return;
+    const initialPool = await fetchRuntimePool(queueMessage);
+    if (initialPool.kind === 'no_accounts_configured') {
+        if (isQuotaPaused())
+            return;
+    }
+    else if (isPoolDeferred(threadId))
+        return;
     if (activeLoops.has(threadId)) {
         sessionLog.info(`Loop already running for thread ${threadId}, skipping`);
         return;
@@ -266,6 +281,34 @@ export async function runInboxLoop(threadId, kbPath, queueMessage, sessionLog) {
             activeLocks.set(threadId, lockToken);
         }
         let currentSessionId = session.session_id || undefined;
+        let currentPin = session.subscription_pin;
+        let deferred = false;
+        let poolIteration = false;
+        const retryCompletedBookkeeping = async (state, liveCompletion = false) => {
+            if (!state.completed)
+                return true;
+            const writes = [];
+            // A live run may expire evidence after clock rollback; replaying an old
+            // completion must preserve observations produced by newer concurrent runs.
+            if (state.completed.rateLimitInfo)
+                writes.push(subscriptionUsageStore().recordPassive(state.pin, state.completed.rateLimitInfo, state.completed.observedAt, liveCompletion ? { expireFutureObservations: true } : undefined));
+            writes.push(lockToken ? updateSessionId(threadId, lockToken, state.sessionId, state.pin) : Promise.reject(new Error('Missing lock')));
+            const outcomes = await Promise.allSettled(writes);
+            if (outcomes.some(outcome => outcome.status === 'rejected' || outcome.value === false)) {
+                sessionLog.warn('[Loop] Inference completed; pool bookkeeping failed and will retry without running the agent');
+                try {
+                    saveInboxRuntime(kbPath, threadId, { ...state, completed: { ...state.completed, retryAt: Date.now() + POOL_RECHECK_MS } });
+                }
+                catch {
+                    sessionLog.warn('[Loop] Could not save bookkeeping retry; completion remains recorded');
+                }
+                return false;
+            }
+            currentSessionId = state.sessionId;
+            currentPin = state.pin;
+            clearInboxRuntime(kbPath, threadId);
+            return true;
+        };
         // PR#308 round-3 review (teamvibe.ai#73): `shuttingDown` (declared below,
         // referenced here via the same forward-reference pattern already used by
         // pollLoop()'s `while (!shuttingDown)`) gates every new iteration of this
@@ -285,6 +328,24 @@ export async function runInboxLoop(threadId, kbPath, queueMessage, sessionLog) {
         // that the existing `if (!message) break` / `if (!hasUnprocessed(...))
         // break` exits already use, so lock release is unaffected.
         while (!shuttingDown) {
+            // Dispatch before ANY pool state/context/model preparation. The explicit
+            // no-pool outcome runs the original spawn and #408 pause gates.
+            const pool = iteration === 0 ? initialPool : await fetchRuntimePool(queueMessage);
+            poolIteration = pool.kind !== 'no_accounts_configured';
+            if (!poolIteration && isQuotaPaused())
+                break;
+            if (poolIteration) {
+                const completion = readInboxRuntime(kbPath, threadId);
+                if (completion?.completed) {
+                    // Completed prompts never enter pickNextMessage/spawn again, even
+                    // when the usage store or the session pin needs a later repair.
+                    markProcessed(kbPath, threadId, completion.filename);
+                    if (Date.now() >= completion.completed.observedAt && Date.now() < completion.completed.retryAt || !await retryCompletedBookkeeping(completion)) {
+                        deferred = hasUnprocessed(kbPath, threadId);
+                        break;
+                    }
+                }
+            }
             const message = pickNextMessage(kbPath, threadId);
             if (!message) {
                 sessionLog.info(`[Loop] No more messages for thread ${threadId}`);
@@ -294,22 +355,91 @@ export async function runInboxLoop(threadId, kbPath, queueMessage, sessionLog) {
             const isFirstMessage = session.message_count === 1 && iteration === 1;
             sessionLog.info(`[Loop] Iteration #${iteration} — processing: ${message.filename}`);
             await ensureBaseBrain();
-            const hasSlackContext = Boolean(queueMessage.response_context.slack?.channel &&
-                queueMessage.response_context.slack?.message_ts);
-            const typing = hasSlackContext
-                ? startTypingIndicator(queueMessage)
-                : undefined;
+            let inboxMsg = queueMessage;
+            let deliveryMsg = queueMessage;
+            let hasSlackContext = Boolean(queueMessage.response_context.slack?.channel && queueMessage.response_context.slack?.message_ts);
+            let typing = !poolIteration && hasSlackContext ? startTypingIndicator(queueMessage) : undefined;
+            const retainedRuntime = readInboxRuntime(kbPath, threadId);
+            let quotaDeferEpisodes = Math.max(retainedRuntime?.filename === message.filename ? retainedRuntime.quotaDeferEpisodes ?? 0 : 0, readPoolDefer(threadId) ? 1 : 0);
+            let quotaPaused = false;
             let succeeded = false;
             let lastExitCode = null;
             let lastKilledByPoller;
             try {
                 // Reconstruct queue message from stored inbox data
                 const storedData = JSON.parse(message.content);
-                const inboxMsg = {
+                inboxMsg = {
                     ...queueMessage,
                     ...storedData,
+                    // Durable inbox routing omits credentials. Preserve live metadata
+                    // missing from older payloads, and always use the live bot token.
+                    teamvibe: { ...queueMessage.teamvibe, ...storedData.teamvibe, botToken: queueMessage.teamvibe.botToken },
                 };
-                const result = await spawnClaudeCode(inboxMsg, sessionLog, kbPath, currentSessionId, isFirstMessage, session.last_message_ts, () => typing?.stop(), typing);
+                // Legacy delivery uses the original routing, just as on main.
+                let result;
+                if (!poolIteration) {
+                    result = await spawnClaudeCode(inboxMsg, sessionLog, kbPath, currentSessionId, isFirstMessage, session.last_message_ts, () => typing?.stop(), typing);
+                }
+                else {
+                    if (storedData.teamvibe && (storedData.teamvibe.workspaceId !== queueMessage.teamvibe.workspaceId || storedData.teamvibe.channelId !== queueMessage.teamvibe.channelId))
+                        throw new PoolDeferredError();
+                    if (!queueMessage.teamvibe.botToken && queueMessage.response_context.slack) {
+                        const secrets = await fetchSecretsForSpawn({ pollerId: getPollerId() || '', workspaceId: queueMessage.teamvibe.workspaceId, channelId: queueMessage.teamvibe.channelId });
+                        const token = secrets.env.unwrap()['SLACK_BOT_TOKEN'];
+                        queueMessage = { ...queueMessage, teamvibe: { ...queueMessage.teamvibe, botToken: typeof token === 'string' ? token : '' } };
+                        inboxMsg = { ...inboxMsg, teamvibe: { ...inboxMsg.teamvibe, botToken: queueMessage.teamvibe.botToken } };
+                    }
+                    deliveryMsg = inboxMsg;
+                    hasSlackContext = Boolean(inboxMsg.response_context.slack?.channel && inboxMsg.response_context.slack?.message_ts);
+                    typing = hasSlackContext ? startTypingIndicator(inboxMsg) : undefined;
+                    try {
+                        saveInboxContext(kbPath, inboxMsg);
+                    }
+                    catch {
+                        throw new PoolDeferredError();
+                    }
+                    const savedRuntime = readInboxRuntime(kbPath, threadId);
+                    if (savedRuntime?.filename === message.filename && savedRuntime.pin.workspaceId === queueMessage.teamvibe.workspaceId && (!currentPin || currentPin.accountId === savedRuntime.pin.accountId)) {
+                        currentSessionId = savedRuntime.sessionId;
+                        currentPin = savedRuntime.pin;
+                    }
+                    result = await spawnPoolClaudeCode(inboxMsg, sessionLog, kbPath, currentSessionId, isFirstMessage, session.last_message_ts, () => typing?.stop(), typing, {
+                        pool,
+                        quotaDeferEpisodes,
+                        attempts: savedRuntime?.filename === message.filename ? savedRuntime.attempts : undefined,
+                        pin: currentPin ? {
+                            ...currentPin,
+                            resumePending: (savedRuntime?.filename === message.filename || !savedRuntime && currentPin.inboxFilename === message.filename) && currentPin.resumePending === true,
+                            sawSendMessageCall: (savedRuntime?.filename === message.filename || !savedRuntime && currentPin.inboxFilename === message.filename) && currentPin.sawSendMessageCall === true,
+                        } : undefined,
+                        shouldStop: () => shuttingDown,
+                        complete: async (id, pin, attempts, rateLimitInfo) => {
+                            const state = { sessionId: id, pin, filename: message.filename, attempts, quotaDeferEpisodes, completed: { observedAt: Date.now(), retryAt: Date.now(), rateLimitInfo } };
+                            // This completion record precedes usage/DDB bookkeeping. Inbox
+                            // deletion also happens before those writes; no defer on failure.
+                            try {
+                                saveInboxRuntime(kbPath, threadId, state);
+                            }
+                            catch {
+                                sessionLog.warn('[Loop] Could not save completion record; removing completed inbox before bookkeeping');
+                            }
+                            markProcessed(kbPath, threadId, message.filename);
+                            currentSessionId = id;
+                            currentPin = pin;
+                            succeeded = true;
+                            clearPoolDefer(threadId);
+                            await retryCompletedBookkeeping(state, true);
+                        },
+                        persist: async (id, pin, attempts) => {
+                            pin = { ...pin, inboxFilename: message.filename };
+                            if (!lockToken || await updateSessionId(threadId, lockToken, id, pin) === false)
+                                throw new PoolDeferredError();
+                            currentSessionId = id;
+                            currentPin = pin;
+                            saveInboxRuntime(kbPath, threadId, { sessionId: id, pin, filename: message.filename, attempts, quotaDeferEpisodes });
+                        },
+                    });
+                }
                 typing?.stop();
                 if (result.newSessionId) {
                     // Update local state first — DynamoDB writes may fail (expired token,
@@ -318,7 +448,9 @@ export async function runInboxLoop(threadId, kbPath, queueMessage, sessionLog) {
                     currentSessionId = result.newSessionId;
                     if (lockToken) {
                         try {
-                            await updateSessionId(threadId, lockToken, result.newSessionId);
+                            await (poolIteration
+                                ? updateSessionId(threadId, lockToken, result.newSessionId, currentPin)
+                                : updateSessionId(threadId, lockToken, result.newSessionId));
                         }
                         catch (err) {
                             sessionLog.error(`[Loop] Failed to persist new session ID to DynamoDB: ${err instanceof Error ? err.message : err}`);
@@ -327,29 +459,33 @@ export async function runInboxLoop(threadId, kbPath, queueMessage, sessionLog) {
                 }
                 if (result.metrics) {
                     sessionLog.info(`[Loop] Iteration #${iteration} complete — ` +
-                        `cost: $${result.metrics.total_cost_usd.toFixed(4)}, ` +
+                        `cost: $${result.metrics.total_cost_usd?.toFixed(4) ?? 'unknown'}, ` +
                         `duration: ${result.metrics.duration_ms}ms`);
                 }
                 if (result.success) {
                     succeeded = true;
+                    clearQuotaPauseAfterSuccess();
                     sessionLog.info(`[Loop] Claude completed iteration #${iteration}`);
-                    if (result.metrics) {
-                        emitEvent('session.completed', queueMessage.teamvibe.channelId, {
-                            threadId: queueMessage.thread_id,
-                            total_cost_usd: result.metrics.total_cost_usd,
-                            input_tokens: result.metrics.input_tokens,
-                            output_tokens: result.metrics.output_tokens,
-                            cache_read_input_tokens: result.metrics.cache_read_input_tokens,
-                            cache_creation_input_tokens: result.metrics.cache_creation_input_tokens,
-                            duration_ms: result.metrics.duration_ms,
-                            duration_api_ms: result.metrics.duration_api_ms,
-                            num_turns: result.metrics.num_turns,
-                            model: result.metrics.model,
+                    if (result.metrics || result.poolAttempts) {
+                        emitEvent('session.completed', deliveryMsg.teamvibe.channelId, {
+                            threadId: deliveryMsg.thread_id,
+                            ...(result.metrics?.context_tokens != null ? { context_tokens: result.metrics.context_tokens } : {}),
+                            ...(result.metrics?.context_window != null ? { context_window: result.metrics.context_window } : {}),
+                            total_cost_usd: result.metrics?.total_cost_usd,
+                            input_tokens: result.metrics?.input_tokens,
+                            output_tokens: result.metrics?.output_tokens,
+                            cache_read_input_tokens: result.metrics?.cache_read_input_tokens,
+                            cache_creation_input_tokens: result.metrics?.cache_creation_input_tokens,
+                            duration_ms: result.metrics?.duration_ms,
+                            duration_api_ms: result.metrics?.duration_api_ms,
+                            num_turns: result.metrics?.num_turns,
+                            model: result.metrics?.model,
                             // Which config layer actually chose the `--model` flag
                             // (settings.json / poller / workspace / channel / env-default)
                             // — teamvibe.ai#235 PR2. Distinct from `model` above, which is
                             // Claude CLI's own report of the model it used.
                             modelSource: result.modelSource,
+                            ...(result.poolAttempts ? { workspaceId: deliveryMsg.teamvibe.workspaceId, ...poolRunTelemetry(result.poolAttempts, quotaDeferEpisodes) } : {}),
                         });
                     }
                     // teamvibe.ai#284 (poller-brain#235): end-of-turn guard. A
@@ -363,23 +499,35 @@ export async function runInboxLoop(threadId, kbPath, queueMessage, sessionLog) {
                     if (fallbackText) {
                         sessionLog.warn(`[Loop] Iteration #${iteration}: no send_message call detected, delivering trailing assistant text via fallback`);
                         try {
-                            await sendSlackMessage(queueMessage, fallbackText);
+                            await sendSlackMessage(deliveryMsg, fallbackText);
                         }
                         catch (err) {
                             sessionLog.error(`[Loop] End-of-turn fallback delivery failed: ${err instanceof Error ? err.message : err}`);
                         }
                         emitError('session', 'End-of-turn guard: delivered assistant text via fallback (no send_message tool call detected)', {
                             severity: 'warn',
-                            channelId: queueMessage.teamvibe.channelId,
-                            context: { threadId: queueMessage.thread_id },
+                            channelId: deliveryMsg.teamvibe.channelId,
+                            context: { threadId: deliveryMsg.thread_id },
                         });
                     }
-                    if (queueMessage.teamvibe.brain?.brainId) {
-                        await pushBrainChanges(kbPath, queueMessage.teamvibe.brain.brainId, queueMessage.teamvibe.workspaceId);
+                    if (deliveryMsg.teamvibe.brain?.brainId) {
+                        await pushBrainChanges(kbPath, deliveryMsg.teamvibe.brain.brainId, deliveryMsg.teamvibe.workspaceId);
                     }
-                    await postMaintenanceReports(kbPath, queueMessage.teamvibe.channelId);
+                    await postMaintenanceReports(kbPath, deliveryMsg.teamvibe.channelId);
                 }
                 else {
+                    // Set before Slack I/O: a delivery failure must not retry or discard
+                    // the saved request. Kill intent still wins over incidental quota.
+                    const decision = classifyFailedIteration(result);
+                    quotaPaused = decision.kind === 'quota';
+                    const notifyQuota = decision.kind === 'quota'
+                        ? pauseForQuota(decision.quota, kbPath, threadId)
+                        : false;
+                    if (quotaPaused && !notifyQuota) {
+                        // Another in-flight request already owns the retained marker.
+                        // Queue this one so the post-pause sweep can find both requests.
+                        requeueMessage(kbPath, threadId, message.filename);
+                    }
                     lastExitCode = result.exitCode;
                     lastKilledByPoller = result.killedByPoller;
                     sessionLog.error(`[Loop] Claude failed iteration #${iteration}: ${result.error}`);
@@ -399,27 +547,70 @@ export async function runInboxLoop(threadId, kbPath, queueMessage, sessionLog) {
                     if (lastKilledByPoller !== 'shutdown') {
                         emitError('session', `Iteration #${iteration} failed for ${message.filename}`, {
                             severity: 'warn',
-                            channelId: queueMessage.teamvibe.channelId,
+                            channelId: deliveryMsg.teamvibe.channelId,
                             context: { threadId, filename: message.filename, iteration, reason: describeFailureReason(lastKilledByPoller, lastExitCode) },
                         });
                     }
-                    if (hasSlackContext) {
-                        const decision = classifyFailedIteration(result);
-                        if (decision.kind === 'neutral') {
-                            await sendSlackMessage(queueMessage, decision.text);
+                    if (hasSlackContext && (decision.kind !== 'quota' || notifyQuota)) {
+                        if (decision.kind === 'neutral' || decision.kind === 'quota') {
+                            await sendSlackMessage(deliveryMsg, decision.text);
                         }
                         else {
-                            await sendSlackError(queueMessage, decision.text);
+                            await sendSlackError(deliveryMsg, decision.text);
                         }
-                        await addReaction(queueMessage, decision.reaction);
+                        await addReaction(deliveryMsg, decision.reaction);
                     }
                 }
             }
             catch (error) {
                 typing?.stop();
-                sessionLog.error(`[Loop] Error in iteration #${iteration}: ${error instanceof Error ? error.message : error}`);
+                if (poolIteration && error instanceof PoolDeferredError) {
+                    quotaPaused = true;
+                    deferred = true;
+                    const pause = deferPool(threadId, error.eligibleAt, error.resetAt);
+                    const retained = readInboxRuntime(kbPath, threadId);
+                    // Rechecks of one exhaustion episode do not inflate the episode count.
+                    quotaDeferEpisodes = Math.max(1, quotaDeferEpisodes);
+                    if (retained)
+                        saveInboxRuntime(kbPath, threadId, { ...retained, quotaDeferEpisodes });
+                    emitEvent('pool.deferred', deliveryMsg.teamvibe.channelId, { workspaceId: deliveryMsg.teamvibe.workspaceId, threadId: deliveryMsg.thread_id, ...poolRunTelemetry(error.attempts ?? [], quotaDeferEpisodes), episodeId: pause.episodeId, eligibleAt: pause.resumeAt, resetAt: pause.resetAt });
+                    if (hasSlackContext && pause.notice !== 'delivered') {
+                        const reconcile = pause.notice === 'claimed' || pause.notice === 'ambiguous';
+                        if (!reconcile && !setPoolNotice(threadId, pause.episodeId, 'claimed'))
+                            throw new PoolDeferredError(error.eligibleAt);
+                        try {
+                            await sendSlackMessage(deliveryMsg, `Claude subscription pool unavailable. ${pause.resetAt ? `Earliest reset: ${pause.resetAt}.` : 'Reset time unavailable.'} Your message is saved. The next inbox sweep at or after ${new Date(pause.resumeAt).toISOString()} will resume pending work.`, { clientMessageId: pause.episodeId, ...(reconcile ? { reconcile: true } : {}) });
+                            setPoolNotice(threadId, pause.episodeId, 'delivered');
+                        }
+                        catch (deliveryError) {
+                            const rejected = deliveryError?.data?.ok === false;
+                            setPoolNotice(threadId, pause.episodeId, rejected && !reconcile ? 'pending' : 'ambiguous');
+                            sessionLog.warn('[Loop] Pool notice delivery uncertain; pending work retained');
+                        }
+                    }
+                }
+                else if (error instanceof QuotaPausedError) {
+                    // No CLI ran: restore queued work without notices or retry counting.
+                    requeueMessage(kbPath, threadId, message.filename);
+                    quotaPaused = true;
+                }
+                else {
+                    sessionLog.error(`[Loop] Error in iteration #${iteration}: ${error instanceof Error ? error.message : error}`);
+                }
+            }
+            if (quotaPaused) {
+                // The durable account pause gates incoming traffic and watcher sweeps.
+                // Keep the original in processing/ so it resumes before queued work.
+                deferred = true;
+                sessionLog.info(`[Loop] Quota pause active; leaving ${message.filename} pending`);
+                break;
             }
             if (succeeded) {
+                if (poolIteration) {
+                    clearPoolDefer(threadId);
+                    if (!readInboxRuntime(kbPath, threadId)?.completed)
+                        clearInboxRuntime(kbPath, threadId);
+                }
                 markProcessed(kbPath, threadId, message.filename);
             }
             else {
@@ -470,6 +661,21 @@ export async function runInboxLoop(threadId, kbPath, queueMessage, sessionLog) {
                         const reasonLabel = describeFailureReason(lastKilledByPoller, lastExitCode);
                         sessionLog.error(`[Loop] Message ${message.filename} failed ${failures} times (last reason: ${reasonLabel}), giving up`);
                         markFailed(kbPath, threadId, message.filename);
+                        if (poolIteration) {
+                            clearInboxRuntime(kbPath, threadId);
+                            clearPoolDefer(threadId);
+                            if (currentPin) {
+                                const { inboxFilename: _filename, ...pin } = currentPin;
+                                currentPin = { ...pin, resumePending: false, sawSendMessageCall: false };
+                                try {
+                                    if (lockToken && currentSessionId)
+                                        await updateSessionId(threadId, lockToken, currentSessionId, currentPin);
+                                }
+                                catch {
+                                    sessionLog.warn('[Loop] Terminal failure cleared local continuation; stale pin is fenced by inbox filename');
+                                }
+                            }
+                        }
                         if (hasSlackContext) {
                             // tv#320 DevGuru round-2 review (BLOCKING): this whole else-if
                             // chain sits structurally after the per-iteration try/catch
@@ -484,16 +690,16 @@ export async function runInboxLoop(threadId, kbPath, queueMessage, sessionLog) {
                             // rate-limit/auth/network error here needs the same explicit
                             // try/catch the end-of-turn-guard fallback above already uses.
                             try {
-                                await sendSlackError(queueMessage, `Giving up after ${failures} attempts (last reason: ${reasonLabel}) — this message was not processed. Please send it again.`);
+                                await sendSlackError(deliveryMsg, `Giving up after ${failures} attempts (last reason: ${reasonLabel}) — this message was not processed. Please send it again.`);
                             }
                             catch (err) {
                                 sessionLog.error(`[Loop] Give-up notice delivery failed: ${err instanceof Error ? err.message : err}`);
                             }
-                            await addReaction(queueMessage, 'x');
+                            await addReaction(deliveryMsg, 'x');
                         }
                         emitError('session', `Message ${message.filename} failed ${failures} times, giving up`, {
                             severity: 'error',
-                            channelId: queueMessage.teamvibe.channelId,
+                            channelId: deliveryMsg.teamvibe.channelId,
                             context: { threadId, filename: message.filename, reason: reasonLabel },
                         });
                     }
@@ -514,6 +720,8 @@ export async function runInboxLoop(threadId, kbPath, queueMessage, sessionLog) {
                     }
                 }
                 else {
+                    if (poolIteration && currentPin?.resumePending)
+                        deferred = true;
                     // PR#308 round-5 review (teamvibe.ai#73): pickNextMessage() moves a
                     // message's file to processing/ as soon as it's picked up, not once
                     // it completes — so a message killed here by shutdown is left
@@ -562,14 +770,25 @@ export async function runInboxLoop(threadId, kbPath, queueMessage, sessionLog) {
             }
             sessionLog.info(`[Loop] More messages found, continuing...`);
         }
+        if (poolIteration && currentPin?.resumePending && hasUnprocessed(kbPath, threadId))
+            deferred = true;
         if (lockToken) {
             const lastMessageTs = queueMessage.response_context.slack?.message_ts;
-            await releaseSessionLock(threadId, lockToken, 'idle', lastMessageTs);
+            await releaseSessionLock(threadId, lockToken, 'idle', poolIteration && deferred ? undefined : lastMessageTs);
             activeLocks.delete(threadId);
         }
         sessionLog.info(`[Loop] Complete. Iterations: ${iteration}`);
     }
     finally {
+        const heldToken = activeLocks.get(threadId);
+        if (heldToken) {
+            try {
+                await releaseSessionLock(threadId, heldToken, 'idle');
+            }
+            catch {
+                sessionLog.warn('[Loop] Could not release session lock; stale-lock recovery will retry');
+            }
+        }
         activeLoops.delete(threadId);
         activeLocks.delete(threadId);
     }
@@ -619,16 +838,8 @@ async function processMessage(received) {
     // Register brain so the maintenance cron knows about it.
     registerBrain(queueMessage);
     // Write raw message to inbox (spawnClaudeCode handles prompt formatting)
-    writeMessage(kbPath, threadId, JSON.stringify({
-        text: queueMessage.text,
-        sender: queueMessage.sender,
-        type: queueMessage.type,
-        source: queueMessage.source,
-        attachments: queueMessage.attachments,
-        approval: queueMessage.approval,
-        button: queueMessage.button,
-        modal: queueMessage.modal,
-    }));
+    const { botToken: _inboxToken, ...inboxTeamvibe } = queueMessage.teamvibe;
+    writeMessage(kbPath, threadId, JSON.stringify({ ...queueMessage, teamvibe: inboxTeamvibe }));
     sessionLog.info(`Wrote message to inbox`);
     // Delete SQS message immediately — it's now persisted in the inbox
     if (receiptHandle) {

package/dist/pool-defer.d.ts

@@ -0,0 +1,13 @@
+export interface PoolDefer {
+    createdAt: number;
+    resumeAt: number;
+    resetAt?: string;
+    episodeId: string;
+    notice: 'pending' | 'claimed' | 'delivered' | 'ambiguous';
+}
+export declare function readPoolDefer(threadId: string): PoolDefer | undefined;
+export declare function isPoolDeferred(threadId: string): boolean;
+export declare function deferPool(threadId: string, eligibleAt: number, resetAt?: string): PoolDefer;
+/** Claim durably before Slack I/O. Ambiguous sends are never blindly replayed. */
+export declare function setPoolNotice(threadId: string, episodeId: string, notice: PoolDefer['notice']): boolean;
+export declare function clearPoolDefer(threadId: string): void;

package/dist/pool-defer.js

@@ -0,0 +1,81 @@
+import { createHash, randomUUID } from 'node:crypto';
+import { mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { config } from './config.js';
+import { logger } from './logger.js';
+import { POOL_RECHECK_MS } from './subscription-selection.js';
+function pathFor(threadId) {
+    return join(config.TEAMVIBE_DATA_DIR, 'pool-defer', `${createHash('sha256').update(threadId).digest('hex')}.json`);
+}
+const warned = new Set();
+export function readPoolDefer(threadId) {
+    const path = pathFor(threadId);
+    try {
+        const state = JSON.parse(readFileSync(path, 'utf8'));
+        if (!Number.isFinite(state.createdAt) || !Number.isFinite(state.resumeAt) || state.resumeAt <= state.createdAt || state.resumeAt > 253402300799000 || typeof state.episodeId !== 'string' || !['pending', 'claimed', 'delivered', 'ambiguous'].includes(state.notice))
+            throw new Error('Invalid defer');
+        warned.delete(path);
+        return state;
+    }
+    catch (error) {
+        if (error.code !== 'ENOENT' && !warned.has(path)) {
+            warned.add(path);
+            logger.warn('[PoolDefer] Unreadable marker treated as expired');
+        }
+        try {
+            rmSync(path, { recursive: true, force: true });
+        }
+        catch {
+            try {
+                renameSync(path, `${path}.invalid`);
+            }
+            catch { }
+        }
+        return undefined;
+    }
+}
+export function isPoolDeferred(threadId) {
+    const state = readPoolDefer(threadId);
+    if (!state)
+        return false;
+    const now = Date.now();
+    return now >= state.createdAt && now < state.resumeAt && state.resumeAt - now <= state.resumeAt - state.createdAt;
+}
+function write(threadId, state) {
+    const path = pathFor(threadId);
+    mkdirSync(join(config.TEAMVIBE_DATA_DIR, 'pool-defer'), { recursive: true, mode: 0o700 });
+    const tmp = `${path}.${randomUUID()}.tmp`;
+    try {
+        writeFileSync(tmp, JSON.stringify(state), { mode: 0o600 });
+        renameSync(tmp, path);
+    }
+    finally {
+        try {
+            rmSync(tmp, { force: true });
+        }
+        catch { }
+    }
+}
+export function deferPool(threadId, eligibleAt, resetAt) {
+    const previous = readPoolDefer(threadId);
+    const now = Date.now();
+    const state = { createdAt: now, resumeAt: Number.isFinite(eligibleAt) && eligibleAt > now ? eligibleAt : now + POOL_RECHECK_MS, episodeId: previous?.episodeId ?? randomUUID(), notice: previous?.notice ?? 'pending', ...(resetAt ? { resetAt } : {}) };
+    write(threadId, state);
+    return state;
+}
+/** Claim durably before Slack I/O. Ambiguous sends are never blindly replayed. */
+export function setPoolNotice(threadId, episodeId, notice) {
+    const state = readPoolDefer(threadId);
+    if (!state || state.episodeId !== episodeId)
+        return false;
+    write(threadId, { ...state, notice });
+    return true;
+}
+export function clearPoolDefer(threadId) {
+    try {
+        rmSync(pathFor(threadId), { recursive: true, force: true });
+    }
+    catch {
+        logger.warn('[PoolDefer] Completed episode cleanup will be retried');
+    }
+}

package/dist/pool-telemetry.d.ts

@@ -0,0 +1,44 @@
+import type { SubscriptionAccount } from './subscription-pool-client.js';
+import { type StoredAccountUsage, type SubscriptionUsageStore } from './subscription-usage-store.js';
+import type { PoolAttempt } from './subscription-runtime.js';
+import type { SessionMetrics, TeamVibeQueueMessage } from './types.js';
+/** Explicit transport DTO: never serialize a pool account/credential envelope. */
+export declare function accountUsageTelemetry(account: SubscriptionAccount, state: StoredAccountUsage): {
+    reactive: {
+        observedAt: number;
+        window: import("./claude-rate-limit.js").ClaudeQuotaWindow | undefined;
+        scope: string;
+        resetAt: number | undefined;
+        eligibleAt: number;
+    }[];
+    lastRateLimitInfo?: import("./claude-rate-limit.js").ClaudeRateLimitInfo | undefined;
+    probeStatus?: import("./subscription-usage-store.js").ProbeStatus | undefined;
+    probeObservedAt?: number | undefined;
+    usage?: import("./subscription-selection.js").UsageObservation | undefined;
+    accountId: string;
+    accountLabel: string;
+    reportedAt: number;
+};
+export declare function emitCandidateUsage(msg: TeamVibeQueueMessage, accounts: readonly SubscriptionAccount[], store: SubscriptionUsageStore): Promise<void>;
+export declare function poolRunTelemetry(attempts: readonly PoolAttempt[], quotaDeferEpisodes?: number): {
+    chainLength: number;
+    switchedAccounts: boolean;
+    quotaDeferEpisodes: number;
+    attempts: {
+        accountId: string;
+        accountLabel: string;
+        sessionId: string | undefined;
+        model: string | undefined;
+        success: boolean;
+        exitCode: number | null;
+        observedAt: number | undefined;
+        quota: import("./claude-quota.js").ClaudeQuotaFailure | undefined;
+        metrics: SessionMetrics | undefined;
+        lastRateLimitInfo: import("./claude-rate-limit.js").ClaudeRateLimitInfo | undefined;
+    }[];
+    accountId?: string | undefined;
+    accountLabel?: string | undefined;
+    model?: string | undefined;
+    observedAt?: number | undefined;
+    lastRateLimitInfo?: import("./claude-rate-limit.js").ClaudeRateLimitInfo | undefined;
+};