@teamvibe/poller 0.7.0 → 0.7.2

package/dist/poller.js

@@ -10,6 +10,7 @@ import { cleanupOrphanWorktreesForAllBrains, isWorktreeIsolationEnabled } from '
 import { postMaintenanceReports } from './maintenance-reporter.js';
 import { initAuth, stopRefresh, getPollerId } from './auth-provider.js';
 import { registerBrain, startMaintenanceLoop, stopMaintenanceLoop } from './maintenance-cron.js';
+import { bootstrapTrackedBrains } from './brains-bootstrap.js';
 import { startEventQueue, stopEventQueue, flush as flushEventQueue, emitEvent, emitError } from './event-queue.js';
 import { startMetricsCron, stopMetricsCron } from './metrics/scheduler.js';
 import { writeMessage, pickNextMessage, markProcessed, markFailed, hasUnprocessed, requeueMessage } from './inbox-manager.js';
@@ -17,6 +18,7 @@ import { startInboxWatcher, stopInboxWatcher } from './inbox-watcher.js';
 import { Cron } from 'croner';
 import { isControlMessage, handleControlMessage } from './control-handler.js';
 import { checkForUpdate } from './auto-update.js';
+import { checkIdleRestart, reconcileBgTasksOnBoot } from './idle-restart.js';
 import { markAutoUpdateJobFired, markAutoUpdateJobSuccess } from './cron-telemetry.js';
 import { exec } from 'child_process';
 import { promisify } from 'util';
@@ -163,6 +165,35 @@ export function isIntentionalKillExitCode(exitCode) {
 export function isPollerInitiatedFailure(result) {
     return Boolean(result.killedByPoller) || isIntentionalKillExitCode(result.exitCode);
 }
+// tv#320 DevGuru round-1 review: runInboxLoop's retry-cap decision (below)
+// no longer calls isPollerInitiatedFailure() directly — since that cap now
+// applies to everything except 'shutdown', the only distinction it needs is
+// `lastKilledByPoller === 'shutdown'`, and `killedByPoller === 'shutdown'`
+// already implies isPollerInitiatedFailure() === true, making a combined
+// check redundant there.
+//
+// tv#320 DevGuru round-2 review: isIntentionalKillExitCode() still has a real
+// caller below (describeFailureReason()) — a kill with NO `killedByPoller`
+// marker that DOES still report the legacy 143/137 exit code is an
+// intentional kill, not an unclassified application failure, and needs to be
+// labeled as such in give-up notices/logs/events (`lastKilledByPoller ??
+// 'application failure'` used to conflate the two).
+//
+// tv#320 DevGuru round-3 review: this only covers the literal 143/137 case.
+// Per the comment above (line ~185), a group-killed process's `close` event
+// reports `code: null`, NOT 143/137 — so a marker-less OOM/external-SIGTERM/
+// group-kill still falls through to "application failure" here, same as
+// before this fix. Closing that gap for real needs claude-spawner.ts's
+// `close` handler to stop discarding its `signal` argument and SpawnResult
+// to carry it (neither does today) — tracked as a follow-up, not attempted
+// here.
+export function describeFailureReason(killedByPoller, exitCode) {
+    if (killedByPoller)
+        return killedByPoller;
+    if (isIntentionalKillExitCode(exitCode))
+        return `killed (exit ${exitCode})`;
+    return 'application failure';
+}
 // teamvibe.ai#284 (poller-brain#235): end-of-turn guard. A successful Claude
 // run can end its turn having only written plain assistant text — no
 // `mcp__slack__send_message` tool call — which the platform otherwise drops
@@ -176,12 +207,27 @@ export function isPollerInitiatedFailure(result) {
 // Deliberately does NOT fire on a failed run (result.success === false):
 // that path already has its own delivery via classifyFailedIteration +
 // sendSlackMessage/sendSlackError, and firing here too would double-deliver.
+//
+// poller-brain#380: a turn whose only tool call is a reaction (or similar)
+// can trigger Claude Code's own forced-continuation nudge ("no visible
+// output, produce a user-visible response"), which previously always fired
+// this same fallback — double-posting even when the reaction was the
+// intended, complete reply. Base-brain CLAUDE.md instructs agents to answer
+// that nudge with this exact sentinel when nothing further is needed; an
+// exact match is treated as "nothing to deliver". Deliberately an exact
+// string match, not a content heuristic — anything else (including real
+// content an agent forgot to send via send_message) still delivers as
+// before, so this can't silently drop a forgotten reply.
+export const END_OF_TURN_NO_FURTHER_RESPONSE_SENTINEL = 'NO_FURTHER_RESPONSE_NEEDED';
 export function endOfTurnFallbackText(hasSlackContext, result) {
     if (!hasSlackContext || !result.success || result.sawSendMessageCall) {
         return undefined;
     }
     const text = result.lastPlainTextContent?.trim();
-    return text ? text : undefined;
+    if (!text || text === END_OF_TURN_NO_FURTHER_RESPONSE_SENTINEL) {
+        return undefined;
+    }
+    return text;
 }
 export async function runInboxLoop(threadId, kbPath, queueMessage, sessionLog) {
     if (activeLoops.has(threadId)) {
@@ -328,6 +374,26 @@ export async function runInboxLoop(threadId, kbPath, queueMessage, sessionLog) {
                     lastExitCode = result.exitCode;
                     lastKilledByPoller = result.killedByPoller;
                     sessionLog.error(`[Loop] Claude failed iteration #${iteration}: ${result.error}`);
+                    // tv#320 DevGuru round-2 review: a fleet-visible record of EVERY
+                    // failed iteration (not just the eventual give-up) means a future
+                    // "was this in one loop or across restarts" question is a DDB query
+                    // (this event, by thread/filename/timestamp) instead of a CloudWatch
+                    // log dig + SSO admin escalation — which is exactly what round-2's
+                    // point 3b hit a dead end on for pb#337.
+                    //
+                    // tv#320 DevGuru round-3 review: excludes 'shutdown' — that fires on
+                    // every poller restart/deploy for every in-flight message fleet-wide
+                    // and isn't a retry-cap-relevant failure (it's not counted toward
+                    // failureCounts either), so it would just be alerting noise for an
+                    // expected event with no new information (the requeue below already
+                    // covers it).
+                    if (lastKilledByPoller !== 'shutdown') {
+                        emitError('session', `Iteration #${iteration} failed for ${message.filename}`, {
+                            severity: 'warn',
+                            channelId: queueMessage.teamvibe.channelId,
+                            context: { threadId, filename: message.filename, iteration, reason: describeFailureReason(lastKilledByPoller, lastExitCode) },
+                        });
+                    }
                     if (hasSlackContext) {
                         const decision = classifyFailedIteration(result);
                         if (decision.kind === 'neutral') {
@@ -349,33 +415,96 @@ export async function runInboxLoop(threadId, kbPath, queueMessage, sessionLog) {
             }
             else {
                 // Poller-initiated kill (timeout/idle/network/zombie/shutdown, via
-                // `killedByPoller`, or the legacy 143/137 exit-code fallback) → never
-                // counts as an application failure, so markFailed/backoff is skipped
-                // below regardless of reason. For timeout/idle/network/zombie this
-                // really does mean "retry immediately" — the `while (!shuttingDown)`
-                // condition above lets the loop come straight back around. For
-                // 'shutdown' specifically, `shuttingDown` is already true by the time
-                // this reads, so the loop condition above stops it from actually
-                // retrying inline — instead the branch below requeues the message
-                // from processing/ back to new/ (PR#308 round-5) so a future boot's
-                // inbox watcher finds it. See isPollerInitiatedFailure() and the
-                // loop-condition comment above.
-                const isKilledSignal = isPollerInitiatedFailure({ exitCode: lastExitCode, killedByPoller: lastKilledByPoller });
-                if (!isKilledSignal) {
+                // `killedByPoller`, or the legacy 143/137 exit-code fallback) never
+                // counts as an *application* failure — but tv#320 (poller-brain#337:
+                // one checkpoint file redelivered 18x/~10min, stopped only by a
+                // manual `rm`) showed that timeout/idle/network/zombie kills can be
+                // just as deterministically tied to this one message's content as a
+                // genuine app failure (e.g. an oversized file that reliably
+                // re-triggers the same idle/timeout watchdog on every immediate
+                // retry). So the same failureCounts/MAX_RETRIES/backoff cap as the
+                // application-failure path now applies to everything except
+                // 'shutdown' — not just those four named reasons: a legacy 143/137
+                // exit with no `killedByPoller` marker at all is just as capable of
+                // repeating forever and there's no reason to treat it differently.
+                // 'shutdown' alone stays excluded: `shuttingDown` is already true by
+                // the time this reads, so the loop condition above stops it from
+                // actually retrying inline — instead the branch below requeues the
+                // message from processing/ back to new/ (PR#308 round-5) so a future
+                // boot's inbox watcher finds it.
+                //
+                // tv#320 DevGuru round-1 review: this branches on `lastKilledByPoller
+                // !== 'shutdown'` directly rather than `isPollerInitiatedFailure(...)`
+                // — `killedByPoller === 'shutdown'` already implies
+                // isPollerInitiatedFailure() === true, so combining the two was a
+                // dead term. isPollerInitiatedFailure() itself is unchanged and still
+                // meaningful (exported, unit-tested in
+                // poller-retry-classification.test.ts) for any caller that needs the
+                // poller-vs-app distinction — this loop just no longer needs it,
+                // since the only distinction that changes behavior here is
+                // shutdown-vs-not.
+                if (lastKilledByPoller !== 'shutdown') {
                     const failures = (failureCounts.get(message.filename) || 0) + 1;
                     failureCounts.set(message.filename, failures);
                     const MAX_RETRIES = 3;
                     if (failures >= MAX_RETRIES) {
-                        sessionLog.error(`[Loop] Message ${message.filename} failed ${failures} times, giving up`);
+                        // tv#320 DevGuru round-1 review: the per-iteration notice sent
+                        // above (classifyFailedIteration) for idle/zombie/network reads
+                        // "send another message to resume/restart/retry" — true before
+                        // this cap existed (retry was unconditional), false now that this
+                        // is the last attempt and the message is about to be dropped.
+                        // Give the user an explicit, accurate final word instead of
+                        // leaving classifyFailedIteration's now-stale promise as the only
+                        // thing they saw, and surface it as a real error event (not just
+                        // a session-log line) so give-up isn't invisible to fleet
+                        // monitoring either.
+                        const reasonLabel = describeFailureReason(lastKilledByPoller, lastExitCode);
+                        sessionLog.error(`[Loop] Message ${message.filename} failed ${failures} times (last reason: ${reasonLabel}), giving up`);
                         markFailed(kbPath, threadId, message.filename);
+                        if (hasSlackContext) {
+                            // tv#320 DevGuru round-2 review (BLOCKING): this whole else-if
+                            // chain sits structurally after the per-iteration try/catch
+                            // above (which closes at the `catch (error)` a few lines up),
+                            // so an unhandled exception here — unlike the per-iteration
+                            // classifyFailedIteration notices, which ARE inside that
+                            // try/catch — would propagate straight out of the `while`
+                            // loop, skip releaseSessionLock() below (it's in the try body,
+                            // not a `finally`), and leave the DDB session lock held until
+                            // TTL. sendSlackError() (slack-client.ts) has no internal catch
+                            // of its own (unlike addReaction(), which does), so a Slack
+                            // rate-limit/auth/network error here needs the same explicit
+                            // try/catch the end-of-turn-guard fallback above already uses.
+                            try {
+                                await sendSlackError(queueMessage, `Giving up after ${failures} attempts (last reason: ${reasonLabel}) — this message was not processed. Please send it again.`);
+                            }
+                            catch (err) {
+                                sessionLog.error(`[Loop] Give-up notice delivery failed: ${err instanceof Error ? err.message : err}`);
+                            }
+                            await addReaction(queueMessage, 'x');
+                        }
+                        emitError('session', `Message ${message.filename} failed ${failures} times, giving up`, {
+                            severity: 'error',
+                            channelId: queueMessage.teamvibe.channelId,
+                            context: { threadId, filename: message.filename, reason: reasonLabel },
+                        });
                     }
                     else {
                         const backoffMs = Math.min(10_000 * Math.pow(2, failures - 1), 60_000);
                         sessionLog.info(`[Loop] Retry ${failures}/${MAX_RETRIES - 1} for ${message.filename}, backing off ${backoffMs}ms`);
-                        await sleep(backoffMs);
+                        // tv#320 DevGuru round-2 nit: this backoff was previously only
+                        // reachable for genuine app failures; the broadened cap above
+                        // means a timeout/idle/network/zombie kill can now land here too,
+                        // including one racing an in-progress shutdown() (e.g. the idle
+                        // watchdog and a shutdown SIGTERM firing close together). Skip
+                        // the up-to-60s wait when already shutting down so the `while
+                        // (!shuttingDown)` condition above can exit on the very next
+                        // check instead of stalling runInboxLoop's return.
+                        if (!shuttingDown) {
+                            await sleep(backoffMs);
+                        }
                     }
                 }
-                else if (lastKilledByPoller === 'shutdown') {
+                else {
                     // PR#308 round-5 review (teamvibe.ai#73): pickNextMessage() moves a
                     // message's file to processing/ as soon as it's picked up, not once
                     // it completes — so a message killed here by shutdown is left
@@ -387,9 +516,15 @@ export async function runInboxLoop(threadId, kbPath, queueMessage, sessionLog) {
                     // shutdown notice text below, classify-failed-iteration.ts)
                     // literally true. It does NOT make resumption automatic: the
                     // watcher's checkInboxForNewSessions() (inbox-watcher.ts) only
-                    // scans brains already in the in-memory trackedBrains map, which
-                    // is empty after a fresh poller start until some SQS activity for
-                    // this brain re-registers it (see teamvibe.ai#310) — hence the
+                    // scans brains already in the in-memory trackedBrains map. Since
+                    // teamvibe.ai#127, that map is best-effort seeded from the
+                    // backend at startup (bootstrapTrackedBrains(), before
+                    // startInboxWatcher() is called) — but it's still not a
+                    // guarantee (network failure, backend outage, brain missing a
+                    // Slack bot/token to resolve) and a brand-new brain from a real
+                    // message that arrives after bootstrap but before its own first
+                    // registerBrain() call is still momentarily untracked. See
+                    // teamvibe.ai#310 for the still-open general case. Hence the
                     // notice text also naming "next message or scheduled run" rather
                     // than claiming automatic resumption.
                     //
@@ -581,6 +716,18 @@ async function pollLoop() {
                 consecutiveAuthFailures = 0;
             }
             if (messages.length === 0) {
+                // tv#317: only worth checking on an otherwise-empty poll — a poll that
+                // returned messages means activeSessions is about to go above 0 anyway.
+                // Routed through the real shutdown() (not a standalone flush+exit) so this
+                // sets shuttingDown, stops the inbox watcher, and releases locks — the same
+                // gate every other exit path goes through. DevGuru review, tv#317: a bare
+                // flushEventQueue()+process.exit(0) here let the inbox watcher spawn a new
+                // session in the window before exit, then kill it mid-work (#308's shape),
+                // and skipped releaseAllActiveLocks(), stranding locks for STALE_LOCK_TIMEOUT_MS.
+                if (checkIdleRestart()) {
+                    logger.info('[IdleRestart] Restarting to let Railway reclaim page-cache memory...');
+                    await shutdown('idle-restart', 0);
+                }
                 continue;
             }
             logger.info(`Received ${messages.length} message(s) from SQS`);
@@ -651,8 +798,17 @@ async function pollLoop() {
         }
     }
 }
+let sleepImpl = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
 function sleep(ms) {
-    return new Promise((resolve) => setTimeout(resolve, ms));
+    return sleepImpl(ms);
+}
+// Exported for testing only (tv#320 DevGuru round-1 nit): lets a test replace
+// the real backoff delay (runInboxLoop's retry-cap path) with an immediate
+// resolve, without a `globalThis.setTimeout` override that would also swallow
+// any other real timer in the same code path. Same pattern as
+// __setShuttingDownForTest above.
+export function __setSleepForTest(impl) {
+    sleepImpl = impl;
 }
 // Graceful shutdown
 let shuttingDown = false;
@@ -749,11 +905,38 @@ export async function startPoller() {
     startEventQueue();
     await ensureDirectories();
     await ensureBaseBrain();
+    reconcileBgTasksOnBoot();
+    if (config.IDLE_RESTART_ENABLED) {
+        // Defaults off (tv#317 follow-up, DevGuru review): a silent disabled gate and an
+        // enabled gate that just hasn't hit its threshold yet look identical in production
+        // (same flat/growing memory graph) — logging only on enabled, matching the
+        // Worktree isolation line below, makes opt-in verifiable from boot logs instead.
+        logger.info('   Idle restart: ENABLED (IDLE_RESTART_ENABLED=true)');
+    }
     if (isWorktreeIsolationEnabled()) {
         logger.info('   Worktree isolation: ENABLED (POLLER_USE_WORKTREES=true)');
         const sweep = await cleanupOrphanWorktreesForAllBrains(config.BRAINS_PATH);
         logger.info(`   Orphan worktree sweep: scanned ${sweep.scanned} brains, removed ${sweep.removed} stale worktrees`);
     }
+    // Seed trackedBrains from the backend's authoritative brain list
+    // (teamvibe.ai#127 Option B) BEFORE the maintenance cron and the inbox
+    // watcher start — both iterate the in-memory trackedBrains map
+    // (maintenance-cron.ts's triggerAllBrains, inbox-watcher.ts's
+    // checkInboxForNewSessions), which is otherwise empty after every
+    // restart until a real message re-registers each brain one at a time.
+    // Best-effort: never throws, so a bootstrap failure here falls back to
+    // the pre-#127 behavior rather than blocking startup.
+    //
+    // This await only covers ONE short (~10s) fetch attempt (DevGuru round-3
+    // review item 2) — real message processing (pollLoop(), below) must not
+    // be blocked for up to ~62s by a backend outage, which is exactly the
+    // condition most likely to coincide with a poller restart (backend
+    // deploys). If that one attempt fails and is retryable, the retry runs
+    // detached in the background and finishes after startPoller() has
+    // already returned — see brains-bootstrap.ts's bootstrapTrackedBrains()
+    // doc comment for why that's safe (neither the maintenance cron nor the
+    // inbox watcher need trackedBrains populated synchronously here).
+    await bootstrapTrackedBrains();
     startMaintenanceLoop(processMessage);
     startAutoUpdateCron();
     startMetricsCron();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@teamvibe/poller",
-  "version": "0.7.0",
+  "version": "0.7.2",
   "type": "module",
   "main": "./dist/index.js",
   "bin": {