@teamvibe/poller 0.7.0 → 0.7.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,45 @@
1
+ /** @internal test-only */
2
+ export declare function __detachedRetryForTest(): Promise<void> | undefined;
3
+ /**
4
+ * Fetch the full list of brains this poller is responsible for from the
5
+ * backend (GET /poller/brains, source of truth = DDB — teamvibe.ai#127
6
+ * Option B) and seed `trackedBrains` (maintenance-cron.ts) before the
7
+ * maintenance/heartbeat cron starts, so a brain that's been quiet since
8
+ * before this poller's most recent restart is still picked up instead of
9
+ * staying invisible until a human happens to message it.
10
+ *
11
+ * Best-effort: on any failure (network, timeout, non-2xx, malformed body,
12
+ * missing pollerId/token) this logs a warning, emits a telemetry event, and
13
+ * returns without throwing. Startup must not hang or crash on this — the
14
+ * pre-#127 behavior (trackedBrains starts empty until real messages
15
+ * trickle in) is the fallback here, not a fatal state.
16
+ *
17
+ * DevGuru round-3 review: the awaited part of this function does exactly
18
+ * ONE fetch attempt, bounded by CRITICAL_PATH_TIMEOUT_MS (~10s) — not the
19
+ * full 30s client timeout, and not the retry. Every poller restart during a
20
+ * backend outage was blocking real user-message processing (pollLoop(),
21
+ * called after this in startPoller()) for up to ~62s (2 attempts * 30s +
22
+ * 2s delay) — and restarts correlate with backend deploys, exactly when
23
+ * this is most likely to happen. If the first attempt fails and is
24
+ * retryable (network error, timeout, or 5xx — not a 4xx or malformed body,
25
+ * which a retry can't fix), the retry is kicked off detached — an
26
+ * immediately-invoked async function whose promise is stored in a
27
+ * module-level variable (see `__detachedRetryForTest` below; test-only,
28
+ * production code never reads it) rather than `void`-discarded, but never
29
+ * awaited by this function — the same fire-and-forget shape
30
+ * maintenance-cron.ts's registerBrain() already uses for its immediate
31
+ * metrics collect+send — so `bootstrapTrackedBrains()` returns and
32
+ * startPoller() can proceed to
33
+ * startMaintenanceLoop/startAutoUpdateCron/startMetricsCron/pollLoop()
34
+ * without waiting on it. This is safe because neither downstream consumer
35
+ * needs bootstrap complete synchronously: the maintenance cron only fires
36
+ * at its own scheduled time (default 03:30, not at startup), and the inbox
37
+ * watcher just needs trackedBrains populated "one tick later" — both read
38
+ * the same shared, mutable Map (getTrackedBrains()) whenever their own tick
39
+ * fires, not a snapshot taken at startup.
40
+ *
41
+ * No-ops in direct (non-token) mode, same gating as sendMetrics
42
+ * (metrics/sender.ts) — there's no backend endpoint to call without
43
+ * TEAMVIBE_API_URL + TEAMVIBE_POLLER_TOKEN.
44
+ */
45
+ export declare function bootstrapTrackedBrains(): Promise<void>;
@@ -0,0 +1,217 @@
1
+ import { config } from './config.js';
2
+ import { logger } from './logger.js';
3
+ import { getPollerId } from './auth-provider.js';
4
+ import { seedTrackedBrains } from './maintenance-cron.js';
5
+ import { emitEvent, emitError } from './event-queue.js';
6
+ // Client-side timeout kept comfortably above the server-side route timeout
7
+ // (infra/poller-auth.ts: GET /poller/brains, 25s) so the poller never aborts
8
+ // a request the Lambda could still have answered (DevGuru round-2 review —
9
+ // with the old 10s client timeout vs a 20s route timeout, a transient slow
10
+ // response aborted client-side while the Lambda kept running to nothing,
11
+ // burning the one bootstrap attempt with no retry until the next restart).
12
+ // Used ONLY for the off-critical-path retry attempt below — DevGuru round-3:
13
+ // "don't touch the 25s/30s numbers themselves", so this stays exactly as
14
+ // round-2 left it, just no longer used for the first attempt.
15
+ const FETCH_TIMEOUT_MS = 30_000;
16
+ // Short deadline for the ONE attempt that's actually awaited on
17
+ // startPoller()'s critical path (DevGuru round-3 review item 2). Judgment
18
+ // call — not spelled out beyond "~10s" — picked 10s exactly as suggested.
19
+ // Rationale for why a short deadline is safe here even though it's shorter
20
+ // than the 25s the Lambda might legitimately take: this is only the FIRST
21
+ // attempt. If it doesn't finish inside 10s, we don't conclude failure and
22
+ // give up — we fall through to the same retry path used for any other
23
+ // transient failure, just detached from startup so a slow-but-alive
24
+ // backend doesn't cost the poller a lost user-message-processing minute.
25
+ const CRITICAL_PATH_TIMEOUT_MS = 10_000;
26
+ // Judgment call (not specified by DevGuru's review, flagged for reviewer):
27
+ // one retry, 2s fixed delay, unchanged from round 2. Since the retry is now
28
+ // off the critical path (see bootstrapTrackedBrains below), the 2s delay no
29
+ // longer costs startup time — it only affects how soon the detached retry
30
+ // fires, which is not time-critical (the maintenance cron doesn't run until
31
+ // its own scheduled time, and the inbox watcher just needs trackedBrains
32
+ // populated "one tick later", not synchronously — DevGuru round-3).
33
+ const RETRY_DELAY_MS = 2_000;
34
+ function sleep(ms) {
35
+ return new Promise(resolve => setTimeout(resolve, ms));
36
+ }
37
+ // Test-only handle to the detached retry promise (see bootstrapTrackedBrains
38
+ // below) kicked off by the most recent call. Undefined until a retryable
39
+ // first-attempt failure schedules one; reset to undefined at the top of
40
+ // every bootstrapTrackedBrains() call so a stale promise from a PRIOR call
41
+ // is never mistaken for the current one. Same test-only-export convention
42
+ // as maintenance-cron.ts's __resetTrackedBrainsForTest() — never read by
43
+ // production code, exists purely so tests can deterministically wait for
44
+ // the detached background work to finish instead of guessing with a fixed
45
+ // sleep (DevGuru review: the fixed-sleep approach was two separate
46
+ // latent-flakiness bugs — see brains-bootstrap.test.ts).
47
+ let detachedRetryForTest;
48
+ /** @internal test-only */
49
+ export function __detachedRetryForTest() {
50
+ return detachedRetryForTest;
51
+ }
52
+ async function attemptFetch(url, token, timeoutMs) {
53
+ const controller = new AbortController();
54
+ const timer = setTimeout(() => controller.abort(), timeoutMs);
55
+ try {
56
+ const response = await fetch(url, {
57
+ headers: { Authorization: `Bearer ${token}` },
58
+ signal: controller.signal,
59
+ });
60
+ if (!response.ok) {
61
+ return {
62
+ ok: false,
63
+ reason: 'non-2xx',
64
+ detail: `status ${response.status}`,
65
+ retryable: response.status >= 500,
66
+ };
67
+ }
68
+ let body;
69
+ try {
70
+ body = await response.json();
71
+ }
72
+ catch (err) {
73
+ return {
74
+ ok: false,
75
+ reason: 'invalid-json',
76
+ detail: err instanceof Error ? err.message : String(err),
77
+ retryable: false,
78
+ };
79
+ }
80
+ if (!body || typeof body !== 'object' || !Array.isArray(body.brains)) {
81
+ return { ok: false, reason: 'malformed-body', detail: 'missing brains array', retryable: false };
82
+ }
83
+ return { ok: true, brains: body.brains };
84
+ }
85
+ catch (err) {
86
+ const isTimeout = err instanceof Error && err.name === 'AbortError';
87
+ return {
88
+ ok: false,
89
+ reason: isTimeout ? 'timeout' : 'network-error',
90
+ detail: err instanceof Error ? err.message : String(err),
91
+ retryable: true,
92
+ };
93
+ }
94
+ finally {
95
+ clearTimeout(timer);
96
+ }
97
+ }
98
+ /** Seed trackedBrains from a successful fetch and emit success telemetry.
99
+ * Shared by both the critical-path attempt and the detached retry below. */
100
+ function applySuccess(brains) {
101
+ const seeded = seedTrackedBrains(brains);
102
+ logger.info(`[Bootstrap] Seeded ${seeded} brain(s) into trackedBrains from backend (${brains.length} returned)`);
103
+ emitEvent('poller.brains_bootstrap.success', 'system', {
104
+ seeded,
105
+ returned: brains.length,
106
+ });
107
+ }
108
+ /** Report a terminal (no-more-retries) failure. Distinct telemetry artifact
109
+ * (DevGuru round-2 review) — from outside, "bootstrap crashed" and
110
+ * "bootstrap ran fine, no quiet brains to seed" were previously
111
+ * indistinguishable (both show seeded=0 in a log line nobody's watching).
112
+ * This feeds the same FLEET_ERROR pipeline as maintenance-cron.ts's
113
+ * emitError call (triggerAllBrains, per-brain maintenance failure) — same
114
+ * category, same shape. Only called once per bootstrap attempt sequence
115
+ * (non-retryable first-attempt failure, or a failed retry) — never for a
116
+ * retryable first-attempt failure, since that one isn't terminal yet. */
117
+ function reportFailure(outcome) {
118
+ emitError('maintenance', `trackedBrains bootstrap failed: ${outcome.reason}`, {
119
+ severity: 'warn',
120
+ context: { reason: outcome.reason, detail: outcome.detail },
121
+ });
122
+ }
123
+ /**
124
+ * Fetch the full list of brains this poller is responsible for from the
125
+ * backend (GET /poller/brains, source of truth = DDB — teamvibe.ai#127
126
+ * Option B) and seed `trackedBrains` (maintenance-cron.ts) before the
127
+ * maintenance/heartbeat cron starts, so a brain that's been quiet since
128
+ * before this poller's most recent restart is still picked up instead of
129
+ * staying invisible until a human happens to message it.
130
+ *
131
+ * Best-effort: on any failure (network, timeout, non-2xx, malformed body,
132
+ * missing pollerId/token) this logs a warning, emits a telemetry event, and
133
+ * returns without throwing. Startup must not hang or crash on this — the
134
+ * pre-#127 behavior (trackedBrains starts empty until real messages
135
+ * trickle in) is the fallback here, not a fatal state.
136
+ *
137
+ * DevGuru round-3 review: the awaited part of this function does exactly
138
+ * ONE fetch attempt, bounded by CRITICAL_PATH_TIMEOUT_MS (~10s) — not the
139
+ * full 30s client timeout, and not the retry. Every poller restart during a
140
+ * backend outage was blocking real user-message processing (pollLoop(),
141
+ * called after this in startPoller()) for up to ~62s (2 attempts * 30s +
142
+ * 2s delay) — and restarts correlate with backend deploys, exactly when
143
+ * this is most likely to happen. If the first attempt fails and is
144
+ * retryable (network error, timeout, or 5xx — not a 4xx or malformed body,
145
+ * which a retry can't fix), the retry is kicked off detached — an
146
+ * immediately-invoked async function whose promise is stored in a
147
+ * module-level variable (see `__detachedRetryForTest` below; test-only,
148
+ * production code never reads it) rather than `void`-discarded, but never
149
+ * awaited by this function — the same fire-and-forget shape
150
+ * maintenance-cron.ts's registerBrain() already uses for its immediate
151
+ * metrics collect+send — so `bootstrapTrackedBrains()` returns and
152
+ * startPoller() can proceed to
153
+ * startMaintenanceLoop/startAutoUpdateCron/startMetricsCron/pollLoop()
154
+ * without waiting on it. This is safe because neither downstream consumer
155
+ * needs bootstrap complete synchronously: the maintenance cron only fires
156
+ * at its own scheduled time (default 03:30, not at startup), and the inbox
157
+ * watcher just needs trackedBrains populated "one tick later" — both read
158
+ * the same shared, mutable Map (getTrackedBrains()) whenever their own tick
159
+ * fires, not a snapshot taken at startup.
160
+ *
161
+ * No-ops in direct (non-token) mode, same gating as sendMetrics
162
+ * (metrics/sender.ts) — there's no backend endpoint to call without
163
+ * TEAMVIBE_API_URL + TEAMVIBE_POLLER_TOKEN.
164
+ */
165
+ export async function bootstrapTrackedBrains() {
166
+ detachedRetryForTest = undefined;
167
+ if (!config.TEAMVIBE_API_URL || !config.TEAMVIBE_POLLER_TOKEN) {
168
+ return;
169
+ }
170
+ const pollerId = getPollerId();
171
+ if (!pollerId) {
172
+ logger.warn('[Bootstrap] Skipping trackedBrains bootstrap: no pollerId (auth not completed?)');
173
+ return;
174
+ }
175
+ const token = config.TEAMVIBE_POLLER_TOKEN;
176
+ const url = `${config.TEAMVIBE_API_URL}/poller/brains?pollerId=${encodeURIComponent(pollerId)}`;
177
+ const outcome = await attemptFetch(url, token, CRITICAL_PATH_TIMEOUT_MS);
178
+ if (outcome.ok) {
179
+ applySuccess(outcome.brains);
180
+ return;
181
+ }
182
+ logger.warn(`[Bootstrap] GET /poller/brains failed on first attempt: ${outcome.reason} (${outcome.detail})`);
183
+ if (!outcome.retryable) {
184
+ reportFailure(outcome);
185
+ return;
186
+ }
187
+ // Off the critical path from here — startPoller() is not waiting on this
188
+ // promise. Wrapped in try/catch (DevGuru round-4 review) rather than just
189
+ // asserting "nothing below throws": attemptFetch() itself always returns
190
+ // an outcome object instead of throwing, but applySuccess() (seedTrackedBrains
191
+ // + emitEvent) and reportFailure() (emitError) are still real function
192
+ // calls — if either ever throws for any reason (a future refactor, an
193
+ // unexpected edge case), an uncaught throw inside this detached function
194
+ // becomes an unhandled promise rejection and crashes the whole poller
195
+ // process. Cheap insurance: catch and log a warning, matching this file's
196
+ // existing logger.warn style, instead of letting it propagate.
197
+ //
198
+ // Stored in detachedRetryForTest (DevGuru round-5 review) rather than
199
+ // `void`-discarded, purely so tests can await it deterministically — see
200
+ // that variable's own comment above. Storing the reference doesn't change
201
+ // runtime behavior here: this function still never awaits it itself.
202
+ detachedRetryForTest = (async () => {
203
+ try {
204
+ await sleep(RETRY_DELAY_MS);
205
+ const retryOutcome = await attemptFetch(url, token, FETCH_TIMEOUT_MS);
206
+ if (retryOutcome.ok) {
207
+ applySuccess(retryOutcome.brains);
208
+ return;
209
+ }
210
+ logger.warn(`[Bootstrap] GET /poller/brains failed on retry: ${retryOutcome.reason} (${retryOutcome.detail})`);
211
+ reportFailure(retryOutcome);
212
+ }
213
+ catch (err) {
214
+ logger.warn(`[Bootstrap] Detached retry threw unexpectedly: ${err instanceof Error ? err.message : String(err)}`);
215
+ }
216
+ })();
217
+ }
@@ -232,6 +232,7 @@ export declare function spawnClaudeCode(msg: TeamVibeQueueMessage, sessionLog: S
232
232
  newSessionId?: string;
233
233
  }>;
234
234
  export declare function getActiveProcessCount(): number;
235
+ export declare function getLastSessionEndTs(): number;
235
236
  /**
236
237
  * Kill a zombie process for a given thread ID.
237
238
  * Returns true if a process was found and killed.
@@ -1116,6 +1116,7 @@ async function runClaudeCode(msg, sessionLog, cwd, brainPath, sessionId, isFirst
1116
1116
  clearNotificationTimers();
1117
1117
  activeProcesses.delete(msg.id);
1118
1118
  activeProcessesByThread.delete(msg.thread_id);
1119
+ lastSessionEndTs = Date.now();
1119
1120
  // #73 gap 2 (flagged independently by two reviews): `close` firing means
1120
1121
  // `claude` itself has exited, but its MCP server children — spawned
1121
1122
  // into the same detached process group, see `spawn()` above — are not
@@ -1178,6 +1179,7 @@ async function runClaudeCode(msg, sessionLog, cwd, brainPath, sessionId, isFirst
1178
1179
  clearInterval(idleWatchdog);
1179
1180
  activeProcesses.delete(msg.id);
1180
1181
  activeProcessesByThread.delete(msg.thread_id);
1182
+ lastSessionEndTs = Date.now();
1181
1183
  emitError('session', `Claude Code process spawn error: ${error.message}`, {
1182
1184
  channelId: msg.teamvibe.channelId,
1183
1185
  context: { threadId: msg.thread_id },
@@ -1275,9 +1277,17 @@ export async function spawnClaudeCode(msg, sessionLog, cwd, sessionId, isFirstMe
1275
1277
  }
1276
1278
  const activeProcesses = new Map();
1277
1279
  const activeProcessesByThread = new Map();
1280
+ // tv#317: when the last `claude` process ended, so the idle-restart gate can require
1281
+ // a quiet period since real work last happened. Starts at module load (poller boot) so
1282
+ // a poller that never runs a session still becomes eligible once IDLE_RESTART_MIN_UPTIME_MS
1283
+ // passes, rather than being stuck at 0/undefined forever.
1284
+ let lastSessionEndTs = Date.now();
1278
1285
  export function getActiveProcessCount() {
1279
1286
  return activeProcesses.size;
1280
1287
  }
1288
+ export function getLastSessionEndTs() {
1289
+ return lastSessionEndTs;
1290
+ }
1281
1291
  /**
1282
1292
  * Kill a zombie process for a given thread ID.
1283
1293
  * Returns true if a process was found and killed.
package/dist/config.d.ts CHANGED
@@ -41,6 +41,10 @@ declare const configSchema: z.ZodObject<{
41
41
  AUTO_UPDATE_SCHEDULE: z.ZodDefault<z.ZodString>;
42
42
  INBOX_WATCH_FALLBACK_INTERVAL_MS: z.ZodDefault<z.ZodNumber>;
43
43
  INBOX_WATCH_DEBOUNCE_MS: z.ZodDefault<z.ZodNumber>;
44
+ IDLE_RESTART_ENABLED: z.ZodEffects<z.ZodDefault<z.ZodString>, boolean, string | undefined>;
45
+ IDLE_RESTART_IDLE_MS: z.ZodDefault<z.ZodNumber>;
46
+ IDLE_RESTART_MIN_UPTIME_MS: z.ZodDefault<z.ZodNumber>;
47
+ IDLE_RESTART_MEMORY_FILE_THRESHOLD_BYTES: z.ZodDefault<z.ZodNumber>;
44
48
  }, "strip", z.ZodTypeAny, {
45
49
  AWS_REGION: string;
46
50
  TEAMVIBE_API_URL: string;
@@ -78,6 +82,10 @@ declare const configSchema: z.ZodObject<{
78
82
  AUTO_UPDATE_SCHEDULE: string;
79
83
  INBOX_WATCH_FALLBACK_INTERVAL_MS: number;
80
84
  INBOX_WATCH_DEBOUNCE_MS: number;
85
+ IDLE_RESTART_ENABLED: boolean;
86
+ IDLE_RESTART_IDLE_MS: number;
87
+ IDLE_RESTART_MIN_UPTIME_MS: number;
88
+ IDLE_RESTART_MEMORY_FILE_THRESHOLD_BYTES: number;
81
89
  SQS_QUEUE_URL?: string | undefined;
82
90
  SESSIONS_TABLE?: string | undefined;
83
91
  TEAMVIBE_POLLER_TOKEN?: string | undefined;
@@ -123,6 +131,10 @@ declare const configSchema: z.ZodObject<{
123
131
  AUTO_UPDATE_SCHEDULE?: string | undefined;
124
132
  INBOX_WATCH_FALLBACK_INTERVAL_MS?: number | undefined;
125
133
  INBOX_WATCH_DEBOUNCE_MS?: number | undefined;
134
+ IDLE_RESTART_ENABLED?: string | undefined;
135
+ IDLE_RESTART_IDLE_MS?: number | undefined;
136
+ IDLE_RESTART_MIN_UPTIME_MS?: number | undefined;
137
+ IDLE_RESTART_MEMORY_FILE_THRESHOLD_BYTES?: number | undefined;
126
138
  }>;
127
139
  export type Config = z.infer<typeof configSchema>;
128
140
  export declare function loadConfig(): Config;
@@ -163,6 +175,10 @@ export declare const config: {
163
175
  AUTO_UPDATE_SCHEDULE: string;
164
176
  INBOX_WATCH_FALLBACK_INTERVAL_MS: number;
165
177
  INBOX_WATCH_DEBOUNCE_MS: number;
178
+ IDLE_RESTART_ENABLED: boolean;
179
+ IDLE_RESTART_IDLE_MS: number;
180
+ IDLE_RESTART_MIN_UPTIME_MS: number;
181
+ IDLE_RESTART_MEMORY_FILE_THRESHOLD_BYTES: number;
166
182
  SQS_QUEUE_URL?: string | undefined;
167
183
  SESSIONS_TABLE?: string | undefined;
168
184
  TEAMVIBE_POLLER_TOKEN?: string | undefined;
package/dist/config.js CHANGED
@@ -76,6 +76,23 @@ const configSchema = z.object({
76
76
  // exist at watcher startup, and prunes empty thread dirs.
77
77
  INBOX_WATCH_FALLBACK_INTERVAL_MS: z.coerce.number().default(300_000), // 5 minutes
78
78
  INBOX_WATCH_DEBOUNCE_MS: z.coerce.number().default(300), // coalesce bursty fs events into one scan
79
+ // Idle restart (tv#317): Railway bills on cgroup memory usage, which includes page
80
+ // cache that Linux only reclaims under real memory pressure — a large Railway memory
81
+ // limit means that pressure never triggers, so cache from heavy sessions (bun install,
82
+ // vite build, apt) accumulates until something forces a restart. Rather than an
83
+ // external supervisor, the poller restarts itself once genuinely idle; Railway's
84
+ // ALWAYS restart policy brings it back up clean and any SQS message that arrives
85
+ // during the few-second restart window just waits in the queue.
86
+ //
87
+ // Defaults OFF: this is a Railway-specific mitigation (the memory-threshold check
88
+ // is gated on cgroup v2 anyway, see readReclaimableFileBytes), but the default
89
+ // should make that explicit rather than lean on an implicit cgroup-absence no-op —
90
+ // deploy targets that also happen to be cgroup v2 Linux shouldn't pick this up by
91
+ // just existing. Set IDLE_RESTART_ENABLED=true on Railway to opt in.
92
+ IDLE_RESTART_ENABLED: z.string().default('false').transform((v) => v.toLowerCase() === 'true'),
93
+ IDLE_RESTART_IDLE_MS: z.coerce.number().default(1_800_000), // 30 minutes since the last session ended
94
+ IDLE_RESTART_MIN_UPTIME_MS: z.coerce.number().default(1_800_000), // crash-loop guard — matches the idle floor above (tv#317 discussion), so it doesn't add its own separate cadence
95
+ IDLE_RESTART_MEMORY_FILE_THRESHOLD_BYTES: z.coerce.number().default(700 * 1024 * 1024), // cgroup memory.stat `file` (reclaimable page cache), not memory.current — gating on raw usage would also fire on a real anon leak and mask it instead of just clearing cache
79
96
  });
80
97
  export function loadConfig() {
81
98
  const result = configSchema.safeParse(process.env);
@@ -0,0 +1,12 @@
1
+ export declare function readReclaimableFileBytes(path?: string): number | null;
2
+ export declare function findLiveBgTask(now?: number): string | null;
3
+ export declare function reconcileBgTasksOnBoot(): void;
4
+ export interface CheckIdleRestartDeps {
5
+ now?: number;
6
+ startTime?: number;
7
+ activeProcessCount?: () => number;
8
+ lastSessionEndTs?: () => number;
9
+ reclaimableBytes?: () => number | null;
10
+ liveBgTask?: () => string | null;
11
+ }
12
+ export declare function checkIdleRestart(deps?: CheckIdleRestartDeps): boolean;
@@ -0,0 +1,200 @@
1
+ import { appendFileSync, readdirSync, readFileSync } from 'fs';
2
+ import { join } from 'path';
3
+ import { config } from './config.js';
4
+ import { logger } from './logger.js';
5
+ import { getActiveProcessCount, getLastSessionEndTs } from './claude-spawner.js';
6
+ import { emitEvent } from './event-queue.js';
7
+ // tv#317: self-restart once genuinely idle, to let Railway reclaim page-cache RAM that
8
+ // Linux never proactively frees under our large memory limit. See config.ts for the
9
+ // IDLE_RESTART_* thresholds this checks against.
10
+ const processStartTime = Date.now();
11
+ // cgroup v2 only. Railway containers run cgroup v2; on anything else (local dev, a
12
+ // future cgroup v1 host) this just returns null and the gate never fires — the memory
13
+ // pressure this exists to relieve is Railway-specific, so failing closed is correct
14
+ // rather than guessing at a v1 path/format.
15
+ const MEMORY_STAT_PATH = '/sys/fs/cgroup/memory.stat';
16
+ export function readReclaimableFileBytes(path = MEMORY_STAT_PATH) {
17
+ let text;
18
+ try {
19
+ text = readFileSync(path, 'utf-8');
20
+ }
21
+ catch {
22
+ return null;
23
+ }
24
+ for (const line of text.split('\n')) {
25
+ const spaceIdx = line.indexOf(' ');
26
+ if (spaceIdx > 0 && line.slice(0, spaceIdx) === 'file') {
27
+ const value = Number(line.slice(spaceIdx + 1));
28
+ return Number.isFinite(value) ? value : null;
29
+ }
30
+ }
31
+ return null;
32
+ }
33
+ // --- bg-task liveness ---------------------------------------------------------------
34
+ //
35
+ // Reimplemented (not imported): bg-task.mjs lives in poller-brain and runs inside a
36
+ // Claude session, not the poller process — there's no shared module to import here.
37
+ // This mirrors its parseStatus/lifecycleOf classification (poller-brain
38
+ // skills/background-task/scripts/bg-task.mjs) rather than a raw /proc scan, which
39
+ // false-negatives on interpreter/path variations and can't distinguish "stuck past TTL"
40
+ // from "healthy" the way the status file's own started/ttl fields can.
41
+ const ABANDON_GRACE_SEC = 60; // mirrors bg-task.mjs's ABANDON_GRACE_SEC
42
+ function parseStatus(text) {
43
+ const kv = {};
44
+ for (const line of text.split('\n')) {
45
+ const i = line.indexOf('=');
46
+ if (i > 0)
47
+ kv[line.slice(0, i)] = line.slice(i + 1);
48
+ }
49
+ return kv;
50
+ }
51
+ function pidAlive(pid) {
52
+ try {
53
+ process.kill(pid, 0);
54
+ return true;
55
+ }
56
+ catch {
57
+ return false;
58
+ }
59
+ }
60
+ // Same four-way classification as bg-task.mjs's lifecycleOf, minus the tests-only
61
+ // `alive` injection point (this module has its own dedicated tests instead).
62
+ function lifecycleOf(kv, now) {
63
+ // Trust the runner's own claim, whatever it says (bg-task.mjs:139) — not just the two
64
+ // values the runner writes today (`finished`/`timed-out`, bg-task-runner.mjs:403).
65
+ // Mirroring the check itself, not just its current output, means the two
66
+ // implementations of this format can't quietly diverge if the runner ever starts
67
+ // writing a state value this list doesn't happen to enumerate — hardcoding a closed
68
+ // set here would silently fall through to the pid/TTL heuristics below instead.
69
+ if (kv.state)
70
+ return kv.state;
71
+ if (kv.runner_crashed)
72
+ return 'crashed';
73
+ if (Object.keys(kv).length === 0)
74
+ return 'unknown';
75
+ const pid = Number(kv.pid);
76
+ if (!Number.isInteger(pid) || pid <= 0 || !pidAlive(pid))
77
+ return 'abandoned';
78
+ const started = kv.started ? Date.parse(kv.started) : NaN;
79
+ const ttl = Number(String(kv.ttl ?? '').replace(/s$/, ''));
80
+ if (Number.isFinite(started) && Number.isFinite(ttl) && now - started > (ttl + ABANDON_GRACE_SEC) * 1000) {
81
+ return 'abandoned';
82
+ }
83
+ return 'running';
84
+ }
85
+ function bgTaskRoot() {
86
+ // Mirrors bg-task.mjs's taskRoot() default exactly, minus the per-channel
87
+ // namespace — the poller checks liveness across every channel it serves, not just
88
+ // one, so it walks the parent directory instead of joining a single channel ID.
89
+ return process.env['BG_TASK_ROOT'] || join(config.PERSISTENT_STORAGE_PATH || '/tmp', 'bg-tasks');
90
+ }
91
+ function listDirs(path) {
92
+ try {
93
+ return readdirSync(path, { withFileTypes: true })
94
+ .filter((entry) => entry.isDirectory())
95
+ .map((entry) => entry.name);
96
+ }
97
+ catch {
98
+ return [];
99
+ }
100
+ }
101
+ // Returns a "channel/taskId" label for the first live (running or unknown-state) task
102
+ // found, or null if none. `unknown` (no status written yet) is treated as live —
103
+ // deliberately conservative, per tv#317: a task a few milliseconds from its first
104
+ // status line looks identical to an orphaned dir, and "skip this idle cycle" is cheap
105
+ // while restarting mid-launch is not recoverable.
106
+ export function findLiveBgTask(now = Date.now()) {
107
+ const root = bgTaskRoot();
108
+ for (const channel of listDirs(root)) {
109
+ const channelPath = join(root, channel);
110
+ for (const taskDir of listDirs(channelPath)) {
111
+ let statusText = '';
112
+ try {
113
+ statusText = readFileSync(join(channelPath, taskDir, 'status'), 'utf-8');
114
+ }
115
+ catch {
116
+ // No status file at all — same "unknown" treatment as an empty one below.
117
+ }
118
+ const state = lifecycleOf(parseStatus(statusText), now);
119
+ if (state === 'running' || state === 'unknown') {
120
+ return `${channel}/${taskDir}`;
121
+ }
122
+ }
123
+ }
124
+ return null;
125
+ }
126
+ // tv#317 "Startup reconciliation": bg-tasks don't survive a poller restart (their
127
+ // runner is a child of the container that just went away), but a status file on a
128
+ // persistent volume has no way to know that on its own. Without this, a task dir
129
+ // whose runner never wrote a terminal line looks identical to a live one — worse,
130
+ // after a restart the fresh container's own low PIDs can coincidentally collide with
131
+ // a stale recorded `pid=`, so pidAlive() alone isn't reliable evidence either. Called
132
+ // once at boot so every pre-restart dir is unambiguously stamped before the idle-restart
133
+ // gate (or `--list`) ever reads it again.
134
+ export function reconcileBgTasksOnBoot() {
135
+ const root = bgTaskRoot();
136
+ let stamped = 0;
137
+ for (const channel of listDirs(root)) {
138
+ const channelPath = join(root, channel);
139
+ for (const taskDir of listDirs(channelPath)) {
140
+ const statusPath = join(channelPath, taskDir, 'status');
141
+ let statusText;
142
+ try {
143
+ statusText = readFileSync(statusPath, 'utf-8');
144
+ }
145
+ catch {
146
+ continue; // no status file yet — nothing to stamp
147
+ }
148
+ const kv = parseStatus(statusText);
149
+ if (kv.state || kv.runner_crashed)
150
+ continue; // already terminal
151
+ try {
152
+ appendFileSync(statusPath, 'runner_crashed=container_restart\n');
153
+ stamped++;
154
+ }
155
+ catch (err) {
156
+ logger.warn(`[IdleRestart] Failed to stamp stale bg-task ${channel}/${taskDir}: ${err instanceof Error ? err.message : err}`);
157
+ }
158
+ }
159
+ }
160
+ if (stamped > 0) {
161
+ logger.info(`[IdleRestart] Reconciled ${stamped} bg-task dir(s) left over from before this boot`);
162
+ }
163
+ }
164
+ export function checkIdleRestart(deps = {}) {
165
+ const now = deps.now ?? Date.now();
166
+ const startTime = deps.startTime ?? processStartTime;
167
+ const activeProcessCount = deps.activeProcessCount ?? getActiveProcessCount;
168
+ const lastSessionEndTs = deps.lastSessionEndTs ?? getLastSessionEndTs;
169
+ const reclaimableBytesFn = deps.reclaimableBytes ?? readReclaimableFileBytes;
170
+ const liveBgTask = deps.liveBgTask ?? findLiveBgTask;
171
+ if (!config.IDLE_RESTART_ENABLED)
172
+ return false;
173
+ const uptimeMs = now - startTime;
174
+ if (uptimeMs < config.IDLE_RESTART_MIN_UPTIME_MS)
175
+ return false;
176
+ if (activeProcessCount() > 0)
177
+ return false;
178
+ const idleMs = now - lastSessionEndTs();
179
+ if (idleMs < config.IDLE_RESTART_IDLE_MS)
180
+ return false;
181
+ const reclaimableBytes = reclaimableBytesFn();
182
+ if (reclaimableBytes === null || reclaimableBytes < config.IDLE_RESTART_MEMORY_FILE_THRESHOLD_BYTES)
183
+ return false;
184
+ if (liveBgTask())
185
+ return false; // skip this cycle, recheck next time — never cut a bg-task short just to free memory sooner
186
+ // Re-verify the two liveness conditions right before signaling restart — a
187
+ // session or bg-task could have started in the time it took to reach this point.
188
+ if (activeProcessCount() > 0)
189
+ return false;
190
+ if (liveBgTask())
191
+ return false;
192
+ logger.info(`[IdleRestart] Idle ${Math.round(idleMs / 60000)}min, reclaimable page cache ` +
193
+ `${Math.round(reclaimableBytes / 1024 / 1024)}MB — restarting to let Railway reclaim it`);
194
+ emitEvent('poller.idle_restart', 'system', {
195
+ idleMs,
196
+ uptimeMs,
197
+ reclaimableBytes,
198
+ });
199
+ return true;
200
+ }
@@ -21,6 +21,72 @@ export declare function getTrackedBrains(): ReadonlyMap<string, TrackedBrainInfo
21
21
  * @internal
22
22
  */
23
23
  export declare function __resetTrackedBrainsForTest(): void;
24
+ /**
25
+ * Shape returned by GET /poller/brains (teamvibe.ai#127 Option B) — the
26
+ * fields registerBrain() needs to build a synthetic maintenance message,
27
+ * minus `lastRealMessageAt` (see seedTrackedBrains below for why that's
28
+ * deliberately not part of the bootstrap payload).
29
+ */
30
+ export interface BootstrapBrainInfo {
31
+ brainId: string;
32
+ gitRepoUrl: string;
33
+ branch: string;
34
+ workspaceId: string;
35
+ botId: string;
36
+ botToken: string;
37
+ channelId: string;
38
+ }
39
+ /**
40
+ * Seed trackedBrains at startup from the backend's authoritative brain list
41
+ * (teamvibe.ai#127) so a "quiet" brain — one that hasn't received a real
42
+ * message since this poller process started — still gets included in the
43
+ * maintenance/heartbeat cron, instead of being silently invisible until a
44
+ * human happens to message it.
45
+ *
46
+ * Deliberately does NOT set `lastRealMessageAt` (left undefined). That
47
+ * field specifically means "we've seen a real message since our own last
48
+ * restart" — shouldSkipMaintenance (maintenance-activity.ts, #131) treats
49
+ * `undefined` as fail-open (never skip) for exactly this reason. Bootstrap
50
+ * seeding isn't a real message; claiming one occurred here would risk the
51
+ * opposite bug — a genuinely-quiet brain wrongly evaluated as "nothing new,
52
+ * skip" on its very first post-restart cron tick.
53
+ *
54
+ * Skips (rather than overwrites) any brainId already present in
55
+ * trackedBrains. This guard is load-bearing, not just defensive (DevGuru
56
+ * review, round 4): since round 3's critical-path detachment
57
+ * (brains-bootstrap.ts's `void (async () => {...})()` retry), the retry
58
+ * branch can finish AFTER startPoller() has already returned and real user
59
+ * messages have started arriving. That means a real registerBrain() call
60
+ * for a brain (which sets `lastRealMessageAt`) can race ahead of — and
61
+ * complete before — the detached bootstrap retry's seedTrackedBrains()
62
+ * call for that same brain. Without this skip, the late-arriving bootstrap
63
+ * seed would silently overwrite the real registration back to a fresh
64
+ * entry with `lastRealMessageAt: undefined`, losing real activity data
65
+ * (and, per the fail-open contract above, artificially reopening the
66
+ * skip-maintenance window for a brain that's actually been active).
67
+ *
68
+ * DOES also register each seeded brain for hourly metrics (see
69
+ * `registerBrainForMetrics` below) — DevGuru review caught that this
70
+ * module's original "metrics has its own disk-walk path" reasoning was
71
+ * wrong: `metrics/scheduler.ts`'s `registered` Map has exactly one
72
+ * production caller (`registerBrain()` below), with no disk-walk discovery
73
+ * of its own. Disk reads only happen in `collector.ts`, and only for a
74
+ * brain already present in that Map. Skipping the metrics seed here would
75
+ * reproduce the exact same restart-amnesia bug for metrics that this whole
76
+ * change exists to fix for maintenance: a quiet brain would get maintenance
77
+ * triggers again, but zero metrics until someone messages it.
78
+ *
79
+ * Deliberately does NOT replicate `registerBrain()`'s *immediate*
80
+ * collect+send side effect (the `if (isNew) { void (async () => ... })()`
81
+ * block below) — only the Map insert. Firing an immediate collect+send for
82
+ * every brain this poller owns, all at once, on every restart would be a
83
+ * fleet-wide burst; the existing hourly cron with its per-brain jitter
84
+ * (`collectAndSendAll`, `jitterMsFor`) already picks up newly-registered
85
+ * brains on its own schedule without a startup spike.
86
+ *
87
+ * Returns the number of brains actually seeded (for startup logging).
88
+ */
89
+ export declare function seedTrackedBrains(brains: readonly BootstrapBrainInfo[]): number;
24
90
  /**
25
91
  * Register a brain from a real message so the maintenance cron knows about it.
26
92
  * Called from processMessage() after brain path is resolved.
@@ -27,6 +27,75 @@ export function __resetTrackedBrainsForTest() {
27
27
  trackedBrains.clear();
28
28
  }
29
29
  let maintenanceJob = null;
30
+ /**
31
+ * Seed trackedBrains at startup from the backend's authoritative brain list
32
+ * (teamvibe.ai#127) so a "quiet" brain — one that hasn't received a real
33
+ * message since this poller process started — still gets included in the
34
+ * maintenance/heartbeat cron, instead of being silently invisible until a
35
+ * human happens to message it.
36
+ *
37
+ * Deliberately does NOT set `lastRealMessageAt` (left undefined). That
38
+ * field specifically means "we've seen a real message since our own last
39
+ * restart" — shouldSkipMaintenance (maintenance-activity.ts, #131) treats
40
+ * `undefined` as fail-open (never skip) for exactly this reason. Bootstrap
41
+ * seeding isn't a real message; claiming one occurred here would risk the
42
+ * opposite bug — a genuinely-quiet brain wrongly evaluated as "nothing new,
43
+ * skip" on its very first post-restart cron tick.
44
+ *
45
+ * Skips (rather than overwrites) any brainId already present in
46
+ * trackedBrains. This guard is load-bearing, not just defensive (DevGuru
47
+ * review, round 4): since round 3's critical-path detachment
48
+ * (brains-bootstrap.ts's `void (async () => {...})()` retry), the retry
49
+ * branch can finish AFTER startPoller() has already returned and real user
50
+ * messages have started arriving. That means a real registerBrain() call
51
+ * for a brain (which sets `lastRealMessageAt`) can race ahead of — and
52
+ * complete before — the detached bootstrap retry's seedTrackedBrains()
53
+ * call for that same brain. Without this skip, the late-arriving bootstrap
54
+ * seed would silently overwrite the real registration back to a fresh
55
+ * entry with `lastRealMessageAt: undefined`, losing real activity data
56
+ * (and, per the fail-open contract above, artificially reopening the
57
+ * skip-maintenance window for a brain that's actually been active).
58
+ *
59
+ * DOES also register each seeded brain for hourly metrics (see
60
+ * `registerBrainForMetrics` below) — DevGuru review caught that this
61
+ * module's original "metrics has its own disk-walk path" reasoning was
62
+ * wrong: `metrics/scheduler.ts`'s `registered` Map has exactly one
63
+ * production caller (`registerBrain()` below), with no disk-walk discovery
64
+ * of its own. Disk reads only happen in `collector.ts`, and only for a
65
+ * brain already present in that Map. Skipping the metrics seed here would
66
+ * reproduce the exact same restart-amnesia bug for metrics that this whole
67
+ * change exists to fix for maintenance: a quiet brain would get maintenance
68
+ * triggers again, but zero metrics until someone messages it.
69
+ *
70
+ * Deliberately does NOT replicate `registerBrain()`'s *immediate*
71
+ * collect+send side effect (the `if (isNew) { void (async () => ... })()`
72
+ * block below) — only the Map insert. Firing an immediate collect+send for
73
+ * every brain this poller owns, all at once, on every restart would be a
74
+ * fleet-wide burst; the existing hourly cron with its per-brain jitter
75
+ * (`collectAndSendAll`, `jitterMsFor`) already picks up newly-registered
76
+ * brains on its own schedule without a startup spike.
77
+ *
78
+ * Returns the number of brains actually seeded (for startup logging).
79
+ */
80
+ export function seedTrackedBrains(brains) {
81
+ let seeded = 0;
82
+ for (const b of brains) {
83
+ if (trackedBrains.has(b.brainId))
84
+ continue;
85
+ trackedBrains.set(b.brainId, {
86
+ brainId: b.brainId,
87
+ gitRepoUrl: b.gitRepoUrl,
88
+ branch: b.branch,
89
+ workspaceId: b.workspaceId,
90
+ botId: b.botId,
91
+ botToken: b.botToken,
92
+ channelId: b.channelId,
93
+ });
94
+ registerBrainForMetrics(b.brainId);
95
+ seeded++;
96
+ }
97
+ return seeded;
98
+ }
30
99
  /**
31
100
  * Register a brain from a real message so the maintenance cron knows about it.
32
101
  * Called from processMessage() after brain path is resolved.
package/dist/poller.d.ts CHANGED
@@ -29,8 +29,11 @@ export declare function isPollerInitiatedFailure(result: {
29
29
  exitCode: number | null;
30
30
  killedByPoller?: KilledByPoller;
31
31
  }): boolean;
32
+ export declare function describeFailureReason(killedByPoller: KilledByPoller | undefined, exitCode: number | null): string;
33
+ export declare const END_OF_TURN_NO_FURTHER_RESPONSE_SENTINEL = "NO_FURTHER_RESPONSE_NEEDED";
32
34
  export declare function endOfTurnFallbackText(hasSlackContext: boolean, result: Pick<SpawnResult, 'success' | 'sawSendMessageCall' | 'lastPlainTextContent'>): string | undefined;
33
35
  export declare function runInboxLoop(threadId: string, kbPath: string, queueMessage: TeamVibeQueueMessage, sessionLog: ReturnType<typeof logger.createSession>): Promise<void>;
36
+ export declare function __setSleepForTest(impl: (ms: number) => Promise<void>): void;
34
37
  export declare function __setShuttingDownForTest(value: boolean): void;
35
38
  export declare function startPoller(): Promise<void>;
36
39
  export {};
package/dist/poller.js CHANGED
@@ -10,6 +10,7 @@ import { cleanupOrphanWorktreesForAllBrains, isWorktreeIsolationEnabled } from '
10
10
  import { postMaintenanceReports } from './maintenance-reporter.js';
11
11
  import { initAuth, stopRefresh, getPollerId } from './auth-provider.js';
12
12
  import { registerBrain, startMaintenanceLoop, stopMaintenanceLoop } from './maintenance-cron.js';
13
+ import { bootstrapTrackedBrains } from './brains-bootstrap.js';
13
14
  import { startEventQueue, stopEventQueue, flush as flushEventQueue, emitEvent, emitError } from './event-queue.js';
14
15
  import { startMetricsCron, stopMetricsCron } from './metrics/scheduler.js';
15
16
  import { writeMessage, pickNextMessage, markProcessed, markFailed, hasUnprocessed, requeueMessage } from './inbox-manager.js';
@@ -17,6 +18,7 @@ import { startInboxWatcher, stopInboxWatcher } from './inbox-watcher.js';
17
18
  import { Cron } from 'croner';
18
19
  import { isControlMessage, handleControlMessage } from './control-handler.js';
19
20
  import { checkForUpdate } from './auto-update.js';
21
+ import { checkIdleRestart, reconcileBgTasksOnBoot } from './idle-restart.js';
20
22
  import { markAutoUpdateJobFired, markAutoUpdateJobSuccess } from './cron-telemetry.js';
21
23
  import { exec } from 'child_process';
22
24
  import { promisify } from 'util';
@@ -163,6 +165,35 @@ export function isIntentionalKillExitCode(exitCode) {
163
165
  export function isPollerInitiatedFailure(result) {
164
166
  return Boolean(result.killedByPoller) || isIntentionalKillExitCode(result.exitCode);
165
167
  }
168
+ // tv#320 DevGuru round-1 review: runInboxLoop's retry-cap decision (below)
169
+ // no longer calls isPollerInitiatedFailure() directly — since that cap now
170
+ // applies to everything except 'shutdown', the only distinction it needs is
171
+ // `lastKilledByPoller === 'shutdown'`, and `killedByPoller === 'shutdown'`
172
+ // already implies isPollerInitiatedFailure() === true, making a combined
173
+ // check redundant there.
174
+ //
175
+ // tv#320 DevGuru round-2 review: isIntentionalKillExitCode() still has a real
176
+ // caller below (describeFailureReason()) — a kill with NO `killedByPoller`
177
+ // marker that DOES still report the legacy 143/137 exit code is an
178
+ // intentional kill, not an unclassified application failure, and needs to be
179
+ // labeled as such in give-up notices/logs/events (`lastKilledByPoller ??
180
+ // 'application failure'` used to conflate the two).
181
+ //
182
+ // tv#320 DevGuru round-3 review: this only covers the literal 143/137 case.
183
+ // Per the comment above (line ~185), a group-killed process's `close` event
184
+ // reports `code: null`, NOT 143/137 — so a marker-less OOM/external-SIGTERM/
185
+ // group-kill still falls through to "application failure" here, same as
186
+ // before this fix. Closing that gap for real needs claude-spawner.ts's
187
+ // `close` handler to stop discarding its `signal` argument and SpawnResult
188
+ // to carry it (neither does today) — tracked as a follow-up, not attempted
189
+ // here.
190
+ export function describeFailureReason(killedByPoller, exitCode) {
191
+ if (killedByPoller)
192
+ return killedByPoller;
193
+ if (isIntentionalKillExitCode(exitCode))
194
+ return `killed (exit ${exitCode})`;
195
+ return 'application failure';
196
+ }
166
197
  // teamvibe.ai#284 (poller-brain#235): end-of-turn guard. A successful Claude
167
198
  // run can end its turn having only written plain assistant text — no
168
199
  // `mcp__slack__send_message` tool call — which the platform otherwise drops
@@ -176,12 +207,27 @@ export function isPollerInitiatedFailure(result) {
176
207
  // Deliberately does NOT fire on a failed run (result.success === false):
177
208
  // that path already has its own delivery via classifyFailedIteration +
178
209
  // sendSlackMessage/sendSlackError, and firing here too would double-deliver.
210
+ //
211
+ // poller-brain#380: a turn whose only tool call is a reaction (or similar)
212
+ // can trigger Claude Code's own forced-continuation nudge ("no visible
213
+ // output, produce a user-visible response"), which previously always fired
214
+ // this same fallback — double-posting even when the reaction was the
215
+ // intended, complete reply. Base-brain CLAUDE.md instructs agents to answer
216
+ // that nudge with this exact sentinel when nothing further is needed; an
217
+ // exact match is treated as "nothing to deliver". Deliberately an exact
218
+ // string match, not a content heuristic — anything else (including real
219
+ // content an agent forgot to send via send_message) still delivers as
220
+ // before, so this can't silently drop a forgotten reply.
221
+ export const END_OF_TURN_NO_FURTHER_RESPONSE_SENTINEL = 'NO_FURTHER_RESPONSE_NEEDED';
179
222
  export function endOfTurnFallbackText(hasSlackContext, result) {
180
223
  if (!hasSlackContext || !result.success || result.sawSendMessageCall) {
181
224
  return undefined;
182
225
  }
183
226
  const text = result.lastPlainTextContent?.trim();
184
- return text ? text : undefined;
227
+ if (!text || text === END_OF_TURN_NO_FURTHER_RESPONSE_SENTINEL) {
228
+ return undefined;
229
+ }
230
+ return text;
185
231
  }
186
232
  export async function runInboxLoop(threadId, kbPath, queueMessage, sessionLog) {
187
233
  if (activeLoops.has(threadId)) {
@@ -328,6 +374,26 @@ export async function runInboxLoop(threadId, kbPath, queueMessage, sessionLog) {
328
374
  lastExitCode = result.exitCode;
329
375
  lastKilledByPoller = result.killedByPoller;
330
376
  sessionLog.error(`[Loop] Claude failed iteration #${iteration}: ${result.error}`);
377
+ // tv#320 DevGuru round-2 review: a fleet-visible record of EVERY
378
+ // failed iteration (not just the eventual give-up) means a future
379
+ // "was this in one loop or across restarts" question is a DDB query
380
+ // (this event, by thread/filename/timestamp) instead of a CloudWatch
381
+ // log dig + SSO admin escalation — which is exactly what round-2's
382
+ // point 3b hit a dead end on for pb#337.
383
+ //
384
+ // tv#320 DevGuru round-3 review: excludes 'shutdown' — that fires on
385
+ // every poller restart/deploy for every in-flight message fleet-wide
386
+ // and isn't a retry-cap-relevant failure (it's not counted toward
387
+ // failureCounts either), so it would just be alerting noise for an
388
+ // expected event with no new information (the requeue below already
389
+ // covers it).
390
+ if (lastKilledByPoller !== 'shutdown') {
391
+ emitError('session', `Iteration #${iteration} failed for ${message.filename}`, {
392
+ severity: 'warn',
393
+ channelId: queueMessage.teamvibe.channelId,
394
+ context: { threadId, filename: message.filename, iteration, reason: describeFailureReason(lastKilledByPoller, lastExitCode) },
395
+ });
396
+ }
331
397
  if (hasSlackContext) {
332
398
  const decision = classifyFailedIteration(result);
333
399
  if (decision.kind === 'neutral') {
@@ -349,33 +415,96 @@ export async function runInboxLoop(threadId, kbPath, queueMessage, sessionLog) {
349
415
  }
350
416
  else {
351
417
  // Poller-initiated kill (timeout/idle/network/zombie/shutdown, via
352
- // `killedByPoller`, or the legacy 143/137 exit-code fallback) → never
353
- // counts as an application failure, so markFailed/backoff is skipped
354
- // below regardless of reason. For timeout/idle/network/zombie this
355
- // really does mean "retry immediately" — the `while (!shuttingDown)`
356
- // condition above lets the loop come straight back around. For
357
- // 'shutdown' specifically, `shuttingDown` is already true by the time
358
- // this reads, so the loop condition above stops it from actually
359
- // retrying inline — instead the branch below requeues the message
360
- // from processing/ back to new/ (PR#308 round-5) so a future boot's
361
- // inbox watcher finds it. See isPollerInitiatedFailure() and the
362
- // loop-condition comment above.
363
- const isKilledSignal = isPollerInitiatedFailure({ exitCode: lastExitCode, killedByPoller: lastKilledByPoller });
364
- if (!isKilledSignal) {
418
+ // `killedByPoller`, or the legacy 143/137 exit-code fallback) never
419
+ // counts as an *application* failure — but tv#320 (poller-brain#337:
420
+ // one checkpoint file redelivered 18x/~10min, stopped only by a
421
+ // manual `rm`) showed that timeout/idle/network/zombie kills can be
422
+ // just as deterministically tied to this one message's content as a
423
+ // genuine app failure (e.g. an oversized file that reliably
424
+ // re-triggers the same idle/timeout watchdog on every immediate
425
+ // retry). So the same failureCounts/MAX_RETRIES/backoff cap as the
426
+ // application-failure path now applies to everything except
427
+ // 'shutdown' — not just those four named reasons: a legacy 143/137
428
+ // exit with no `killedByPoller` marker at all is just as capable of
429
+ // repeating forever and there's no reason to treat it differently.
430
+ // 'shutdown' alone stays excluded: `shuttingDown` is already true by
431
+ // the time this reads, so the loop condition above stops it from
432
+ // actually retrying inline — instead the branch below requeues the
433
+ // message from processing/ back to new/ (PR#308 round-5) so a future
434
+ // boot's inbox watcher finds it.
435
+ //
436
+ // tv#320 DevGuru round-1 review: this branches on `lastKilledByPoller
437
+ // !== 'shutdown'` directly rather than `isPollerInitiatedFailure(...)`
438
+ // — `killedByPoller === 'shutdown'` already implies
439
+ // isPollerInitiatedFailure() === true, so combining the two was a
440
+ // dead term. isPollerInitiatedFailure() itself is unchanged and still
441
+ // meaningful (exported, unit-tested in
442
+ // poller-retry-classification.test.ts) for any caller that needs the
443
+ // poller-vs-app distinction — this loop just no longer needs it,
444
+ // since the only distinction that changes behavior here is
445
+ // shutdown-vs-not.
446
+ if (lastKilledByPoller !== 'shutdown') {
365
447
  const failures = (failureCounts.get(message.filename) || 0) + 1;
366
448
  failureCounts.set(message.filename, failures);
367
449
  const MAX_RETRIES = 3;
368
450
  if (failures >= MAX_RETRIES) {
369
- sessionLog.error(`[Loop] Message ${message.filename} failed ${failures} times, giving up`);
451
+ // tv#320 DevGuru round-1 review: the per-iteration notice sent
452
+ // above (classifyFailedIteration) for idle/zombie/network reads
453
+ // "send another message to resume/restart/retry" — true before
454
+ // this cap existed (retry was unconditional), false now that this
455
+ // is the last attempt and the message is about to be dropped.
456
+ // Give the user an explicit, accurate final word instead of
457
+ // leaving classifyFailedIteration's now-stale promise as the only
458
+ // thing they saw, and surface it as a real error event (not just
459
+ // a session-log line) so give-up isn't invisible to fleet
460
+ // monitoring either.
461
+ const reasonLabel = describeFailureReason(lastKilledByPoller, lastExitCode);
462
+ sessionLog.error(`[Loop] Message ${message.filename} failed ${failures} times (last reason: ${reasonLabel}), giving up`);
370
463
  markFailed(kbPath, threadId, message.filename);
464
+ if (hasSlackContext) {
465
+ // tv#320 DevGuru round-2 review (BLOCKING): this whole else-if
466
+ // chain sits structurally after the per-iteration try/catch
467
+ // above (which closes at the `catch (error)` a few lines up),
468
+ // so an unhandled exception here — unlike the per-iteration
469
+ // classifyFailedIteration notices, which ARE inside that
470
+ // try/catch — would propagate straight out of the `while`
471
+ // loop, skip releaseSessionLock() below (it's in the try body,
472
+ // not a `finally`), and leave the DDB session lock held until
473
+ // TTL. sendSlackError() (slack-client.ts) has no internal catch
474
+ // of its own (unlike addReaction(), which does), so a Slack
475
+ // rate-limit/auth/network error here needs the same explicit
476
+ // try/catch the end-of-turn-guard fallback above already uses.
477
+ try {
478
+ await sendSlackError(queueMessage, `Giving up after ${failures} attempts (last reason: ${reasonLabel}) — this message was not processed. Please send it again.`);
479
+ }
480
+ catch (err) {
481
+ sessionLog.error(`[Loop] Give-up notice delivery failed: ${err instanceof Error ? err.message : err}`);
482
+ }
483
+ await addReaction(queueMessage, 'x');
484
+ }
485
+ emitError('session', `Message ${message.filename} failed ${failures} times, giving up`, {
486
+ severity: 'error',
487
+ channelId: queueMessage.teamvibe.channelId,
488
+ context: { threadId, filename: message.filename, reason: reasonLabel },
489
+ });
371
490
  }
372
491
  else {
373
492
  const backoffMs = Math.min(10_000 * Math.pow(2, failures - 1), 60_000);
374
493
  sessionLog.info(`[Loop] Retry ${failures}/${MAX_RETRIES - 1} for ${message.filename}, backing off ${backoffMs}ms`);
375
- await sleep(backoffMs);
494
+ // tv#320 DevGuru round-2 nit: this backoff was previously only
495
+ // reachable for genuine app failures; the broadened cap above
496
+ // means a timeout/idle/network/zombie kill can now land here too,
497
+ // including one racing an in-progress shutdown() (e.g. the idle
498
+ // watchdog and a shutdown SIGTERM firing close together). Skip
499
+ // the up-to-60s wait when already shutting down so the `while
500
+ // (!shuttingDown)` condition above can exit on the very next
501
+ // check instead of stalling runInboxLoop's return.
502
+ if (!shuttingDown) {
503
+ await sleep(backoffMs);
504
+ }
376
505
  }
377
506
  }
378
- else if (lastKilledByPoller === 'shutdown') {
507
+ else {
379
508
  // PR#308 round-5 review (teamvibe.ai#73): pickNextMessage() moves a
380
509
  // message's file to processing/ as soon as it's picked up, not once
381
510
  // it completes — so a message killed here by shutdown is left
@@ -387,9 +516,15 @@ export async function runInboxLoop(threadId, kbPath, queueMessage, sessionLog) {
387
516
  // shutdown notice text below, classify-failed-iteration.ts)
388
517
  // literally true. It does NOT make resumption automatic: the
389
518
  // watcher's checkInboxForNewSessions() (inbox-watcher.ts) only
390
- // scans brains already in the in-memory trackedBrains map, which
391
- // is empty after a fresh poller start until some SQS activity for
392
- // this brain re-registers it (see teamvibe.ai#310) — hence the
519
+ // scans brains already in the in-memory trackedBrains map. Since
520
+ // teamvibe.ai#127, that map is best-effort seeded from the
521
+ // backend at startup (bootstrapTrackedBrains(), before
522
+ // startInboxWatcher() is called) — but it's still not a
523
+ // guarantee (network failure, backend outage, brain missing a
524
+ // Slack bot/token to resolve) and a brand-new brain from a real
525
+ // message that arrives after bootstrap but before its own first
526
+ // registerBrain() call is still momentarily untracked. See
527
+ // teamvibe.ai#310 for the still-open general case. Hence the
393
528
  // notice text also naming "next message or scheduled run" rather
394
529
  // than claiming automatic resumption.
395
530
  //
@@ -581,6 +716,18 @@ async function pollLoop() {
581
716
  consecutiveAuthFailures = 0;
582
717
  }
583
718
  if (messages.length === 0) {
719
+ // tv#317: only worth checking on an otherwise-empty poll — a poll that
720
+ // returned messages means activeSessions is about to go above 0 anyway.
721
+ // Routed through the real shutdown() (not a standalone flush+exit) so this
722
+ // sets shuttingDown, stops the inbox watcher, and releases locks — the same
723
+ // gate every other exit path goes through. DevGuru review, tv#317: a bare
724
+ // flushEventQueue()+process.exit(0) here let the inbox watcher spawn a new
725
+ // session in the window before exit, then kill it mid-work (#308's shape),
726
+ // and skipped releaseAllActiveLocks(), stranding locks for STALE_LOCK_TIMEOUT_MS.
727
+ if (checkIdleRestart()) {
728
+ logger.info('[IdleRestart] Restarting to let Railway reclaim page-cache memory...');
729
+ await shutdown('idle-restart', 0);
730
+ }
584
731
  continue;
585
732
  }
586
733
  logger.info(`Received ${messages.length} message(s) from SQS`);
@@ -651,8 +798,17 @@ async function pollLoop() {
651
798
  }
652
799
  }
653
800
  }
801
+ let sleepImpl = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
654
802
  function sleep(ms) {
655
- return new Promise((resolve) => setTimeout(resolve, ms));
803
+ return sleepImpl(ms);
804
+ }
805
+ // Exported for testing only (tv#320 DevGuru round-1 nit): lets a test replace
806
+ // the real backoff delay (runInboxLoop's retry-cap path) with an immediate
807
+ // resolve, without a `globalThis.setTimeout` override that would also swallow
808
+ // any other real timer in the same code path. Same pattern as
809
+ // __setShuttingDownForTest above.
810
+ export function __setSleepForTest(impl) {
811
+ sleepImpl = impl;
656
812
  }
657
813
  // Graceful shutdown
658
814
  let shuttingDown = false;
@@ -749,11 +905,38 @@ export async function startPoller() {
749
905
  startEventQueue();
750
906
  await ensureDirectories();
751
907
  await ensureBaseBrain();
908
+ reconcileBgTasksOnBoot();
909
+ if (config.IDLE_RESTART_ENABLED) {
910
+ // Defaults off (tv#317 follow-up, DevGuru review): a silent disabled gate and an
911
+ // enabled gate that just hasn't hit its threshold yet look identical in production
912
+ // (same flat/growing memory graph) — logging only on enabled, matching the
913
+ // Worktree isolation line below, makes opt-in verifiable from boot logs instead.
914
+ logger.info(' Idle restart: ENABLED (IDLE_RESTART_ENABLED=true)');
915
+ }
752
916
  if (isWorktreeIsolationEnabled()) {
753
917
  logger.info(' Worktree isolation: ENABLED (POLLER_USE_WORKTREES=true)');
754
918
  const sweep = await cleanupOrphanWorktreesForAllBrains(config.BRAINS_PATH);
755
919
  logger.info(` Orphan worktree sweep: scanned ${sweep.scanned} brains, removed ${sweep.removed} stale worktrees`);
756
920
  }
921
+ // Seed trackedBrains from the backend's authoritative brain list
922
+ // (teamvibe.ai#127 Option B) BEFORE the maintenance cron and the inbox
923
+ // watcher start — both iterate the in-memory trackedBrains map
924
+ // (maintenance-cron.ts's triggerAllBrains, inbox-watcher.ts's
925
+ // checkInboxForNewSessions), which is otherwise empty after every
926
+ // restart until a real message re-registers each brain one at a time.
927
+ // Best-effort: never throws, so a bootstrap failure here falls back to
928
+ // the pre-#127 behavior rather than blocking startup.
929
+ //
930
+ // This await only covers ONE short (~10s) fetch attempt (DevGuru round-3
931
+ // review item 2) — real message processing (pollLoop(), below) must not
932
+ // be blocked for up to ~62s by a backend outage, which is exactly the
933
+ // condition most likely to coincide with a poller restart (backend
934
+ // deploys). If that one attempt fails and is retryable, the retry runs
935
+ // detached in the background and finishes after startPoller() has
936
+ // already returned — see brains-bootstrap.ts's bootstrapTrackedBrains()
937
+ // doc comment for why that's safe (neither the maintenance cron nor the
938
+ // inbox watcher need trackedBrains populated synchronously here).
939
+ await bootstrapTrackedBrains();
757
940
  startMaintenanceLoop(processMessage);
758
941
  startAutoUpdateCron();
759
942
  startMetricsCron();
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@teamvibe/poller",
3
- "version": "0.7.0",
3
+ "version": "0.7.1",
4
4
  "type": "module",
5
5
  "main": "./dist/index.js",
6
6
  "bin": {