@teamvibe/poller 0.7.0 → 0.7.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -18,6 +18,14 @@ interface BrainConfig {
18
18
  * pushBrainChanges below.
19
19
  */
20
20
  export declare function cleanupStaleLocks(brainDir: string): Promise<void>;
21
+ /**
22
+ * Write the local (uncommitted) merge-driver attributes for a brain checkout.
23
+ * Idempotent and safe to call on every `getBrainPath` invocation, not just on
24
+ * first clone — this also self-heals brains that were checked out before
25
+ * this existed. Exported for testing — production call sites go through
26
+ * `getBrainPath`.
27
+ */
28
+ export declare function ensureGitAttributes(brainDir: string): Promise<void>;
21
29
  export declare function getBrainPath(brain?: BrainConfig, channelId?: string, workspaceId?: string): Promise<string>;
22
30
  /**
23
31
  * Update a brain repo (respects cooldown). Exported for testing — production
@@ -1,7 +1,7 @@
1
1
  import { exec } from 'child_process';
2
2
  import { promisify } from 'util';
3
3
  import { existsSync, mkdirSync } from 'fs';
4
- import { readdir, stat, unlink } from 'fs/promises';
4
+ import { readdir, stat, unlink, mkdir, writeFile } from 'fs/promises';
5
5
  import { join, basename } from 'path';
6
6
  import { config } from './config.js';
7
7
  import { logger } from './logger.js';
@@ -128,6 +128,47 @@ async function reportBrainSync(brainId, workspaceId, event) {
128
128
  emitError('brain', `Brain sync report error: ${error instanceof Error ? error.message : String(error)}`, { severity: 'warn', context: { brainId } });
129
129
  }
130
130
  }
131
+ // tv#247: local-only (never committed, never synced) merge-driver config for
132
+ // a brain checkout. `merge=union` is one of git's built-in low-level merge
133
+ // drivers (alongside `text`/`binary`) — an attributes entry alone is enough,
134
+ // no `[merge "union"]` section in git config is needed (verified: a rebase
135
+ // across two branches with independent appends to a `merge=union`-attributed
136
+ // file merges both lines automatically instead of conflicting).
137
+ //
138
+ // Scoped ONLY to genuinely pure-append, line-independent files. Deliberately
139
+ // does NOT cover MEM_REGISTRY.md, MEM_REGISTRY_ARCHIVE.md, semantic/*, or
140
+ // episodic/* — consolidation edits those in place (promotes/removes
141
+ // entries), so a union merge there would resurrect already-promoted-and-
142
+ // removed content instead of resolving the conflict, rather than concatenate
143
+ // two genuinely independent additions.
144
+ //
145
+ // Written to `.git/info/attributes` (not a committed `.gitattributes`)
146
+ // because this is poller-local git behavior, not brain content — channel
147
+ // brain repos are written to only by their own sessions, never by the
148
+ // platform directly.
149
+ const GIT_ATTRIBUTES_CONTENT = ['memory/TODAY.md merge=union', 'memory/daily/*.md merge=union', ''].join('\n');
150
+ /**
151
+ * Write the local (uncommitted) merge-driver attributes for a brain checkout.
152
+ * Idempotent and safe to call on every `getBrainPath` invocation, not just on
153
+ * first clone — this also self-heals brains that were checked out before
154
+ * this existed. Exported for testing — production call sites go through
155
+ * `getBrainPath`.
156
+ */
157
+ export async function ensureGitAttributes(brainDir) {
158
+ try {
159
+ const gitDir = join(brainDir, '.git');
160
+ if (!existsSync(gitDir))
161
+ return; // not a git repo yet (defensive — getBrainPath always clones first)
162
+ const infoDir = join(gitDir, 'info');
163
+ await mkdir(infoDir, { recursive: true });
164
+ await writeFile(join(infoDir, 'attributes'), GIT_ATTRIBUTES_CONTENT);
165
+ }
166
+ catch (error) {
167
+ // Best-effort: worst case a future push-race on these files needs manual
168
+ // resolution same as before this existed. Not a reason to fail the caller.
169
+ logger.warn(`Brain ${brainDir}: failed to write .git/info/attributes: ${error instanceof Error ? error.message : error}`);
170
+ }
171
+ }
131
172
  export async function getBrainPath(brain, channelId, workspaceId) {
132
173
  if (!brain?.gitRepoUrl) {
133
174
  if (channelId) {
@@ -164,6 +205,9 @@ export async function getBrainPath(brain, channelId, workspaceId) {
164
205
  if (workspaceId)
165
206
  reportBrainSync(brain.brainId, workspaceId, 'pulled');
166
207
  }
208
+ // Idempotent, local-only — covers both the fresh-clone and existing-dir
209
+ // branches above, and self-heals brains cloned before this existed.
210
+ await ensureGitAttributes(brainDir);
167
211
  return brainDir;
168
212
  }
169
213
  /**
@@ -263,6 +307,52 @@ async function gitStashList(brainDir) {
263
307
  const { stdout } = await gitExec('git stash list', { cwd: brainDir });
264
308
  return stdout.trim();
265
309
  }
310
+ // Per-brainId rescue-ref name, seeded with a timestamp the first time a given
311
+ // brainId hits an unresolved push-race conflict, then reused (force-pushed
312
+ // to) on every later pushBrainChangesLocked invocation that still can't
313
+ // converge — e.g. the next message's push attempt while the conflict remains
314
+ // unresolved on disk. Without this, each retry would mint its own
315
+ // `conflict/<brainId>-<ts>` ref and spam origin with one throwaway branch per
316
+ // retry instead of one per actual incident. Cleared on the next successful
317
+ // push for that brainId, since any future conflict after that point is a new,
318
+ // unrelated incident that deserves its own ref.
319
+ const rescueRefNames = new Map();
320
+ /**
321
+ * Best-effort durability net for a `PUSH_RACE_CONFLICT`: push whatever HEAD
322
+ * currently points to onto a throwaway ref on origin before the caller
323
+ * reports the conflict, so the session's commits survive even if this
324
+ * checkout is lost (crash, re-clone, TTL'd container) before a human/AI does
325
+ * the manual resolution. Never throws — a failed rescue push just means no
326
+ * safety net for this occurrence, not a reason to mask the real
327
+ * PUSH_RACE_CONFLICT error the caller is about to report.
328
+ *
329
+ * Callers MUST ensure `HEAD` is the safe, non-mid-rebase content to rescue
330
+ * before calling this (e.g. after `git rebase --abort` restores the
331
+ * pre-rebase local branch) — this function pushes HEAD as-is.
332
+ */
333
+ async function pushRescueRef(brainDir, brainId) {
334
+ let refName = rescueRefNames.get(brainId);
335
+ if (!refName) {
336
+ // Unix seconds (not ms) — matches the <timestamp> format documented on
337
+ // the consuming side (poller-brain#443's MAINTENANCE.md `oldestAgeDays`
338
+ // derivation), so both sides agree on how to parse the ref name.
339
+ refName = `conflict/${brainId}-${Math.floor(Date.now() / 1000)}`;
340
+ rescueRefNames.set(brainId, refName);
341
+ }
342
+ try {
343
+ await gitExec(`git push --force origin HEAD:refs/heads/${refName}`, { cwd: brainDir });
344
+ logger.info(`Brain ${brainId} push-race: rescued local commits to ${refName}`);
345
+ return { refName, pushed: true };
346
+ }
347
+ catch (error) {
348
+ // Distinct sub-case from a successful rescue (same failure class as the
349
+ // known pb#440 issue — e.g. broken git credentials) — logged separately
350
+ // so it's never mistaken for "commits are safe on origin".
351
+ const message = error instanceof Error ? error.message : String(error);
352
+ logger.error(`Brain ${brainId} push-race: rescue-ref push to ${refName} itself FAILED, local commits remain only on disk: ${message}`);
353
+ return { refName, pushed: false };
354
+ }
355
+ }
266
356
  /**
267
357
  * Scan the commits about to be pushed for blobs over `maxBytes`. Returns the
268
358
  * offending paths/sizes, or an empty array if everything is under the limit.
@@ -382,6 +472,10 @@ async function pushBrainChangesLocked(brainDir, brainId, workspaceId, retryOptio
382
472
  attempt++;
383
473
  try {
384
474
  await gitExec(`git push -u origin ${branch}`, { cwd: brainDir });
475
+ // A successful push means any earlier unresolved conflict for this
476
+ // brainId (if any) is now resolved — a future conflict is a new,
477
+ // unrelated incident, so stop reusing that rescue ref name.
478
+ rescueRefNames.delete(brainId);
385
479
  if (attempt > 1) {
386
480
  logger.info(`Brain ${brainId} push-race auto-resolved after ${attempt} attempts`);
387
481
  emitError('brain', `Brain ${brainId} push-race auto-resolved after ${attempt} attempts`, {
@@ -427,7 +521,15 @@ async function pushBrainChangesLocked(brainDir, brainId, workspaceId, retryOptio
427
521
  const stashAfter = await gitStashList(brainDir);
428
522
  if (stashAfter !== stashBefore) {
429
523
  logger.error(`Brain ${brainId} push-race: autostash pop left changes stashed after rebase, treating as conflict`);
430
- emitError('brain', `Brain ${brainId} push-race autostash pop conflict — manual sync needed (stash: ${stashAfter.split('\n')[0]})`, { severity: 'error', context: { brainId, errorCode: 'PUSH_RACE_CONFLICT', attempts: attempt, reason: 'autostash_pop_conflict' } });
524
+ // The rebase itself (of committed local work onto origin) succeeded
525
+ // here — HEAD is the safe, non-mid-rebase content to rescue; only
526
+ // the separate stash-pop of uncommitted changes is what's stuck.
527
+ const { refName: rescueRef, pushed: rescuePushed } = await pushRescueRef(brainDir, brainId);
528
+ emitError('brain', `Brain ${brainId} push-race autostash pop conflict — manual sync needed (stash: ${stashAfter.split('\n')[0]})` +
529
+ (rescuePushed ? `, committed local work rescued to ${rescueRef}` : `, rescue push to ${rescueRef} also FAILED`), {
530
+ severity: 'error',
531
+ context: { brainId, errorCode: 'PUSH_RACE_CONFLICT', attempts: attempt, reason: 'autostash_pop_conflict', rescueRef, rescuePushed },
532
+ });
431
533
  return;
432
534
  }
433
535
  }
@@ -442,13 +544,21 @@ async function pushBrainChangesLocked(brainDir, brainId, workspaceId, retryOptio
442
544
  severity: 'error',
443
545
  context: { brainId, errorCode: 'PUSH_RACE_ABORT_FAILED', attempts: attempt },
444
546
  });
547
+ // Repo is left mid-rebase / in an uncertain state — do NOT push
548
+ // HEAD to a rescue ref here, that would rescue the wrong
549
+ // (partially-applied) content instead of the pre-conflict local work.
445
550
  return;
446
551
  }
552
+ // `git rebase --abort` above restored HEAD to the pre-rebase local
553
+ // branch — i.e. exactly the unpushed, pre-conflict work we need to
554
+ // keep durable, not whatever mid-rebase state existed a moment ago.
555
+ const { refName: rescueRef, pushed: rescuePushed } = await pushRescueRef(brainDir, brainId);
447
556
  const rebaseMessage = rebaseError instanceof Error ? rebaseError.message : String(rebaseError);
448
557
  logger.error(`Brain ${brainId} push-race rebase conflict: ${rebaseMessage}`);
449
- emitError('brain', `Brain ${brainId} push-race rebase conflict — manual sync needed: ${rebaseMessage}`, {
558
+ emitError('brain', `Brain ${brainId} push-race rebase conflict — manual sync needed: ${rebaseMessage}` +
559
+ (rescuePushed ? `, local commits rescued to ${rescueRef}` : `, rescue push to ${rescueRef} also FAILED`), {
450
560
  severity: 'error',
451
- context: { brainId, errorCode: 'PUSH_RACE_CONFLICT', attempts: attempt },
561
+ context: { brainId, errorCode: 'PUSH_RACE_CONFLICT', attempts: attempt, rescueRef, rescuePushed },
452
562
  });
453
563
  return;
454
564
  }
@@ -457,9 +567,25 @@ async function pushBrainChangesLocked(brainDir, brainId, workspaceId, retryOptio
457
567
  }
458
568
  const errorCode = NON_FF_REJECTION_RE.test(lastErrorMessage) ? 'PUSH_RACE_UNRESOLVED' : 'PUSH_FAILED';
459
569
  logger.warn(`Failed to push brain ${brainId} changes: ${lastErrorMessage}`);
460
- emitError('brain', `Failed to push brain ${brainId} changes: ${lastErrorMessage}`, {
570
+ // PUSH_RACE_UNRESOLVED means the retry budget ran out while HEAD was still
571
+ // in the safe, non-mid-rebase state left by the last successful rebase
572
+ // (the loop only reaches here via the `attempt >= maxAttempts` break right
573
+ // after a push rejection, never after a failed rebase — those return
574
+ // earlier). Same "local commits only on disk" risk as the two conflict
575
+ // cases above, just a different exhaustion cause — rescue it too.
576
+ // PUSH_FAILED (non-NON_FF failures, e.g. auth/network) deliberately
577
+ // skipped: a rescue push would almost always fail for the identical
578
+ // reason as the push it's rescuing.
579
+ let rescueRef;
580
+ let rescuePushed;
581
+ if (errorCode === 'PUSH_RACE_UNRESOLVED') {
582
+ ;
583
+ ({ refName: rescueRef, pushed: rescuePushed } = await pushRescueRef(brainDir, brainId));
584
+ }
585
+ emitError('brain', `Failed to push brain ${brainId} changes: ${lastErrorMessage}` +
586
+ (errorCode === 'PUSH_RACE_UNRESOLVED' ? (rescuePushed ? `, local commits rescued to ${rescueRef}` : `, rescue push to ${rescueRef} also FAILED`) : ''), {
461
587
  severity: errorCode === 'PUSH_RACE_UNRESOLVED' ? 'error' : 'warn',
462
- context: { brainId, errorCode, attempts: attempt },
588
+ context: { brainId, errorCode, attempts: attempt, ...(errorCode === 'PUSH_RACE_UNRESOLVED' ? { rescueRef, rescuePushed } : {}) },
463
589
  });
464
590
  }
465
591
  catch (error) {
@@ -0,0 +1,45 @@
1
+ /** @internal test-only */
2
+ export declare function __detachedRetryForTest(): Promise<void> | undefined;
3
+ /**
4
+ * Fetch the full list of brains this poller is responsible for from the
5
+ * backend (GET /poller/brains, source of truth = DDB — teamvibe.ai#127
6
+ * Option B) and seed `trackedBrains` (maintenance-cron.ts) before the
7
+ * maintenance/heartbeat cron starts, so a brain that's been quiet since
8
+ * before this poller's most recent restart is still picked up instead of
9
+ * staying invisible until a human happens to message it.
10
+ *
11
+ * Best-effort: on any failure (network, timeout, non-2xx, malformed body,
12
+ * missing pollerId/token) this logs a warning, emits a telemetry event, and
13
+ * returns without throwing. Startup must not hang or crash on this — the
14
+ * pre-#127 behavior (trackedBrains starts empty until real messages
15
+ * trickle in) is the fallback here, not a fatal state.
16
+ *
17
+ * DevGuru round-3 review: the awaited part of this function does exactly
18
+ * ONE fetch attempt, bounded by CRITICAL_PATH_TIMEOUT_MS (~10s) — not the
19
+ * full 30s client timeout, and not the retry. Every poller restart during a
20
+ * backend outage was blocking real user-message processing (pollLoop(),
21
+ * called after this in startPoller()) for up to ~62s (2 attempts * 30s +
22
+ * 2s delay) — and restarts correlate with backend deploys, exactly when
23
+ * this is most likely to happen. If the first attempt fails and is
24
+ * retryable (network error, timeout, or 5xx — not a 4xx or malformed body,
25
+ * which a retry can't fix), the retry is kicked off detached — an
26
+ * immediately-invoked async function whose promise is stored in a
27
+ * module-level variable (see `__detachedRetryForTest` below; test-only,
28
+ * production code never reads it) rather than `void`-discarded, but never
29
+ * awaited by this function — the same fire-and-forget shape
30
+ * maintenance-cron.ts's registerBrain() already uses for its immediate
31
+ * metrics collect+send — so `bootstrapTrackedBrains()` returns and
32
+ * startPoller() can proceed to
33
+ * startMaintenanceLoop/startAutoUpdateCron/startMetricsCron/pollLoop()
34
+ * without waiting on it. This is safe because neither downstream consumer
35
+ * needs bootstrap complete synchronously: the maintenance cron only fires
36
+ * at its own scheduled time (default 03:30, not at startup), and the inbox
37
+ * watcher just needs trackedBrains populated "one tick later" — both read
38
+ * the same shared, mutable Map (getTrackedBrains()) whenever their own tick
39
+ * fires, not a snapshot taken at startup.
40
+ *
41
+ * No-ops in direct (non-token) mode, same gating as sendMetrics
42
+ * (metrics/sender.ts) — there's no backend endpoint to call without
43
+ * TEAMVIBE_API_URL + TEAMVIBE_POLLER_TOKEN.
44
+ */
45
+ export declare function bootstrapTrackedBrains(): Promise<void>;
@@ -0,0 +1,217 @@
1
+ import { config } from './config.js';
2
+ import { logger } from './logger.js';
3
+ import { getPollerId } from './auth-provider.js';
4
+ import { seedTrackedBrains } from './maintenance-cron.js';
5
+ import { emitEvent, emitError } from './event-queue.js';
6
+ // Client-side timeout kept comfortably above the server-side route timeout
7
+ // (infra/poller-auth.ts: GET /poller/brains, 25s) so the poller never aborts
8
+ // a request the Lambda could still have answered (DevGuru round-2 review —
9
+ // with the old 10s client timeout vs a 20s route timeout, a transient slow
10
+ // response aborted client-side while the Lambda kept running to nothing,
11
+ // burning the one bootstrap attempt with no retry until the next restart).
12
+ // Used ONLY for the off-critical-path retry attempt below — DevGuru round-3:
13
+ // "don't touch the 25s/30s numbers themselves", so this stays exactly as
14
+ // round-2 left it, just no longer used for the first attempt.
15
+ const FETCH_TIMEOUT_MS = 30_000;
16
+ // Short deadline for the ONE attempt that's actually awaited on
17
+ // startPoller()'s critical path (DevGuru round-3 review item 2). Judgment
18
+ // call — not spelled out beyond "~10s" — picked 10s exactly as suggested.
19
+ // Rationale for why a short deadline is safe here even though it's shorter
20
+ // than the 25s the Lambda might legitimately take: this is only the FIRST
21
+ // attempt. If it doesn't finish inside 10s, we don't conclude failure and
22
+ // give up — we fall through to the same retry path used for any other
23
+ // transient failure, just detached from startup so a slow-but-alive
24
+ // backend doesn't cost the poller a lost user-message-processing minute.
25
+ const CRITICAL_PATH_TIMEOUT_MS = 10_000;
26
+ // Judgment call (not specified by DevGuru's review, flagged for reviewer):
27
+ // one retry, 2s fixed delay, unchanged from round 2. Since the retry is now
28
+ // off the critical path (see bootstrapTrackedBrains below), the 2s delay no
29
+ // longer costs startup time — it only affects how soon the detached retry
30
+ // fires, which is not time-critical (the maintenance cron doesn't run until
31
+ // its own scheduled time, and the inbox watcher just needs trackedBrains
32
+ // populated "one tick later", not synchronously — DevGuru round-3).
33
+ const RETRY_DELAY_MS = 2_000;
34
+ function sleep(ms) {
35
+ return new Promise(resolve => setTimeout(resolve, ms));
36
+ }
37
+ // Test-only handle to the detached retry promise (see bootstrapTrackedBrains
38
+ // below) kicked off by the most recent call. Undefined until a retryable
39
+ // first-attempt failure schedules one; reset to undefined at the top of
40
+ // every bootstrapTrackedBrains() call so a stale promise from a PRIOR call
41
+ // is never mistaken for the current one. Same test-only-export convention
42
+ // as maintenance-cron.ts's __resetTrackedBrainsForTest() — never read by
43
+ // production code, exists purely so tests can deterministically wait for
44
+ // the detached background work to finish instead of guessing with a fixed
45
+ // sleep (DevGuru review: the fixed-sleep approach was two separate
46
+ // latent-flakiness bugs — see brains-bootstrap.test.ts).
47
+ let detachedRetryForTest;
48
+ /** @internal test-only */
49
+ export function __detachedRetryForTest() {
50
+ return detachedRetryForTest;
51
+ }
52
+ async function attemptFetch(url, token, timeoutMs) {
53
+ const controller = new AbortController();
54
+ const timer = setTimeout(() => controller.abort(), timeoutMs);
55
+ try {
56
+ const response = await fetch(url, {
57
+ headers: { Authorization: `Bearer ${token}` },
58
+ signal: controller.signal,
59
+ });
60
+ if (!response.ok) {
61
+ return {
62
+ ok: false,
63
+ reason: 'non-2xx',
64
+ detail: `status ${response.status}`,
65
+ retryable: response.status >= 500,
66
+ };
67
+ }
68
+ let body;
69
+ try {
70
+ body = await response.json();
71
+ }
72
+ catch (err) {
73
+ return {
74
+ ok: false,
75
+ reason: 'invalid-json',
76
+ detail: err instanceof Error ? err.message : String(err),
77
+ retryable: false,
78
+ };
79
+ }
80
+ if (!body || typeof body !== 'object' || !Array.isArray(body.brains)) {
81
+ return { ok: false, reason: 'malformed-body', detail: 'missing brains array', retryable: false };
82
+ }
83
+ return { ok: true, brains: body.brains };
84
+ }
85
+ catch (err) {
86
+ const isTimeout = err instanceof Error && err.name === 'AbortError';
87
+ return {
88
+ ok: false,
89
+ reason: isTimeout ? 'timeout' : 'network-error',
90
+ detail: err instanceof Error ? err.message : String(err),
91
+ retryable: true,
92
+ };
93
+ }
94
+ finally {
95
+ clearTimeout(timer);
96
+ }
97
+ }
98
+ /** Seed trackedBrains from a successful fetch and emit success telemetry.
99
+ * Shared by both the critical-path attempt and the detached retry below. */
100
+ function applySuccess(brains) {
101
+ const seeded = seedTrackedBrains(brains);
102
+ logger.info(`[Bootstrap] Seeded ${seeded} brain(s) into trackedBrains from backend (${brains.length} returned)`);
103
+ emitEvent('poller.brains_bootstrap.success', 'system', {
104
+ seeded,
105
+ returned: brains.length,
106
+ });
107
+ }
108
+ /** Report a terminal (no-more-retries) failure. Distinct telemetry artifact
109
+ * (DevGuru round-2 review) — from outside, "bootstrap crashed" and
110
+ * "bootstrap ran fine, no quiet brains to seed" were previously
111
+ * indistinguishable (both show seeded=0 in a log line nobody's watching).
112
+ * This feeds the same FLEET_ERROR pipeline as maintenance-cron.ts's
113
+ * emitError call (triggerAllBrains, per-brain maintenance failure) — same
114
+ * category, same shape. Only called once per bootstrap attempt sequence
115
+ * (non-retryable first-attempt failure, or a failed retry) — never for a
116
+ * retryable first-attempt failure, since that one isn't terminal yet. */
117
+ function reportFailure(outcome) {
118
+ emitError('maintenance', `trackedBrains bootstrap failed: ${outcome.reason}`, {
119
+ severity: 'warn',
120
+ context: { reason: outcome.reason, detail: outcome.detail },
121
+ });
122
+ }
123
+ /**
124
+ * Fetch the full list of brains this poller is responsible for from the
125
+ * backend (GET /poller/brains, source of truth = DDB — teamvibe.ai#127
126
+ * Option B) and seed `trackedBrains` (maintenance-cron.ts) before the
127
+ * maintenance/heartbeat cron starts, so a brain that's been quiet since
128
+ * before this poller's most recent restart is still picked up instead of
129
+ * staying invisible until a human happens to message it.
130
+ *
131
+ * Best-effort: on any failure (network, timeout, non-2xx, malformed body,
132
+ * missing pollerId/token) this logs a warning, emits a telemetry event, and
133
+ * returns without throwing. Startup must not hang or crash on this — the
134
+ * pre-#127 behavior (trackedBrains starts empty until real messages
135
+ * trickle in) is the fallback here, not a fatal state.
136
+ *
137
+ * DevGuru round-3 review: the awaited part of this function does exactly
138
+ * ONE fetch attempt, bounded by CRITICAL_PATH_TIMEOUT_MS (~10s) — not the
139
+ * full 30s client timeout, and not the retry. Every poller restart during a
140
+ * backend outage was blocking real user-message processing (pollLoop(),
141
+ * called after this in startPoller()) for up to ~62s (2 attempts * 30s +
142
+ * 2s delay) — and restarts correlate with backend deploys, exactly when
143
+ * this is most likely to happen. If the first attempt fails and is
144
+ * retryable (network error, timeout, or 5xx — not a 4xx or malformed body,
145
+ * which a retry can't fix), the retry is kicked off detached — an
146
+ * immediately-invoked async function whose promise is stored in a
147
+ * module-level variable (see `__detachedRetryForTest` below; test-only,
148
+ * production code never reads it) rather than `void`-discarded, but never
149
+ * awaited by this function — the same fire-and-forget shape
150
+ * maintenance-cron.ts's registerBrain() already uses for its immediate
151
+ * metrics collect+send — so `bootstrapTrackedBrains()` returns and
152
+ * startPoller() can proceed to
153
+ * startMaintenanceLoop/startAutoUpdateCron/startMetricsCron/pollLoop()
154
+ * without waiting on it. This is safe because neither downstream consumer
155
+ * needs bootstrap complete synchronously: the maintenance cron only fires
156
+ * at its own scheduled time (default 03:30, not at startup), and the inbox
157
+ * watcher just needs trackedBrains populated "one tick later" — both read
158
+ * the same shared, mutable Map (getTrackedBrains()) whenever their own tick
159
+ * fires, not a snapshot taken at startup.
160
+ *
161
+ * No-ops in direct (non-token) mode, same gating as sendMetrics
162
+ * (metrics/sender.ts) — there's no backend endpoint to call without
163
+ * TEAMVIBE_API_URL + TEAMVIBE_POLLER_TOKEN.
164
+ */
165
+ export async function bootstrapTrackedBrains() {
166
+ detachedRetryForTest = undefined;
167
+ if (!config.TEAMVIBE_API_URL || !config.TEAMVIBE_POLLER_TOKEN) {
168
+ return;
169
+ }
170
+ const pollerId = getPollerId();
171
+ if (!pollerId) {
172
+ logger.warn('[Bootstrap] Skipping trackedBrains bootstrap: no pollerId (auth not completed?)');
173
+ return;
174
+ }
175
+ const token = config.TEAMVIBE_POLLER_TOKEN;
176
+ const url = `${config.TEAMVIBE_API_URL}/poller/brains?pollerId=${encodeURIComponent(pollerId)}`;
177
+ const outcome = await attemptFetch(url, token, CRITICAL_PATH_TIMEOUT_MS);
178
+ if (outcome.ok) {
179
+ applySuccess(outcome.brains);
180
+ return;
181
+ }
182
+ logger.warn(`[Bootstrap] GET /poller/brains failed on first attempt: ${outcome.reason} (${outcome.detail})`);
183
+ if (!outcome.retryable) {
184
+ reportFailure(outcome);
185
+ return;
186
+ }
187
+ // Off the critical path from here — startPoller() is not waiting on this
188
+ // promise. Wrapped in try/catch (DevGuru round-4 review) rather than just
189
+ // asserting "nothing below throws": attemptFetch() itself always returns
190
+ // an outcome object instead of throwing, but applySuccess() (seedTrackedBrains
191
+ // + emitEvent) and reportFailure() (emitError) are still real function
192
+ // calls — if either ever throws for any reason (a future refactor, an
193
+ // unexpected edge case), an uncaught throw inside this detached function
194
+ // becomes an unhandled promise rejection and crashes the whole poller
195
+ // process. Cheap insurance: catch and log a warning, matching this file's
196
+ // existing logger.warn style, instead of letting it propagate.
197
+ //
198
+ // Stored in detachedRetryForTest (DevGuru round-5 review) rather than
199
+ // `void`-discarded, purely so tests can await it deterministically — see
200
+ // that variable's own comment above. Storing the reference doesn't change
201
+ // runtime behavior here: this function still never awaits it itself.
202
+ detachedRetryForTest = (async () => {
203
+ try {
204
+ await sleep(RETRY_DELAY_MS);
205
+ const retryOutcome = await attemptFetch(url, token, FETCH_TIMEOUT_MS);
206
+ if (retryOutcome.ok) {
207
+ applySuccess(retryOutcome.brains);
208
+ return;
209
+ }
210
+ logger.warn(`[Bootstrap] GET /poller/brains failed on retry: ${retryOutcome.reason} (${retryOutcome.detail})`);
211
+ reportFailure(retryOutcome);
212
+ }
213
+ catch (err) {
214
+ logger.warn(`[Bootstrap] Detached retry threw unexpectedly: ${err instanceof Error ? err.message : String(err)}`);
215
+ }
216
+ })();
217
+ }
@@ -232,6 +232,7 @@ export declare function spawnClaudeCode(msg: TeamVibeQueueMessage, sessionLog: S
232
232
  newSessionId?: string;
233
233
  }>;
234
234
  export declare function getActiveProcessCount(): number;
235
+ export declare function getLastSessionEndTs(): number;
235
236
  /**
236
237
  * Kill a zombie process for a given thread ID.
237
238
  * Returns true if a process was found and killed.
@@ -1116,6 +1116,7 @@ async function runClaudeCode(msg, sessionLog, cwd, brainPath, sessionId, isFirst
1116
1116
  clearNotificationTimers();
1117
1117
  activeProcesses.delete(msg.id);
1118
1118
  activeProcessesByThread.delete(msg.thread_id);
1119
+ lastSessionEndTs = Date.now();
1119
1120
  // #73 gap 2 (flagged independently by two reviews): `close` firing means
1120
1121
  // `claude` itself has exited, but its MCP server children — spawned
1121
1122
  // into the same detached process group, see `spawn()` above — are not
@@ -1178,6 +1179,7 @@ async function runClaudeCode(msg, sessionLog, cwd, brainPath, sessionId, isFirst
1178
1179
  clearInterval(idleWatchdog);
1179
1180
  activeProcesses.delete(msg.id);
1180
1181
  activeProcessesByThread.delete(msg.thread_id);
1182
+ lastSessionEndTs = Date.now();
1181
1183
  emitError('session', `Claude Code process spawn error: ${error.message}`, {
1182
1184
  channelId: msg.teamvibe.channelId,
1183
1185
  context: { threadId: msg.thread_id },
@@ -1275,9 +1277,17 @@ export async function spawnClaudeCode(msg, sessionLog, cwd, sessionId, isFirstMe
1275
1277
  }
1276
1278
  const activeProcesses = new Map();
1277
1279
  const activeProcessesByThread = new Map();
1280
+ // tv#317: when the last `claude` process ended, so the idle-restart gate can require
1281
+ // a quiet period since real work last happened. Starts at module load (poller boot) so
1282
+ // a poller that never runs a session still becomes eligible once IDLE_RESTART_MIN_UPTIME_MS
1283
+ // passes, rather than being stuck at 0/undefined forever.
1284
+ let lastSessionEndTs = Date.now();
1278
1285
  export function getActiveProcessCount() {
1279
1286
  return activeProcesses.size;
1280
1287
  }
1288
+ export function getLastSessionEndTs() {
1289
+ return lastSessionEndTs;
1290
+ }
1281
1291
  /**
1282
1292
  * Kill a zombie process for a given thread ID.
1283
1293
  * Returns true if a process was found and killed.
package/dist/config.d.ts CHANGED
@@ -41,6 +41,10 @@ declare const configSchema: z.ZodObject<{
41
41
  AUTO_UPDATE_SCHEDULE: z.ZodDefault<z.ZodString>;
42
42
  INBOX_WATCH_FALLBACK_INTERVAL_MS: z.ZodDefault<z.ZodNumber>;
43
43
  INBOX_WATCH_DEBOUNCE_MS: z.ZodDefault<z.ZodNumber>;
44
+ IDLE_RESTART_ENABLED: z.ZodEffects<z.ZodDefault<z.ZodString>, boolean, string | undefined>;
45
+ IDLE_RESTART_IDLE_MS: z.ZodDefault<z.ZodNumber>;
46
+ IDLE_RESTART_MIN_UPTIME_MS: z.ZodDefault<z.ZodNumber>;
47
+ IDLE_RESTART_MEMORY_FILE_THRESHOLD_BYTES: z.ZodDefault<z.ZodNumber>;
44
48
  }, "strip", z.ZodTypeAny, {
45
49
  AWS_REGION: string;
46
50
  TEAMVIBE_API_URL: string;
@@ -78,6 +82,10 @@ declare const configSchema: z.ZodObject<{
78
82
  AUTO_UPDATE_SCHEDULE: string;
79
83
  INBOX_WATCH_FALLBACK_INTERVAL_MS: number;
80
84
  INBOX_WATCH_DEBOUNCE_MS: number;
85
+ IDLE_RESTART_ENABLED: boolean;
86
+ IDLE_RESTART_IDLE_MS: number;
87
+ IDLE_RESTART_MIN_UPTIME_MS: number;
88
+ IDLE_RESTART_MEMORY_FILE_THRESHOLD_BYTES: number;
81
89
  SQS_QUEUE_URL?: string | undefined;
82
90
  SESSIONS_TABLE?: string | undefined;
83
91
  TEAMVIBE_POLLER_TOKEN?: string | undefined;
@@ -123,6 +131,10 @@ declare const configSchema: z.ZodObject<{
123
131
  AUTO_UPDATE_SCHEDULE?: string | undefined;
124
132
  INBOX_WATCH_FALLBACK_INTERVAL_MS?: number | undefined;
125
133
  INBOX_WATCH_DEBOUNCE_MS?: number | undefined;
134
+ IDLE_RESTART_ENABLED?: string | undefined;
135
+ IDLE_RESTART_IDLE_MS?: number | undefined;
136
+ IDLE_RESTART_MIN_UPTIME_MS?: number | undefined;
137
+ IDLE_RESTART_MEMORY_FILE_THRESHOLD_BYTES?: number | undefined;
126
138
  }>;
127
139
  export type Config = z.infer<typeof configSchema>;
128
140
  export declare function loadConfig(): Config;
@@ -163,6 +175,10 @@ export declare const config: {
163
175
  AUTO_UPDATE_SCHEDULE: string;
164
176
  INBOX_WATCH_FALLBACK_INTERVAL_MS: number;
165
177
  INBOX_WATCH_DEBOUNCE_MS: number;
178
+ IDLE_RESTART_ENABLED: boolean;
179
+ IDLE_RESTART_IDLE_MS: number;
180
+ IDLE_RESTART_MIN_UPTIME_MS: number;
181
+ IDLE_RESTART_MEMORY_FILE_THRESHOLD_BYTES: number;
166
182
  SQS_QUEUE_URL?: string | undefined;
167
183
  SESSIONS_TABLE?: string | undefined;
168
184
  TEAMVIBE_POLLER_TOKEN?: string | undefined;
package/dist/config.js CHANGED
@@ -76,6 +76,23 @@ const configSchema = z.object({
76
76
  // exist at watcher startup, and prunes empty thread dirs.
77
77
  INBOX_WATCH_FALLBACK_INTERVAL_MS: z.coerce.number().default(300_000), // 5 minutes
78
78
  INBOX_WATCH_DEBOUNCE_MS: z.coerce.number().default(300), // coalesce bursty fs events into one scan
79
+ // Idle restart (tv#317): Railway bills on cgroup memory usage, which includes page
80
+ // cache that Linux only reclaims under real memory pressure — a large Railway memory
81
+ // limit means that pressure never triggers, so cache from heavy sessions (bun install,
82
+ // vite build, apt) accumulates until something forces a restart. Rather than an
83
+ // external supervisor, the poller restarts itself once genuinely idle; Railway's
84
+ // ALWAYS restart policy brings it back up clean and any SQS message that arrives
85
+ // during the few-second restart window just waits in the queue.
86
+ //
87
+ // Defaults OFF: this is a Railway-specific mitigation (the memory-threshold check
88
+ // is gated on cgroup v2 anyway, see readReclaimableFileBytes), but the default
89
+ // should make that explicit rather than lean on an implicit cgroup-absence no-op —
90
+ // deploy targets that also happen to be cgroup v2 Linux shouldn't pick this up by
91
+ // just existing. Set IDLE_RESTART_ENABLED=true on Railway to opt in.
92
+ IDLE_RESTART_ENABLED: z.string().default('false').transform((v) => v.toLowerCase() === 'true'),
93
+ IDLE_RESTART_IDLE_MS: z.coerce.number().default(1_800_000), // 30 minutes since the last session ended
94
+ IDLE_RESTART_MIN_UPTIME_MS: z.coerce.number().default(1_800_000), // crash-loop guard — matches the idle floor above (tv#317 discussion), so it doesn't add its own separate cadence
95
+ IDLE_RESTART_MEMORY_FILE_THRESHOLD_BYTES: z.coerce.number().default(700 * 1024 * 1024), // cgroup memory.stat `file` (reclaimable page cache), not memory.current — gating on raw usage would also fire on a real anon leak and mask it instead of just clearing cache
79
96
  });
80
97
  export function loadConfig() {
81
98
  const result = configSchema.safeParse(process.env);
@@ -0,0 +1,12 @@
1
+ export declare function readReclaimableFileBytes(path?: string): number | null;
2
+ export declare function findLiveBgTask(now?: number): string | null;
3
+ export declare function reconcileBgTasksOnBoot(): void;
4
+ export interface CheckIdleRestartDeps {
5
+ now?: number;
6
+ startTime?: number;
7
+ activeProcessCount?: () => number;
8
+ lastSessionEndTs?: () => number;
9
+ reclaimableBytes?: () => number | null;
10
+ liveBgTask?: () => string | null;
11
+ }
12
+ export declare function checkIdleRestart(deps?: CheckIdleRestartDeps): boolean;