@teamvibe/poller 0.7.0 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/brains-bootstrap.d.ts +45 -0
- package/dist/brains-bootstrap.js +217 -0
- package/dist/claude-spawner.d.ts +1 -0
- package/dist/claude-spawner.js +10 -0
- package/dist/config.d.ts +16 -0
- package/dist/config.js +17 -0
- package/dist/idle-restart.d.ts +12 -0
- package/dist/idle-restart.js +200 -0
- package/dist/maintenance-cron.d.ts +66 -0
- package/dist/maintenance-cron.js +69 -0
- package/dist/poller.d.ts +3 -0
- package/dist/poller.js +204 -21
- package/package.json +1 -1
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
/** @internal test-only */
|
|
2
|
+
export declare function __detachedRetryForTest(): Promise<void> | undefined;
|
|
3
|
+
/**
|
|
4
|
+
* Fetch the full list of brains this poller is responsible for from the
|
|
5
|
+
* backend (GET /poller/brains, source of truth = DDB — teamvibe.ai#127
|
|
6
|
+
* Option B) and seed `trackedBrains` (maintenance-cron.ts) before the
|
|
7
|
+
* maintenance/heartbeat cron starts, so a brain that's been quiet since
|
|
8
|
+
* before this poller's most recent restart is still picked up instead of
|
|
9
|
+
* staying invisible until a human happens to message it.
|
|
10
|
+
*
|
|
11
|
+
* Best-effort: on any failure (network, timeout, non-2xx, malformed body,
|
|
12
|
+
* missing pollerId/token) this logs a warning, emits a telemetry event, and
|
|
13
|
+
* returns without throwing. Startup must not hang or crash on this — the
|
|
14
|
+
* pre-#127 behavior (trackedBrains starts empty until real messages
|
|
15
|
+
* trickle in) is the fallback here, not a fatal state.
|
|
16
|
+
*
|
|
17
|
+
* DevGuru round-3 review: the awaited part of this function does exactly
|
|
18
|
+
* ONE fetch attempt, bounded by CRITICAL_PATH_TIMEOUT_MS (~10s) — not the
|
|
19
|
+
* full 30s client timeout, and not the retry. Every poller restart during a
|
|
20
|
+
* backend outage was blocking real user-message processing (pollLoop(),
|
|
21
|
+
* called after this in startPoller()) for up to ~62s (2 attempts * 30s +
|
|
22
|
+
* 2s delay) — and restarts correlate with backend deploys, exactly when
|
|
23
|
+
* this is most likely to happen. If the first attempt fails and is
|
|
24
|
+
* retryable (network error, timeout, or 5xx — not a 4xx or malformed body,
|
|
25
|
+
* which a retry can't fix), the retry is kicked off detached — an
|
|
26
|
+
* immediately-invoked async function whose promise is stored in a
|
|
27
|
+
* module-level variable (see `__detachedRetryForTest` below; test-only,
|
|
28
|
+
* production code never reads it) rather than `void`-discarded, but never
|
|
29
|
+
* awaited by this function — the same fire-and-forget shape
|
|
30
|
+
* maintenance-cron.ts's registerBrain() already uses for its immediate
|
|
31
|
+
* metrics collect+send — so `bootstrapTrackedBrains()` returns and
|
|
32
|
+
* startPoller() can proceed to
|
|
33
|
+
* startMaintenanceLoop/startAutoUpdateCron/startMetricsCron/pollLoop()
|
|
34
|
+
* without waiting on it. This is safe because neither downstream consumer
|
|
35
|
+
* needs bootstrap complete synchronously: the maintenance cron only fires
|
|
36
|
+
* at its own scheduled time (default 03:30, not at startup), and the inbox
|
|
37
|
+
* watcher just needs trackedBrains populated "one tick later" — both read
|
|
38
|
+
* the same shared, mutable Map (getTrackedBrains()) whenever their own tick
|
|
39
|
+
* fires, not a snapshot taken at startup.
|
|
40
|
+
*
|
|
41
|
+
* No-ops in direct (non-token) mode, same gating as sendMetrics
|
|
42
|
+
* (metrics/sender.ts) — there's no backend endpoint to call without
|
|
43
|
+
* TEAMVIBE_API_URL + TEAMVIBE_POLLER_TOKEN.
|
|
44
|
+
*/
|
|
45
|
+
export declare function bootstrapTrackedBrains(): Promise<void>;
|
|
@@ -0,0 +1,217 @@
|
|
|
1
|
+
import { config } from './config.js';
|
|
2
|
+
import { logger } from './logger.js';
|
|
3
|
+
import { getPollerId } from './auth-provider.js';
|
|
4
|
+
import { seedTrackedBrains } from './maintenance-cron.js';
|
|
5
|
+
import { emitEvent, emitError } from './event-queue.js';
|
|
6
|
+
// Client-side timeout kept comfortably above the server-side route timeout
|
|
7
|
+
// (infra/poller-auth.ts: GET /poller/brains, 25s) so the poller never aborts
|
|
8
|
+
// a request the Lambda could still have answered (DevGuru round-2 review —
|
|
9
|
+
// with the old 10s client timeout vs a 20s route timeout, a transient slow
|
|
10
|
+
// response aborted client-side while the Lambda kept running to nothing,
|
|
11
|
+
// burning the one bootstrap attempt with no retry until the next restart).
|
|
12
|
+
// Used ONLY for the off-critical-path retry attempt below — DevGuru round-3:
|
|
13
|
+
// "don't touch the 25s/30s numbers themselves", so this stays exactly as
|
|
14
|
+
// round-2 left it, just no longer used for the first attempt.
|
|
15
|
+
const FETCH_TIMEOUT_MS = 30_000;
|
|
16
|
+
// Short deadline for the ONE attempt that's actually awaited on
|
|
17
|
+
// startPoller()'s critical path (DevGuru round-3 review item 2). Judgment
|
|
18
|
+
// call — not spelled out beyond "~10s" — picked 10s exactly as suggested.
|
|
19
|
+
// Rationale for why a short deadline is safe here even though it's shorter
|
|
20
|
+
// than the 25s the Lambda might legitimately take: this is only the FIRST
|
|
21
|
+
// attempt. If it doesn't finish inside 10s, we don't conclude failure and
|
|
22
|
+
// give up — we fall through to the same retry path used for any other
|
|
23
|
+
// transient failure, just detached from startup so a slow-but-alive
|
|
24
|
+
// backend doesn't cost the poller a lost user-message-processing minute.
|
|
25
|
+
const CRITICAL_PATH_TIMEOUT_MS = 10_000;
|
|
26
|
+
// Judgment call (not specified by DevGuru's review, flagged for reviewer):
|
|
27
|
+
// one retry, 2s fixed delay, unchanged from round 2. Since the retry is now
|
|
28
|
+
// off the critical path (see bootstrapTrackedBrains below), the 2s delay no
|
|
29
|
+
// longer costs startup time — it only affects how soon the detached retry
|
|
30
|
+
// fires, which is not time-critical (the maintenance cron doesn't run until
|
|
31
|
+
// its own scheduled time, and the inbox watcher just needs trackedBrains
|
|
32
|
+
// populated "one tick later", not synchronously — DevGuru round-3).
|
|
33
|
+
const RETRY_DELAY_MS = 2_000;
|
|
34
|
+
function sleep(ms) {
|
|
35
|
+
return new Promise(resolve => setTimeout(resolve, ms));
|
|
36
|
+
}
|
|
37
|
+
// Test-only handle to the detached retry promise (see bootstrapTrackedBrains
|
|
38
|
+
// below) kicked off by the most recent call. Undefined until a retryable
|
|
39
|
+
// first-attempt failure schedules one; reset to undefined at the top of
|
|
40
|
+
// every bootstrapTrackedBrains() call so a stale promise from a PRIOR call
|
|
41
|
+
// is never mistaken for the current one. Same test-only-export convention
|
|
42
|
+
// as maintenance-cron.ts's __resetTrackedBrainsForTest() — never read by
|
|
43
|
+
// production code, exists purely so tests can deterministically wait for
|
|
44
|
+
// the detached background work to finish instead of guessing with a fixed
|
|
45
|
+
// sleep (DevGuru review: the fixed-sleep approach was two separate
|
|
46
|
+
// latent-flakiness bugs — see brains-bootstrap.test.ts).
|
|
47
|
+
let detachedRetryForTest;
|
|
48
|
+
/** @internal test-only */
|
|
49
|
+
export function __detachedRetryForTest() {
|
|
50
|
+
return detachedRetryForTest;
|
|
51
|
+
}
|
|
52
|
+
async function attemptFetch(url, token, timeoutMs) {
|
|
53
|
+
const controller = new AbortController();
|
|
54
|
+
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
55
|
+
try {
|
|
56
|
+
const response = await fetch(url, {
|
|
57
|
+
headers: { Authorization: `Bearer ${token}` },
|
|
58
|
+
signal: controller.signal,
|
|
59
|
+
});
|
|
60
|
+
if (!response.ok) {
|
|
61
|
+
return {
|
|
62
|
+
ok: false,
|
|
63
|
+
reason: 'non-2xx',
|
|
64
|
+
detail: `status ${response.status}`,
|
|
65
|
+
retryable: response.status >= 500,
|
|
66
|
+
};
|
|
67
|
+
}
|
|
68
|
+
let body;
|
|
69
|
+
try {
|
|
70
|
+
body = await response.json();
|
|
71
|
+
}
|
|
72
|
+
catch (err) {
|
|
73
|
+
return {
|
|
74
|
+
ok: false,
|
|
75
|
+
reason: 'invalid-json',
|
|
76
|
+
detail: err instanceof Error ? err.message : String(err),
|
|
77
|
+
retryable: false,
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
if (!body || typeof body !== 'object' || !Array.isArray(body.brains)) {
|
|
81
|
+
return { ok: false, reason: 'malformed-body', detail: 'missing brains array', retryable: false };
|
|
82
|
+
}
|
|
83
|
+
return { ok: true, brains: body.brains };
|
|
84
|
+
}
|
|
85
|
+
catch (err) {
|
|
86
|
+
const isTimeout = err instanceof Error && err.name === 'AbortError';
|
|
87
|
+
return {
|
|
88
|
+
ok: false,
|
|
89
|
+
reason: isTimeout ? 'timeout' : 'network-error',
|
|
90
|
+
detail: err instanceof Error ? err.message : String(err),
|
|
91
|
+
retryable: true,
|
|
92
|
+
};
|
|
93
|
+
}
|
|
94
|
+
finally {
|
|
95
|
+
clearTimeout(timer);
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
/** Seed trackedBrains from a successful fetch and emit success telemetry.
|
|
99
|
+
* Shared by both the critical-path attempt and the detached retry below. */
|
|
100
|
+
function applySuccess(brains) {
|
|
101
|
+
const seeded = seedTrackedBrains(brains);
|
|
102
|
+
logger.info(`[Bootstrap] Seeded ${seeded} brain(s) into trackedBrains from backend (${brains.length} returned)`);
|
|
103
|
+
emitEvent('poller.brains_bootstrap.success', 'system', {
|
|
104
|
+
seeded,
|
|
105
|
+
returned: brains.length,
|
|
106
|
+
});
|
|
107
|
+
}
|
|
108
|
+
/** Report a terminal (no-more-retries) failure. Distinct telemetry artifact
|
|
109
|
+
* (DevGuru round-2 review) — from outside, "bootstrap crashed" and
|
|
110
|
+
* "bootstrap ran fine, no quiet brains to seed" were previously
|
|
111
|
+
* indistinguishable (both show seeded=0 in a log line nobody's watching).
|
|
112
|
+
* This feeds the same FLEET_ERROR pipeline as maintenance-cron.ts's
|
|
113
|
+
* emitError call (triggerAllBrains, per-brain maintenance failure) — same
|
|
114
|
+
* category, same shape. Only called once per bootstrap attempt sequence
|
|
115
|
+
* (non-retryable first-attempt failure, or a failed retry) — never for a
|
|
116
|
+
* retryable first-attempt failure, since that one isn't terminal yet. */
|
|
117
|
+
function reportFailure(outcome) {
|
|
118
|
+
emitError('maintenance', `trackedBrains bootstrap failed: ${outcome.reason}`, {
|
|
119
|
+
severity: 'warn',
|
|
120
|
+
context: { reason: outcome.reason, detail: outcome.detail },
|
|
121
|
+
});
|
|
122
|
+
}
|
|
123
|
+
/**
|
|
124
|
+
* Fetch the full list of brains this poller is responsible for from the
|
|
125
|
+
* backend (GET /poller/brains, source of truth = DDB — teamvibe.ai#127
|
|
126
|
+
* Option B) and seed `trackedBrains` (maintenance-cron.ts) before the
|
|
127
|
+
* maintenance/heartbeat cron starts, so a brain that's been quiet since
|
|
128
|
+
* before this poller's most recent restart is still picked up instead of
|
|
129
|
+
* staying invisible until a human happens to message it.
|
|
130
|
+
*
|
|
131
|
+
* Best-effort: on any failure (network, timeout, non-2xx, malformed body,
|
|
132
|
+
* missing pollerId/token) this logs a warning, emits a telemetry event, and
|
|
133
|
+
* returns without throwing. Startup must not hang or crash on this — the
|
|
134
|
+
* pre-#127 behavior (trackedBrains starts empty until real messages
|
|
135
|
+
* trickle in) is the fallback here, not a fatal state.
|
|
136
|
+
*
|
|
137
|
+
* DevGuru round-3 review: the awaited part of this function does exactly
|
|
138
|
+
* ONE fetch attempt, bounded by CRITICAL_PATH_TIMEOUT_MS (~10s) — not the
|
|
139
|
+
* full 30s client timeout, and not the retry. Every poller restart during a
|
|
140
|
+
* backend outage was blocking real user-message processing (pollLoop(),
|
|
141
|
+
* called after this in startPoller()) for up to ~62s (2 attempts * 30s +
|
|
142
|
+
* 2s delay) — and restarts correlate with backend deploys, exactly when
|
|
143
|
+
* this is most likely to happen. If the first attempt fails and is
|
|
144
|
+
* retryable (network error, timeout, or 5xx — not a 4xx or malformed body,
|
|
145
|
+
* which a retry can't fix), the retry is kicked off detached — an
|
|
146
|
+
* immediately-invoked async function whose promise is stored in a
|
|
147
|
+
* module-level variable (see `__detachedRetryForTest` below; test-only,
|
|
148
|
+
* production code never reads it) rather than `void`-discarded, but never
|
|
149
|
+
* awaited by this function — the same fire-and-forget shape
|
|
150
|
+
* maintenance-cron.ts's registerBrain() already uses for its immediate
|
|
151
|
+
* metrics collect+send — so `bootstrapTrackedBrains()` returns and
|
|
152
|
+
* startPoller() can proceed to
|
|
153
|
+
* startMaintenanceLoop/startAutoUpdateCron/startMetricsCron/pollLoop()
|
|
154
|
+
* without waiting on it. This is safe because neither downstream consumer
|
|
155
|
+
* needs bootstrap complete synchronously: the maintenance cron only fires
|
|
156
|
+
* at its own scheduled time (default 03:30, not at startup), and the inbox
|
|
157
|
+
* watcher just needs trackedBrains populated "one tick later" — both read
|
|
158
|
+
* the same shared, mutable Map (getTrackedBrains()) whenever their own tick
|
|
159
|
+
* fires, not a snapshot taken at startup.
|
|
160
|
+
*
|
|
161
|
+
* No-ops in direct (non-token) mode, same gating as sendMetrics
|
|
162
|
+
* (metrics/sender.ts) — there's no backend endpoint to call without
|
|
163
|
+
* TEAMVIBE_API_URL + TEAMVIBE_POLLER_TOKEN.
|
|
164
|
+
*/
|
|
165
|
+
export async function bootstrapTrackedBrains() {
|
|
166
|
+
detachedRetryForTest = undefined;
|
|
167
|
+
if (!config.TEAMVIBE_API_URL || !config.TEAMVIBE_POLLER_TOKEN) {
|
|
168
|
+
return;
|
|
169
|
+
}
|
|
170
|
+
const pollerId = getPollerId();
|
|
171
|
+
if (!pollerId) {
|
|
172
|
+
logger.warn('[Bootstrap] Skipping trackedBrains bootstrap: no pollerId (auth not completed?)');
|
|
173
|
+
return;
|
|
174
|
+
}
|
|
175
|
+
const token = config.TEAMVIBE_POLLER_TOKEN;
|
|
176
|
+
const url = `${config.TEAMVIBE_API_URL}/poller/brains?pollerId=${encodeURIComponent(pollerId)}`;
|
|
177
|
+
const outcome = await attemptFetch(url, token, CRITICAL_PATH_TIMEOUT_MS);
|
|
178
|
+
if (outcome.ok) {
|
|
179
|
+
applySuccess(outcome.brains);
|
|
180
|
+
return;
|
|
181
|
+
}
|
|
182
|
+
logger.warn(`[Bootstrap] GET /poller/brains failed on first attempt: ${outcome.reason} (${outcome.detail})`);
|
|
183
|
+
if (!outcome.retryable) {
|
|
184
|
+
reportFailure(outcome);
|
|
185
|
+
return;
|
|
186
|
+
}
|
|
187
|
+
// Off the critical path from here — startPoller() is not waiting on this
|
|
188
|
+
// promise. Wrapped in try/catch (DevGuru round-4 review) rather than just
|
|
189
|
+
// asserting "nothing below throws": attemptFetch() itself always returns
|
|
190
|
+
// an outcome object instead of throwing, but applySuccess() (seedTrackedBrains
|
|
191
|
+
// + emitEvent) and reportFailure() (emitError) are still real function
|
|
192
|
+
// calls — if either ever throws for any reason (a future refactor, an
|
|
193
|
+
// unexpected edge case), an uncaught throw inside this detached function
|
|
194
|
+
// becomes an unhandled promise rejection and crashes the whole poller
|
|
195
|
+
// process. Cheap insurance: catch and log a warning, matching this file's
|
|
196
|
+
// existing logger.warn style, instead of letting it propagate.
|
|
197
|
+
//
|
|
198
|
+
// Stored in detachedRetryForTest (DevGuru round-5 review) rather than
|
|
199
|
+
// `void`-discarded, purely so tests can await it deterministically — see
|
|
200
|
+
// that variable's own comment above. Storing the reference doesn't change
|
|
201
|
+
// runtime behavior here: this function still never awaits it itself.
|
|
202
|
+
detachedRetryForTest = (async () => {
|
|
203
|
+
try {
|
|
204
|
+
await sleep(RETRY_DELAY_MS);
|
|
205
|
+
const retryOutcome = await attemptFetch(url, token, FETCH_TIMEOUT_MS);
|
|
206
|
+
if (retryOutcome.ok) {
|
|
207
|
+
applySuccess(retryOutcome.brains);
|
|
208
|
+
return;
|
|
209
|
+
}
|
|
210
|
+
logger.warn(`[Bootstrap] GET /poller/brains failed on retry: ${retryOutcome.reason} (${retryOutcome.detail})`);
|
|
211
|
+
reportFailure(retryOutcome);
|
|
212
|
+
}
|
|
213
|
+
catch (err) {
|
|
214
|
+
logger.warn(`[Bootstrap] Detached retry threw unexpectedly: ${err instanceof Error ? err.message : String(err)}`);
|
|
215
|
+
}
|
|
216
|
+
})();
|
|
217
|
+
}
|
package/dist/claude-spawner.d.ts
CHANGED
|
@@ -232,6 +232,7 @@ export declare function spawnClaudeCode(msg: TeamVibeQueueMessage, sessionLog: S
|
|
|
232
232
|
newSessionId?: string;
|
|
233
233
|
}>;
|
|
234
234
|
export declare function getActiveProcessCount(): number;
|
|
235
|
+
export declare function getLastSessionEndTs(): number;
|
|
235
236
|
/**
|
|
236
237
|
* Kill a zombie process for a given thread ID.
|
|
237
238
|
* Returns true if a process was found and killed.
|
package/dist/claude-spawner.js
CHANGED
|
@@ -1116,6 +1116,7 @@ async function runClaudeCode(msg, sessionLog, cwd, brainPath, sessionId, isFirst
|
|
|
1116
1116
|
clearNotificationTimers();
|
|
1117
1117
|
activeProcesses.delete(msg.id);
|
|
1118
1118
|
activeProcessesByThread.delete(msg.thread_id);
|
|
1119
|
+
lastSessionEndTs = Date.now();
|
|
1119
1120
|
// #73 gap 2 (flagged independently by two reviews): `close` firing means
|
|
1120
1121
|
// `claude` itself has exited, but its MCP server children — spawned
|
|
1121
1122
|
// into the same detached process group, see `spawn()` above — are not
|
|
@@ -1178,6 +1179,7 @@ async function runClaudeCode(msg, sessionLog, cwd, brainPath, sessionId, isFirst
|
|
|
1178
1179
|
clearInterval(idleWatchdog);
|
|
1179
1180
|
activeProcesses.delete(msg.id);
|
|
1180
1181
|
activeProcessesByThread.delete(msg.thread_id);
|
|
1182
|
+
lastSessionEndTs = Date.now();
|
|
1181
1183
|
emitError('session', `Claude Code process spawn error: ${error.message}`, {
|
|
1182
1184
|
channelId: msg.teamvibe.channelId,
|
|
1183
1185
|
context: { threadId: msg.thread_id },
|
|
@@ -1275,9 +1277,17 @@ export async function spawnClaudeCode(msg, sessionLog, cwd, sessionId, isFirstMe
|
|
|
1275
1277
|
}
|
|
1276
1278
|
const activeProcesses = new Map();
|
|
1277
1279
|
const activeProcessesByThread = new Map();
|
|
1280
|
+
// tv#317: when the last `claude` process ended, so the idle-restart gate can require
|
|
1281
|
+
// a quiet period since real work last happened. Starts at module load (poller boot) so
|
|
1282
|
+
// a poller that never runs a session still becomes eligible once IDLE_RESTART_MIN_UPTIME_MS
|
|
1283
|
+
// passes, rather than being stuck at 0/undefined forever.
|
|
1284
|
+
let lastSessionEndTs = Date.now();
|
|
1278
1285
|
export function getActiveProcessCount() {
|
|
1279
1286
|
return activeProcesses.size;
|
|
1280
1287
|
}
|
|
1288
|
+
export function getLastSessionEndTs() {
|
|
1289
|
+
return lastSessionEndTs;
|
|
1290
|
+
}
|
|
1281
1291
|
/**
|
|
1282
1292
|
* Kill a zombie process for a given thread ID.
|
|
1283
1293
|
* Returns true if a process was found and killed.
|
package/dist/config.d.ts
CHANGED
|
@@ -41,6 +41,10 @@ declare const configSchema: z.ZodObject<{
|
|
|
41
41
|
AUTO_UPDATE_SCHEDULE: z.ZodDefault<z.ZodString>;
|
|
42
42
|
INBOX_WATCH_FALLBACK_INTERVAL_MS: z.ZodDefault<z.ZodNumber>;
|
|
43
43
|
INBOX_WATCH_DEBOUNCE_MS: z.ZodDefault<z.ZodNumber>;
|
|
44
|
+
IDLE_RESTART_ENABLED: z.ZodEffects<z.ZodDefault<z.ZodString>, boolean, string | undefined>;
|
|
45
|
+
IDLE_RESTART_IDLE_MS: z.ZodDefault<z.ZodNumber>;
|
|
46
|
+
IDLE_RESTART_MIN_UPTIME_MS: z.ZodDefault<z.ZodNumber>;
|
|
47
|
+
IDLE_RESTART_MEMORY_FILE_THRESHOLD_BYTES: z.ZodDefault<z.ZodNumber>;
|
|
44
48
|
}, "strip", z.ZodTypeAny, {
|
|
45
49
|
AWS_REGION: string;
|
|
46
50
|
TEAMVIBE_API_URL: string;
|
|
@@ -78,6 +82,10 @@ declare const configSchema: z.ZodObject<{
|
|
|
78
82
|
AUTO_UPDATE_SCHEDULE: string;
|
|
79
83
|
INBOX_WATCH_FALLBACK_INTERVAL_MS: number;
|
|
80
84
|
INBOX_WATCH_DEBOUNCE_MS: number;
|
|
85
|
+
IDLE_RESTART_ENABLED: boolean;
|
|
86
|
+
IDLE_RESTART_IDLE_MS: number;
|
|
87
|
+
IDLE_RESTART_MIN_UPTIME_MS: number;
|
|
88
|
+
IDLE_RESTART_MEMORY_FILE_THRESHOLD_BYTES: number;
|
|
81
89
|
SQS_QUEUE_URL?: string | undefined;
|
|
82
90
|
SESSIONS_TABLE?: string | undefined;
|
|
83
91
|
TEAMVIBE_POLLER_TOKEN?: string | undefined;
|
|
@@ -123,6 +131,10 @@ declare const configSchema: z.ZodObject<{
|
|
|
123
131
|
AUTO_UPDATE_SCHEDULE?: string | undefined;
|
|
124
132
|
INBOX_WATCH_FALLBACK_INTERVAL_MS?: number | undefined;
|
|
125
133
|
INBOX_WATCH_DEBOUNCE_MS?: number | undefined;
|
|
134
|
+
IDLE_RESTART_ENABLED?: string | undefined;
|
|
135
|
+
IDLE_RESTART_IDLE_MS?: number | undefined;
|
|
136
|
+
IDLE_RESTART_MIN_UPTIME_MS?: number | undefined;
|
|
137
|
+
IDLE_RESTART_MEMORY_FILE_THRESHOLD_BYTES?: number | undefined;
|
|
126
138
|
}>;
|
|
127
139
|
export type Config = z.infer<typeof configSchema>;
|
|
128
140
|
export declare function loadConfig(): Config;
|
|
@@ -163,6 +175,10 @@ export declare const config: {
|
|
|
163
175
|
AUTO_UPDATE_SCHEDULE: string;
|
|
164
176
|
INBOX_WATCH_FALLBACK_INTERVAL_MS: number;
|
|
165
177
|
INBOX_WATCH_DEBOUNCE_MS: number;
|
|
178
|
+
IDLE_RESTART_ENABLED: boolean;
|
|
179
|
+
IDLE_RESTART_IDLE_MS: number;
|
|
180
|
+
IDLE_RESTART_MIN_UPTIME_MS: number;
|
|
181
|
+
IDLE_RESTART_MEMORY_FILE_THRESHOLD_BYTES: number;
|
|
166
182
|
SQS_QUEUE_URL?: string | undefined;
|
|
167
183
|
SESSIONS_TABLE?: string | undefined;
|
|
168
184
|
TEAMVIBE_POLLER_TOKEN?: string | undefined;
|
package/dist/config.js
CHANGED
|
@@ -76,6 +76,23 @@ const configSchema = z.object({
|
|
|
76
76
|
// exist at watcher startup, and prunes empty thread dirs.
|
|
77
77
|
INBOX_WATCH_FALLBACK_INTERVAL_MS: z.coerce.number().default(300_000), // 5 minutes
|
|
78
78
|
INBOX_WATCH_DEBOUNCE_MS: z.coerce.number().default(300), // coalesce bursty fs events into one scan
|
|
79
|
+
// Idle restart (tv#317): Railway bills on cgroup memory usage, which includes page
|
|
80
|
+
// cache that Linux only reclaims under real memory pressure — a large Railway memory
|
|
81
|
+
// limit means that pressure never triggers, so cache from heavy sessions (bun install,
|
|
82
|
+
// vite build, apt) accumulates until something forces a restart. Rather than an
|
|
83
|
+
// external supervisor, the poller restarts itself once genuinely idle; Railway's
|
|
84
|
+
// ALWAYS restart policy brings it back up clean and any SQS message that arrives
|
|
85
|
+
// during the few-second restart window just waits in the queue.
|
|
86
|
+
//
|
|
87
|
+
// Defaults OFF: this is a Railway-specific mitigation (the memory-threshold check
|
|
88
|
+
// is gated on cgroup v2 anyway, see readReclaimableFileBytes), but the default
|
|
89
|
+
// should make that explicit rather than lean on an implicit cgroup-absence no-op —
|
|
90
|
+
// deploy targets that also happen to be cgroup v2 Linux shouldn't pick this up by
|
|
91
|
+
// just existing. Set IDLE_RESTART_ENABLED=true on Railway to opt in.
|
|
92
|
+
IDLE_RESTART_ENABLED: z.string().default('false').transform((v) => v.toLowerCase() === 'true'),
|
|
93
|
+
IDLE_RESTART_IDLE_MS: z.coerce.number().default(1_800_000), // 30 minutes since the last session ended
|
|
94
|
+
IDLE_RESTART_MIN_UPTIME_MS: z.coerce.number().default(1_800_000), // crash-loop guard — matches the idle floor above (tv#317 discussion), so it doesn't add its own separate cadence
|
|
95
|
+
IDLE_RESTART_MEMORY_FILE_THRESHOLD_BYTES: z.coerce.number().default(700 * 1024 * 1024), // cgroup memory.stat `file` (reclaimable page cache), not memory.current — gating on raw usage would also fire on a real anon leak and mask it instead of just clearing cache
|
|
79
96
|
});
|
|
80
97
|
export function loadConfig() {
|
|
81
98
|
const result = configSchema.safeParse(process.env);
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
export declare function readReclaimableFileBytes(path?: string): number | null;
|
|
2
|
+
export declare function findLiveBgTask(now?: number): string | null;
|
|
3
|
+
export declare function reconcileBgTasksOnBoot(): void;
|
|
4
|
+
export interface CheckIdleRestartDeps {
|
|
5
|
+
now?: number;
|
|
6
|
+
startTime?: number;
|
|
7
|
+
activeProcessCount?: () => number;
|
|
8
|
+
lastSessionEndTs?: () => number;
|
|
9
|
+
reclaimableBytes?: () => number | null;
|
|
10
|
+
liveBgTask?: () => string | null;
|
|
11
|
+
}
|
|
12
|
+
export declare function checkIdleRestart(deps?: CheckIdleRestartDeps): boolean;
|
|
@@ -0,0 +1,200 @@
|
|
|
1
|
+
import { appendFileSync, readdirSync, readFileSync } from 'fs';
|
|
2
|
+
import { join } from 'path';
|
|
3
|
+
import { config } from './config.js';
|
|
4
|
+
import { logger } from './logger.js';
|
|
5
|
+
import { getActiveProcessCount, getLastSessionEndTs } from './claude-spawner.js';
|
|
6
|
+
import { emitEvent } from './event-queue.js';
|
|
7
|
+
// tv#317: self-restart once genuinely idle, to let Railway reclaim page-cache RAM that
|
|
8
|
+
// Linux never proactively frees under our large memory limit. See config.ts for the
|
|
9
|
+
// IDLE_RESTART_* thresholds this checks against.
|
|
10
|
+
const processStartTime = Date.now();
|
|
11
|
+
// cgroup v2 only. Railway containers run cgroup v2; on anything else (local dev, a
|
|
12
|
+
// future cgroup v1 host) this just returns null and the gate never fires — the memory
|
|
13
|
+
// pressure this exists to relieve is Railway-specific, so failing closed is correct
|
|
14
|
+
// rather than guessing at a v1 path/format.
|
|
15
|
+
const MEMORY_STAT_PATH = '/sys/fs/cgroup/memory.stat';
|
|
16
|
+
export function readReclaimableFileBytes(path = MEMORY_STAT_PATH) {
|
|
17
|
+
let text;
|
|
18
|
+
try {
|
|
19
|
+
text = readFileSync(path, 'utf-8');
|
|
20
|
+
}
|
|
21
|
+
catch {
|
|
22
|
+
return null;
|
|
23
|
+
}
|
|
24
|
+
for (const line of text.split('\n')) {
|
|
25
|
+
const spaceIdx = line.indexOf(' ');
|
|
26
|
+
if (spaceIdx > 0 && line.slice(0, spaceIdx) === 'file') {
|
|
27
|
+
const value = Number(line.slice(spaceIdx + 1));
|
|
28
|
+
return Number.isFinite(value) ? value : null;
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
return null;
|
|
32
|
+
}
|
|
33
|
+
// --- bg-task liveness ---------------------------------------------------------------
|
|
34
|
+
//
|
|
35
|
+
// Reimplemented (not imported): bg-task.mjs lives in poller-brain and runs inside a
|
|
36
|
+
// Claude session, not the poller process — there's no shared module to import here.
|
|
37
|
+
// This mirrors its parseStatus/lifecycleOf classification (poller-brain
|
|
38
|
+
// skills/background-task/scripts/bg-task.mjs) rather than a raw /proc scan, which
|
|
39
|
+
// false-negatives on interpreter/path variations and can't distinguish "stuck past TTL"
|
|
40
|
+
// from "healthy" the way the status file's own started/ttl fields can.
|
|
41
|
+
const ABANDON_GRACE_SEC = 60; // mirrors bg-task.mjs's ABANDON_GRACE_SEC
|
|
42
|
+
function parseStatus(text) {
|
|
43
|
+
const kv = {};
|
|
44
|
+
for (const line of text.split('\n')) {
|
|
45
|
+
const i = line.indexOf('=');
|
|
46
|
+
if (i > 0)
|
|
47
|
+
kv[line.slice(0, i)] = line.slice(i + 1);
|
|
48
|
+
}
|
|
49
|
+
return kv;
|
|
50
|
+
}
|
|
51
|
+
function pidAlive(pid) {
|
|
52
|
+
try {
|
|
53
|
+
process.kill(pid, 0);
|
|
54
|
+
return true;
|
|
55
|
+
}
|
|
56
|
+
catch {
|
|
57
|
+
return false;
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
// Same four-way classification as bg-task.mjs's lifecycleOf, minus the tests-only
|
|
61
|
+
// `alive` injection point (this module has its own dedicated tests instead).
|
|
62
|
+
function lifecycleOf(kv, now) {
|
|
63
|
+
// Trust the runner's own claim, whatever it says (bg-task.mjs:139) — not just the two
|
|
64
|
+
// values the runner writes today (`finished`/`timed-out`, bg-task-runner.mjs:403).
|
|
65
|
+
// Mirroring the check itself, not just its current output, means the two
|
|
66
|
+
// implementations of this format can't quietly diverge if the runner ever starts
|
|
67
|
+
// writing a state value this list doesn't happen to enumerate — hardcoding a closed
|
|
68
|
+
// set here would silently fall through to the pid/TTL heuristics below instead.
|
|
69
|
+
if (kv.state)
|
|
70
|
+
return kv.state;
|
|
71
|
+
if (kv.runner_crashed)
|
|
72
|
+
return 'crashed';
|
|
73
|
+
if (Object.keys(kv).length === 0)
|
|
74
|
+
return 'unknown';
|
|
75
|
+
const pid = Number(kv.pid);
|
|
76
|
+
if (!Number.isInteger(pid) || pid <= 0 || !pidAlive(pid))
|
|
77
|
+
return 'abandoned';
|
|
78
|
+
const started = kv.started ? Date.parse(kv.started) : NaN;
|
|
79
|
+
const ttl = Number(String(kv.ttl ?? '').replace(/s$/, ''));
|
|
80
|
+
if (Number.isFinite(started) && Number.isFinite(ttl) && now - started > (ttl + ABANDON_GRACE_SEC) * 1000) {
|
|
81
|
+
return 'abandoned';
|
|
82
|
+
}
|
|
83
|
+
return 'running';
|
|
84
|
+
}
|
|
85
|
+
function bgTaskRoot() {
|
|
86
|
+
// Mirrors bg-task.mjs's taskRoot() default exactly, minus the per-channel
|
|
87
|
+
// namespace — the poller checks liveness across every channel it serves, not just
|
|
88
|
+
// one, so it walks the parent directory instead of joining a single channel ID.
|
|
89
|
+
return process.env['BG_TASK_ROOT'] || join(config.PERSISTENT_STORAGE_PATH || '/tmp', 'bg-tasks');
|
|
90
|
+
}
|
|
91
|
+
function listDirs(path) {
|
|
92
|
+
try {
|
|
93
|
+
return readdirSync(path, { withFileTypes: true })
|
|
94
|
+
.filter((entry) => entry.isDirectory())
|
|
95
|
+
.map((entry) => entry.name);
|
|
96
|
+
}
|
|
97
|
+
catch {
|
|
98
|
+
return [];
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
// Returns a "channel/taskId" label for the first live (running or unknown-state) task
|
|
102
|
+
// found, or null if none. `unknown` (no status written yet) is treated as live —
|
|
103
|
+
// deliberately conservative, per tv#317: a task a few milliseconds from its first
|
|
104
|
+
// status line looks identical to an orphaned dir, and "skip this idle cycle" is cheap
|
|
105
|
+
// while restarting mid-launch is not recoverable.
|
|
106
|
+
export function findLiveBgTask(now = Date.now()) {
|
|
107
|
+
const root = bgTaskRoot();
|
|
108
|
+
for (const channel of listDirs(root)) {
|
|
109
|
+
const channelPath = join(root, channel);
|
|
110
|
+
for (const taskDir of listDirs(channelPath)) {
|
|
111
|
+
let statusText = '';
|
|
112
|
+
try {
|
|
113
|
+
statusText = readFileSync(join(channelPath, taskDir, 'status'), 'utf-8');
|
|
114
|
+
}
|
|
115
|
+
catch {
|
|
116
|
+
// No status file at all — same "unknown" treatment as an empty one below.
|
|
117
|
+
}
|
|
118
|
+
const state = lifecycleOf(parseStatus(statusText), now);
|
|
119
|
+
if (state === 'running' || state === 'unknown') {
|
|
120
|
+
return `${channel}/${taskDir}`;
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
return null;
|
|
125
|
+
}
|
|
126
|
+
// tv#317 "Startup reconciliation": bg-tasks don't survive a poller restart (their
|
|
127
|
+
// runner is a child of the container that just went away), but a status file on a
|
|
128
|
+
// persistent volume has no way to know that on its own. Without this, a task dir
|
|
129
|
+
// whose runner never wrote a terminal line looks identical to a live one — worse,
|
|
130
|
+
// after a restart the fresh container's own low PIDs can coincidentally collide with
|
|
131
|
+
// a stale recorded `pid=`, so pidAlive() alone isn't reliable evidence either. Called
|
|
132
|
+
// once at boot so every pre-restart dir is unambiguously stamped before the idle-restart
|
|
133
|
+
// gate (or `--list`) ever reads it again.
|
|
134
|
+
export function reconcileBgTasksOnBoot() {
|
|
135
|
+
const root = bgTaskRoot();
|
|
136
|
+
let stamped = 0;
|
|
137
|
+
for (const channel of listDirs(root)) {
|
|
138
|
+
const channelPath = join(root, channel);
|
|
139
|
+
for (const taskDir of listDirs(channelPath)) {
|
|
140
|
+
const statusPath = join(channelPath, taskDir, 'status');
|
|
141
|
+
let statusText;
|
|
142
|
+
try {
|
|
143
|
+
statusText = readFileSync(statusPath, 'utf-8');
|
|
144
|
+
}
|
|
145
|
+
catch {
|
|
146
|
+
continue; // no status file yet — nothing to stamp
|
|
147
|
+
}
|
|
148
|
+
const kv = parseStatus(statusText);
|
|
149
|
+
if (kv.state || kv.runner_crashed)
|
|
150
|
+
continue; // already terminal
|
|
151
|
+
try {
|
|
152
|
+
appendFileSync(statusPath, 'runner_crashed=container_restart\n');
|
|
153
|
+
stamped++;
|
|
154
|
+
}
|
|
155
|
+
catch (err) {
|
|
156
|
+
logger.warn(`[IdleRestart] Failed to stamp stale bg-task ${channel}/${taskDir}: ${err instanceof Error ? err.message : err}`);
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
if (stamped > 0) {
|
|
161
|
+
logger.info(`[IdleRestart] Reconciled ${stamped} bg-task dir(s) left over from before this boot`);
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
export function checkIdleRestart(deps = {}) {
|
|
165
|
+
const now = deps.now ?? Date.now();
|
|
166
|
+
const startTime = deps.startTime ?? processStartTime;
|
|
167
|
+
const activeProcessCount = deps.activeProcessCount ?? getActiveProcessCount;
|
|
168
|
+
const lastSessionEndTs = deps.lastSessionEndTs ?? getLastSessionEndTs;
|
|
169
|
+
const reclaimableBytesFn = deps.reclaimableBytes ?? readReclaimableFileBytes;
|
|
170
|
+
const liveBgTask = deps.liveBgTask ?? findLiveBgTask;
|
|
171
|
+
if (!config.IDLE_RESTART_ENABLED)
|
|
172
|
+
return false;
|
|
173
|
+
const uptimeMs = now - startTime;
|
|
174
|
+
if (uptimeMs < config.IDLE_RESTART_MIN_UPTIME_MS)
|
|
175
|
+
return false;
|
|
176
|
+
if (activeProcessCount() > 0)
|
|
177
|
+
return false;
|
|
178
|
+
const idleMs = now - lastSessionEndTs();
|
|
179
|
+
if (idleMs < config.IDLE_RESTART_IDLE_MS)
|
|
180
|
+
return false;
|
|
181
|
+
const reclaimableBytes = reclaimableBytesFn();
|
|
182
|
+
if (reclaimableBytes === null || reclaimableBytes < config.IDLE_RESTART_MEMORY_FILE_THRESHOLD_BYTES)
|
|
183
|
+
return false;
|
|
184
|
+
if (liveBgTask())
|
|
185
|
+
return false; // skip this cycle, recheck next time — never cut a bg-task short just to free memory sooner
|
|
186
|
+
// Re-verify the two liveness conditions right before signaling restart — a
|
|
187
|
+
// session or bg-task could have started in the time it took to reach this point.
|
|
188
|
+
if (activeProcessCount() > 0)
|
|
189
|
+
return false;
|
|
190
|
+
if (liveBgTask())
|
|
191
|
+
return false;
|
|
192
|
+
logger.info(`[IdleRestart] Idle ${Math.round(idleMs / 60000)}min, reclaimable page cache ` +
|
|
193
|
+
`${Math.round(reclaimableBytes / 1024 / 1024)}MB — restarting to let Railway reclaim it`);
|
|
194
|
+
emitEvent('poller.idle_restart', 'system', {
|
|
195
|
+
idleMs,
|
|
196
|
+
uptimeMs,
|
|
197
|
+
reclaimableBytes,
|
|
198
|
+
});
|
|
199
|
+
return true;
|
|
200
|
+
}
|
|
@@ -21,6 +21,72 @@ export declare function getTrackedBrains(): ReadonlyMap<string, TrackedBrainInfo
|
|
|
21
21
|
* @internal
|
|
22
22
|
*/
|
|
23
23
|
export declare function __resetTrackedBrainsForTest(): void;
|
|
24
|
+
/**
|
|
25
|
+
* Shape returned by GET /poller/brains (teamvibe.ai#127 Option B) — the
|
|
26
|
+
* fields registerBrain() needs to build a synthetic maintenance message,
|
|
27
|
+
* minus `lastRealMessageAt` (see seedTrackedBrains below for why that's
|
|
28
|
+
* deliberately not part of the bootstrap payload).
|
|
29
|
+
*/
|
|
30
|
+
export interface BootstrapBrainInfo {
|
|
31
|
+
brainId: string;
|
|
32
|
+
gitRepoUrl: string;
|
|
33
|
+
branch: string;
|
|
34
|
+
workspaceId: string;
|
|
35
|
+
botId: string;
|
|
36
|
+
botToken: string;
|
|
37
|
+
channelId: string;
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Seed trackedBrains at startup from the backend's authoritative brain list
|
|
41
|
+
* (teamvibe.ai#127) so a "quiet" brain — one that hasn't received a real
|
|
42
|
+
* message since this poller process started — still gets included in the
|
|
43
|
+
* maintenance/heartbeat cron, instead of being silently invisible until a
|
|
44
|
+
* human happens to message it.
|
|
45
|
+
*
|
|
46
|
+
* Deliberately does NOT set `lastRealMessageAt` (left undefined). That
|
|
47
|
+
* field specifically means "we've seen a real message since our own last
|
|
48
|
+
* restart" — shouldSkipMaintenance (maintenance-activity.ts, #131) treats
|
|
49
|
+
* `undefined` as fail-open (never skip) for exactly this reason. Bootstrap
|
|
50
|
+
* seeding isn't a real message; claiming one occurred here would risk the
|
|
51
|
+
* opposite bug — a genuinely-quiet brain wrongly evaluated as "nothing new,
|
|
52
|
+
* skip" on its very first post-restart cron tick.
|
|
53
|
+
*
|
|
54
|
+
* Skips (rather than overwrites) any brainId already present in
|
|
55
|
+
* trackedBrains. This guard is load-bearing, not just defensive (DevGuru
|
|
56
|
+
* review, round 4): since round 3's critical-path detachment
|
|
57
|
+
* (brains-bootstrap.ts's `void (async () => {...})()` retry), the retry
|
|
58
|
+
* branch can finish AFTER startPoller() has already returned and real user
|
|
59
|
+
* messages have started arriving. That means a real registerBrain() call
|
|
60
|
+
* for a brain (which sets `lastRealMessageAt`) can race ahead of — and
|
|
61
|
+
* complete before — the detached bootstrap retry's seedTrackedBrains()
|
|
62
|
+
* call for that same brain. Without this skip, the late-arriving bootstrap
|
|
63
|
+
* seed would silently overwrite the real registration back to a fresh
|
|
64
|
+
* entry with `lastRealMessageAt: undefined`, losing real activity data
|
|
65
|
+
* (and, per the fail-open contract above, artificially reopening the
|
|
66
|
+
* skip-maintenance window for a brain that's actually been active).
|
|
67
|
+
*
|
|
68
|
+
* DOES also register each seeded brain for hourly metrics (see
|
|
69
|
+
* `registerBrainForMetrics` below) — DevGuru review caught that this
|
|
70
|
+
* module's original "metrics has its own disk-walk path" reasoning was
|
|
71
|
+
* wrong: `metrics/scheduler.ts`'s `registered` Map has exactly one
|
|
72
|
+
* production caller (`registerBrain()` below), with no disk-walk discovery
|
|
73
|
+
* of its own. Disk reads only happen in `collector.ts`, and only for a
|
|
74
|
+
* brain already present in that Map. Skipping the metrics seed here would
|
|
75
|
+
* reproduce the exact same restart-amnesia bug for metrics that this whole
|
|
76
|
+
* change exists to fix for maintenance: a quiet brain would get maintenance
|
|
77
|
+
* triggers again, but zero metrics until someone messages it.
|
|
78
|
+
*
|
|
79
|
+
* Deliberately does NOT replicate `registerBrain()`'s *immediate*
|
|
80
|
+
* collect+send side effect (the `if (isNew) { void (async () => ... })()`
|
|
81
|
+
* block below) — only the Map insert. Firing an immediate collect+send for
|
|
82
|
+
* every brain this poller owns, all at once, on every restart would be a
|
|
83
|
+
* fleet-wide burst; the existing hourly cron with its per-brain jitter
|
|
84
|
+
* (`collectAndSendAll`, `jitterMsFor`) already picks up newly-registered
|
|
85
|
+
* brains on its own schedule without a startup spike.
|
|
86
|
+
*
|
|
87
|
+
* Returns the number of brains actually seeded (for startup logging).
|
|
88
|
+
*/
|
|
89
|
+
export declare function seedTrackedBrains(brains: readonly BootstrapBrainInfo[]): number;
|
|
24
90
|
/**
|
|
25
91
|
* Register a brain from a real message so the maintenance cron knows about it.
|
|
26
92
|
* Called from processMessage() after brain path is resolved.
|
package/dist/maintenance-cron.js
CHANGED
|
@@ -27,6 +27,75 @@ export function __resetTrackedBrainsForTest() {
|
|
|
27
27
|
trackedBrains.clear();
|
|
28
28
|
}
|
|
29
29
|
let maintenanceJob = null;
|
|
30
|
+
/**
|
|
31
|
+
* Seed trackedBrains at startup from the backend's authoritative brain list
|
|
32
|
+
* (teamvibe.ai#127) so a "quiet" brain — one that hasn't received a real
|
|
33
|
+
* message since this poller process started — still gets included in the
|
|
34
|
+
* maintenance/heartbeat cron, instead of being silently invisible until a
|
|
35
|
+
* human happens to message it.
|
|
36
|
+
*
|
|
37
|
+
* Deliberately does NOT set `lastRealMessageAt` (left undefined). That
|
|
38
|
+
* field specifically means "we've seen a real message since our own last
|
|
39
|
+
* restart" — shouldSkipMaintenance (maintenance-activity.ts, #131) treats
|
|
40
|
+
* `undefined` as fail-open (never skip) for exactly this reason. Bootstrap
|
|
41
|
+
* seeding isn't a real message; claiming one occurred here would risk the
|
|
42
|
+
* opposite bug — a genuinely-quiet brain wrongly evaluated as "nothing new,
|
|
43
|
+
* skip" on its very first post-restart cron tick.
|
|
44
|
+
*
|
|
45
|
+
* Skips (rather than overwrites) any brainId already present in
|
|
46
|
+
* trackedBrains. This guard is load-bearing, not just defensive (DevGuru
|
|
47
|
+
* review, round 4): since round 3's critical-path detachment
|
|
48
|
+
* (brains-bootstrap.ts's `void (async () => {...})()` retry), the retry
|
|
49
|
+
* branch can finish AFTER startPoller() has already returned and real user
|
|
50
|
+
* messages have started arriving. That means a real registerBrain() call
|
|
51
|
+
* for a brain (which sets `lastRealMessageAt`) can race ahead of — and
|
|
52
|
+
* complete before — the detached bootstrap retry's seedTrackedBrains()
|
|
53
|
+
* call for that same brain. Without this skip, the late-arriving bootstrap
|
|
54
|
+
* seed would silently overwrite the real registration back to a fresh
|
|
55
|
+
* entry with `lastRealMessageAt: undefined`, losing real activity data
|
|
56
|
+
* (and, per the fail-open contract above, artificially reopening the
|
|
57
|
+
* skip-maintenance window for a brain that's actually been active).
|
|
58
|
+
*
|
|
59
|
+
* DOES also register each seeded brain for hourly metrics (see
|
|
60
|
+
* `registerBrainForMetrics` below) — DevGuru review caught that this
|
|
61
|
+
* module's original "metrics has its own disk-walk path" reasoning was
|
|
62
|
+
* wrong: `metrics/scheduler.ts`'s `registered` Map has exactly one
|
|
63
|
+
* production caller (`registerBrain()` below), with no disk-walk discovery
|
|
64
|
+
* of its own. Disk reads only happen in `collector.ts`, and only for a
|
|
65
|
+
* brain already present in that Map. Skipping the metrics seed here would
|
|
66
|
+
* reproduce the exact same restart-amnesia bug for metrics that this whole
|
|
67
|
+
* change exists to fix for maintenance: a quiet brain would get maintenance
|
|
68
|
+
* triggers again, but zero metrics until someone messages it.
|
|
69
|
+
*
|
|
70
|
+
* Deliberately does NOT replicate `registerBrain()`'s *immediate*
|
|
71
|
+
* collect+send side effect (the `if (isNew) { void (async () => ... })()`
|
|
72
|
+
* block below) — only the Map insert. Firing an immediate collect+send for
|
|
73
|
+
* every brain this poller owns, all at once, on every restart would be a
|
|
74
|
+
* fleet-wide burst; the existing hourly cron with its per-brain jitter
|
|
75
|
+
* (`collectAndSendAll`, `jitterMsFor`) already picks up newly-registered
|
|
76
|
+
* brains on its own schedule without a startup spike.
|
|
77
|
+
*
|
|
78
|
+
* Returns the number of brains actually seeded (for startup logging).
|
|
79
|
+
*/
|
|
80
|
+
export function seedTrackedBrains(brains) {
|
|
81
|
+
let seeded = 0;
|
|
82
|
+
for (const b of brains) {
|
|
83
|
+
if (trackedBrains.has(b.brainId))
|
|
84
|
+
continue;
|
|
85
|
+
trackedBrains.set(b.brainId, {
|
|
86
|
+
brainId: b.brainId,
|
|
87
|
+
gitRepoUrl: b.gitRepoUrl,
|
|
88
|
+
branch: b.branch,
|
|
89
|
+
workspaceId: b.workspaceId,
|
|
90
|
+
botId: b.botId,
|
|
91
|
+
botToken: b.botToken,
|
|
92
|
+
channelId: b.channelId,
|
|
93
|
+
});
|
|
94
|
+
registerBrainForMetrics(b.brainId);
|
|
95
|
+
seeded++;
|
|
96
|
+
}
|
|
97
|
+
return seeded;
|
|
98
|
+
}
|
|
30
99
|
/**
|
|
31
100
|
* Register a brain from a real message so the maintenance cron knows about it.
|
|
32
101
|
* Called from processMessage() after brain path is resolved.
|
package/dist/poller.d.ts
CHANGED
|
@@ -29,8 +29,11 @@ export declare function isPollerInitiatedFailure(result: {
|
|
|
29
29
|
exitCode: number | null;
|
|
30
30
|
killedByPoller?: KilledByPoller;
|
|
31
31
|
}): boolean;
|
|
32
|
+
export declare function describeFailureReason(killedByPoller: KilledByPoller | undefined, exitCode: number | null): string;
|
|
33
|
+
export declare const END_OF_TURN_NO_FURTHER_RESPONSE_SENTINEL = "NO_FURTHER_RESPONSE_NEEDED";
|
|
32
34
|
export declare function endOfTurnFallbackText(hasSlackContext: boolean, result: Pick<SpawnResult, 'success' | 'sawSendMessageCall' | 'lastPlainTextContent'>): string | undefined;
|
|
33
35
|
export declare function runInboxLoop(threadId: string, kbPath: string, queueMessage: TeamVibeQueueMessage, sessionLog: ReturnType<typeof logger.createSession>): Promise<void>;
|
|
36
|
+
export declare function __setSleepForTest(impl: (ms: number) => Promise<void>): void;
|
|
34
37
|
export declare function __setShuttingDownForTest(value: boolean): void;
|
|
35
38
|
export declare function startPoller(): Promise<void>;
|
|
36
39
|
export {};
|
package/dist/poller.js
CHANGED
|
@@ -10,6 +10,7 @@ import { cleanupOrphanWorktreesForAllBrains, isWorktreeIsolationEnabled } from '
|
|
|
10
10
|
import { postMaintenanceReports } from './maintenance-reporter.js';
|
|
11
11
|
import { initAuth, stopRefresh, getPollerId } from './auth-provider.js';
|
|
12
12
|
import { registerBrain, startMaintenanceLoop, stopMaintenanceLoop } from './maintenance-cron.js';
|
|
13
|
+
import { bootstrapTrackedBrains } from './brains-bootstrap.js';
|
|
13
14
|
import { startEventQueue, stopEventQueue, flush as flushEventQueue, emitEvent, emitError } from './event-queue.js';
|
|
14
15
|
import { startMetricsCron, stopMetricsCron } from './metrics/scheduler.js';
|
|
15
16
|
import { writeMessage, pickNextMessage, markProcessed, markFailed, hasUnprocessed, requeueMessage } from './inbox-manager.js';
|
|
@@ -17,6 +18,7 @@ import { startInboxWatcher, stopInboxWatcher } from './inbox-watcher.js';
|
|
|
17
18
|
import { Cron } from 'croner';
|
|
18
19
|
import { isControlMessage, handleControlMessage } from './control-handler.js';
|
|
19
20
|
import { checkForUpdate } from './auto-update.js';
|
|
21
|
+
import { checkIdleRestart, reconcileBgTasksOnBoot } from './idle-restart.js';
|
|
20
22
|
import { markAutoUpdateJobFired, markAutoUpdateJobSuccess } from './cron-telemetry.js';
|
|
21
23
|
import { exec } from 'child_process';
|
|
22
24
|
import { promisify } from 'util';
|
|
@@ -163,6 +165,35 @@ export function isIntentionalKillExitCode(exitCode) {
|
|
|
163
165
|
export function isPollerInitiatedFailure(result) {
|
|
164
166
|
return Boolean(result.killedByPoller) || isIntentionalKillExitCode(result.exitCode);
|
|
165
167
|
}
|
|
168
|
+
// tv#320 DevGuru round-1 review: runInboxLoop's retry-cap decision (below)
|
|
169
|
+
// no longer calls isPollerInitiatedFailure() directly — since that cap now
|
|
170
|
+
// applies to everything except 'shutdown', the only distinction it needs is
|
|
171
|
+
// `lastKilledByPoller === 'shutdown'`, and `killedByPoller === 'shutdown'`
|
|
172
|
+
// already implies isPollerInitiatedFailure() === true, making a combined
|
|
173
|
+
// check redundant there.
|
|
174
|
+
//
|
|
175
|
+
// tv#320 DevGuru round-2 review: isIntentionalKillExitCode() still has a real
|
|
176
|
+
// caller below (describeFailureReason()) — a kill with NO `killedByPoller`
|
|
177
|
+
// marker that DOES still report the legacy 143/137 exit code is an
|
|
178
|
+
// intentional kill, not an unclassified application failure, and needs to be
|
|
179
|
+
// labeled as such in give-up notices/logs/events (`lastKilledByPoller ??
|
|
180
|
+
// 'application failure'` used to conflate the two).
|
|
181
|
+
//
|
|
182
|
+
// tv#320 DevGuru round-3 review: this only covers the literal 143/137 case.
|
|
183
|
+
// Per the comment above (line ~185), a group-killed process's `close` event
|
|
184
|
+
// reports `code: null`, NOT 143/137 — so a marker-less OOM/external-SIGTERM/
|
|
185
|
+
// group-kill still falls through to "application failure" here, same as
|
|
186
|
+
// before this fix. Closing that gap for real needs claude-spawner.ts's
|
|
187
|
+
// `close` handler to stop discarding its `signal` argument and SpawnResult
|
|
188
|
+
// to carry it (neither does today) — tracked as a follow-up, not attempted
|
|
189
|
+
// here.
|
|
190
|
+
export function describeFailureReason(killedByPoller, exitCode) {
|
|
191
|
+
if (killedByPoller)
|
|
192
|
+
return killedByPoller;
|
|
193
|
+
if (isIntentionalKillExitCode(exitCode))
|
|
194
|
+
return `killed (exit ${exitCode})`;
|
|
195
|
+
return 'application failure';
|
|
196
|
+
}
|
|
166
197
|
// teamvibe.ai#284 (poller-brain#235): end-of-turn guard. A successful Claude
|
|
167
198
|
// run can end its turn having only written plain assistant text — no
|
|
168
199
|
// `mcp__slack__send_message` tool call — which the platform otherwise drops
|
|
@@ -176,12 +207,27 @@ export function isPollerInitiatedFailure(result) {
|
|
|
176
207
|
// Deliberately does NOT fire on a failed run (result.success === false):
|
|
177
208
|
// that path already has its own delivery via classifyFailedIteration +
|
|
178
209
|
// sendSlackMessage/sendSlackError, and firing here too would double-deliver.
|
|
210
|
+
//
|
|
211
|
+
// poller-brain#380: a turn whose only tool call is a reaction (or similar)
|
|
212
|
+
// can trigger Claude Code's own forced-continuation nudge ("no visible
|
|
213
|
+
// output, produce a user-visible response"), which previously always fired
|
|
214
|
+
// this same fallback — double-posting even when the reaction was the
|
|
215
|
+
// intended, complete reply. Base-brain CLAUDE.md instructs agents to answer
|
|
216
|
+
// that nudge with this exact sentinel when nothing further is needed; an
|
|
217
|
+
// exact match is treated as "nothing to deliver". Deliberately an exact
|
|
218
|
+
// string match, not a content heuristic — anything else (including real
|
|
219
|
+
// content an agent forgot to send via send_message) still delivers as
|
|
220
|
+
// before, so this can't silently drop a forgotten reply.
|
|
221
|
+
export const END_OF_TURN_NO_FURTHER_RESPONSE_SENTINEL = 'NO_FURTHER_RESPONSE_NEEDED';
|
|
179
222
|
export function endOfTurnFallbackText(hasSlackContext, result) {
|
|
180
223
|
if (!hasSlackContext || !result.success || result.sawSendMessageCall) {
|
|
181
224
|
return undefined;
|
|
182
225
|
}
|
|
183
226
|
const text = result.lastPlainTextContent?.trim();
|
|
184
|
-
|
|
227
|
+
if (!text || text === END_OF_TURN_NO_FURTHER_RESPONSE_SENTINEL) {
|
|
228
|
+
return undefined;
|
|
229
|
+
}
|
|
230
|
+
return text;
|
|
185
231
|
}
|
|
186
232
|
export async function runInboxLoop(threadId, kbPath, queueMessage, sessionLog) {
|
|
187
233
|
if (activeLoops.has(threadId)) {
|
|
@@ -328,6 +374,26 @@ export async function runInboxLoop(threadId, kbPath, queueMessage, sessionLog) {
|
|
|
328
374
|
lastExitCode = result.exitCode;
|
|
329
375
|
lastKilledByPoller = result.killedByPoller;
|
|
330
376
|
sessionLog.error(`[Loop] Claude failed iteration #${iteration}: ${result.error}`);
|
|
377
|
+
// tv#320 DevGuru round-2 review: a fleet-visible record of EVERY
|
|
378
|
+
// failed iteration (not just the eventual give-up) means a future
|
|
379
|
+
// "was this in one loop or across restarts" question is a DDB query
|
|
380
|
+
// (this event, by thread/filename/timestamp) instead of a CloudWatch
|
|
381
|
+
// log dig + SSO admin escalation — which is exactly what round-2's
|
|
382
|
+
// point 3b hit a dead end on for pb#337.
|
|
383
|
+
//
|
|
384
|
+
// tv#320 DevGuru round-3 review: excludes 'shutdown' — that fires on
|
|
385
|
+
// every poller restart/deploy for every in-flight message fleet-wide
|
|
386
|
+
// and isn't a retry-cap-relevant failure (it's not counted toward
|
|
387
|
+
// failureCounts either), so it would just be alerting noise for an
|
|
388
|
+
// expected event with no new information (the requeue below already
|
|
389
|
+
// covers it).
|
|
390
|
+
if (lastKilledByPoller !== 'shutdown') {
|
|
391
|
+
emitError('session', `Iteration #${iteration} failed for ${message.filename}`, {
|
|
392
|
+
severity: 'warn',
|
|
393
|
+
channelId: queueMessage.teamvibe.channelId,
|
|
394
|
+
context: { threadId, filename: message.filename, iteration, reason: describeFailureReason(lastKilledByPoller, lastExitCode) },
|
|
395
|
+
});
|
|
396
|
+
}
|
|
331
397
|
if (hasSlackContext) {
|
|
332
398
|
const decision = classifyFailedIteration(result);
|
|
333
399
|
if (decision.kind === 'neutral') {
|
|
@@ -349,33 +415,96 @@ export async function runInboxLoop(threadId, kbPath, queueMessage, sessionLog) {
|
|
|
349
415
|
}
|
|
350
416
|
else {
|
|
351
417
|
// Poller-initiated kill (timeout/idle/network/zombie/shutdown, via
|
|
352
|
-
// `killedByPoller`, or the legacy 143/137 exit-code fallback)
|
|
353
|
-
// counts as an application failure
|
|
354
|
-
//
|
|
355
|
-
//
|
|
356
|
-
//
|
|
357
|
-
//
|
|
358
|
-
//
|
|
359
|
-
//
|
|
360
|
-
//
|
|
361
|
-
//
|
|
362
|
-
//
|
|
363
|
-
|
|
364
|
-
|
|
418
|
+
// `killedByPoller`, or the legacy 143/137 exit-code fallback) never
|
|
419
|
+
// counts as an *application* failure — but tv#320 (poller-brain#337:
|
|
420
|
+
// one checkpoint file redelivered 18x/~10min, stopped only by a
|
|
421
|
+
// manual `rm`) showed that timeout/idle/network/zombie kills can be
|
|
422
|
+
// just as deterministically tied to this one message's content as a
|
|
423
|
+
// genuine app failure (e.g. an oversized file that reliably
|
|
424
|
+
// re-triggers the same idle/timeout watchdog on every immediate
|
|
425
|
+
// retry). So the same failureCounts/MAX_RETRIES/backoff cap as the
|
|
426
|
+
// application-failure path now applies to everything except
|
|
427
|
+
// 'shutdown' — not just those four named reasons: a legacy 143/137
|
|
428
|
+
// exit with no `killedByPoller` marker at all is just as capable of
|
|
429
|
+
// repeating forever and there's no reason to treat it differently.
|
|
430
|
+
// 'shutdown' alone stays excluded: `shuttingDown` is already true by
|
|
431
|
+
// the time this reads, so the loop condition above stops it from
|
|
432
|
+
// actually retrying inline — instead the branch below requeues the
|
|
433
|
+
// message from processing/ back to new/ (PR#308 round-5) so a future
|
|
434
|
+
// boot's inbox watcher finds it.
|
|
435
|
+
//
|
|
436
|
+
// tv#320 DevGuru round-1 review: this branches on `lastKilledByPoller
|
|
437
|
+
// !== 'shutdown'` directly rather than `isPollerInitiatedFailure(...)`
|
|
438
|
+
// — `killedByPoller === 'shutdown'` already implies
|
|
439
|
+
// isPollerInitiatedFailure() === true, so combining the two was a
|
|
440
|
+
// dead term. isPollerInitiatedFailure() itself is unchanged and still
|
|
441
|
+
// meaningful (exported, unit-tested in
|
|
442
|
+
// poller-retry-classification.test.ts) for any caller that needs the
|
|
443
|
+
// poller-vs-app distinction — this loop just no longer needs it,
|
|
444
|
+
// since the only distinction that changes behavior here is
|
|
445
|
+
// shutdown-vs-not.
|
|
446
|
+
if (lastKilledByPoller !== 'shutdown') {
|
|
365
447
|
const failures = (failureCounts.get(message.filename) || 0) + 1;
|
|
366
448
|
failureCounts.set(message.filename, failures);
|
|
367
449
|
const MAX_RETRIES = 3;
|
|
368
450
|
if (failures >= MAX_RETRIES) {
|
|
369
|
-
|
|
451
|
+
// tv#320 DevGuru round-1 review: the per-iteration notice sent
|
|
452
|
+
// above (classifyFailedIteration) for idle/zombie/network reads
|
|
453
|
+
// "send another message to resume/restart/retry" — true before
|
|
454
|
+
// this cap existed (retry was unconditional), false now that this
|
|
455
|
+
// is the last attempt and the message is about to be dropped.
|
|
456
|
+
// Give the user an explicit, accurate final word instead of
|
|
457
|
+
// leaving classifyFailedIteration's now-stale promise as the only
|
|
458
|
+
// thing they saw, and surface it as a real error event (not just
|
|
459
|
+
// a session-log line) so give-up isn't invisible to fleet
|
|
460
|
+
// monitoring either.
|
|
461
|
+
const reasonLabel = describeFailureReason(lastKilledByPoller, lastExitCode);
|
|
462
|
+
sessionLog.error(`[Loop] Message ${message.filename} failed ${failures} times (last reason: ${reasonLabel}), giving up`);
|
|
370
463
|
markFailed(kbPath, threadId, message.filename);
|
|
464
|
+
if (hasSlackContext) {
|
|
465
|
+
// tv#320 DevGuru round-2 review (BLOCKING): this whole else-if
|
|
466
|
+
// chain sits structurally after the per-iteration try/catch
|
|
467
|
+
// above (which closes at the `catch (error)` a few lines up),
|
|
468
|
+
// so an unhandled exception here — unlike the per-iteration
|
|
469
|
+
// classifyFailedIteration notices, which ARE inside that
|
|
470
|
+
// try/catch — would propagate straight out of the `while`
|
|
471
|
+
// loop, skip releaseSessionLock() below (it's in the try body,
|
|
472
|
+
// not a `finally`), and leave the DDB session lock held until
|
|
473
|
+
// TTL. sendSlackError() (slack-client.ts) has no internal catch
|
|
474
|
+
// of its own (unlike addReaction(), which does), so a Slack
|
|
475
|
+
// rate-limit/auth/network error here needs the same explicit
|
|
476
|
+
// try/catch the end-of-turn-guard fallback above already uses.
|
|
477
|
+
try {
|
|
478
|
+
await sendSlackError(queueMessage, `Giving up after ${failures} attempts (last reason: ${reasonLabel}) — this message was not processed. Please send it again.`);
|
|
479
|
+
}
|
|
480
|
+
catch (err) {
|
|
481
|
+
sessionLog.error(`[Loop] Give-up notice delivery failed: ${err instanceof Error ? err.message : err}`);
|
|
482
|
+
}
|
|
483
|
+
await addReaction(queueMessage, 'x');
|
|
484
|
+
}
|
|
485
|
+
emitError('session', `Message ${message.filename} failed ${failures} times, giving up`, {
|
|
486
|
+
severity: 'error',
|
|
487
|
+
channelId: queueMessage.teamvibe.channelId,
|
|
488
|
+
context: { threadId, filename: message.filename, reason: reasonLabel },
|
|
489
|
+
});
|
|
371
490
|
}
|
|
372
491
|
else {
|
|
373
492
|
const backoffMs = Math.min(10_000 * Math.pow(2, failures - 1), 60_000);
|
|
374
493
|
sessionLog.info(`[Loop] Retry ${failures}/${MAX_RETRIES - 1} for ${message.filename}, backing off ${backoffMs}ms`);
|
|
375
|
-
|
|
494
|
+
// tv#320 DevGuru round-2 nit: this backoff was previously only
|
|
495
|
+
// reachable for genuine app failures; the broadened cap above
|
|
496
|
+
// means a timeout/idle/network/zombie kill can now land here too,
|
|
497
|
+
// including one racing an in-progress shutdown() (e.g. the idle
|
|
498
|
+
// watchdog and a shutdown SIGTERM firing close together). Skip
|
|
499
|
+
// the up-to-60s wait when already shutting down so the `while
|
|
500
|
+
// (!shuttingDown)` condition above can exit on the very next
|
|
501
|
+
// check instead of stalling runInboxLoop's return.
|
|
502
|
+
if (!shuttingDown) {
|
|
503
|
+
await sleep(backoffMs);
|
|
504
|
+
}
|
|
376
505
|
}
|
|
377
506
|
}
|
|
378
|
-
else
|
|
507
|
+
else {
|
|
379
508
|
// PR#308 round-5 review (teamvibe.ai#73): pickNextMessage() moves a
|
|
380
509
|
// message's file to processing/ as soon as it's picked up, not once
|
|
381
510
|
// it completes — so a message killed here by shutdown is left
|
|
@@ -387,9 +516,15 @@ export async function runInboxLoop(threadId, kbPath, queueMessage, sessionLog) {
|
|
|
387
516
|
// shutdown notice text below, classify-failed-iteration.ts)
|
|
388
517
|
// literally true. It does NOT make resumption automatic: the
|
|
389
518
|
// watcher's checkInboxForNewSessions() (inbox-watcher.ts) only
|
|
390
|
-
// scans brains already in the in-memory trackedBrains map
|
|
391
|
-
//
|
|
392
|
-
//
|
|
519
|
+
// scans brains already in the in-memory trackedBrains map. Since
|
|
520
|
+
// teamvibe.ai#127, that map is best-effort seeded from the
|
|
521
|
+
// backend at startup (bootstrapTrackedBrains(), before
|
|
522
|
+
// startInboxWatcher() is called) — but it's still not a
|
|
523
|
+
// guarantee (network failure, backend outage, brain missing a
|
|
524
|
+
// Slack bot/token to resolve) and a brand-new brain from a real
|
|
525
|
+
// message that arrives after bootstrap but before its own first
|
|
526
|
+
// registerBrain() call is still momentarily untracked. See
|
|
527
|
+
// teamvibe.ai#310 for the still-open general case. Hence the
|
|
393
528
|
// notice text also naming "next message or scheduled run" rather
|
|
394
529
|
// than claiming automatic resumption.
|
|
395
530
|
//
|
|
@@ -581,6 +716,18 @@ async function pollLoop() {
|
|
|
581
716
|
consecutiveAuthFailures = 0;
|
|
582
717
|
}
|
|
583
718
|
if (messages.length === 0) {
|
|
719
|
+
// tv#317: only worth checking on an otherwise-empty poll — a poll that
|
|
720
|
+
// returned messages means activeSessions is about to go above 0 anyway.
|
|
721
|
+
// Routed through the real shutdown() (not a standalone flush+exit) so this
|
|
722
|
+
// sets shuttingDown, stops the inbox watcher, and releases locks — the same
|
|
723
|
+
// gate every other exit path goes through. DevGuru review, tv#317: a bare
|
|
724
|
+
// flushEventQueue()+process.exit(0) here let the inbox watcher spawn a new
|
|
725
|
+
// session in the window before exit, then kill it mid-work (#308's shape),
|
|
726
|
+
// and skipped releaseAllActiveLocks(), stranding locks for STALE_LOCK_TIMEOUT_MS.
|
|
727
|
+
if (checkIdleRestart()) {
|
|
728
|
+
logger.info('[IdleRestart] Restarting to let Railway reclaim page-cache memory...');
|
|
729
|
+
await shutdown('idle-restart', 0);
|
|
730
|
+
}
|
|
584
731
|
continue;
|
|
585
732
|
}
|
|
586
733
|
logger.info(`Received ${messages.length} message(s) from SQS`);
|
|
@@ -651,8 +798,17 @@ async function pollLoop() {
|
|
|
651
798
|
}
|
|
652
799
|
}
|
|
653
800
|
}
|
|
801
|
+
let sleepImpl = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
654
802
|
function sleep(ms) {
|
|
655
|
-
return
|
|
803
|
+
return sleepImpl(ms);
|
|
804
|
+
}
|
|
805
|
+
// Exported for testing only (tv#320 DevGuru round-1 nit): lets a test replace
|
|
806
|
+
// the real backoff delay (runInboxLoop's retry-cap path) with an immediate
|
|
807
|
+
// resolve, without a `globalThis.setTimeout` override that would also swallow
|
|
808
|
+
// any other real timer in the same code path. Same pattern as
|
|
809
|
+
// __setShuttingDownForTest above.
|
|
810
|
+
export function __setSleepForTest(impl) {
|
|
811
|
+
sleepImpl = impl;
|
|
656
812
|
}
|
|
657
813
|
// Graceful shutdown
|
|
658
814
|
let shuttingDown = false;
|
|
@@ -749,11 +905,38 @@ export async function startPoller() {
|
|
|
749
905
|
startEventQueue();
|
|
750
906
|
await ensureDirectories();
|
|
751
907
|
await ensureBaseBrain();
|
|
908
|
+
reconcileBgTasksOnBoot();
|
|
909
|
+
if (config.IDLE_RESTART_ENABLED) {
|
|
910
|
+
// Defaults off (tv#317 follow-up, DevGuru review): a silent disabled gate and an
|
|
911
|
+
// enabled gate that just hasn't hit its threshold yet look identical in production
|
|
912
|
+
// (same flat/growing memory graph) — logging only on enabled, matching the
|
|
913
|
+
// Worktree isolation line below, makes opt-in verifiable from boot logs instead.
|
|
914
|
+
logger.info(' Idle restart: ENABLED (IDLE_RESTART_ENABLED=true)');
|
|
915
|
+
}
|
|
752
916
|
if (isWorktreeIsolationEnabled()) {
|
|
753
917
|
logger.info(' Worktree isolation: ENABLED (POLLER_USE_WORKTREES=true)');
|
|
754
918
|
const sweep = await cleanupOrphanWorktreesForAllBrains(config.BRAINS_PATH);
|
|
755
919
|
logger.info(` Orphan worktree sweep: scanned ${sweep.scanned} brains, removed ${sweep.removed} stale worktrees`);
|
|
756
920
|
}
|
|
921
|
+
// Seed trackedBrains from the backend's authoritative brain list
|
|
922
|
+
// (teamvibe.ai#127 Option B) BEFORE the maintenance cron and the inbox
|
|
923
|
+
// watcher start — both iterate the in-memory trackedBrains map
|
|
924
|
+
// (maintenance-cron.ts's triggerAllBrains, inbox-watcher.ts's
|
|
925
|
+
// checkInboxForNewSessions), which is otherwise empty after every
|
|
926
|
+
// restart until a real message re-registers each brain one at a time.
|
|
927
|
+
// Best-effort: never throws, so a bootstrap failure here falls back to
|
|
928
|
+
// the pre-#127 behavior rather than blocking startup.
|
|
929
|
+
//
|
|
930
|
+
// This await only covers ONE short (~10s) fetch attempt (DevGuru round-3
|
|
931
|
+
// review item 2) — real message processing (pollLoop(), below) must not
|
|
932
|
+
// be blocked for up to ~62s by a backend outage, which is exactly the
|
|
933
|
+
// condition most likely to coincide with a poller restart (backend
|
|
934
|
+
// deploys). If that one attempt fails and is retryable, the retry runs
|
|
935
|
+
// detached in the background and finishes after startPoller() has
|
|
936
|
+
// already returned — see brains-bootstrap.ts's bootstrapTrackedBrains()
|
|
937
|
+
// doc comment for why that's safe (neither the maintenance cron nor the
|
|
938
|
+
// inbox watcher need trackedBrains populated synchronously here).
|
|
939
|
+
await bootstrapTrackedBrains();
|
|
757
940
|
startMaintenanceLoop(processMessage);
|
|
758
941
|
startAutoUpdateCron();
|
|
759
942
|
startMetricsCron();
|