@teamvibe/poller 0.6.2 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/brain-manager.d.ts +43 -1
- package/dist/brain-manager.js +259 -6
- package/dist/brains-bootstrap.d.ts +45 -0
- package/dist/brains-bootstrap.js +217 -0
- package/dist/classify-failed-iteration.d.ts +1 -1
- package/dist/classify-failed-iteration.js +33 -1
- package/dist/claude-spawner.d.ts +179 -1
- package/dist/claude-spawner.js +697 -92
- package/dist/config.d.ts +40 -0
- package/dist/config.js +32 -2
- package/dist/cron-telemetry.d.ts +3 -0
- package/dist/cron-telemetry.js +16 -0
- package/dist/event-queue.d.ts +30 -0
- package/dist/event-queue.js +4 -0
- package/dist/idle-restart.d.ts +12 -0
- package/dist/idle-restart.js +200 -0
- package/dist/inbox-manager.d.ts +25 -0
- package/dist/inbox-manager.js +47 -0
- package/dist/maintenance-activity.d.ts +18 -0
- package/dist/maintenance-activity.js +28 -0
- package/dist/maintenance-cron.d.ts +73 -0
- package/dist/maintenance-cron.js +85 -1
- package/dist/metrics/collector.d.ts +2 -0
- package/dist/metrics/collector.js +7 -0
- package/dist/poller.d.ts +12 -0
- package/dist/poller.js +345 -8
- package/dist/types.d.ts +6 -0
- package/package.json +1 -1
package/dist/poller.js
CHANGED
|
@@ -10,13 +10,15 @@ import { cleanupOrphanWorktreesForAllBrains, isWorktreeIsolationEnabled } from '
|
|
|
10
10
|
import { postMaintenanceReports } from './maintenance-reporter.js';
|
|
11
11
|
import { initAuth, stopRefresh, getPollerId } from './auth-provider.js';
|
|
12
12
|
import { registerBrain, startMaintenanceLoop, stopMaintenanceLoop } from './maintenance-cron.js';
|
|
13
|
+
import { bootstrapTrackedBrains } from './brains-bootstrap.js';
|
|
13
14
|
import { startEventQueue, stopEventQueue, flush as flushEventQueue, emitEvent, emitError } from './event-queue.js';
|
|
14
15
|
import { startMetricsCron, stopMetricsCron } from './metrics/scheduler.js';
|
|
15
|
-
import { writeMessage, pickNextMessage, markProcessed, markFailed, hasUnprocessed } from './inbox-manager.js';
|
|
16
|
+
import { writeMessage, pickNextMessage, markProcessed, markFailed, hasUnprocessed, requeueMessage } from './inbox-manager.js';
|
|
16
17
|
import { startInboxWatcher, stopInboxWatcher } from './inbox-watcher.js';
|
|
17
18
|
import { Cron } from 'croner';
|
|
18
19
|
import { isControlMessage, handleControlMessage } from './control-handler.js';
|
|
19
20
|
import { checkForUpdate } from './auto-update.js';
|
|
21
|
+
import { checkIdleRestart, reconcileBgTasksOnBoot } from './idle-restart.js';
|
|
20
22
|
import { markAutoUpdateJobFired, markAutoUpdateJobSuccess } from './cron-telemetry.js';
|
|
21
23
|
import { exec } from 'child_process';
|
|
22
24
|
import { promisify } from 'util';
|
|
@@ -132,6 +134,101 @@ const activeLoops = new Set();
|
|
|
132
134
|
export function isInboxLoopActive(threadId) {
|
|
133
135
|
return activeLoops.has(threadId);
|
|
134
136
|
}
|
|
137
|
+
// Exported for testing (teamvibe.ai#32). SIGTERM (143) / SIGKILL (137) exit
|
|
138
|
+
// codes are the classic 128+signal convention for "the poller itself killed
|
|
139
|
+
// this process on purpose" (timeout, idle, network, zombie, or — as of #32 —
|
|
140
|
+
// a compaction-ceiling watchdog kill). Historically all of these set
|
|
141
|
+
// `killedByPoller` via the same `proc.kill('SIGTERM')` path in
|
|
142
|
+
// claude-spawner.ts and were observed to surface as 143/137 here.
|
|
143
|
+
//
|
|
144
|
+
// PR#308 round-2 review (teamvibe.ai#73): this exit-code convention is
|
|
145
|
+
// fragile on its own — the Claude CLI can legitimately exit 143 for
|
|
146
|
+
// unrelated reasons, and #73's move to group-kill (`process.kill(-pid, ...)`
|
|
147
|
+
// instead of a direct `proc.kill()`) means a signal-terminated close now
|
|
148
|
+
// reports `code: null` with a separate `signal` field (verified empirically
|
|
149
|
+
// against a synthetic detached child in this repo's environment — Node's
|
|
150
|
+
// `close` event reports `code: null` for any signal-terminated process,
|
|
151
|
+
// group-killed or not). Kept as a defensive fallback only — see
|
|
152
|
+
// isPollerInitiatedFailure() below, which is what callers should actually use.
|
|
153
|
+
export function isIntentionalKillExitCode(exitCode) {
|
|
154
|
+
return exitCode === 143 || exitCode === 137;
|
|
155
|
+
}
|
|
156
|
+
// The actual retry-classification signal (PR#308 round-2 review item 1):
|
|
157
|
+
// `killedByPoller` is the purpose-built marker for "the poller killed this
|
|
158
|
+
// process on purpose, not an application failure" (see SpawnResult /
|
|
159
|
+
// classify-failed-iteration.ts) and is set on every poller-initiated kill
|
|
160
|
+
// path this branch touches — timeout, idle watchdog, network watchdog,
|
|
161
|
+
// zombie-kill, and (added by this fix) poller shutdown. Prefer it, and fall
|
|
162
|
+
// back to the exit-code heuristic only when it's absent (e.g. hand-built
|
|
163
|
+
// SpawnResult literals in older/unrelated call sites) so behavior doesn't
|
|
164
|
+
// regress for anything not yet covered by the marker.
|
|
165
|
+
export function isPollerInitiatedFailure(result) {
|
|
166
|
+
return Boolean(result.killedByPoller) || isIntentionalKillExitCode(result.exitCode);
|
|
167
|
+
}
|
|
168
|
+
// tv#320 DevGuru round-1 review: runInboxLoop's retry-cap decision (below)
|
|
169
|
+
// no longer calls isPollerInitiatedFailure() directly — since that cap now
|
|
170
|
+
// applies to everything except 'shutdown', the only distinction it needs is
|
|
171
|
+
// `lastKilledByPoller === 'shutdown'`, and `killedByPoller === 'shutdown'`
|
|
172
|
+
// already implies isPollerInitiatedFailure() === true, making a combined
|
|
173
|
+
// check redundant there.
|
|
174
|
+
//
|
|
175
|
+
// tv#320 DevGuru round-2 review: isIntentionalKillExitCode() still has a real
|
|
176
|
+
// caller below (describeFailureReason()) — a kill with NO `killedByPoller`
|
|
177
|
+
// marker that DOES still report the legacy 143/137 exit code is an
|
|
178
|
+
// intentional kill, not an unclassified application failure, and needs to be
|
|
179
|
+
// labeled as such in give-up notices/logs/events (`lastKilledByPoller ??
|
|
180
|
+
// 'application failure'` used to conflate the two).
|
|
181
|
+
//
|
|
182
|
+
// tv#320 DevGuru round-3 review: this only covers the literal 143/137 case.
|
|
183
|
+
// Per the comment above (line ~185), a group-killed process's `close` event
|
|
184
|
+
// reports `code: null`, NOT 143/137 — so a marker-less OOM/external-SIGTERM/
|
|
185
|
+
// group-kill still falls through to "application failure" here, same as
|
|
186
|
+
// before this fix. Closing that gap for real needs claude-spawner.ts's
|
|
187
|
+
// `close` handler to stop discarding its `signal` argument and SpawnResult
|
|
188
|
+
// to carry it (neither does today) — tracked as a follow-up, not attempted
|
|
189
|
+
// here.
|
|
190
|
+
export function describeFailureReason(killedByPoller, exitCode) {
|
|
191
|
+
if (killedByPoller)
|
|
192
|
+
return killedByPoller;
|
|
193
|
+
if (isIntentionalKillExitCode(exitCode))
|
|
194
|
+
return `killed (exit ${exitCode})`;
|
|
195
|
+
return 'application failure';
|
|
196
|
+
}
|
|
197
|
+
// teamvibe.ai#284 (poller-brain#235): end-of-turn guard. A successful Claude
|
|
198
|
+
// run can end its turn having only written plain assistant text — no
|
|
199
|
+
// `mcp__slack__send_message` tool call — which the platform otherwise drops
|
|
200
|
+
// silently (no error, no signal; see poller-brain#235's 4x-in-one-session
|
|
201
|
+
// recurrence). Decides whether this run needs the deterministic fallback
|
|
202
|
+
// delivery, and if so, what text to send.
|
|
203
|
+
//
|
|
204
|
+
// Pure/exported (mirrors classifyFailedIteration's pattern) so the gating
|
|
205
|
+
// logic — "Slack context AND no send_message call AND non-empty trailing
|
|
206
|
+
// text" — is unit-testable without mocking spawnClaudeCode or sendSlackMessage.
|
|
207
|
+
// Deliberately does NOT fire on a failed run (result.success === false):
|
|
208
|
+
// that path already has its own delivery via classifyFailedIteration +
|
|
209
|
+
// sendSlackMessage/sendSlackError, and firing here too would double-deliver.
|
|
210
|
+
//
|
|
211
|
+
// poller-brain#380: a turn whose only tool call is a reaction (or similar)
|
|
212
|
+
// can trigger Claude Code's own forced-continuation nudge ("no visible
|
|
213
|
+
// output, produce a user-visible response"), which previously always fired
|
|
214
|
+
// this same fallback — double-posting even when the reaction was the
|
|
215
|
+
// intended, complete reply. Base-brain CLAUDE.md instructs agents to answer
|
|
216
|
+
// that nudge with this exact sentinel when nothing further is needed; an
|
|
217
|
+
// exact match is treated as "nothing to deliver". Deliberately an exact
|
|
218
|
+
// string match, not a content heuristic — anything else (including real
|
|
219
|
+
// content an agent forgot to send via send_message) still delivers as
|
|
220
|
+
// before, so this can't silently drop a forgotten reply.
|
|
221
|
+
export const END_OF_TURN_NO_FURTHER_RESPONSE_SENTINEL = 'NO_FURTHER_RESPONSE_NEEDED';
|
|
222
|
+
export function endOfTurnFallbackText(hasSlackContext, result) {
|
|
223
|
+
if (!hasSlackContext || !result.success || result.sawSendMessageCall) {
|
|
224
|
+
return undefined;
|
|
225
|
+
}
|
|
226
|
+
const text = result.lastPlainTextContent?.trim();
|
|
227
|
+
if (!text || text === END_OF_TURN_NO_FURTHER_RESPONSE_SENTINEL) {
|
|
228
|
+
return undefined;
|
|
229
|
+
}
|
|
230
|
+
return text;
|
|
231
|
+
}
|
|
135
232
|
export async function runInboxLoop(threadId, kbPath, queueMessage, sessionLog) {
|
|
136
233
|
if (activeLoops.has(threadId)) {
|
|
137
234
|
sessionLog.info(`Loop already running for thread ${threadId}, skipping`);
|
|
@@ -160,7 +257,25 @@ export async function runInboxLoop(threadId, kbPath, queueMessage, sessionLog) {
|
|
|
160
257
|
activeLocks.set(threadId, lockToken);
|
|
161
258
|
}
|
|
162
259
|
let currentSessionId = session.session_id || undefined;
|
|
163
|
-
|
|
260
|
+
// PR#308 round-3 review (teamvibe.ai#73): `shuttingDown` (declared below,
|
|
261
|
+
// referenced here via the same forward-reference pattern already used by
|
|
262
|
+
// pollLoop()'s `while (!shuttingDown)`) gates every new iteration of this
|
|
263
|
+
// loop, not just the retry-classification decision inside it. Without
|
|
264
|
+
// this, a message killed with `killedByPoller: 'shutdown'` (added by this
|
|
265
|
+
// branch) falls into the same "poller-initiated, retry immediately, no
|
|
266
|
+
// backoff" bucket as timeout/idle/network/zombie (see
|
|
267
|
+
// isPollerInitiatedFailure() below) — correctly skipping markProcessed/
|
|
268
|
+
// markFailed so the message stays unprocessed for a future boot, but with
|
|
269
|
+
// nothing to stop the loop from immediately calling pickNextMessage()
|
|
270
|
+
// again and spawning a NEW `claude` process for that same message, even
|
|
271
|
+
// though killAllActiveProcesses() (shutdown()'s first step) already ran.
|
|
272
|
+
// Gating the loop *condition* itself (rather than special-casing
|
|
273
|
+
// 'shutdown' inside the retry decision) closes this for every
|
|
274
|
+
// killedByPoller reason uniformly, and — unlike a `break` — falls through
|
|
275
|
+
// to the exact same post-loop cleanup (releaseSessionLock etc., below)
|
|
276
|
+
// that the existing `if (!message) break` / `if (!hasUnprocessed(...))
|
|
277
|
+
// break` exits already use, so lock release is unaffected.
|
|
278
|
+
while (!shuttingDown) {
|
|
164
279
|
const message = pickNextMessage(kbPath, threadId);
|
|
165
280
|
if (!message) {
|
|
166
281
|
sessionLog.info(`[Loop] No more messages for thread ${threadId}`);
|
|
@@ -177,6 +292,7 @@ export async function runInboxLoop(threadId, kbPath, queueMessage, sessionLog) {
|
|
|
177
292
|
: undefined;
|
|
178
293
|
let succeeded = false;
|
|
179
294
|
let lastExitCode = null;
|
|
295
|
+
let lastKilledByPoller;
|
|
180
296
|
try {
|
|
181
297
|
// Reconstruct queue message from stored inbox data
|
|
182
298
|
const storedData = JSON.parse(message.content);
|
|
@@ -227,6 +343,28 @@ export async function runInboxLoop(threadId, kbPath, queueMessage, sessionLog) {
|
|
|
227
343
|
modelSource: result.modelSource,
|
|
228
344
|
});
|
|
229
345
|
}
|
|
346
|
+
// teamvibe.ai#284 (poller-brain#235): end-of-turn guard. A
|
|
347
|
+
// successful run that never called send_message but left trailing
|
|
348
|
+
// plain text would otherwise vanish silently — deliver it via the
|
|
349
|
+
// same sendSlackMessage() helper used for the failure-path neutral
|
|
350
|
+
// notices, and emit a warning-level event so the near-miss is
|
|
351
|
+
// visible (today session.completed looks identical whether or not
|
|
352
|
+
// this fired).
|
|
353
|
+
const fallbackText = endOfTurnFallbackText(hasSlackContext, result);
|
|
354
|
+
if (fallbackText) {
|
|
355
|
+
sessionLog.warn(`[Loop] Iteration #${iteration}: no send_message call detected, delivering trailing assistant text via fallback`);
|
|
356
|
+
try {
|
|
357
|
+
await sendSlackMessage(queueMessage, fallbackText);
|
|
358
|
+
}
|
|
359
|
+
catch (err) {
|
|
360
|
+
sessionLog.error(`[Loop] End-of-turn fallback delivery failed: ${err instanceof Error ? err.message : err}`);
|
|
361
|
+
}
|
|
362
|
+
emitError('session', 'End-of-turn guard: delivered assistant text via fallback (no send_message tool call detected)', {
|
|
363
|
+
severity: 'warn',
|
|
364
|
+
channelId: queueMessage.teamvibe.channelId,
|
|
365
|
+
context: { threadId: queueMessage.thread_id },
|
|
366
|
+
});
|
|
367
|
+
}
|
|
230
368
|
if (queueMessage.teamvibe.brain?.brainId) {
|
|
231
369
|
await pushBrainChanges(kbPath, queueMessage.teamvibe.brain.brainId, queueMessage.teamvibe.workspaceId);
|
|
232
370
|
}
|
|
@@ -234,7 +372,28 @@ export async function runInboxLoop(threadId, kbPath, queueMessage, sessionLog) {
|
|
|
234
372
|
}
|
|
235
373
|
else {
|
|
236
374
|
lastExitCode = result.exitCode;
|
|
375
|
+
lastKilledByPoller = result.killedByPoller;
|
|
237
376
|
sessionLog.error(`[Loop] Claude failed iteration #${iteration}: ${result.error}`);
|
|
377
|
+
// tv#320 DevGuru round-2 review: a fleet-visible record of EVERY
|
|
378
|
+
// failed iteration (not just the eventual give-up) means a future
|
|
379
|
+
// "was this in one loop or across restarts" question is a DDB query
|
|
380
|
+
// (this event, by thread/filename/timestamp) instead of a CloudWatch
|
|
381
|
+
// log dig + SSO admin escalation — which is exactly what round-2's
|
|
382
|
+
// point 3b hit a dead end on for pb#337.
|
|
383
|
+
//
|
|
384
|
+
// tv#320 DevGuru round-3 review: excludes 'shutdown' — that fires on
|
|
385
|
+
// every poller restart/deploy for every in-flight message fleet-wide
|
|
386
|
+
// and isn't a retry-cap-relevant failure (it's not counted toward
|
|
387
|
+
// failureCounts either), so it would just be alerting noise for an
|
|
388
|
+
// expected event with no new information (the requeue below already
|
|
389
|
+
// covers it).
|
|
390
|
+
if (lastKilledByPoller !== 'shutdown') {
|
|
391
|
+
emitError('session', `Iteration #${iteration} failed for ${message.filename}`, {
|
|
392
|
+
severity: 'warn',
|
|
393
|
+
channelId: queueMessage.teamvibe.channelId,
|
|
394
|
+
context: { threadId, filename: message.filename, iteration, reason: describeFailureReason(lastKilledByPoller, lastExitCode) },
|
|
395
|
+
});
|
|
396
|
+
}
|
|
238
397
|
if (hasSlackContext) {
|
|
239
398
|
const decision = classifyFailedIteration(result);
|
|
240
399
|
if (decision.kind === 'neutral') {
|
|
@@ -255,22 +414,139 @@ export async function runInboxLoop(threadId, kbPath, queueMessage, sessionLog) {
|
|
|
255
414
|
markProcessed(kbPath, threadId, message.filename);
|
|
256
415
|
}
|
|
257
416
|
else {
|
|
258
|
-
//
|
|
259
|
-
|
|
260
|
-
|
|
417
|
+
// Poller-initiated kill (timeout/idle/network/zombie/shutdown, via
|
|
418
|
+
// `killedByPoller`, or the legacy 143/137 exit-code fallback) never
|
|
419
|
+
// counts as an *application* failure — but tv#320 (poller-brain#337:
|
|
420
|
+
// one checkpoint file redelivered 18x/~10min, stopped only by a
|
|
421
|
+
// manual `rm`) showed that timeout/idle/network/zombie kills can be
|
|
422
|
+
// just as deterministically tied to this one message's content as a
|
|
423
|
+
// genuine app failure (e.g. an oversized file that reliably
|
|
424
|
+
// re-triggers the same idle/timeout watchdog on every immediate
|
|
425
|
+
// retry). So the same failureCounts/MAX_RETRIES/backoff cap as the
|
|
426
|
+
// application-failure path now applies to everything except
|
|
427
|
+
// 'shutdown' — not just those four named reasons: a legacy 143/137
|
|
428
|
+
// exit with no `killedByPoller` marker at all is just as capable of
|
|
429
|
+
// repeating forever and there's no reason to treat it differently.
|
|
430
|
+
// 'shutdown' alone stays excluded: `shuttingDown` is already true by
|
|
431
|
+
// the time this reads, so the loop condition above stops it from
|
|
432
|
+
// actually retrying inline — instead the branch below requeues the
|
|
433
|
+
// message from processing/ back to new/ (PR#308 round-5) so a future
|
|
434
|
+
// boot's inbox watcher finds it.
|
|
435
|
+
//
|
|
436
|
+
// tv#320 DevGuru round-1 review: this branches on `lastKilledByPoller
|
|
437
|
+
// !== 'shutdown'` directly rather than `isPollerInitiatedFailure(...)`
|
|
438
|
+
// — `killedByPoller === 'shutdown'` already implies
|
|
439
|
+
// isPollerInitiatedFailure() === true, so combining the two was a
|
|
440
|
+
// dead term. isPollerInitiatedFailure() itself is unchanged and still
|
|
441
|
+
// meaningful (exported, unit-tested in
|
|
442
|
+
// poller-retry-classification.test.ts) for any caller that needs the
|
|
443
|
+
// poller-vs-app distinction — this loop just no longer needs it,
|
|
444
|
+
// since the only distinction that changes behavior here is
|
|
445
|
+
// shutdown-vs-not.
|
|
446
|
+
if (lastKilledByPoller !== 'shutdown') {
|
|
261
447
|
const failures = (failureCounts.get(message.filename) || 0) + 1;
|
|
262
448
|
failureCounts.set(message.filename, failures);
|
|
263
449
|
const MAX_RETRIES = 3;
|
|
264
450
|
if (failures >= MAX_RETRIES) {
|
|
265
|
-
|
|
451
|
+
// tv#320 DevGuru round-1 review: the per-iteration notice sent
|
|
452
|
+
// above (classifyFailedIteration) for idle/zombie/network reads
|
|
453
|
+
// "send another message to resume/restart/retry" — true before
|
|
454
|
+
// this cap existed (retry was unconditional), false now that this
|
|
455
|
+
// is the last attempt and the message is about to be dropped.
|
|
456
|
+
// Give the user an explicit, accurate final word instead of
|
|
457
|
+
// leaving classifyFailedIteration's now-stale promise as the only
|
|
458
|
+
// thing they saw, and surface it as a real error event (not just
|
|
459
|
+
// a session-log line) so give-up isn't invisible to fleet
|
|
460
|
+
// monitoring either.
|
|
461
|
+
const reasonLabel = describeFailureReason(lastKilledByPoller, lastExitCode);
|
|
462
|
+
sessionLog.error(`[Loop] Message ${message.filename} failed ${failures} times (last reason: ${reasonLabel}), giving up`);
|
|
266
463
|
markFailed(kbPath, threadId, message.filename);
|
|
464
|
+
if (hasSlackContext) {
|
|
465
|
+
// tv#320 DevGuru round-2 review (BLOCKING): this whole else-if
|
|
466
|
+
// chain sits structurally after the per-iteration try/catch
|
|
467
|
+
// above (which closes at the `catch (error)` a few lines up),
|
|
468
|
+
// so an unhandled exception here — unlike the per-iteration
|
|
469
|
+
// classifyFailedIteration notices, which ARE inside that
|
|
470
|
+
// try/catch — would propagate straight out of the `while`
|
|
471
|
+
// loop, skip releaseSessionLock() below (it's in the try body,
|
|
472
|
+
// not a `finally`), and leave the DDB session lock held until
|
|
473
|
+
// TTL. sendSlackError() (slack-client.ts) has no internal catch
|
|
474
|
+
// of its own (unlike addReaction(), which does), so a Slack
|
|
475
|
+
// rate-limit/auth/network error here needs the same explicit
|
|
476
|
+
// try/catch the end-of-turn-guard fallback above already uses.
|
|
477
|
+
try {
|
|
478
|
+
await sendSlackError(queueMessage, `Giving up after ${failures} attempts (last reason: ${reasonLabel}) — this message was not processed. Please send it again.`);
|
|
479
|
+
}
|
|
480
|
+
catch (err) {
|
|
481
|
+
sessionLog.error(`[Loop] Give-up notice delivery failed: ${err instanceof Error ? err.message : err}`);
|
|
482
|
+
}
|
|
483
|
+
await addReaction(queueMessage, 'x');
|
|
484
|
+
}
|
|
485
|
+
emitError('session', `Message ${message.filename} failed ${failures} times, giving up`, {
|
|
486
|
+
severity: 'error',
|
|
487
|
+
channelId: queueMessage.teamvibe.channelId,
|
|
488
|
+
context: { threadId, filename: message.filename, reason: reasonLabel },
|
|
489
|
+
});
|
|
267
490
|
}
|
|
268
491
|
else {
|
|
269
492
|
const backoffMs = Math.min(10_000 * Math.pow(2, failures - 1), 60_000);
|
|
270
493
|
sessionLog.info(`[Loop] Retry ${failures}/${MAX_RETRIES - 1} for ${message.filename}, backing off ${backoffMs}ms`);
|
|
271
|
-
|
|
494
|
+
// tv#320 DevGuru round-2 nit: this backoff was previously only
|
|
495
|
+
// reachable for genuine app failures; the broadened cap above
|
|
496
|
+
// means a timeout/idle/network/zombie kill can now land here too,
|
|
497
|
+
// including one racing an in-progress shutdown() (e.g. the idle
|
|
498
|
+
// watchdog and a shutdown SIGTERM firing close together). Skip
|
|
499
|
+
// the up-to-60s wait when already shutting down so the `while
|
|
500
|
+
// (!shuttingDown)` condition above can exit on the very next
|
|
501
|
+
// check instead of stalling runInboxLoop's return.
|
|
502
|
+
if (!shuttingDown) {
|
|
503
|
+
await sleep(backoffMs);
|
|
504
|
+
}
|
|
272
505
|
}
|
|
273
506
|
}
|
|
507
|
+
else {
|
|
508
|
+
// PR#308 round-5 review (teamvibe.ai#73): pickNextMessage() moves a
|
|
509
|
+
// message's file to processing/ as soon as it's picked up, not once
|
|
510
|
+
// it completes — so a message killed here by shutdown is left
|
|
511
|
+
// sitting in processing/, not new/. The watcher's wake condition
|
|
512
|
+
// (hasNewMessage(), inbox-watcher.ts) only scans new/, so nothing
|
|
513
|
+
// would start a fresh runInboxLoop() for this thread on next boot.
|
|
514
|
+
// Requeue it back to new/ so a future boot's watcher scan actually
|
|
515
|
+
// finds it — this is what makes "your message is saved" (the
|
|
516
|
+
// shutdown notice text below, classify-failed-iteration.ts)
|
|
517
|
+
// literally true. It does NOT make resumption automatic: the
|
|
518
|
+
// watcher's checkInboxForNewSessions() (inbox-watcher.ts) only
|
|
519
|
+
// scans brains already in the in-memory trackedBrains map. Since
|
|
520
|
+
// teamvibe.ai#127, that map is best-effort seeded from the
|
|
521
|
+
// backend at startup (bootstrapTrackedBrains(), before
|
|
522
|
+
// startInboxWatcher() is called) — but it's still not a
|
|
523
|
+
// guarantee (network failure, backend outage, brain missing a
|
|
524
|
+
// Slack bot/token to resolve) and a brand-new brain from a real
|
|
525
|
+
// message that arrives after bootstrap but before its own first
|
|
526
|
+
// registerBrain() call is still momentarily untracked. See
|
|
527
|
+
// teamvibe.ai#310 for the still-open general case. Hence the
|
|
528
|
+
// notice text also naming "next message or scheduled run" rather
|
|
529
|
+
// than claiming automatic resumption.
|
|
530
|
+
//
|
|
531
|
+
// Scoped to 'shutdown' only, deliberately not the other four
|
|
532
|
+
// poller-initiated reasons (timeout/idle/network/zombie) — those
|
|
533
|
+
// retry immediately within this same still-running loop iteration
|
|
534
|
+
// (the `while (!shuttingDown)` condition above lets it come
|
|
535
|
+
// straight back around), so the existing processing/
|
|
536
|
+
// crash-recovery branch inside pickNextMessage() already covers
|
|
537
|
+
// them; requeueing those too would be redundant and could
|
|
538
|
+
// double-process.
|
|
539
|
+
//
|
|
540
|
+
// Safe against waking a brand-new loop for this thread before this
|
|
541
|
+
// poller process actually exits: stopInboxWatcher() runs earlier in
|
|
542
|
+
// shutdown() (poller.ts), BEFORE killAllActiveProcesses() (whose
|
|
543
|
+
// completion is what unblocks this iteration and reaches this
|
|
544
|
+
// branch) — and stopInboxWatcher() closes the real fs.watch
|
|
545
|
+
// handlers themselves (inbox-watcher.ts stopInboxWatcher(), not
|
|
546
|
+
// just an interval/poll timer), so by the time this requeue runs,
|
|
547
|
+
// nothing is watching new/ to react to the file reappearing there.
|
|
548
|
+
requeueMessage(kbPath, threadId, message.filename);
|
|
549
|
+
}
|
|
274
550
|
}
|
|
275
551
|
if (!hasUnprocessed(kbPath, threadId)) {
|
|
276
552
|
break;
|
|
@@ -440,6 +716,18 @@ async function pollLoop() {
|
|
|
440
716
|
consecutiveAuthFailures = 0;
|
|
441
717
|
}
|
|
442
718
|
if (messages.length === 0) {
|
|
719
|
+
// tv#317: only worth checking on an otherwise-empty poll — a poll that
|
|
720
|
+
// returned messages means activeSessions is about to go above 0 anyway.
|
|
721
|
+
// Routed through the real shutdown() (not a standalone flush+exit) so this
|
|
722
|
+
// sets shuttingDown, stops the inbox watcher, and releases locks — the same
|
|
723
|
+
// gate every other exit path goes through. DevGuru review, tv#317: a bare
|
|
724
|
+
// flushEventQueue()+process.exit(0) here let the inbox watcher spawn a new
|
|
725
|
+
// session in the window before exit, then kill it mid-work (#308's shape),
|
|
726
|
+
// and skipped releaseAllActiveLocks(), stranding locks for STALE_LOCK_TIMEOUT_MS.
|
|
727
|
+
if (checkIdleRestart()) {
|
|
728
|
+
logger.info('[IdleRestart] Restarting to let Railway reclaim page-cache memory...');
|
|
729
|
+
await shutdown('idle-restart', 0);
|
|
730
|
+
}
|
|
443
731
|
continue;
|
|
444
732
|
}
|
|
445
733
|
logger.info(`Received ${messages.length} message(s) from SQS`);
|
|
@@ -510,11 +798,33 @@ async function pollLoop() {
|
|
|
510
798
|
}
|
|
511
799
|
}
|
|
512
800
|
}
|
|
801
|
+
let sleepImpl = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
513
802
|
function sleep(ms) {
|
|
514
|
-
return
|
|
803
|
+
return sleepImpl(ms);
|
|
804
|
+
}
|
|
805
|
+
// Exported for testing only (tv#320 DevGuru round-1 nit): lets a test replace
|
|
806
|
+
// the real backoff delay (runInboxLoop's retry-cap path) with an immediate
|
|
807
|
+
// resolve, without a `globalThis.setTimeout` override that would also swallow
|
|
808
|
+
// any other real timer in the same code path. Same pattern as
|
|
809
|
+
// __setShuttingDownForTest above.
|
|
810
|
+
export function __setSleepForTest(impl) {
|
|
811
|
+
sleepImpl = impl;
|
|
515
812
|
}
|
|
516
813
|
// Graceful shutdown
|
|
517
814
|
let shuttingDown = false;
|
|
815
|
+
// Exported for testing only (PR#308 round-3 review item 1). `shuttingDown`
|
|
816
|
+
// has no other external mutator — production code only ever flips it via
|
|
817
|
+
// the real shutdown() below, which also runs killAllActiveProcesses(),
|
|
818
|
+
// releaseAllActiveLocks(), the drain loop, and process.exit(). This setter
|
|
819
|
+
// lets a test put runInboxLoop's `while (!shuttingDown)` gate into the
|
|
820
|
+
// "shutdown in progress" state directly (mirroring the exact race
|
|
821
|
+
// shutdown() itself creates — killAllActiveProcesses() kills the in-flight
|
|
822
|
+
// spawn while its promise is still pending) without invoking any of those
|
|
823
|
+
// real side effects. Same pattern as __resetCronTelemetryForTest /
|
|
824
|
+
// __resetTrackedBrainsForTest elsewhere in this package.
|
|
825
|
+
export function __setShuttingDownForTest(value) {
|
|
826
|
+
shuttingDown = value;
|
|
827
|
+
}
|
|
518
828
|
async function shutdown(signal, exitCode = 0) {
|
|
519
829
|
if (shuttingDown)
|
|
520
830
|
return;
|
|
@@ -595,11 +905,38 @@ export async function startPoller() {
|
|
|
595
905
|
startEventQueue();
|
|
596
906
|
await ensureDirectories();
|
|
597
907
|
await ensureBaseBrain();
|
|
908
|
+
reconcileBgTasksOnBoot();
|
|
909
|
+
if (config.IDLE_RESTART_ENABLED) {
|
|
910
|
+
// Defaults off (tv#317 follow-up, DevGuru review): a silent disabled gate and an
|
|
911
|
+
// enabled gate that just hasn't hit its threshold yet look identical in production
|
|
912
|
+
// (same flat/growing memory graph) — logging only on enabled, matching the
|
|
913
|
+
// Worktree isolation line below, makes opt-in verifiable from boot logs instead.
|
|
914
|
+
logger.info(' Idle restart: ENABLED (IDLE_RESTART_ENABLED=true)');
|
|
915
|
+
}
|
|
598
916
|
if (isWorktreeIsolationEnabled()) {
|
|
599
917
|
logger.info(' Worktree isolation: ENABLED (POLLER_USE_WORKTREES=true)');
|
|
600
918
|
const sweep = await cleanupOrphanWorktreesForAllBrains(config.BRAINS_PATH);
|
|
601
919
|
logger.info(` Orphan worktree sweep: scanned ${sweep.scanned} brains, removed ${sweep.removed} stale worktrees`);
|
|
602
920
|
}
|
|
921
|
+
// Seed trackedBrains from the backend's authoritative brain list
|
|
922
|
+
// (teamvibe.ai#127 Option B) BEFORE the maintenance cron and the inbox
|
|
923
|
+
// watcher start — both iterate the in-memory trackedBrains map
|
|
924
|
+
// (maintenance-cron.ts's triggerAllBrains, inbox-watcher.ts's
|
|
925
|
+
// checkInboxForNewSessions), which is otherwise empty after every
|
|
926
|
+
// restart until a real message re-registers each brain one at a time.
|
|
927
|
+
// Best-effort: never throws, so a bootstrap failure here falls back to
|
|
928
|
+
// the pre-#127 behavior rather than blocking startup.
|
|
929
|
+
//
|
|
930
|
+
// This await only covers ONE short (~10s) fetch attempt (DevGuru round-3
|
|
931
|
+
// review item 2) — real message processing (pollLoop(), below) must not
|
|
932
|
+
// be blocked for up to ~62s by a backend outage, which is exactly the
|
|
933
|
+
// condition most likely to coincide with a poller restart (backend
|
|
934
|
+
// deploys). If that one attempt fails and is retryable, the retry runs
|
|
935
|
+
// detached in the background and finishes after startPoller() has
|
|
936
|
+
// already returned — see brains-bootstrap.ts's bootstrapTrackedBrains()
|
|
937
|
+
// doc comment for why that's safe (neither the maintenance cron nor the
|
|
938
|
+
// inbox watcher need trackedBrains populated synchronously here).
|
|
939
|
+
await bootstrapTrackedBrains();
|
|
603
940
|
startMaintenanceLoop(processMessage);
|
|
604
941
|
startAutoUpdateCron();
|
|
605
942
|
startMetricsCron();
|
package/dist/types.d.ts
CHANGED
|
@@ -19,6 +19,12 @@ export interface TeamVibeQueueMessage {
|
|
|
19
19
|
filename: string;
|
|
20
20
|
mimetype: string;
|
|
21
21
|
}>;
|
|
22
|
+
inlineImages?: Array<{
|
|
23
|
+
mediaType: string;
|
|
24
|
+
data?: string;
|
|
25
|
+
url?: string;
|
|
26
|
+
expiresAt?: string;
|
|
27
|
+
}>;
|
|
22
28
|
response_context: {
|
|
23
29
|
slack?: {
|
|
24
30
|
channel: string;
|