@teamvibe/poller 0.6.2 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/brain-manager.d.ts +43 -1
- package/dist/brain-manager.js +259 -6
- package/dist/classify-failed-iteration.d.ts +1 -1
- package/dist/classify-failed-iteration.js +33 -1
- package/dist/claude-spawner.d.ts +178 -1
- package/dist/claude-spawner.js +687 -92
- package/dist/config.d.ts +24 -0
- package/dist/config.js +15 -2
- package/dist/cron-telemetry.d.ts +3 -0
- package/dist/cron-telemetry.js +16 -0
- package/dist/event-queue.d.ts +30 -0
- package/dist/event-queue.js +4 -0
- package/dist/inbox-manager.d.ts +25 -0
- package/dist/inbox-manager.js +47 -0
- package/dist/maintenance-activity.d.ts +18 -0
- package/dist/maintenance-activity.js +28 -0
- package/dist/maintenance-cron.d.ts +7 -0
- package/dist/maintenance-cron.js +16 -1
- package/dist/metrics/collector.d.ts +2 -0
- package/dist/metrics/collector.js +7 -0
- package/dist/poller.d.ts +9 -0
- package/dist/poller.js +158 -4
- package/dist/types.d.ts +6 -0
- package/package.json +1 -1
package/dist/poller.js
CHANGED
|
@@ -12,7 +12,7 @@ import { initAuth, stopRefresh, getPollerId } from './auth-provider.js';
|
|
|
12
12
|
import { registerBrain, startMaintenanceLoop, stopMaintenanceLoop } from './maintenance-cron.js';
|
|
13
13
|
import { startEventQueue, stopEventQueue, flush as flushEventQueue, emitEvent, emitError } from './event-queue.js';
|
|
14
14
|
import { startMetricsCron, stopMetricsCron } from './metrics/scheduler.js';
|
|
15
|
-
import { writeMessage, pickNextMessage, markProcessed, markFailed, hasUnprocessed } from './inbox-manager.js';
|
|
15
|
+
import { writeMessage, pickNextMessage, markProcessed, markFailed, hasUnprocessed, requeueMessage } from './inbox-manager.js';
|
|
16
16
|
import { startInboxWatcher, stopInboxWatcher } from './inbox-watcher.js';
|
|
17
17
|
import { Cron } from 'croner';
|
|
18
18
|
import { isControlMessage, handleControlMessage } from './control-handler.js';
|
|
@@ -132,6 +132,57 @@ const activeLoops = new Set();
|
|
|
132
132
|
export function isInboxLoopActive(threadId) {
|
|
133
133
|
return activeLoops.has(threadId);
|
|
134
134
|
}
|
|
135
|
+
// Exported for testing (teamvibe.ai#32). SIGTERM (143) / SIGKILL (137) exit
|
|
136
|
+
// codes are the classic 128+signal convention for "the poller itself killed
|
|
137
|
+
// this process on purpose" (timeout, idle, network, zombie, or — as of #32 —
|
|
138
|
+
// a compaction-ceiling watchdog kill). Historically all of these set
|
|
139
|
+
// `killedByPoller` via the same `proc.kill('SIGTERM')` path in
|
|
140
|
+
// claude-spawner.ts and were observed to surface as 143/137 here.
|
|
141
|
+
//
|
|
142
|
+
// PR#308 round-2 review (teamvibe.ai#73): this exit-code convention is
|
|
143
|
+
// fragile on its own — the Claude CLI can legitimately exit 143 for
|
|
144
|
+
// unrelated reasons, and #73's move to group-kill (`process.kill(-pid, ...)`
|
|
145
|
+
// instead of a direct `proc.kill()`) means a signal-terminated close now
|
|
146
|
+
// reports `code: null` with a separate `signal` field (verified empirically
|
|
147
|
+
// against a synthetic detached child in this repo's environment — Node's
|
|
148
|
+
// `close` event reports `code: null` for any signal-terminated process,
|
|
149
|
+
// group-killed or not). Kept as a defensive fallback only — see
|
|
150
|
+
// isPollerInitiatedFailure() below, which is what callers should actually use.
|
|
151
|
+
export function isIntentionalKillExitCode(exitCode) {
|
|
152
|
+
return exitCode === 143 || exitCode === 137;
|
|
153
|
+
}
|
|
154
|
+
// The actual retry-classification signal (PR#308 round-2 review item 1):
|
|
155
|
+
// `killedByPoller` is the purpose-built marker for "the poller killed this
|
|
156
|
+
// process on purpose, not an application failure" (see SpawnResult /
|
|
157
|
+
// classify-failed-iteration.ts) and is set on every poller-initiated kill
|
|
158
|
+
// path this branch touches — timeout, idle watchdog, network watchdog,
|
|
159
|
+
// zombie-kill, and (added by this fix) poller shutdown. Prefer it, and fall
|
|
160
|
+
// back to the exit-code heuristic only when it's absent (e.g. hand-built
|
|
161
|
+
// SpawnResult literals in older/unrelated call sites) so behavior doesn't
|
|
162
|
+
// regress for anything not yet covered by the marker.
|
|
163
|
+
export function isPollerInitiatedFailure(result) {
|
|
164
|
+
return Boolean(result.killedByPoller) || isIntentionalKillExitCode(result.exitCode);
|
|
165
|
+
}
|
|
166
|
+
// teamvibe.ai#284 (poller-brain#235): end-of-turn guard. A successful Claude
|
|
167
|
+
// run can end its turn having only written plain assistant text — no
|
|
168
|
+
// `mcp__slack__send_message` tool call — which the platform otherwise drops
|
|
169
|
+
// silently (no error, no signal; see poller-brain#235's 4x-in-one-session
|
|
170
|
+
// recurrence). Decides whether this run needs the deterministic fallback
|
|
171
|
+
// delivery, and if so, what text to send.
|
|
172
|
+
//
|
|
173
|
+
// Pure/exported (mirrors classifyFailedIteration's pattern) so the gating
|
|
174
|
+
// logic — "Slack context AND no send_message call AND non-empty trailing
|
|
175
|
+
// text" — is unit-testable without mocking spawnClaudeCode or sendSlackMessage.
|
|
176
|
+
// Deliberately does NOT fire on a failed run (result.success === false):
|
|
177
|
+
// that path already has its own delivery via classifyFailedIteration +
|
|
178
|
+
// sendSlackMessage/sendSlackError, and firing here too would double-deliver.
|
|
179
|
+
export function endOfTurnFallbackText(hasSlackContext, result) {
|
|
180
|
+
if (!hasSlackContext || !result.success || result.sawSendMessageCall) {
|
|
181
|
+
return undefined;
|
|
182
|
+
}
|
|
183
|
+
const text = result.lastPlainTextContent?.trim();
|
|
184
|
+
return text ? text : undefined;
|
|
185
|
+
}
|
|
135
186
|
export async function runInboxLoop(threadId, kbPath, queueMessage, sessionLog) {
|
|
136
187
|
if (activeLoops.has(threadId)) {
|
|
137
188
|
sessionLog.info(`Loop already running for thread ${threadId}, skipping`);
|
|
@@ -160,7 +211,25 @@ export async function runInboxLoop(threadId, kbPath, queueMessage, sessionLog) {
|
|
|
160
211
|
activeLocks.set(threadId, lockToken);
|
|
161
212
|
}
|
|
162
213
|
let currentSessionId = session.session_id || undefined;
|
|
163
|
-
|
|
214
|
+
// PR#308 round-3 review (teamvibe.ai#73): `shuttingDown` (declared below,
|
|
215
|
+
// referenced here via the same forward-reference pattern already used by
|
|
216
|
+
// pollLoop()'s `while (!shuttingDown)`) gates every new iteration of this
|
|
217
|
+
// loop, not just the retry-classification decision inside it. Without
|
|
218
|
+
// this, a message killed with `killedByPoller: 'shutdown'` (added by this
|
|
219
|
+
// branch) falls into the same "poller-initiated, retry immediately, no
|
|
220
|
+
// backoff" bucket as timeout/idle/network/zombie (see
|
|
221
|
+
// isPollerInitiatedFailure() below) — correctly skipping markProcessed/
|
|
222
|
+
// markFailed so the message stays unprocessed for a future boot, but with
|
|
223
|
+
// nothing to stop the loop from immediately calling pickNextMessage()
|
|
224
|
+
// again and spawning a NEW `claude` process for that same message, even
|
|
225
|
+
// though killAllActiveProcesses() (shutdown()'s first step) already ran.
|
|
226
|
+
// Gating the loop *condition* itself (rather than special-casing
|
|
227
|
+
// 'shutdown' inside the retry decision) closes this for every
|
|
228
|
+
// killedByPoller reason uniformly, and — unlike a `break` — falls through
|
|
229
|
+
// to the exact same post-loop cleanup (releaseSessionLock etc., below)
|
|
230
|
+
// that the existing `if (!message) break` / `if (!hasUnprocessed(...))
|
|
231
|
+
// break` exits already use, so lock release is unaffected.
|
|
232
|
+
while (!shuttingDown) {
|
|
164
233
|
const message = pickNextMessage(kbPath, threadId);
|
|
165
234
|
if (!message) {
|
|
166
235
|
sessionLog.info(`[Loop] No more messages for thread ${threadId}`);
|
|
@@ -177,6 +246,7 @@ export async function runInboxLoop(threadId, kbPath, queueMessage, sessionLog) {
|
|
|
177
246
|
: undefined;
|
|
178
247
|
let succeeded = false;
|
|
179
248
|
let lastExitCode = null;
|
|
249
|
+
let lastKilledByPoller;
|
|
180
250
|
try {
|
|
181
251
|
// Reconstruct queue message from stored inbox data
|
|
182
252
|
const storedData = JSON.parse(message.content);
|
|
@@ -227,6 +297,28 @@ export async function runInboxLoop(threadId, kbPath, queueMessage, sessionLog) {
|
|
|
227
297
|
modelSource: result.modelSource,
|
|
228
298
|
});
|
|
229
299
|
}
|
|
300
|
+
// teamvibe.ai#284 (poller-brain#235): end-of-turn guard. A
|
|
301
|
+
// successful run that never called send_message but left trailing
|
|
302
|
+
// plain text would otherwise vanish silently — deliver it via the
|
|
303
|
+
// same sendSlackMessage() helper used for the failure-path neutral
|
|
304
|
+
// notices, and emit a warning-level event so the near-miss is
|
|
305
|
+
// visible (today session.completed looks identical whether or not
|
|
306
|
+
// this fired).
|
|
307
|
+
const fallbackText = endOfTurnFallbackText(hasSlackContext, result);
|
|
308
|
+
if (fallbackText) {
|
|
309
|
+
sessionLog.warn(`[Loop] Iteration #${iteration}: no send_message call detected, delivering trailing assistant text via fallback`);
|
|
310
|
+
try {
|
|
311
|
+
await sendSlackMessage(queueMessage, fallbackText);
|
|
312
|
+
}
|
|
313
|
+
catch (err) {
|
|
314
|
+
sessionLog.error(`[Loop] End-of-turn fallback delivery failed: ${err instanceof Error ? err.message : err}`);
|
|
315
|
+
}
|
|
316
|
+
emitError('session', 'End-of-turn guard: delivered assistant text via fallback (no send_message tool call detected)', {
|
|
317
|
+
severity: 'warn',
|
|
318
|
+
channelId: queueMessage.teamvibe.channelId,
|
|
319
|
+
context: { threadId: queueMessage.thread_id },
|
|
320
|
+
});
|
|
321
|
+
}
|
|
230
322
|
if (queueMessage.teamvibe.brain?.brainId) {
|
|
231
323
|
await pushBrainChanges(kbPath, queueMessage.teamvibe.brain.brainId, queueMessage.teamvibe.workspaceId);
|
|
232
324
|
}
|
|
@@ -234,6 +326,7 @@ export async function runInboxLoop(threadId, kbPath, queueMessage, sessionLog) {
|
|
|
234
326
|
}
|
|
235
327
|
else {
|
|
236
328
|
lastExitCode = result.exitCode;
|
|
329
|
+
lastKilledByPoller = result.killedByPoller;
|
|
237
330
|
sessionLog.error(`[Loop] Claude failed iteration #${iteration}: ${result.error}`);
|
|
238
331
|
if (hasSlackContext) {
|
|
239
332
|
const decision = classifyFailedIteration(result);
|
|
@@ -255,8 +348,19 @@ export async function runInboxLoop(threadId, kbPath, queueMessage, sessionLog) {
|
|
|
255
348
|
markProcessed(kbPath, threadId, message.filename);
|
|
256
349
|
}
|
|
257
350
|
else {
|
|
258
|
-
//
|
|
259
|
-
|
|
351
|
+
// Poller-initiated kill (timeout/idle/network/zombie/shutdown, via
|
|
352
|
+
// `killedByPoller`, or the legacy 143/137 exit-code fallback) → never
|
|
353
|
+
// counts as an application failure, so markFailed/backoff is skipped
|
|
354
|
+
// below regardless of reason. For timeout/idle/network/zombie this
|
|
355
|
+
// really does mean "retry immediately" — the `while (!shuttingDown)`
|
|
356
|
+
// condition above lets the loop come straight back around. For
|
|
357
|
+
// 'shutdown' specifically, `shuttingDown` is already true by the time
|
|
358
|
+
// this reads, so the loop condition above stops it from actually
|
|
359
|
+
// retrying inline — instead the branch below requeues the message
|
|
360
|
+
// from processing/ back to new/ (PR#308 round-5) so a future boot's
|
|
361
|
+
// inbox watcher finds it. See isPollerInitiatedFailure() and the
|
|
362
|
+
// loop-condition comment above.
|
|
363
|
+
const isKilledSignal = isPollerInitiatedFailure({ exitCode: lastExitCode, killedByPoller: lastKilledByPoller });
|
|
260
364
|
if (!isKilledSignal) {
|
|
261
365
|
const failures = (failureCounts.get(message.filename) || 0) + 1;
|
|
262
366
|
failureCounts.set(message.filename, failures);
|
|
@@ -271,6 +375,43 @@ export async function runInboxLoop(threadId, kbPath, queueMessage, sessionLog) {
|
|
|
271
375
|
await sleep(backoffMs);
|
|
272
376
|
}
|
|
273
377
|
}
|
|
378
|
+
else if (lastKilledByPoller === 'shutdown') {
|
|
379
|
+
// PR#308 round-5 review (teamvibe.ai#73): pickNextMessage() moves a
|
|
380
|
+
// message's file to processing/ as soon as it's picked up, not once
|
|
381
|
+
// it completes — so a message killed here by shutdown is left
|
|
382
|
+
// sitting in processing/, not new/. The watcher's wake condition
|
|
383
|
+
// (hasNewMessage(), inbox-watcher.ts) only scans new/, so nothing
|
|
384
|
+
// would start a fresh runInboxLoop() for this thread on next boot.
|
|
385
|
+
// Requeue it back to new/ so a future boot's watcher scan actually
|
|
386
|
+
// finds it — this is what makes "your message is saved" (the
|
|
387
|
+
// shutdown notice text below, classify-failed-iteration.ts)
|
|
388
|
+
// literally true. It does NOT make resumption automatic: the
|
|
389
|
+
// watcher's checkInboxForNewSessions() (inbox-watcher.ts) only
|
|
390
|
+
// scans brains already in the in-memory trackedBrains map, which
|
|
391
|
+
// is empty after a fresh poller start until some SQS activity for
|
|
392
|
+
// this brain re-registers it (see teamvibe.ai#310) — hence the
|
|
393
|
+
// notice text also naming "next message or scheduled run" rather
|
|
394
|
+
// than claiming automatic resumption.
|
|
395
|
+
//
|
|
396
|
+
// Scoped to 'shutdown' only, deliberately not the other four
|
|
397
|
+
// poller-initiated reasons (timeout/idle/network/zombie) — those
|
|
398
|
+
// retry immediately within this same still-running loop iteration
|
|
399
|
+
// (the `while (!shuttingDown)` condition above lets it come
|
|
400
|
+
// straight back around), so the existing processing/
|
|
401
|
+
// crash-recovery branch inside pickNextMessage() already covers
|
|
402
|
+
// them; requeueing those too would be redundant and could
|
|
403
|
+
// double-process.
|
|
404
|
+
//
|
|
405
|
+
// Safe against waking a brand-new loop for this thread before this
|
|
406
|
+
// poller process actually exits: stopInboxWatcher() runs earlier in
|
|
407
|
+
// shutdown() (poller.ts), BEFORE killAllActiveProcesses() (whose
|
|
408
|
+
// completion is what unblocks this iteration and reaches this
|
|
409
|
+
// branch) — and stopInboxWatcher() closes the real fs.watch
|
|
410
|
+
// handlers themselves (inbox-watcher.ts stopInboxWatcher(), not
|
|
411
|
+
// just an interval/poll timer), so by the time this requeue runs,
|
|
412
|
+
// nothing is watching new/ to react to the file reappearing there.
|
|
413
|
+
requeueMessage(kbPath, threadId, message.filename);
|
|
414
|
+
}
|
|
274
415
|
}
|
|
275
416
|
if (!hasUnprocessed(kbPath, threadId)) {
|
|
276
417
|
break;
|
|
@@ -515,6 +656,19 @@ function sleep(ms) {
|
|
|
515
656
|
}
|
|
516
657
|
// Graceful shutdown
|
|
517
658
|
let shuttingDown = false;
|
|
659
|
+
// Exported for testing only (PR#308 round-3 review item 1). `shuttingDown`
|
|
660
|
+
// has no other external mutator — production code only ever flips it via
|
|
661
|
+
// the real shutdown() below, which also runs killAllActiveProcesses(),
|
|
662
|
+
// releaseAllActiveLocks(), the drain loop, and process.exit(). This setter
|
|
663
|
+
// lets a test put runInboxLoop's `while (!shuttingDown)` gate into the
|
|
664
|
+
// "shutdown in progress" state directly (mirroring the exact race
|
|
665
|
+
// shutdown() itself creates — killAllActiveProcesses() kills the in-flight
|
|
666
|
+
// spawn while its promise is still pending) without invoking any of those
|
|
667
|
+
// real side effects. Same pattern as __resetCronTelemetryForTest /
|
|
668
|
+
// __resetTrackedBrainsForTest elsewhere in this package.
|
|
669
|
+
export function __setShuttingDownForTest(value) {
|
|
670
|
+
shuttingDown = value;
|
|
671
|
+
}
|
|
518
672
|
async function shutdown(signal, exitCode = 0) {
|
|
519
673
|
if (shuttingDown)
|
|
520
674
|
return;
|
package/dist/types.d.ts
CHANGED
|
@@ -19,6 +19,12 @@ export interface TeamVibeQueueMessage {
|
|
|
19
19
|
filename: string;
|
|
20
20
|
mimetype: string;
|
|
21
21
|
}>;
|
|
22
|
+
inlineImages?: Array<{
|
|
23
|
+
mediaType: string;
|
|
24
|
+
data?: string;
|
|
25
|
+
url?: string;
|
|
26
|
+
expiresAt?: string;
|
|
27
|
+
}>;
|
|
22
28
|
response_context: {
|
|
23
29
|
slack?: {
|
|
24
30
|
channel: string;
|