@teamvibe/poller 0.7.0 → 0.7.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/brain-manager.d.ts +8 -0
- package/dist/brain-manager.js +132 -6
- package/dist/brains-bootstrap.d.ts +45 -0
- package/dist/brains-bootstrap.js +217 -0
- package/dist/claude-spawner.d.ts +1 -0
- package/dist/claude-spawner.js +10 -0
- package/dist/config.d.ts +16 -0
- package/dist/config.js +17 -0
- package/dist/idle-restart.d.ts +12 -0
- package/dist/idle-restart.js +200 -0
- package/dist/maintenance-cron.d.ts +66 -0
- package/dist/maintenance-cron.js +69 -0
- package/dist/poller.d.ts +3 -0
- package/dist/poller.js +204 -21
- package/package.json +1 -1
package/dist/brain-manager.d.ts
CHANGED
|
@@ -18,6 +18,14 @@ interface BrainConfig {
|
|
|
18
18
|
* pushBrainChanges below.
|
|
19
19
|
*/
|
|
20
20
|
export declare function cleanupStaleLocks(brainDir: string): Promise<void>;
|
|
21
|
+
/**
|
|
22
|
+
* Write the local (uncommitted) merge-driver attributes for a brain checkout.
|
|
23
|
+
* Idempotent and safe to call on every `getBrainPath` invocation, not just on
|
|
24
|
+
* first clone — this also self-heals brains that were checked out before
|
|
25
|
+
* this existed. Exported for testing — production call sites go through
|
|
26
|
+
* `getBrainPath`.
|
|
27
|
+
*/
|
|
28
|
+
export declare function ensureGitAttributes(brainDir: string): Promise<void>;
|
|
21
29
|
export declare function getBrainPath(brain?: BrainConfig, channelId?: string, workspaceId?: string): Promise<string>;
|
|
22
30
|
/**
|
|
23
31
|
* Update a brain repo (respects cooldown). Exported for testing — production
|
package/dist/brain-manager.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { exec } from 'child_process';
|
|
2
2
|
import { promisify } from 'util';
|
|
3
3
|
import { existsSync, mkdirSync } from 'fs';
|
|
4
|
-
import { readdir, stat, unlink } from 'fs/promises';
|
|
4
|
+
import { readdir, stat, unlink, mkdir, writeFile } from 'fs/promises';
|
|
5
5
|
import { join, basename } from 'path';
|
|
6
6
|
import { config } from './config.js';
|
|
7
7
|
import { logger } from './logger.js';
|
|
@@ -128,6 +128,47 @@ async function reportBrainSync(brainId, workspaceId, event) {
|
|
|
128
128
|
emitError('brain', `Brain sync report error: ${error instanceof Error ? error.message : String(error)}`, { severity: 'warn', context: { brainId } });
|
|
129
129
|
}
|
|
130
130
|
}
|
|
131
|
+
// tv#247: local-only (never committed, never synced) merge-driver config for
|
|
132
|
+
// a brain checkout. `merge=union` is one of git's built-in low-level merge
|
|
133
|
+
// drivers (alongside `text`/`binary`) — an attributes entry alone is enough,
|
|
134
|
+
// no `[merge "union"]` section in git config is needed (verified: a rebase
|
|
135
|
+
// across two branches with independent appends to a `merge=union`-attributed
|
|
136
|
+
// file merges both lines automatically instead of conflicting).
|
|
137
|
+
//
|
|
138
|
+
// Scoped ONLY to genuinely pure-append, line-independent files. Deliberately
|
|
139
|
+
// does NOT cover MEM_REGISTRY.md, MEM_REGISTRY_ARCHIVE.md, semantic/*, or
|
|
140
|
+
// episodic/* — consolidation edits those in place (promotes/removes
|
|
141
|
+
// entries), so a union merge there would resurrect already-promoted-and-
|
|
142
|
+
// removed content instead of resolving the conflict, rather than concatenate
|
|
143
|
+
// two genuinely independent additions.
|
|
144
|
+
//
|
|
145
|
+
// Written to `.git/info/attributes` (not a committed `.gitattributes`)
|
|
146
|
+
// because this is poller-local git behavior, not brain content — channel
|
|
147
|
+
// brain repos are written to only by their own sessions, never by the
|
|
148
|
+
// platform directly.
|
|
149
|
+
const GIT_ATTRIBUTES_CONTENT = ['memory/TODAY.md merge=union', 'memory/daily/*.md merge=union', ''].join('\n');
|
|
150
|
+
/**
|
|
151
|
+
* Write the local (uncommitted) merge-driver attributes for a brain checkout.
|
|
152
|
+
* Idempotent and safe to call on every `getBrainPath` invocation, not just on
|
|
153
|
+
* first clone — this also self-heals brains that were checked out before
|
|
154
|
+
* this existed. Exported for testing — production call sites go through
|
|
155
|
+
* `getBrainPath`.
|
|
156
|
+
*/
|
|
157
|
+
export async function ensureGitAttributes(brainDir) {
|
|
158
|
+
try {
|
|
159
|
+
const gitDir = join(brainDir, '.git');
|
|
160
|
+
if (!existsSync(gitDir))
|
|
161
|
+
return; // not a git repo yet (defensive — getBrainPath always clones first)
|
|
162
|
+
const infoDir = join(gitDir, 'info');
|
|
163
|
+
await mkdir(infoDir, { recursive: true });
|
|
164
|
+
await writeFile(join(infoDir, 'attributes'), GIT_ATTRIBUTES_CONTENT);
|
|
165
|
+
}
|
|
166
|
+
catch (error) {
|
|
167
|
+
// Best-effort: worst case a future push-race on these files needs manual
|
|
168
|
+
// resolution same as before this existed. Not a reason to fail the caller.
|
|
169
|
+
logger.warn(`Brain ${brainDir}: failed to write .git/info/attributes: ${error instanceof Error ? error.message : error}`);
|
|
170
|
+
}
|
|
171
|
+
}
|
|
131
172
|
export async function getBrainPath(brain, channelId, workspaceId) {
|
|
132
173
|
if (!brain?.gitRepoUrl) {
|
|
133
174
|
if (channelId) {
|
|
@@ -164,6 +205,9 @@ export async function getBrainPath(brain, channelId, workspaceId) {
|
|
|
164
205
|
if (workspaceId)
|
|
165
206
|
reportBrainSync(brain.brainId, workspaceId, 'pulled');
|
|
166
207
|
}
|
|
208
|
+
// Idempotent, local-only — covers both the fresh-clone and existing-dir
|
|
209
|
+
// branches above, and self-heals brains cloned before this existed.
|
|
210
|
+
await ensureGitAttributes(brainDir);
|
|
167
211
|
return brainDir;
|
|
168
212
|
}
|
|
169
213
|
/**
|
|
@@ -263,6 +307,52 @@ async function gitStashList(brainDir) {
|
|
|
263
307
|
const { stdout } = await gitExec('git stash list', { cwd: brainDir });
|
|
264
308
|
return stdout.trim();
|
|
265
309
|
}
|
|
310
|
+
// Per-brainId rescue-ref name, seeded with a timestamp the first time a given
|
|
311
|
+
// brainId hits an unresolved push-race conflict, then reused (force-pushed
|
|
312
|
+
// to) on every later pushBrainChangesLocked invocation that still can't
|
|
313
|
+
// converge — e.g. the next message's push attempt while the conflict remains
|
|
314
|
+
// unresolved on disk. Without this, each retry would mint its own
|
|
315
|
+
// `conflict/<brainId>-<ts>` ref and spam origin with one throwaway branch per
|
|
316
|
+
// retry instead of one per actual incident. Cleared on the next successful
|
|
317
|
+
// push for that brainId, since any future conflict after that point is a new,
|
|
318
|
+
// unrelated incident that deserves its own ref.
|
|
319
|
+
const rescueRefNames = new Map();
|
|
320
|
+
/**
|
|
321
|
+
* Best-effort durability net for a `PUSH_RACE_CONFLICT`: push whatever HEAD
|
|
322
|
+
* currently points to onto a throwaway ref on origin before the caller
|
|
323
|
+
* reports the conflict, so the session's commits survive even if this
|
|
324
|
+
* checkout is lost (crash, re-clone, TTL'd container) before a human/AI does
|
|
325
|
+
* the manual resolution. Never throws — a failed rescue push just means no
|
|
326
|
+
* safety net for this occurrence, not a reason to mask the real
|
|
327
|
+
* PUSH_RACE_CONFLICT error the caller is about to report.
|
|
328
|
+
*
|
|
329
|
+
* Callers MUST ensure `HEAD` is the safe, non-mid-rebase content to rescue
|
|
330
|
+
* before calling this (e.g. after `git rebase --abort` restores the
|
|
331
|
+
* pre-rebase local branch) — this function pushes HEAD as-is.
|
|
332
|
+
*/
|
|
333
|
+
async function pushRescueRef(brainDir, brainId) {
|
|
334
|
+
let refName = rescueRefNames.get(brainId);
|
|
335
|
+
if (!refName) {
|
|
336
|
+
// Unix seconds (not ms) — matches the <timestamp> format documented on
|
|
337
|
+
// the consuming side (poller-brain#443's MAINTENANCE.md `oldestAgeDays`
|
|
338
|
+
// derivation), so both sides agree on how to parse the ref name.
|
|
339
|
+
refName = `conflict/${brainId}-${Math.floor(Date.now() / 1000)}`;
|
|
340
|
+
rescueRefNames.set(brainId, refName);
|
|
341
|
+
}
|
|
342
|
+
try {
|
|
343
|
+
await gitExec(`git push --force origin HEAD:refs/heads/${refName}`, { cwd: brainDir });
|
|
344
|
+
logger.info(`Brain ${brainId} push-race: rescued local commits to ${refName}`);
|
|
345
|
+
return { refName, pushed: true };
|
|
346
|
+
}
|
|
347
|
+
catch (error) {
|
|
348
|
+
// Distinct sub-case from a successful rescue (same failure class as the
|
|
349
|
+
// known pb#440 issue — e.g. broken git credentials) — logged separately
|
|
350
|
+
// so it's never mistaken for "commits are safe on origin".
|
|
351
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
352
|
+
logger.error(`Brain ${brainId} push-race: rescue-ref push to ${refName} itself FAILED, local commits remain only on disk: ${message}`);
|
|
353
|
+
return { refName, pushed: false };
|
|
354
|
+
}
|
|
355
|
+
}
|
|
266
356
|
/**
|
|
267
357
|
* Scan the commits about to be pushed for blobs over `maxBytes`. Returns the
|
|
268
358
|
* offending paths/sizes, or an empty array if everything is under the limit.
|
|
@@ -382,6 +472,10 @@ async function pushBrainChangesLocked(brainDir, brainId, workspaceId, retryOptio
|
|
|
382
472
|
attempt++;
|
|
383
473
|
try {
|
|
384
474
|
await gitExec(`git push -u origin ${branch}`, { cwd: brainDir });
|
|
475
|
+
// A successful push means any earlier unresolved conflict for this
|
|
476
|
+
// brainId (if any) is now resolved — a future conflict is a new,
|
|
477
|
+
// unrelated incident, so stop reusing that rescue ref name.
|
|
478
|
+
rescueRefNames.delete(brainId);
|
|
385
479
|
if (attempt > 1) {
|
|
386
480
|
logger.info(`Brain ${brainId} push-race auto-resolved after ${attempt} attempts`);
|
|
387
481
|
emitError('brain', `Brain ${brainId} push-race auto-resolved after ${attempt} attempts`, {
|
|
@@ -427,7 +521,15 @@ async function pushBrainChangesLocked(brainDir, brainId, workspaceId, retryOptio
|
|
|
427
521
|
const stashAfter = await gitStashList(brainDir);
|
|
428
522
|
if (stashAfter !== stashBefore) {
|
|
429
523
|
logger.error(`Brain ${brainId} push-race: autostash pop left changes stashed after rebase, treating as conflict`);
|
|
430
|
-
|
|
524
|
+
// The rebase itself (of committed local work onto origin) succeeded
|
|
525
|
+
// here — HEAD is the safe, non-mid-rebase content to rescue; only
|
|
526
|
+
// the separate stash-pop of uncommitted changes is what's stuck.
|
|
527
|
+
const { refName: rescueRef, pushed: rescuePushed } = await pushRescueRef(brainDir, brainId);
|
|
528
|
+
emitError('brain', `Brain ${brainId} push-race autostash pop conflict — manual sync needed (stash: ${stashAfter.split('\n')[0]})` +
|
|
529
|
+
(rescuePushed ? `, committed local work rescued to ${rescueRef}` : `, rescue push to ${rescueRef} also FAILED`), {
|
|
530
|
+
severity: 'error',
|
|
531
|
+
context: { brainId, errorCode: 'PUSH_RACE_CONFLICT', attempts: attempt, reason: 'autostash_pop_conflict', rescueRef, rescuePushed },
|
|
532
|
+
});
|
|
431
533
|
return;
|
|
432
534
|
}
|
|
433
535
|
}
|
|
@@ -442,13 +544,21 @@ async function pushBrainChangesLocked(brainDir, brainId, workspaceId, retryOptio
|
|
|
442
544
|
severity: 'error',
|
|
443
545
|
context: { brainId, errorCode: 'PUSH_RACE_ABORT_FAILED', attempts: attempt },
|
|
444
546
|
});
|
|
547
|
+
// Repo is left mid-rebase / in an uncertain state — do NOT push
|
|
548
|
+
// HEAD to a rescue ref here, that would rescue the wrong
|
|
549
|
+
// (partially-applied) content instead of the pre-conflict local work.
|
|
445
550
|
return;
|
|
446
551
|
}
|
|
552
|
+
// `git rebase --abort` above restored HEAD to the pre-rebase local
|
|
553
|
+
// branch — i.e. exactly the unpushed, pre-conflict work we need to
|
|
554
|
+
// keep durable, not whatever mid-rebase state existed a moment ago.
|
|
555
|
+
const { refName: rescueRef, pushed: rescuePushed } = await pushRescueRef(brainDir, brainId);
|
|
447
556
|
const rebaseMessage = rebaseError instanceof Error ? rebaseError.message : String(rebaseError);
|
|
448
557
|
logger.error(`Brain ${brainId} push-race rebase conflict: ${rebaseMessage}`);
|
|
449
|
-
emitError('brain', `Brain ${brainId} push-race rebase conflict — manual sync needed: ${rebaseMessage}
|
|
558
|
+
emitError('brain', `Brain ${brainId} push-race rebase conflict — manual sync needed: ${rebaseMessage}` +
|
|
559
|
+
(rescuePushed ? `, local commits rescued to ${rescueRef}` : `, rescue push to ${rescueRef} also FAILED`), {
|
|
450
560
|
severity: 'error',
|
|
451
|
-
context: { brainId, errorCode: 'PUSH_RACE_CONFLICT', attempts: attempt },
|
|
561
|
+
context: { brainId, errorCode: 'PUSH_RACE_CONFLICT', attempts: attempt, rescueRef, rescuePushed },
|
|
452
562
|
});
|
|
453
563
|
return;
|
|
454
564
|
}
|
|
@@ -457,9 +567,25 @@ async function pushBrainChangesLocked(brainDir, brainId, workspaceId, retryOptio
|
|
|
457
567
|
}
|
|
458
568
|
const errorCode = NON_FF_REJECTION_RE.test(lastErrorMessage) ? 'PUSH_RACE_UNRESOLVED' : 'PUSH_FAILED';
|
|
459
569
|
logger.warn(`Failed to push brain ${brainId} changes: ${lastErrorMessage}`);
|
|
460
|
-
|
|
570
|
+
// PUSH_RACE_UNRESOLVED means the retry budget ran out while HEAD was still
|
|
571
|
+
// in the safe, non-mid-rebase state left by the last successful rebase
|
|
572
|
+
// (the loop only reaches here via the `attempt >= maxAttempts` break right
|
|
573
|
+
// after a push rejection, never after a failed rebase — those return
|
|
574
|
+
// earlier). Same "local commits only on disk" risk as the two conflict
|
|
575
|
+
// cases above, just a different exhaustion cause — rescue it too.
|
|
576
|
+
// PUSH_FAILED (non-NON_FF failures, e.g. auth/network) deliberately
|
|
577
|
+
// skipped: a rescue push would almost always fail for the identical
|
|
578
|
+
// reason as the push it's rescuing.
|
|
579
|
+
let rescueRef;
|
|
580
|
+
let rescuePushed;
|
|
581
|
+
if (errorCode === 'PUSH_RACE_UNRESOLVED') {
|
|
582
|
+
;
|
|
583
|
+
({ refName: rescueRef, pushed: rescuePushed } = await pushRescueRef(brainDir, brainId));
|
|
584
|
+
}
|
|
585
|
+
emitError('brain', `Failed to push brain ${brainId} changes: ${lastErrorMessage}` +
|
|
586
|
+
(errorCode === 'PUSH_RACE_UNRESOLVED' ? (rescuePushed ? `, local commits rescued to ${rescueRef}` : `, rescue push to ${rescueRef} also FAILED`) : ''), {
|
|
461
587
|
severity: errorCode === 'PUSH_RACE_UNRESOLVED' ? 'error' : 'warn',
|
|
462
|
-
context: { brainId, errorCode, attempts: attempt },
|
|
588
|
+
context: { brainId, errorCode, attempts: attempt, ...(errorCode === 'PUSH_RACE_UNRESOLVED' ? { rescueRef, rescuePushed } : {}) },
|
|
463
589
|
});
|
|
464
590
|
}
|
|
465
591
|
catch (error) {
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
/** @internal test-only */
|
|
2
|
+
export declare function __detachedRetryForTest(): Promise<void> | undefined;
|
|
3
|
+
/**
|
|
4
|
+
* Fetch the full list of brains this poller is responsible for from the
|
|
5
|
+
* backend (GET /poller/brains, source of truth = DDB — teamvibe.ai#127
|
|
6
|
+
* Option B) and seed `trackedBrains` (maintenance-cron.ts) before the
|
|
7
|
+
* maintenance/heartbeat cron starts, so a brain that's been quiet since
|
|
8
|
+
* before this poller's most recent restart is still picked up instead of
|
|
9
|
+
* staying invisible until a human happens to message it.
|
|
10
|
+
*
|
|
11
|
+
* Best-effort: on any failure (network, timeout, non-2xx, malformed body,
|
|
12
|
+
* missing pollerId/token) this logs a warning, emits a telemetry event, and
|
|
13
|
+
* returns without throwing. Startup must not hang or crash on this — the
|
|
14
|
+
* pre-#127 behavior (trackedBrains starts empty until real messages
|
|
15
|
+
* trickle in) is the fallback here, not a fatal state.
|
|
16
|
+
*
|
|
17
|
+
* DevGuru round-3 review: the awaited part of this function does exactly
|
|
18
|
+
* ONE fetch attempt, bounded by CRITICAL_PATH_TIMEOUT_MS (~10s) — not the
|
|
19
|
+
* full 30s client timeout, and not the retry. Every poller restart during a
|
|
20
|
+
* backend outage was blocking real user-message processing (pollLoop(),
|
|
21
|
+
* called after this in startPoller()) for up to ~62s (2 attempts * 30s +
|
|
22
|
+
* 2s delay) — and restarts correlate with backend deploys, exactly when
|
|
23
|
+
* this is most likely to happen. If the first attempt fails and is
|
|
24
|
+
* retryable (network error, timeout, or 5xx — not a 4xx or malformed body,
|
|
25
|
+
* which a retry can't fix), the retry is kicked off detached — an
|
|
26
|
+
* immediately-invoked async function whose promise is stored in a
|
|
27
|
+
* module-level variable (see `__detachedRetryForTest` below; test-only,
|
|
28
|
+
* production code never reads it) rather than `void`-discarded, but never
|
|
29
|
+
* awaited by this function — the same fire-and-forget shape
|
|
30
|
+
* maintenance-cron.ts's registerBrain() already uses for its immediate
|
|
31
|
+
* metrics collect+send — so `bootstrapTrackedBrains()` returns and
|
|
32
|
+
* startPoller() can proceed to
|
|
33
|
+
* startMaintenanceLoop/startAutoUpdateCron/startMetricsCron/pollLoop()
|
|
34
|
+
* without waiting on it. This is safe because neither downstream consumer
|
|
35
|
+
* needs bootstrap complete synchronously: the maintenance cron only fires
|
|
36
|
+
* at its own scheduled time (default 03:30, not at startup), and the inbox
|
|
37
|
+
* watcher just needs trackedBrains populated "one tick later" — both read
|
|
38
|
+
* the same shared, mutable Map (getTrackedBrains()) whenever their own tick
|
|
39
|
+
* fires, not a snapshot taken at startup.
|
|
40
|
+
*
|
|
41
|
+
* No-ops in direct (non-token) mode, same gating as sendMetrics
|
|
42
|
+
* (metrics/sender.ts) — there's no backend endpoint to call without
|
|
43
|
+
* TEAMVIBE_API_URL + TEAMVIBE_POLLER_TOKEN.
|
|
44
|
+
*/
|
|
45
|
+
export declare function bootstrapTrackedBrains(): Promise<void>;
|
|
@@ -0,0 +1,217 @@
|
|
|
1
|
+
import { config } from './config.js';
|
|
2
|
+
import { logger } from './logger.js';
|
|
3
|
+
import { getPollerId } from './auth-provider.js';
|
|
4
|
+
import { seedTrackedBrains } from './maintenance-cron.js';
|
|
5
|
+
import { emitEvent, emitError } from './event-queue.js';
|
|
6
|
+
// Client-side timeout kept comfortably above the server-side route timeout
|
|
7
|
+
// (infra/poller-auth.ts: GET /poller/brains, 25s) so the poller never aborts
|
|
8
|
+
// a request the Lambda could still have answered (DevGuru round-2 review —
|
|
9
|
+
// with the old 10s client timeout vs a 20s route timeout, a transient slow
|
|
10
|
+
// response aborted client-side while the Lambda kept running to nothing,
|
|
11
|
+
// burning the one bootstrap attempt with no retry until the next restart).
|
|
12
|
+
// Used ONLY for the off-critical-path retry attempt below — DevGuru round-3:
|
|
13
|
+
// "don't touch the 25s/30s numbers themselves", so this stays exactly as
|
|
14
|
+
// round-2 left it, just no longer used for the first attempt.
|
|
15
|
+
const FETCH_TIMEOUT_MS = 30_000;
|
|
16
|
+
// Short deadline for the ONE attempt that's actually awaited on
|
|
17
|
+
// startPoller()'s critical path (DevGuru round-3 review item 2). Judgment
|
|
18
|
+
// call — not spelled out beyond "~10s" — picked 10s exactly as suggested.
|
|
19
|
+
// Rationale for why a short deadline is safe here even though it's shorter
|
|
20
|
+
// than the 25s the Lambda might legitimately take: this is only the FIRST
|
|
21
|
+
// attempt. If it doesn't finish inside 10s, we don't conclude failure and
|
|
22
|
+
// give up — we fall through to the same retry path used for any other
|
|
23
|
+
// transient failure, just detached from startup so a slow-but-alive
|
|
24
|
+
// backend doesn't cost the poller a lost user-message-processing minute.
|
|
25
|
+
const CRITICAL_PATH_TIMEOUT_MS = 10_000;
|
|
26
|
+
// Judgment call (not specified by DevGuru's review, flagged for reviewer):
|
|
27
|
+
// one retry, 2s fixed delay, unchanged from round 2. Since the retry is now
|
|
28
|
+
// off the critical path (see bootstrapTrackedBrains below), the 2s delay no
|
|
29
|
+
// longer costs startup time — it only affects how soon the detached retry
|
|
30
|
+
// fires, which is not time-critical (the maintenance cron doesn't run until
|
|
31
|
+
// its own scheduled time, and the inbox watcher just needs trackedBrains
|
|
32
|
+
// populated "one tick later", not synchronously — DevGuru round-3).
|
|
33
|
+
const RETRY_DELAY_MS = 2_000;
|
|
34
|
+
function sleep(ms) {
|
|
35
|
+
return new Promise(resolve => setTimeout(resolve, ms));
|
|
36
|
+
}
|
|
37
|
+
// Test-only handle to the detached retry promise (see bootstrapTrackedBrains
|
|
38
|
+
// below) kicked off by the most recent call. Undefined until a retryable
|
|
39
|
+
// first-attempt failure schedules one; reset to undefined at the top of
|
|
40
|
+
// every bootstrapTrackedBrains() call so a stale promise from a PRIOR call
|
|
41
|
+
// is never mistaken for the current one. Same test-only-export convention
|
|
42
|
+
// as maintenance-cron.ts's __resetTrackedBrainsForTest() — never read by
|
|
43
|
+
// production code, exists purely so tests can deterministically wait for
|
|
44
|
+
// the detached background work to finish instead of guessing with a fixed
|
|
45
|
+
// sleep (DevGuru review: the fixed-sleep approach was two separate
|
|
46
|
+
// latent-flakiness bugs — see brains-bootstrap.test.ts).
|
|
47
|
+
let detachedRetryForTest;
|
|
48
|
+
/** @internal test-only */
|
|
49
|
+
export function __detachedRetryForTest() {
|
|
50
|
+
return detachedRetryForTest;
|
|
51
|
+
}
|
|
52
|
+
async function attemptFetch(url, token, timeoutMs) {
|
|
53
|
+
const controller = new AbortController();
|
|
54
|
+
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
55
|
+
try {
|
|
56
|
+
const response = await fetch(url, {
|
|
57
|
+
headers: { Authorization: `Bearer ${token}` },
|
|
58
|
+
signal: controller.signal,
|
|
59
|
+
});
|
|
60
|
+
if (!response.ok) {
|
|
61
|
+
return {
|
|
62
|
+
ok: false,
|
|
63
|
+
reason: 'non-2xx',
|
|
64
|
+
detail: `status ${response.status}`,
|
|
65
|
+
retryable: response.status >= 500,
|
|
66
|
+
};
|
|
67
|
+
}
|
|
68
|
+
let body;
|
|
69
|
+
try {
|
|
70
|
+
body = await response.json();
|
|
71
|
+
}
|
|
72
|
+
catch (err) {
|
|
73
|
+
return {
|
|
74
|
+
ok: false,
|
|
75
|
+
reason: 'invalid-json',
|
|
76
|
+
detail: err instanceof Error ? err.message : String(err),
|
|
77
|
+
retryable: false,
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
if (!body || typeof body !== 'object' || !Array.isArray(body.brains)) {
|
|
81
|
+
return { ok: false, reason: 'malformed-body', detail: 'missing brains array', retryable: false };
|
|
82
|
+
}
|
|
83
|
+
return { ok: true, brains: body.brains };
|
|
84
|
+
}
|
|
85
|
+
catch (err) {
|
|
86
|
+
const isTimeout = err instanceof Error && err.name === 'AbortError';
|
|
87
|
+
return {
|
|
88
|
+
ok: false,
|
|
89
|
+
reason: isTimeout ? 'timeout' : 'network-error',
|
|
90
|
+
detail: err instanceof Error ? err.message : String(err),
|
|
91
|
+
retryable: true,
|
|
92
|
+
};
|
|
93
|
+
}
|
|
94
|
+
finally {
|
|
95
|
+
clearTimeout(timer);
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
/** Seed trackedBrains from a successful fetch and emit success telemetry.
|
|
99
|
+
* Shared by both the critical-path attempt and the detached retry below. */
|
|
100
|
+
function applySuccess(brains) {
|
|
101
|
+
const seeded = seedTrackedBrains(brains);
|
|
102
|
+
logger.info(`[Bootstrap] Seeded ${seeded} brain(s) into trackedBrains from backend (${brains.length} returned)`);
|
|
103
|
+
emitEvent('poller.brains_bootstrap.success', 'system', {
|
|
104
|
+
seeded,
|
|
105
|
+
returned: brains.length,
|
|
106
|
+
});
|
|
107
|
+
}
|
|
108
|
+
/** Report a terminal (no-more-retries) failure. Distinct telemetry artifact
|
|
109
|
+
* (DevGuru round-2 review) — from outside, "bootstrap crashed" and
|
|
110
|
+
* "bootstrap ran fine, no quiet brains to seed" were previously
|
|
111
|
+
* indistinguishable (both show seeded=0 in a log line nobody's watching).
|
|
112
|
+
* This feeds the same FLEET_ERROR pipeline as maintenance-cron.ts's
|
|
113
|
+
* emitError call (triggerAllBrains, per-brain maintenance failure) — same
|
|
114
|
+
* category, same shape. Only called once per bootstrap attempt sequence
|
|
115
|
+
* (non-retryable first-attempt failure, or a failed retry) — never for a
|
|
116
|
+
* retryable first-attempt failure, since that one isn't terminal yet. */
|
|
117
|
+
function reportFailure(outcome) {
|
|
118
|
+
emitError('maintenance', `trackedBrains bootstrap failed: ${outcome.reason}`, {
|
|
119
|
+
severity: 'warn',
|
|
120
|
+
context: { reason: outcome.reason, detail: outcome.detail },
|
|
121
|
+
});
|
|
122
|
+
}
|
|
123
|
+
/**
|
|
124
|
+
* Fetch the full list of brains this poller is responsible for from the
|
|
125
|
+
* backend (GET /poller/brains, source of truth = DDB — teamvibe.ai#127
|
|
126
|
+
* Option B) and seed `trackedBrains` (maintenance-cron.ts) before the
|
|
127
|
+
* maintenance/heartbeat cron starts, so a brain that's been quiet since
|
|
128
|
+
* before this poller's most recent restart is still picked up instead of
|
|
129
|
+
* staying invisible until a human happens to message it.
|
|
130
|
+
*
|
|
131
|
+
* Best-effort: on any failure (network, timeout, non-2xx, malformed body,
|
|
132
|
+
* missing pollerId/token) this logs a warning, emits a telemetry event, and
|
|
133
|
+
* returns without throwing. Startup must not hang or crash on this — the
|
|
134
|
+
* pre-#127 behavior (trackedBrains starts empty until real messages
|
|
135
|
+
* trickle in) is the fallback here, not a fatal state.
|
|
136
|
+
*
|
|
137
|
+
* DevGuru round-3 review: the awaited part of this function does exactly
|
|
138
|
+
* ONE fetch attempt, bounded by CRITICAL_PATH_TIMEOUT_MS (~10s) — not the
|
|
139
|
+
* full 30s client timeout, and not the retry. Every poller restart during a
|
|
140
|
+
* backend outage was blocking real user-message processing (pollLoop(),
|
|
141
|
+
* called after this in startPoller()) for up to ~62s (2 attempts * 30s +
|
|
142
|
+
* 2s delay) — and restarts correlate with backend deploys, exactly when
|
|
143
|
+
* this is most likely to happen. If the first attempt fails and is
|
|
144
|
+
* retryable (network error, timeout, or 5xx — not a 4xx or malformed body,
|
|
145
|
+
* which a retry can't fix), the retry is kicked off detached — an
|
|
146
|
+
* immediately-invoked async function whose promise is stored in a
|
|
147
|
+
* module-level variable (see `__detachedRetryForTest` below; test-only,
|
|
148
|
+
* production code never reads it) rather than `void`-discarded, but never
|
|
149
|
+
* awaited by this function — the same fire-and-forget shape
|
|
150
|
+
* maintenance-cron.ts's registerBrain() already uses for its immediate
|
|
151
|
+
* metrics collect+send — so `bootstrapTrackedBrains()` returns and
|
|
152
|
+
* startPoller() can proceed to
|
|
153
|
+
* startMaintenanceLoop/startAutoUpdateCron/startMetricsCron/pollLoop()
|
|
154
|
+
* without waiting on it. This is safe because neither downstream consumer
|
|
155
|
+
* needs bootstrap complete synchronously: the maintenance cron only fires
|
|
156
|
+
* at its own scheduled time (default 03:30, not at startup), and the inbox
|
|
157
|
+
* watcher just needs trackedBrains populated "one tick later" — both read
|
|
158
|
+
* the same shared, mutable Map (getTrackedBrains()) whenever their own tick
|
|
159
|
+
* fires, not a snapshot taken at startup.
|
|
160
|
+
*
|
|
161
|
+
* No-ops in direct (non-token) mode, same gating as sendMetrics
|
|
162
|
+
* (metrics/sender.ts) — there's no backend endpoint to call without
|
|
163
|
+
* TEAMVIBE_API_URL + TEAMVIBE_POLLER_TOKEN.
|
|
164
|
+
*/
|
|
165
|
+
export async function bootstrapTrackedBrains() {
|
|
166
|
+
detachedRetryForTest = undefined;
|
|
167
|
+
if (!config.TEAMVIBE_API_URL || !config.TEAMVIBE_POLLER_TOKEN) {
|
|
168
|
+
return;
|
|
169
|
+
}
|
|
170
|
+
const pollerId = getPollerId();
|
|
171
|
+
if (!pollerId) {
|
|
172
|
+
logger.warn('[Bootstrap] Skipping trackedBrains bootstrap: no pollerId (auth not completed?)');
|
|
173
|
+
return;
|
|
174
|
+
}
|
|
175
|
+
const token = config.TEAMVIBE_POLLER_TOKEN;
|
|
176
|
+
const url = `${config.TEAMVIBE_API_URL}/poller/brains?pollerId=${encodeURIComponent(pollerId)}`;
|
|
177
|
+
const outcome = await attemptFetch(url, token, CRITICAL_PATH_TIMEOUT_MS);
|
|
178
|
+
if (outcome.ok) {
|
|
179
|
+
applySuccess(outcome.brains);
|
|
180
|
+
return;
|
|
181
|
+
}
|
|
182
|
+
logger.warn(`[Bootstrap] GET /poller/brains failed on first attempt: ${outcome.reason} (${outcome.detail})`);
|
|
183
|
+
if (!outcome.retryable) {
|
|
184
|
+
reportFailure(outcome);
|
|
185
|
+
return;
|
|
186
|
+
}
|
|
187
|
+
// Off the critical path from here — startPoller() is not waiting on this
|
|
188
|
+
// promise. Wrapped in try/catch (DevGuru round-4 review) rather than just
|
|
189
|
+
// asserting "nothing below throws": attemptFetch() itself always returns
|
|
190
|
+
// an outcome object instead of throwing, but applySuccess() (seedTrackedBrains
|
|
191
|
+
// + emitEvent) and reportFailure() (emitError) are still real function
|
|
192
|
+
// calls — if either ever throws for any reason (a future refactor, an
|
|
193
|
+
// unexpected edge case), an uncaught throw inside this detached function
|
|
194
|
+
// becomes an unhandled promise rejection and crashes the whole poller
|
|
195
|
+
// process. Cheap insurance: catch and log a warning, matching this file's
|
|
196
|
+
// existing logger.warn style, instead of letting it propagate.
|
|
197
|
+
//
|
|
198
|
+
// Stored in detachedRetryForTest (DevGuru round-5 review) rather than
|
|
199
|
+
// `void`-discarded, purely so tests can await it deterministically — see
|
|
200
|
+
// that variable's own comment above. Storing the reference doesn't change
|
|
201
|
+
// runtime behavior here: this function still never awaits it itself.
|
|
202
|
+
detachedRetryForTest = (async () => {
|
|
203
|
+
try {
|
|
204
|
+
await sleep(RETRY_DELAY_MS);
|
|
205
|
+
const retryOutcome = await attemptFetch(url, token, FETCH_TIMEOUT_MS);
|
|
206
|
+
if (retryOutcome.ok) {
|
|
207
|
+
applySuccess(retryOutcome.brains);
|
|
208
|
+
return;
|
|
209
|
+
}
|
|
210
|
+
logger.warn(`[Bootstrap] GET /poller/brains failed on retry: ${retryOutcome.reason} (${retryOutcome.detail})`);
|
|
211
|
+
reportFailure(retryOutcome);
|
|
212
|
+
}
|
|
213
|
+
catch (err) {
|
|
214
|
+
logger.warn(`[Bootstrap] Detached retry threw unexpectedly: ${err instanceof Error ? err.message : String(err)}`);
|
|
215
|
+
}
|
|
216
|
+
})();
|
|
217
|
+
}
|
package/dist/claude-spawner.d.ts
CHANGED
|
@@ -232,6 +232,7 @@ export declare function spawnClaudeCode(msg: TeamVibeQueueMessage, sessionLog: S
|
|
|
232
232
|
newSessionId?: string;
|
|
233
233
|
}>;
|
|
234
234
|
export declare function getActiveProcessCount(): number;
|
|
235
|
+
export declare function getLastSessionEndTs(): number;
|
|
235
236
|
/**
|
|
236
237
|
* Kill a zombie process for a given thread ID.
|
|
237
238
|
* Returns true if a process was found and killed.
|
package/dist/claude-spawner.js
CHANGED
|
@@ -1116,6 +1116,7 @@ async function runClaudeCode(msg, sessionLog, cwd, brainPath, sessionId, isFirst
|
|
|
1116
1116
|
clearNotificationTimers();
|
|
1117
1117
|
activeProcesses.delete(msg.id);
|
|
1118
1118
|
activeProcessesByThread.delete(msg.thread_id);
|
|
1119
|
+
lastSessionEndTs = Date.now();
|
|
1119
1120
|
// #73 gap 2 (flagged independently by two reviews): `close` firing means
|
|
1120
1121
|
// `claude` itself has exited, but its MCP server children — spawned
|
|
1121
1122
|
// into the same detached process group, see `spawn()` above — are not
|
|
@@ -1178,6 +1179,7 @@ async function runClaudeCode(msg, sessionLog, cwd, brainPath, sessionId, isFirst
|
|
|
1178
1179
|
clearInterval(idleWatchdog);
|
|
1179
1180
|
activeProcesses.delete(msg.id);
|
|
1180
1181
|
activeProcessesByThread.delete(msg.thread_id);
|
|
1182
|
+
lastSessionEndTs = Date.now();
|
|
1181
1183
|
emitError('session', `Claude Code process spawn error: ${error.message}`, {
|
|
1182
1184
|
channelId: msg.teamvibe.channelId,
|
|
1183
1185
|
context: { threadId: msg.thread_id },
|
|
@@ -1275,9 +1277,17 @@ export async function spawnClaudeCode(msg, sessionLog, cwd, sessionId, isFirstMe
|
|
|
1275
1277
|
}
|
|
1276
1278
|
const activeProcesses = new Map();
|
|
1277
1279
|
const activeProcessesByThread = new Map();
|
|
1280
|
+
// tv#317: when the last `claude` process ended, so the idle-restart gate can require
|
|
1281
|
+
// a quiet period since real work last happened. Starts at module load (poller boot) so
|
|
1282
|
+
// a poller that never runs a session still becomes eligible once IDLE_RESTART_MIN_UPTIME_MS
|
|
1283
|
+
// passes, rather than being stuck at 0/undefined forever.
|
|
1284
|
+
let lastSessionEndTs = Date.now();
|
|
1278
1285
|
export function getActiveProcessCount() {
|
|
1279
1286
|
return activeProcesses.size;
|
|
1280
1287
|
}
|
|
1288
|
+
export function getLastSessionEndTs() {
|
|
1289
|
+
return lastSessionEndTs;
|
|
1290
|
+
}
|
|
1281
1291
|
/**
|
|
1282
1292
|
* Kill a zombie process for a given thread ID.
|
|
1283
1293
|
* Returns true if a process was found and killed.
|
package/dist/config.d.ts
CHANGED
|
@@ -41,6 +41,10 @@ declare const configSchema: z.ZodObject<{
|
|
|
41
41
|
AUTO_UPDATE_SCHEDULE: z.ZodDefault<z.ZodString>;
|
|
42
42
|
INBOX_WATCH_FALLBACK_INTERVAL_MS: z.ZodDefault<z.ZodNumber>;
|
|
43
43
|
INBOX_WATCH_DEBOUNCE_MS: z.ZodDefault<z.ZodNumber>;
|
|
44
|
+
IDLE_RESTART_ENABLED: z.ZodEffects<z.ZodDefault<z.ZodString>, boolean, string | undefined>;
|
|
45
|
+
IDLE_RESTART_IDLE_MS: z.ZodDefault<z.ZodNumber>;
|
|
46
|
+
IDLE_RESTART_MIN_UPTIME_MS: z.ZodDefault<z.ZodNumber>;
|
|
47
|
+
IDLE_RESTART_MEMORY_FILE_THRESHOLD_BYTES: z.ZodDefault<z.ZodNumber>;
|
|
44
48
|
}, "strip", z.ZodTypeAny, {
|
|
45
49
|
AWS_REGION: string;
|
|
46
50
|
TEAMVIBE_API_URL: string;
|
|
@@ -78,6 +82,10 @@ declare const configSchema: z.ZodObject<{
|
|
|
78
82
|
AUTO_UPDATE_SCHEDULE: string;
|
|
79
83
|
INBOX_WATCH_FALLBACK_INTERVAL_MS: number;
|
|
80
84
|
INBOX_WATCH_DEBOUNCE_MS: number;
|
|
85
|
+
IDLE_RESTART_ENABLED: boolean;
|
|
86
|
+
IDLE_RESTART_IDLE_MS: number;
|
|
87
|
+
IDLE_RESTART_MIN_UPTIME_MS: number;
|
|
88
|
+
IDLE_RESTART_MEMORY_FILE_THRESHOLD_BYTES: number;
|
|
81
89
|
SQS_QUEUE_URL?: string | undefined;
|
|
82
90
|
SESSIONS_TABLE?: string | undefined;
|
|
83
91
|
TEAMVIBE_POLLER_TOKEN?: string | undefined;
|
|
@@ -123,6 +131,10 @@ declare const configSchema: z.ZodObject<{
|
|
|
123
131
|
AUTO_UPDATE_SCHEDULE?: string | undefined;
|
|
124
132
|
INBOX_WATCH_FALLBACK_INTERVAL_MS?: number | undefined;
|
|
125
133
|
INBOX_WATCH_DEBOUNCE_MS?: number | undefined;
|
|
134
|
+
IDLE_RESTART_ENABLED?: string | undefined;
|
|
135
|
+
IDLE_RESTART_IDLE_MS?: number | undefined;
|
|
136
|
+
IDLE_RESTART_MIN_UPTIME_MS?: number | undefined;
|
|
137
|
+
IDLE_RESTART_MEMORY_FILE_THRESHOLD_BYTES?: number | undefined;
|
|
126
138
|
}>;
|
|
127
139
|
export type Config = z.infer<typeof configSchema>;
|
|
128
140
|
export declare function loadConfig(): Config;
|
|
@@ -163,6 +175,10 @@ export declare const config: {
|
|
|
163
175
|
AUTO_UPDATE_SCHEDULE: string;
|
|
164
176
|
INBOX_WATCH_FALLBACK_INTERVAL_MS: number;
|
|
165
177
|
INBOX_WATCH_DEBOUNCE_MS: number;
|
|
178
|
+
IDLE_RESTART_ENABLED: boolean;
|
|
179
|
+
IDLE_RESTART_IDLE_MS: number;
|
|
180
|
+
IDLE_RESTART_MIN_UPTIME_MS: number;
|
|
181
|
+
IDLE_RESTART_MEMORY_FILE_THRESHOLD_BYTES: number;
|
|
166
182
|
SQS_QUEUE_URL?: string | undefined;
|
|
167
183
|
SESSIONS_TABLE?: string | undefined;
|
|
168
184
|
TEAMVIBE_POLLER_TOKEN?: string | undefined;
|
package/dist/config.js
CHANGED
|
@@ -76,6 +76,23 @@ const configSchema = z.object({
|
|
|
76
76
|
// exist at watcher startup, and prunes empty thread dirs.
|
|
77
77
|
INBOX_WATCH_FALLBACK_INTERVAL_MS: z.coerce.number().default(300_000), // 5 minutes
|
|
78
78
|
INBOX_WATCH_DEBOUNCE_MS: z.coerce.number().default(300), // coalesce bursty fs events into one scan
|
|
79
|
+
// Idle restart (tv#317): Railway bills on cgroup memory usage, which includes page
|
|
80
|
+
// cache that Linux only reclaims under real memory pressure — a large Railway memory
|
|
81
|
+
// limit means that pressure never triggers, so cache from heavy sessions (bun install,
|
|
82
|
+
// vite build, apt) accumulates until something forces a restart. Rather than an
|
|
83
|
+
// external supervisor, the poller restarts itself once genuinely idle; Railway's
|
|
84
|
+
// ALWAYS restart policy brings it back up clean and any SQS message that arrives
|
|
85
|
+
// during the few-second restart window just waits in the queue.
|
|
86
|
+
//
|
|
87
|
+
// Defaults OFF: this is a Railway-specific mitigation (the memory-threshold check
|
|
88
|
+
// is gated on cgroup v2 anyway, see readReclaimableFileBytes), but the default
|
|
89
|
+
// should make that explicit rather than lean on an implicit cgroup-absence no-op —
|
|
90
|
+
// deploy targets that also happen to be cgroup v2 Linux shouldn't pick this up by
|
|
91
|
+
// just existing. Set IDLE_RESTART_ENABLED=true on Railway to opt in.
|
|
92
|
+
IDLE_RESTART_ENABLED: z.string().default('false').transform((v) => v.toLowerCase() === 'true'),
|
|
93
|
+
IDLE_RESTART_IDLE_MS: z.coerce.number().default(1_800_000), // 30 minutes since the last session ended
|
|
94
|
+
IDLE_RESTART_MIN_UPTIME_MS: z.coerce.number().default(1_800_000), // crash-loop guard — matches the idle floor above (tv#317 discussion), so it doesn't add its own separate cadence
|
|
95
|
+
IDLE_RESTART_MEMORY_FILE_THRESHOLD_BYTES: z.coerce.number().default(700 * 1024 * 1024), // cgroup memory.stat `file` (reclaimable page cache), not memory.current — gating on raw usage would also fire on a real anon leak and mask it instead of just clearing cache
|
|
79
96
|
});
|
|
80
97
|
export function loadConfig() {
|
|
81
98
|
const result = configSchema.safeParse(process.env);
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
export declare function readReclaimableFileBytes(path?: string): number | null;
|
|
2
|
+
export declare function findLiveBgTask(now?: number): string | null;
|
|
3
|
+
export declare function reconcileBgTasksOnBoot(): void;
|
|
4
|
+
export interface CheckIdleRestartDeps {
|
|
5
|
+
now?: number;
|
|
6
|
+
startTime?: number;
|
|
7
|
+
activeProcessCount?: () => number;
|
|
8
|
+
lastSessionEndTs?: () => number;
|
|
9
|
+
reclaimableBytes?: () => number | null;
|
|
10
|
+
liveBgTask?: () => string | null;
|
|
11
|
+
}
|
|
12
|
+
export declare function checkIdleRestart(deps?: CheckIdleRestartDeps): boolean;
|