@teamvibe/poller 0.6.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,8 +1,23 @@
1
+ export declare function withBrainLock<T>(brainId: string, fn: () => Promise<T>): Promise<T>;
1
2
  interface BrainConfig {
2
3
  brainId: string;
3
4
  gitRepoUrl: string;
4
5
  branch: string;
5
6
  }
7
+ /**
8
+ * Remove stale git lock files (tv#76) so an orphaned lock from a killed
9
+ * process (e.g. the 2026-05-01 outage) doesn't block brain updates/pushes
10
+ * indefinitely. Fresh locks (younger than the threshold) are left alone —
11
+ * they may belong to a git operation that's genuinely still running.
12
+ * A missing lock (stat fails) is a silent no-op. A genuine removal failure
13
+ * (unlink fails, e.g. permission denied) is logged via logger.warn with the
14
+ * path and error — that's the exact "lock exists but can't be cleared"
15
+ * scenario this fix exists to make visible.
16
+ *
17
+ * Exported for testing — production call sites go through updateBrain /
18
+ * pushBrainChanges below.
19
+ */
20
+ export declare function cleanupStaleLocks(brainDir: string): Promise<void>;
6
21
  export declare function getBrainPath(brain?: BrainConfig, channelId?: string, workspaceId?: string): Promise<string>;
7
22
  /**
8
23
  * Update a brain repo (respects cooldown). Exported for testing — production
@@ -18,11 +33,38 @@ export declare function ensureBaseBrain(): Promise<void>;
18
33
  * Get the path to the base brain directory.
19
34
  */
20
35
  export declare function getBaseBrainPath(): string;
36
+ /**
37
+ * Scan the commits about to be pushed for blobs over `maxBytes`. Returns the
38
+ * offending paths/sizes, or an empty array if everything is under the limit.
39
+ *
40
+ * Scans `@{u}..HEAD` when an upstream exists (the actual new content being
41
+ * pushed); falls back to all objects reachable from HEAD for a branch's first
42
+ * push (no upstream yet), which is equivalent since nothing has been pushed.
43
+ */
44
+ export declare function findOversizedBlobs(brainDir: string, maxBytes: number): Promise<Array<{
45
+ path: string;
46
+ size: number;
47
+ }>>;
21
48
  /**
22
49
  * Commit and push any changes in a brain repo.
23
50
  * Silently skips if there are no changes or no remote.
51
+ *
52
+ * On non-fast-forward rejection (tv#232 push-race — remote advanced between
53
+ * local commit and push, e.g. a concurrent session or scheduled-tick already
54
+ * pushed), fetch+rebase and retry, bounded by BRAIN_PUSH_RETRY_MAX. A genuine
55
+ * rebase conflict (or an autostash pop that silently leaves changes stashed —
56
+ * git does not surface that as a rebase error) is treated as a hard failure
57
+ * needing manual sync, not retried further.
58
+ *
59
+ * `retryOptions` defaults to the BRAIN_PUSH_RETRY_* config; exposed as a
60
+ * parameter (rather than reading `config` directly inline) so tests can
61
+ * exercise the retry-exhaustion boundary deterministically without relying on
62
+ * process-wide env/module-load-order tricks.
24
63
  */
25
- export declare function pushBrainChanges(brainDir: string, brainId: string, workspaceId?: string): Promise<void>;
64
+ export declare function pushBrainChanges(brainDir: string, brainId: string, workspaceId?: string, retryOptions?: {
65
+ maxAttempts?: number;
66
+ backoffMs?: number;
67
+ }): Promise<void>;
26
68
  /**
27
69
  * Ensure base paths exist
28
70
  */
@@ -1,7 +1,8 @@
1
1
  import { exec } from 'child_process';
2
2
  import { promisify } from 'util';
3
3
  import { existsSync, mkdirSync } from 'fs';
4
- import { join } from 'path';
4
+ import { readdir, stat, unlink } from 'fs/promises';
5
+ import { join, basename } from 'path';
5
6
  import { config } from './config.js';
6
7
  import { logger } from './logger.js';
7
8
  import { emitError } from './event-queue.js';
@@ -20,8 +21,84 @@ if (config.PERSISTENT_STORAGE_PATH) {
20
21
  function gitExec(command, options) {
21
22
  return execAsync(command, { ...options, env: gitEnv });
22
23
  }
24
+ // Node's child_process.exec() defaults to a 1MB stdout cap. This bounds the
25
+ // text output of a git plumbing command (e.g. one line per object listed),
26
+ // not any single blob's size -- it has no relationship to
27
+ // config.BRAIN_MAX_BLOB_BYTES and must stay its own named constant so the
28
+ // two don't drift into looking coupled just because they're both 50MB today.
29
+ const GIT_SCAN_MAX_BUFFER_BYTES = 50 * 1024 * 1024;
23
30
  // Cooldown tracker per brain
24
31
  const lastUpdateTimes = new Map();
32
+ // Per-brainId mutex serializing updateBrain/pushBrainChanges for a given
33
+ // brain. Two threads of the SAME brain running concurrently in this process
34
+ // share one brainDir (tv#76/tv#277 review) — without this, cleanupStaleLocks()
35
+ // can unlink a git lock file that a still-in-flight sibling call's own git
36
+ // operation legitimately holds, corrupting the index. Poller instances never
37
+ // share a physical brainDir across processes/machines (BRAINS_PATH is
38
+ // $HOME-derived and each poller is a single process — see tv#287), so this
39
+ // in-process mutex fully closes the race; no DynamoDB-level lock is needed.
40
+ const brainLocks = new Map();
41
+ // Exported for testing — production call sites go through updateBrain / pushBrainChanges above.
42
+ export function withBrainLock(brainId, fn) {
43
+ const prior = brainLocks.get(brainId) ?? Promise.resolve();
44
+ const run = prior.then(fn, fn);
45
+ brainLocks.set(brainId, run.then(() => undefined, () => undefined));
46
+ return run;
47
+ }
48
+ /**
49
+ * Remove stale git lock files (tv#76) so an orphaned lock from a killed
50
+ * process (e.g. the 2026-05-01 outage) doesn't block brain updates/pushes
51
+ * indefinitely. Fresh locks (younger than the threshold) are left alone —
52
+ * they may belong to a git operation that's genuinely still running.
53
+ * A missing lock (stat fails) is a silent no-op. A genuine removal failure
54
+ * (unlink fails, e.g. permission denied) is logged via logger.warn with the
55
+ * path and error — that's the exact "lock exists but can't be cleared"
56
+ * scenario this fix exists to make visible.
57
+ *
58
+ * Exported for testing — production call sites go through updateBrain /
59
+ * pushBrainChanges below.
60
+ */
61
+ export async function cleanupStaleLocks(brainDir) {
62
+ const candidates = [
63
+ { path: join(brainDir, '.git', 'index.lock'), maxAgeMs: config.GIT_LOCK_MAX_AGE_MS },
64
+ ];
65
+ const modulesDir = join(brainDir, '.git', 'modules');
66
+ try {
67
+ const entries = await readdir(modulesDir, { recursive: true });
68
+ for (const entry of entries) {
69
+ if (basename(entry) === 'config.lock') {
70
+ candidates.push({ path: join(modulesDir, entry), maxAgeMs: config.GIT_SUBMODULE_LOCK_MAX_AGE_MS });
71
+ }
72
+ }
73
+ }
74
+ catch {
75
+ // No .git/modules dir (no submodules yet) — nothing to scan.
76
+ }
77
+ for (const { path: lockPath, maxAgeMs } of candidates) {
78
+ let lockStat;
79
+ try {
80
+ lockStat = await stat(lockPath);
81
+ }
82
+ catch {
83
+ // Lock doesn't exist — normal no-op, nothing worth logging.
84
+ continue;
85
+ }
86
+ const ageMs = Date.now() - lockStat.mtimeMs;
87
+ if (ageMs <= maxAgeMs)
88
+ continue;
89
+ try {
90
+ await unlink(lockPath);
91
+ logger.warn(`Removed stale git lock: ${lockPath} (age: ${Math.round(ageMs / 1000)}s, threshold: ${Math.round(maxAgeMs / 1000)}s)`);
92
+ }
93
+ catch (error) {
94
+ // Real removal failure (e.g. permission denied) — this is the exact
95
+ // 2026-05-01 outage scenario, so it must leave a diagnostic trail
96
+ // instead of silently skipping like a plain "lock doesn't exist".
97
+ const errorMessage = error instanceof Error ? error.message : String(error);
98
+ logger.warn(`Failed to remove stale git lock: ${lockPath} (age: ${Math.round(ageMs / 1000)}s): ${errorMessage}`);
99
+ }
100
+ }
101
+ }
25
102
  /**
26
103
  * Get or clone brain repo. Returns the working directory path.
27
104
  */
@@ -94,6 +171,9 @@ export async function getBrainPath(brain, channelId, workspaceId) {
94
171
  * call sites should go through `getBrainPath` / `ensureBaseBrain`.
95
172
  */
96
173
  export async function updateBrain(brainDir, brainId, branch) {
174
+ return withBrainLock(brainId, () => updateBrainLocked(brainDir, brainId, branch));
175
+ }
176
+ async function updateBrainLocked(brainDir, brainId, branch) {
97
177
  const now = Date.now();
98
178
  const lastUpdate = lastUpdateTimes.get(brainId) || 0;
99
179
  if (now - lastUpdate < config.BRAIN_UPDATE_COOLDOWN_MS) {
@@ -137,6 +217,7 @@ export async function updateBrain(brainDir, brainId, branch) {
137
217
  logger.warn(`Brain ${brainId} pre-check failed, skipping update: ${errorMessage}`);
138
218
  return;
139
219
  }
220
+ await cleanupStaleLocks(brainDir);
140
221
  try {
141
222
  logger.info(`Updating brain ${brainId} at ${brainDir}...`);
142
223
  await gitExec(`git fetch origin ${branch} && git reset --hard origin/${branch} && git submodule update --init --recursive`, {
@@ -173,11 +254,73 @@ export async function ensureBaseBrain() {
173
254
  export function getBaseBrainPath() {
174
255
  return config.BASE_BRAIN_PATH;
175
256
  }
257
+ // Matches git's non-fast-forward push rejection ("! [rejected] ... (fetch first)"
258
+ // or "... (non-fast-forward)"). Distinguishes a push-race (remote advanced,
259
+ // reconcilable via rebase) from other push failures (network, auth, oversized
260
+ // blob pre-receive decline — see tv#246) which retrying can never fix.
261
+ const NON_FF_REJECTION_RE = /\[rejected\][\s\S]*(fetch first|non-fast-forward)/;
262
+ async function gitStashList(brainDir) {
263
+ const { stdout } = await gitExec('git stash list', { cwd: brainDir });
264
+ return stdout.trim();
265
+ }
266
+ /**
267
+ * Scan the commits about to be pushed for blobs over `maxBytes`. Returns the
268
+ * offending paths/sizes, or an empty array if everything is under the limit.
269
+ *
270
+ * Scans `@{u}..HEAD` when an upstream exists (the actual new content being
271
+ * pushed); falls back to all objects reachable from HEAD for a branch's first
272
+ * push (no upstream yet), which is equivalent since nothing has been pushed.
273
+ */
274
+ export async function findOversizedBlobs(brainDir, maxBytes) {
275
+ let range = 'HEAD';
276
+ try {
277
+ await gitExec('git rev-parse @{u}', { cwd: brainDir });
278
+ range = '@{u}..HEAD';
279
+ }
280
+ catch {
281
+ // No upstream tracking ref yet — first push for this branch, scan all of HEAD.
282
+ }
283
+ // A full-HEAD scan (the fallback path this guard needs to work on) can
284
+ // exceed Node's default 1MB stdout cap on a repo with many objects.
285
+ // Without a larger maxBuffer the scan throws ENOBUFS and the guard
286
+ // silently disables itself (see catch below) on exactly the histories most
287
+ // likely to contain an oversized blob.
288
+ const { stdout } = await gitExec(`git rev-list --objects ${range} | git cat-file --batch-check='%(objecttype) %(objectname) %(objectsize) %(rest)'`, { cwd: brainDir, maxBuffer: GIT_SCAN_MAX_BUFFER_BYTES });
289
+ const oversized = [];
290
+ for (const line of stdout.split('\n')) {
291
+ if (!line.trim())
292
+ continue;
293
+ const [type, , sizeStr, ...pathParts] = line.split(' ');
294
+ if (type !== 'blob')
295
+ continue;
296
+ const size = Number(sizeStr);
297
+ if (size > maxBytes) {
298
+ oversized.push({ path: pathParts.join(' ') || '(unknown path)', size });
299
+ }
300
+ }
301
+ return oversized;
302
+ }
176
303
  /**
177
304
  * Commit and push any changes in a brain repo.
178
305
  * Silently skips if there are no changes or no remote.
306
+ *
307
+ * On non-fast-forward rejection (tv#232 push-race — remote advanced between
308
+ * local commit and push, e.g. a concurrent session or scheduled-tick already
309
+ * pushed), fetch+rebase and retry, bounded by BRAIN_PUSH_RETRY_MAX. A genuine
310
+ * rebase conflict (or an autostash pop that silently leaves changes stashed —
311
+ * git does not surface that as a rebase error) is treated as a hard failure
312
+ * needing manual sync, not retried further.
313
+ *
314
+ * `retryOptions` defaults to the BRAIN_PUSH_RETRY_* config; exposed as a
315
+ * parameter (rather than reading `config` directly inline) so tests can
316
+ * exercise the retry-exhaustion boundary deterministically without relying on
317
+ * process-wide env/module-load-order tricks.
179
318
  */
180
- export async function pushBrainChanges(brainDir, brainId, workspaceId) {
319
+ export async function pushBrainChanges(brainDir, brainId, workspaceId, retryOptions) {
320
+ return withBrainLock(brainId, () => pushBrainChangesLocked(brainDir, brainId, workspaceId, retryOptions));
321
+ }
322
+ async function pushBrainChangesLocked(brainDir, brainId, workspaceId, retryOptions) {
323
+ await cleanupStaleLocks(brainDir);
181
324
  try {
182
325
  // Check if this is a git repo with a remote
183
326
  try {
@@ -203,11 +346,121 @@ export async function pushBrainChanges(brainDir, brainId, workspaceId) {
203
346
  logger.debug(`Brain ${brainId} has no unpushed commits`);
204
347
  return;
205
348
  }
349
+ // Guard: refuse to push if any blob about to be pushed exceeds the size limit.
350
+ // GitHub hard-rejects blobs over its limit, and once such a blob is committed
351
+ // the push can never succeed by retrying — it needs a history rewrite (tv#246).
352
+ // Skipping the push here keeps the brain retrying safely (and alerting) instead
353
+ // of hammering GitHub with a doomed push and losing the failure into poller.error.
354
+ try {
355
+ const oversized = await findOversizedBlobs(brainDir, config.BRAIN_MAX_BLOB_BYTES);
356
+ if (oversized.length > 0) {
357
+ const limitMb = (config.BRAIN_MAX_BLOB_BYTES / (1024 * 1024)).toFixed(0);
358
+ const details = oversized.map((o) => `${o.path} (${(o.size / (1024 * 1024)).toFixed(1)} MB)`).join(', ');
359
+ const message = `Brain ${brainId} push blocked — blob(s) over ${limitMb}MB limit in unpushed commits: ${details}`;
360
+ logger.error(message);
361
+ emitError('brain', message, { context: { brainId, oversizedPaths: oversized.map((o) => o.path) } });
362
+ return;
363
+ }
364
+ }
365
+ catch (error) {
366
+ // Guard itself failed (e.g. git plumbing hiccup) — don't block a legitimate
367
+ // push on that; fall through to the push attempt as before this guard existed.
368
+ // Still surface it as an error (not just a warn log): a silently-disabled
369
+ // guard defeats the whole point of tv#246, so it needs the same alerting
370
+ // visibility as the case it's meant to catch.
371
+ const errorMessage = error instanceof Error ? error.message : String(error);
372
+ const message = `Brain ${brainId} oversized-blob guard failed, proceeding without it: ${errorMessage}`;
373
+ logger.error(message);
374
+ emitError('brain', message, { context: { brainId } });
375
+ }
206
376
  const branch = (await gitExec('git branch --show-current', { cwd: brainDir })).stdout.trim() || 'main';
207
- await gitExec(`git push -u origin ${branch}`, { cwd: brainDir });
208
- logger.info(`Brain ${brainId} changes pushed successfully`);
209
- if (workspaceId)
210
- reportBrainSync(brainId, workspaceId, 'pushed');
377
+ const maxAttempts = retryOptions?.maxAttempts ?? config.BRAIN_PUSH_RETRY_MAX;
378
+ const backoffMs = retryOptions?.backoffMs ?? config.BRAIN_PUSH_RETRY_BACKOFF_MS;
379
+ let attempt = 0;
380
+ let lastErrorMessage = '';
381
+ while (attempt < maxAttempts) {
382
+ attempt++;
383
+ try {
384
+ await gitExec(`git push -u origin ${branch}`, { cwd: brainDir });
385
+ if (attempt > 1) {
386
+ logger.info(`Brain ${brainId} push-race auto-resolved after ${attempt} attempts`);
387
+ emitError('brain', `Brain ${brainId} push-race auto-resolved after ${attempt} attempts`, {
388
+ severity: 'warn',
389
+ context: { brainId, errorCode: 'PUSH_RACE_CONVERGED', attempts: attempt },
390
+ });
391
+ }
392
+ else {
393
+ logger.info(`Brain ${brainId} changes pushed successfully`);
394
+ }
395
+ if (workspaceId)
396
+ reportBrainSync(brainId, workspaceId, 'pushed');
397
+ return;
398
+ }
399
+ catch (error) {
400
+ lastErrorMessage = error instanceof Error ? error.message : String(error);
401
+ if (!NON_FF_REJECTION_RE.test(lastErrorMessage) || attempt >= maxAttempts)
402
+ break;
403
+ logger.info(`Brain ${brainId} push rejected (non-fast-forward), rebasing and retrying (attempt ${attempt}/${maxAttempts})...`);
404
+ try {
405
+ await gitExec(`git fetch origin ${branch}`, { cwd: brainDir });
406
+ }
407
+ catch (fetchError) {
408
+ // No rebase has started yet at this point, so `git rebase --abort`
409
+ // below would itself fail ("no rebase in progress") and mislabel a
410
+ // plain network/fetch failure as PUSH_RACE_ABORT_FAILED. Keep fetch
411
+ // failures in their own errorCode instead.
412
+ const fetchMessage = fetchError instanceof Error ? fetchError.message : String(fetchError);
413
+ logger.error(`Brain ${brainId} push-race: git fetch failed: ${fetchMessage}`);
414
+ emitError('brain', `Brain ${brainId} push-race: fetch failed, cannot rebase: ${fetchMessage}`, {
415
+ severity: 'error',
416
+ context: { brainId, errorCode: 'PUSH_RACE_FETCH_FAILED', attempts: attempt },
417
+ });
418
+ return;
419
+ }
420
+ const stashBefore = await gitStashList(brainDir);
421
+ try {
422
+ await gitExec(`git rebase --autostash origin/${branch}`, { cwd: brainDir });
423
+ // `git rebase --autostash` does NOT surface a stash-pop conflict as a
424
+ // rebase error — it just leaves the change stashed and warns on
425
+ // stderr. Detect that silently-failed pop by diffing the stash list;
426
+ // otherwise the loop would proceed to push with local work missing.
427
+ const stashAfter = await gitStashList(brainDir);
428
+ if (stashAfter !== stashBefore) {
429
+ logger.error(`Brain ${brainId} push-race: autostash pop left changes stashed after rebase, treating as conflict`);
430
+ emitError('brain', `Brain ${brainId} push-race autostash pop conflict — manual sync needed (stash: ${stashAfter.split('\n')[0]})`, { severity: 'error', context: { brainId, errorCode: 'PUSH_RACE_CONFLICT', attempts: attempt, reason: 'autostash_pop_conflict' } });
431
+ return;
432
+ }
433
+ }
434
+ catch (rebaseError) {
435
+ try {
436
+ await gitExec('git rebase --abort', { cwd: brainDir });
437
+ }
438
+ catch (abortError) {
439
+ const abortMessage = abortError instanceof Error ? abortError.message : String(abortError);
440
+ logger.error(`Brain ${brainId} push-race: git rebase --abort itself failed, repo may be left mid-rebase: ${abortMessage}`);
441
+ emitError('brain', `Brain ${brainId} push-race: rebase abort failed, repo possibly stuck mid-rebase: ${abortMessage}`, {
442
+ severity: 'error',
443
+ context: { brainId, errorCode: 'PUSH_RACE_ABORT_FAILED', attempts: attempt },
444
+ });
445
+ return;
446
+ }
447
+ const rebaseMessage = rebaseError instanceof Error ? rebaseError.message : String(rebaseError);
448
+ logger.error(`Brain ${brainId} push-race rebase conflict: ${rebaseMessage}`);
449
+ emitError('brain', `Brain ${brainId} push-race rebase conflict — manual sync needed: ${rebaseMessage}`, {
450
+ severity: 'error',
451
+ context: { brainId, errorCode: 'PUSH_RACE_CONFLICT', attempts: attempt },
452
+ });
453
+ return;
454
+ }
455
+ await new Promise((resolve) => setTimeout(resolve, backoffMs * attempt));
456
+ }
457
+ }
458
+ const errorCode = NON_FF_REJECTION_RE.test(lastErrorMessage) ? 'PUSH_RACE_UNRESOLVED' : 'PUSH_FAILED';
459
+ logger.warn(`Failed to push brain ${brainId} changes: ${lastErrorMessage}`);
460
+ emitError('brain', `Failed to push brain ${brainId} changes: ${lastErrorMessage}`, {
461
+ severity: errorCode === 'PUSH_RACE_UNRESOLVED' ? 'error' : 'warn',
462
+ context: { brainId, errorCode, attempts: attempt },
463
+ });
211
464
  }
212
465
  catch (error) {
213
466
  const errorMessage = error instanceof Error ? error.message : String(error);
@@ -1,4 +1,4 @@
1
- export type KilledByPoller = 'timeout' | 'idle' | 'zombie' | 'network';
1
+ export type KilledByPoller = 'timeout' | 'idle' | 'zombie' | 'network' | 'shutdown';
2
2
  export interface FailedIterationInput {
3
3
  error?: string;
4
4
  exitCode: number | null;
@@ -1,7 +1,7 @@
1
1
  // Decides how to surface a failed claude-spawner iteration to the user.
2
2
  //
3
3
  // Background (teamvibe.ai#193): poller-induced kills (timeout, idle watchdog,
4
- // network watchdog, zombie cleanup) used to leak as generic
4
+ // network watchdog, zombie cleanup, poller shutdown) used to leak as generic
5
5
  // "Error processing your request: Process exited with code 143" via
6
6
  // sendSlackError. The explicit `killedByPoller` intent marker (set by the
7
7
  // spawner before SIGTERM) lets us emit a neutral 1-sentence notice instead —
@@ -33,6 +33,38 @@ export function classifyFailedIteration(result) {
33
33
  text: 'Network connectivity lost — send another message to retry.',
34
34
  reaction: 'satellite_antenna',
35
35
  };
36
+ case 'shutdown':
37
+ // PR#308 round-6 review: "will resume automatically" (round-5 wording)
38
+ // is still not reliably true even after the requeue-to-new/ fix in
39
+ // poller.ts. Requeuing correctly makes the message file findable in
40
+ // new/ again, but checkInboxForNewSessions() (inbox-watcher.ts) never
41
+ // scans the filesystem — it iterates getTrackedBrains(), an in-memory
42
+ // Map populated only by registerBrain() (maintenance-cron.ts), which
43
+ // only runs when the poller processes an SQS message for that brain
44
+ // (poller.ts ~line 578). On a fresh poller start, that map is empty
45
+ // and nothing hydrates it from disk, so a brain with no cron job and
46
+ // no incoming traffic can have its requeued message sit untouched
47
+ // until *some* SQS activity for that brain re-registers it — tracked
48
+ // separately as teamvibe.ai#310 (trackedBrains not hydrated from disk
49
+ // at startup). Deliberately not "fixed" here per DevGuru round-6: that
50
+ // hydration gap is a separate, deeper issue.
51
+ //
52
+ // "Some SQS activity" isn't only a human message — a user-created
53
+ // scheduled message (`source: 'cron'`, e.g. create_scheduled_message)
54
+ // also goes through the same registerBrain() call, so it re-registers
55
+ // the brain and would surface the requeued message just as well. This
56
+ // does NOT extend to the poller's own internal maintenance-cron jobs
57
+ // (`source: 'maintenance'`) — registerBrain() explicitly skips those
58
+ // (maintenance-cron.ts) — so a maintenance run alone will not unstick
59
+ // a requeued message. The text below names the two real paths (human
60
+ // message, user-scheduled run) instead of implying a human reply is
61
+ // the only way, while still not claiming guaranteed/automatic
62
+ // resumption for a brain with neither.
63
+ return {
64
+ kind: 'neutral',
65
+ text: "Poller is restarting — your message is saved and will resume with your next message, or this brain's next scheduled run.",
66
+ reaction: 'arrows_counterclockwise',
67
+ };
36
68
  case 'timeout':
37
69
  case undefined: {
38
70
  // 'timeout' keeps the legacy ":warning: Session timed out after N
@@ -1,8 +1,11 @@
1
+ import { type ChildProcess } from 'child_process';
1
2
  import type { TeamVibeQueueMessage, SessionMetrics } from './types.js';
2
3
  import { type SessionLogger } from './logger.js';
3
- import type { TypingIndicator } from './slack-client.js';
4
+ import { type TypingIndicator } from './slack-client.js';
4
5
  import { type ResolvedModelConfig } from './config-client.js';
5
6
  import type { KilledByPoller } from './classify-failed-iteration.js';
7
+ export declare function progressNotificationMarks(estimateMinutes: number): number[];
8
+ export declare function computeCompactionKillDeadline(previousActivityTime: number, now: number, idleKillMs: number, graceMs: number): number;
6
9
  export interface ContextHintParams {
7
10
  isFirstMessage: boolean;
8
11
  lastMessageTs?: string;
@@ -22,6 +25,8 @@ export interface SpawnResult {
22
25
  error?: string;
23
26
  metrics?: SessionMetrics;
24
27
  killedByPoller?: KilledByPoller;
28
+ sawSendMessageCall?: boolean;
29
+ lastPlainTextContent?: string;
25
30
  modelSource?: ModelSource;
26
31
  }
27
32
  /**
@@ -54,7 +59,175 @@ export declare function resolveModel(brainSettingsModel: string | undefined, ddb
54
59
  model: string | undefined;
55
60
  modelSource: ModelSource;
56
61
  };
62
+ /**
63
+ * teamvibeai/teamvibe.ai#73: `claude` is spawned with `detached: true`, which
64
+ * (via `setsid`) makes it the leader of its own new process group, distinct
65
+ * from the poller's. `claude` itself spawns MCP server child processes under
66
+ * that same group. Signaling `proc` alone (`proc.kill()`) only reaches
67
+ * `claude` — its MCP children are never signaled and can survive as orphans,
68
+ * leaking RAM (the Railway staircase-memory pattern that motivated #73).
69
+ *
70
+ * This sends the signal to the whole process group instead (negative pid is
71
+ * the `kill(2)` convention for "signal the group", see `man 2 kill`). The
72
+ * group id equals `claude`'s own pid because `setsid` was used to create it.
73
+ *
74
+ * Group membership survives the leader's own exit — as long as at least one
75
+ * process (e.g. a lingering MCP server) is still alive in the group, `-pid`
76
+ * remains a valid target even after `claude` itself has already exited. This
77
+ * is what makes it safe to call from the `close` handler too (#73 gap 2),
78
+ * not just from the pre-exit timeout/idle/shutdown kill paths.
79
+ *
80
+ * Unlike `ChildProcess#kill()` (a safe no-op on an already-dead process),
81
+ * `process.kill(-pid, ...)` throws `ESRCH` once the whole group is gone —
82
+ * expected and common (the group is often already fully reaped by the time
83
+ * the follow-up SIGKILL sweep fires), so it's caught and swallowed here
84
+ * rather than propagating into an uncaught exception that would crash the
85
+ * poller.
86
+ */
87
+ export declare function killProcessGroup(proc: ChildProcess, signal: NodeJS.Signals): void;
88
+ /**
89
+ * DevGuru PR#308 round-2 review (item 2): the group-SIGKILL sweeps below
90
+ * (`killAllActiveProcesses()` and the `close`-handler sweep) used to always
91
+ * pay their full fixed grace period (5s / `CLOSE_SWEEP_GRACE_MS`) before
92
+ * resolving, even in the overwhelmingly common case where `claude` exited
93
+ * cleanly and took its MCP children down with it — i.e. the group is already
94
+ * empty and there is nothing left to sweep. That matters specifically for
95
+ * `killAllActiveProcesses()`: `poller.ts`'s `shutdown()` awaits it BEFORE
96
+ * `releaseAllActiveLocks()`, and delaying that release costs the next
97
+ * message on the same thread a 15-minute stale-lock wait.
98
+ *
99
+ * Cheap fast-path: probe the group with signal `0` — a pure existence check
100
+ * per `man 2 kill`, it never actually signals anything. `ESRCH` means the
101
+ * group is provably already gone, so the caller can skip the grace-period
102
+ * wait entirely instead of always paying it. Any other outcome (including an
103
+ * unexpected errno) is treated as "still alive" — fail toward the existing,
104
+ * safe wait+SIGKILL behavior rather than skipping a real cleanup wait.
105
+ *
106
+ * Known limitation (DevGuru PR#308 round-4 review, deliberately deferred to
107
+ * teamvibeai/teamvibe.ai#309 rather than fixed here): a zombie (defunct,
108
+ * unreaped) process still answers this signal-0 probe without `ESRCH` — the
109
+ * process table entry is still present, it just has no live task behind it —
110
+ * so a group containing only zombies is still reported "alive." That makes
111
+ * the fast-path above silently miss and fall back to the fixed-cost
112
+ * wait+SIGKILL sequence instead of resolving immediately. This is a safe
113
+ * fallback (no hang, just no speedup), not a correctness bug — but it does
114
+ * mean the fast-path quietly stops paying off on a container that has
115
+ * accumulated zombies. What actually removes the limitation is giving the
116
+ * container a real init process (e.g. `tini`) as PID 1: unlike a bare `node`
117
+ * entrypoint, a real init reaps orphaned zombies as they appear, so they
118
+ * never accumulate and confuse this probe in the first place. See #309 for
119
+ * the tracked Docker-bake/tini follow-up.
120
+ */
121
+ export declare function isProcessGroupAlive(pid: number): boolean;
122
+ export interface SweepProcessGroupOptions {
123
+ onGraceExpired?: () => void;
124
+ unref?: boolean;
125
+ }
126
+ /**
127
+ * Shared group-kill grace-period sweep: SIGTERM, then either resolve
128
+ * immediately via the `isProcessGroupAlive()` fast-path (group already
129
+ * provably empty — see its doc comment) or wait up to `graceMs` before
130
+ * escalating to SIGKILL and resolving. Extracted so both call sites
131
+ * (`killAllActiveProcesses()` and the `close`-handler sweep) share one
132
+ * tested implementation instead of duplicating the wait+escalate logic.
133
+ *
134
+ * PR#308 round-3 review item 2: the up-front `isProcessGroupAlive()` probe
135
+ * above only fast-paths the "already dead before we even started" case. The
136
+ * case that matters most — the group IS alive right this instant, which is
137
+ * always true during a real shutdown with an active session, exactly when
138
+ * `releaseAllActiveLocks()` is waiting on this — used to still unconditionally
139
+ * pay the *entire* fixed `graceMs` even if `claude` died 50ms after its
140
+ * SIGTERM. Fixed by polling `isProcessGroupAlive()` every
141
+ * `PROCESS_GROUP_POLL_INTERVAL_MS` after the SIGTERM instead of relying on a
142
+ * single fixed-delay timer to both wait AND escalate: whichever fires first —
143
+ * a poll observing ESRCH, or the grace period elapsing — resolves/escalates.
144
+ * `graceMs` remains the hard upper bound on how long this ever waits (a
145
+ * safety ceiling, unchanged from before); this only shortens the *common*
146
+ * case where the group dies well before that ceiling.
147
+ */
148
+ export declare function sweepProcessGroup(proc: ChildProcess, graceMs: number, options?: SweepProcessGroupOptions): Promise<void>;
149
+ export interface BuildSpawnEnvParams {
150
+ spawnSecretsEnv: NodeJS.ProcessEnv;
151
+ processEnv: NodeJS.ProcessEnv;
152
+ envPath: string | undefined;
153
+ claudeConfigDir: string;
154
+ persistentStoragePath?: string;
155
+ slackContext?: {
156
+ channel: string;
157
+ thread_ts: string;
158
+ message_ts: string;
159
+ };
160
+ teamvibeApiUrl: string;
161
+ teamvibePollerToken: string;
162
+ teamvibeWorkspaceId: string;
163
+ teamvibeChannelId: string;
164
+ teamvibeBotId: string;
165
+ teamvibePollerId: string;
166
+ inboxThreadId: string;
167
+ brainPath: string;
168
+ }
169
+ /**
170
+ * Pure construction of the env object passed to the spawned `claude` child
171
+ * process. Extracted out of the `spawn()` call site so the "what does the
172
+ * child actually inherit" logic is unit-testable without spinning up a real
173
+ * process tree or mocking `fetchSecretsForSpawn`/DDB config.
174
+ *
175
+ * `NODE_OPTIONS` is explicitly stripped (same convention as `CLAUDECODE`
176
+ * below): the poller's own Railway start command sets
177
+ * `NODE_OPTIONS='--max-old-space-size=128 --max-semi-space-size=2'` to cap
178
+ * the *poller's* own idle V8 heap — a thin SQS-listener process is fine at
179
+ * 128MB. That var lives in `process.env` at the poller-process level, so
180
+ * without this override it would leak via the `...processEnv` spread below
181
+ * into the spawned `claude` CLI process. `claude` needs real headroom to do
182
+ * actual agentic work (parsing files, holding conversation context, etc.) —
183
+ * the poller's own tuned-for-idle heap cap is simply the wrong number for
184
+ * what that process does, independent of what any specific OOM failure
185
+ * would look like if left uncapped.
186
+ */
187
+ export declare function buildSpawnEnv(params: BuildSpawnEnvParams): NodeJS.ProcessEnv;
57
188
  export declare function buildPrompt(msg: TeamVibeQueueMessage, contextHint?: string): string;
189
+ export interface SpawnStdinPlan {
190
+ /** Extra CLI args to append. Empty for the plain-text path — this is the exact pre-tv293-bug2 behavior, unconditionally. */
191
+ extraArgs: string[];
192
+ /** Exact chunks written to the child process's stdin, in order, before `.end()`. */
193
+ stdinChunks: string[];
194
+ }
195
+ /**
196
+ * Decides how the built prompt reaches the `claude` CLI: the default
197
+ * plain-text stdin path (unchanged from before teamvibe.ai#293 bug 2 — this
198
+ * covers essentially all production traffic: every Slack message, every
199
+ * text-only email, cron, maintenance) or a `stream-json` structured-content
200
+ * path carrying inline image blocks, used only when `inlineImages` is
201
+ * present on the incoming message (populated by email-received.ts' Part A
202
+ * — see packages/modules-teamvibe/src/utils/email-inline-images.ts).
203
+ *
204
+ * Pure function — no child_process/spawn access — so the no-image path's
205
+ * exact output (and thus "no behavior change for non-image sessions") is
206
+ * directly unit-testable without mocking `spawn` at all. See
207
+ * claude-spawner.test.ts's "no-image path is unchanged" coverage.
208
+ *
209
+ * `now` is injectable (defaults to `Date.now()`) purely so tests can assert
210
+ * both sides of the `expiresAt` check deterministically — see below.
211
+ */
212
+ export declare function buildSpawnStdinPlan(prompt: string, inlineImages: TeamVibeQueueMessage['inlineImages'], now?: number): SpawnStdinPlan;
213
+ export declare function extractAssistantText(event: any): string | undefined;
214
+ export declare function assistantCalledTool(event: any, toolName: string): boolean;
215
+ /**
216
+ * Decides whether a spawn attempt should report its session ID back to the
217
+ * caller (poller.ts) for persistence to DynamoDB via updateSessionId().
218
+ *
219
+ * First-message spawns always mint a fresh session ID locally (see
220
+ * spawnClaudeCode below, teamvibe.ai#77) instead of recycling whatever
221
+ * placeholder DynamoDB currently holds — so on ANY successful first-message
222
+ * spawn (not just the "already in use" collision-retry path), the ID actually
223
+ * used by Claude Code must be surfaced here. Otherwise DynamoDB keeps the
224
+ * createSession() placeholder forever and the thread's next message does
225
+ * `--resume <id-claude-code-never-used>`, which fails 100% of the time.
226
+ * (teamvibe.ai#77, DevGuru review round)
227
+ */
228
+ export declare function finalizeSpawnResult(retryResult: SpawnResult, isFirstMessage: boolean, effectiveSessionId: string | undefined): SpawnResult & {
229
+ newSessionId?: string;
230
+ };
58
231
  export declare function spawnClaudeCode(msg: TeamVibeQueueMessage, sessionLog: SessionLogger, cwd: string, sessionId?: string, isFirstMessage?: boolean, lastMessageTs?: string, onMessageSent?: () => void, typingIndicator?: TypingIndicator): Promise<SpawnResult & {
59
232
  newSessionId?: string;
60
233
  }>;
@@ -66,3 +239,7 @@ export declare function getActiveProcessCount(): number;
66
239
  export declare function killZombieForThread(threadId: string): boolean;
67
240
  export declare function isAtCapacity(): boolean;
68
241
  export declare function killAllActiveProcesses(): Promise<void>;
242
+ export declare function __setActiveProcessForTest(id: string, proc: ChildProcess): void;
243
+ export declare function __clearActiveProcessesForTest(): void;
244
+ export declare function __setKillReasonForTest(proc: ChildProcess, reason: KilledByPoller): void;
245
+ export declare function __getKillReasonForTest(proc: ChildProcess): KilledByPoller | undefined;