@teamvibe/poller 0.6.2 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/brain-manager.d.ts +43 -1
- package/dist/brain-manager.js +259 -6
- package/dist/classify-failed-iteration.d.ts +1 -1
- package/dist/classify-failed-iteration.js +33 -1
- package/dist/claude-spawner.d.ts +178 -1
- package/dist/claude-spawner.js +687 -92
- package/dist/config.d.ts +24 -0
- package/dist/config.js +15 -2
- package/dist/cron-telemetry.d.ts +3 -0
- package/dist/cron-telemetry.js +16 -0
- package/dist/event-queue.d.ts +30 -0
- package/dist/event-queue.js +4 -0
- package/dist/inbox-manager.d.ts +25 -0
- package/dist/inbox-manager.js +47 -0
- package/dist/maintenance-activity.d.ts +18 -0
- package/dist/maintenance-activity.js +28 -0
- package/dist/maintenance-cron.d.ts +7 -0
- package/dist/maintenance-cron.js +16 -1
- package/dist/metrics/collector.d.ts +2 -0
- package/dist/metrics/collector.js +7 -0
- package/dist/poller.d.ts +9 -0
- package/dist/poller.js +158 -4
- package/dist/types.d.ts +6 -0
- package/package.json +1 -1
package/dist/brain-manager.d.ts
CHANGED
|
@@ -1,8 +1,23 @@
|
|
|
1
|
+
export declare function withBrainLock<T>(brainId: string, fn: () => Promise<T>): Promise<T>;
|
|
1
2
|
interface BrainConfig {
|
|
2
3
|
brainId: string;
|
|
3
4
|
gitRepoUrl: string;
|
|
4
5
|
branch: string;
|
|
5
6
|
}
|
|
7
|
+
/**
|
|
8
|
+
* Remove stale git lock files (tv#76) so an orphaned lock from a killed
|
|
9
|
+
* process (e.g. the 2026-05-01 outage) doesn't block brain updates/pushes
|
|
10
|
+
* indefinitely. Fresh locks (younger than the threshold) are left alone —
|
|
11
|
+
* they may belong to a git operation that's genuinely still running.
|
|
12
|
+
* A missing lock (stat fails) is a silent no-op. A genuine removal failure
|
|
13
|
+
* (unlink fails, e.g. permission denied) is logged via logger.warn with the
|
|
14
|
+
* path and error — that's the exact "lock exists but can't be cleared"
|
|
15
|
+
* scenario this fix exists to make visible.
|
|
16
|
+
*
|
|
17
|
+
* Exported for testing — production call sites go through updateBrain /
|
|
18
|
+
* pushBrainChanges below.
|
|
19
|
+
*/
|
|
20
|
+
export declare function cleanupStaleLocks(brainDir: string): Promise<void>;
|
|
6
21
|
export declare function getBrainPath(brain?: BrainConfig, channelId?: string, workspaceId?: string): Promise<string>;
|
|
7
22
|
/**
|
|
8
23
|
* Update a brain repo (respects cooldown). Exported for testing — production
|
|
@@ -18,11 +33,38 @@ export declare function ensureBaseBrain(): Promise<void>;
|
|
|
18
33
|
* Get the path to the base brain directory.
|
|
19
34
|
*/
|
|
20
35
|
export declare function getBaseBrainPath(): string;
|
|
36
|
+
/**
|
|
37
|
+
* Scan the commits about to be pushed for blobs over `maxBytes`. Returns the
|
|
38
|
+
* offending paths/sizes, or an empty array if everything is under the limit.
|
|
39
|
+
*
|
|
40
|
+
* Scans `@{u}..HEAD` when an upstream exists (the actual new content being
|
|
41
|
+
* pushed); falls back to all objects reachable from HEAD for a branch's first
|
|
42
|
+
* push (no upstream yet), which is equivalent since nothing has been pushed.
|
|
43
|
+
*/
|
|
44
|
+
export declare function findOversizedBlobs(brainDir: string, maxBytes: number): Promise<Array<{
|
|
45
|
+
path: string;
|
|
46
|
+
size: number;
|
|
47
|
+
}>>;
|
|
21
48
|
/**
|
|
22
49
|
* Commit and push any changes in a brain repo.
|
|
23
50
|
* Silently skips if there are no changes or no remote.
|
|
51
|
+
*
|
|
52
|
+
* On non-fast-forward rejection (tv#232 push-race — remote advanced between
|
|
53
|
+
* local commit and push, e.g. a concurrent session or scheduled-tick already
|
|
54
|
+
* pushed), fetch+rebase and retry, bounded by BRAIN_PUSH_RETRY_MAX. A genuine
|
|
55
|
+
* rebase conflict (or an autostash pop that silently leaves changes stashed —
|
|
56
|
+
* git does not surface that as a rebase error) is treated as a hard failure
|
|
57
|
+
* needing manual sync, not retried further.
|
|
58
|
+
*
|
|
59
|
+
* `retryOptions` defaults to the BRAIN_PUSH_RETRY_* config; exposed as a
|
|
60
|
+
* parameter (rather than reading `config` directly inline) so tests can
|
|
61
|
+
* exercise the retry-exhaustion boundary deterministically without relying on
|
|
62
|
+
* process-wide env/module-load-order tricks.
|
|
24
63
|
*/
|
|
25
|
-
export declare function pushBrainChanges(brainDir: string, brainId: string, workspaceId?: string
|
|
64
|
+
export declare function pushBrainChanges(brainDir: string, brainId: string, workspaceId?: string, retryOptions?: {
|
|
65
|
+
maxAttempts?: number;
|
|
66
|
+
backoffMs?: number;
|
|
67
|
+
}): Promise<void>;
|
|
26
68
|
/**
|
|
27
69
|
* Ensure base paths exist
|
|
28
70
|
*/
|
package/dist/brain-manager.js
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import { exec } from 'child_process';
|
|
2
2
|
import { promisify } from 'util';
|
|
3
3
|
import { existsSync, mkdirSync } from 'fs';
|
|
4
|
-
import {
|
|
4
|
+
import { readdir, stat, unlink } from 'fs/promises';
|
|
5
|
+
import { join, basename } from 'path';
|
|
5
6
|
import { config } from './config.js';
|
|
6
7
|
import { logger } from './logger.js';
|
|
7
8
|
import { emitError } from './event-queue.js';
|
|
@@ -20,8 +21,84 @@ if (config.PERSISTENT_STORAGE_PATH) {
|
|
|
20
21
|
function gitExec(command, options) {
|
|
21
22
|
return execAsync(command, { ...options, env: gitEnv });
|
|
22
23
|
}
|
|
24
|
+
// Node's child_process.exec() defaults to a 1MB stdout cap. This bounds the
|
|
25
|
+
// text output of a git plumbing command (e.g. one line per object listed),
|
|
26
|
+
// not any single blob's size -- it has no relationship to
|
|
27
|
+
// config.BRAIN_MAX_BLOB_BYTES and must stay its own named constant so the
|
|
28
|
+
// two don't drift into looking coupled just because they're both 50MB today.
|
|
29
|
+
const GIT_SCAN_MAX_BUFFER_BYTES = 50 * 1024 * 1024;
|
|
23
30
|
// Cooldown tracker per brain
|
|
24
31
|
const lastUpdateTimes = new Map();
|
|
32
|
+
// Per-brainId mutex serializing updateBrain/pushBrainChanges for a given
|
|
33
|
+
// brain. Two threads of the SAME brain running concurrently in this process
|
|
34
|
+
// share one brainDir (tv#76/tv#277 review) — without this, cleanupStaleLocks()
|
|
35
|
+
// can unlink a git lock file that a still-in-flight sibling call's own git
|
|
36
|
+
// operation legitimately holds, corrupting the index. Poller instances never
|
|
37
|
+
// share a physical brainDir across processes/machines (BRAINS_PATH is
|
|
38
|
+
// $HOME-derived and each poller is a single process — see tv#287), so this
|
|
39
|
+
// in-process mutex fully closes the race; no DynamoDB-level lock is needed.
|
|
40
|
+
const brainLocks = new Map();
|
|
41
|
+
// Exported for testing — production call sites go through updateBrain / pushBrainChanges above.
|
|
42
|
+
export function withBrainLock(brainId, fn) {
|
|
43
|
+
const prior = brainLocks.get(brainId) ?? Promise.resolve();
|
|
44
|
+
const run = prior.then(fn, fn);
|
|
45
|
+
brainLocks.set(brainId, run.then(() => undefined, () => undefined));
|
|
46
|
+
return run;
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* Remove stale git lock files (tv#76) so an orphaned lock from a killed
|
|
50
|
+
* process (e.g. the 2026-05-01 outage) doesn't block brain updates/pushes
|
|
51
|
+
* indefinitely. Fresh locks (younger than the threshold) are left alone —
|
|
52
|
+
* they may belong to a git operation that's genuinely still running.
|
|
53
|
+
* A missing lock (stat fails) is a silent no-op. A genuine removal failure
|
|
54
|
+
* (unlink fails, e.g. permission denied) is logged via logger.warn with the
|
|
55
|
+
* path and error — that's the exact "lock exists but can't be cleared"
|
|
56
|
+
* scenario this fix exists to make visible.
|
|
57
|
+
*
|
|
58
|
+
* Exported for testing — production call sites go through updateBrain /
|
|
59
|
+
* pushBrainChanges below.
|
|
60
|
+
*/
|
|
61
|
+
export async function cleanupStaleLocks(brainDir) {
|
|
62
|
+
const candidates = [
|
|
63
|
+
{ path: join(brainDir, '.git', 'index.lock'), maxAgeMs: config.GIT_LOCK_MAX_AGE_MS },
|
|
64
|
+
];
|
|
65
|
+
const modulesDir = join(brainDir, '.git', 'modules');
|
|
66
|
+
try {
|
|
67
|
+
const entries = await readdir(modulesDir, { recursive: true });
|
|
68
|
+
for (const entry of entries) {
|
|
69
|
+
if (basename(entry) === 'config.lock') {
|
|
70
|
+
candidates.push({ path: join(modulesDir, entry), maxAgeMs: config.GIT_SUBMODULE_LOCK_MAX_AGE_MS });
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
catch {
|
|
75
|
+
// No .git/modules dir (no submodules yet) — nothing to scan.
|
|
76
|
+
}
|
|
77
|
+
for (const { path: lockPath, maxAgeMs } of candidates) {
|
|
78
|
+
let lockStat;
|
|
79
|
+
try {
|
|
80
|
+
lockStat = await stat(lockPath);
|
|
81
|
+
}
|
|
82
|
+
catch {
|
|
83
|
+
// Lock doesn't exist — normal no-op, nothing worth logging.
|
|
84
|
+
continue;
|
|
85
|
+
}
|
|
86
|
+
const ageMs = Date.now() - lockStat.mtimeMs;
|
|
87
|
+
if (ageMs <= maxAgeMs)
|
|
88
|
+
continue;
|
|
89
|
+
try {
|
|
90
|
+
await unlink(lockPath);
|
|
91
|
+
logger.warn(`Removed stale git lock: ${lockPath} (age: ${Math.round(ageMs / 1000)}s, threshold: ${Math.round(maxAgeMs / 1000)}s)`);
|
|
92
|
+
}
|
|
93
|
+
catch (error) {
|
|
94
|
+
// Real removal failure (e.g. permission denied) — this is the exact
|
|
95
|
+
// 2026-05-01 outage scenario, so it must leave a diagnostic trail
|
|
96
|
+
// instead of silently skipping like a plain "lock doesn't exist".
|
|
97
|
+
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
98
|
+
logger.warn(`Failed to remove stale git lock: ${lockPath} (age: ${Math.round(ageMs / 1000)}s): ${errorMessage}`);
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
}
|
|
25
102
|
/**
|
|
26
103
|
* Get or clone brain repo. Returns the working directory path.
|
|
27
104
|
*/
|
|
@@ -94,6 +171,9 @@ export async function getBrainPath(brain, channelId, workspaceId) {
|
|
|
94
171
|
* call sites should go through `getBrainPath` / `ensureBaseBrain`.
|
|
95
172
|
*/
|
|
96
173
|
export async function updateBrain(brainDir, brainId, branch) {
|
|
174
|
+
return withBrainLock(brainId, () => updateBrainLocked(brainDir, brainId, branch));
|
|
175
|
+
}
|
|
176
|
+
async function updateBrainLocked(brainDir, brainId, branch) {
|
|
97
177
|
const now = Date.now();
|
|
98
178
|
const lastUpdate = lastUpdateTimes.get(brainId) || 0;
|
|
99
179
|
if (now - lastUpdate < config.BRAIN_UPDATE_COOLDOWN_MS) {
|
|
@@ -137,6 +217,7 @@ export async function updateBrain(brainDir, brainId, branch) {
|
|
|
137
217
|
logger.warn(`Brain ${brainId} pre-check failed, skipping update: ${errorMessage}`);
|
|
138
218
|
return;
|
|
139
219
|
}
|
|
220
|
+
await cleanupStaleLocks(brainDir);
|
|
140
221
|
try {
|
|
141
222
|
logger.info(`Updating brain ${brainId} at ${brainDir}...`);
|
|
142
223
|
await gitExec(`git fetch origin ${branch} && git reset --hard origin/${branch} && git submodule update --init --recursive`, {
|
|
@@ -173,11 +254,73 @@ export async function ensureBaseBrain() {
|
|
|
173
254
|
export function getBaseBrainPath() {
|
|
174
255
|
return config.BASE_BRAIN_PATH;
|
|
175
256
|
}
|
|
257
|
+
// Matches git's non-fast-forward push rejection ("! [rejected] ... (fetch first)"
|
|
258
|
+
// or "... (non-fast-forward)"). Distinguishes a push-race (remote advanced,
|
|
259
|
+
// reconcilable via rebase) from other push failures (network, auth, oversized
|
|
260
|
+
// blob pre-receive decline — see tv#246) which retrying can never fix.
|
|
261
|
+
const NON_FF_REJECTION_RE = /\[rejected\][\s\S]*(fetch first|non-fast-forward)/;
|
|
262
|
+
async function gitStashList(brainDir) {
|
|
263
|
+
const { stdout } = await gitExec('git stash list', { cwd: brainDir });
|
|
264
|
+
return stdout.trim();
|
|
265
|
+
}
|
|
266
|
+
/**
|
|
267
|
+
* Scan the commits about to be pushed for blobs over `maxBytes`. Returns the
|
|
268
|
+
* offending paths/sizes, or an empty array if everything is under the limit.
|
|
269
|
+
*
|
|
270
|
+
* Scans `@{u}..HEAD` when an upstream exists (the actual new content being
|
|
271
|
+
* pushed); falls back to all objects reachable from HEAD for a branch's first
|
|
272
|
+
* push (no upstream yet), which is equivalent since nothing has been pushed.
|
|
273
|
+
*/
|
|
274
|
+
export async function findOversizedBlobs(brainDir, maxBytes) {
|
|
275
|
+
let range = 'HEAD';
|
|
276
|
+
try {
|
|
277
|
+
await gitExec('git rev-parse @{u}', { cwd: brainDir });
|
|
278
|
+
range = '@{u}..HEAD';
|
|
279
|
+
}
|
|
280
|
+
catch {
|
|
281
|
+
// No upstream tracking ref yet — first push for this branch, scan all of HEAD.
|
|
282
|
+
}
|
|
283
|
+
// A full-HEAD scan (the fallback path this guard needs to work on) can
|
|
284
|
+
// exceed Node's default 1MB stdout cap on a repo with many objects.
|
|
285
|
+
// Without a larger maxBuffer the scan throws ENOBUFS and the guard
|
|
286
|
+
// silently disables itself (see catch below) on exactly the histories most
|
|
287
|
+
// likely to contain an oversized blob.
|
|
288
|
+
const { stdout } = await gitExec(`git rev-list --objects ${range} | git cat-file --batch-check='%(objecttype) %(objectname) %(objectsize) %(rest)'`, { cwd: brainDir, maxBuffer: GIT_SCAN_MAX_BUFFER_BYTES });
|
|
289
|
+
const oversized = [];
|
|
290
|
+
for (const line of stdout.split('\n')) {
|
|
291
|
+
if (!line.trim())
|
|
292
|
+
continue;
|
|
293
|
+
const [type, , sizeStr, ...pathParts] = line.split(' ');
|
|
294
|
+
if (type !== 'blob')
|
|
295
|
+
continue;
|
|
296
|
+
const size = Number(sizeStr);
|
|
297
|
+
if (size > maxBytes) {
|
|
298
|
+
oversized.push({ path: pathParts.join(' ') || '(unknown path)', size });
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
return oversized;
|
|
302
|
+
}
|
|
176
303
|
/**
|
|
177
304
|
* Commit and push any changes in a brain repo.
|
|
178
305
|
* Silently skips if there are no changes or no remote.
|
|
306
|
+
*
|
|
307
|
+
* On non-fast-forward rejection (tv#232 push-race — remote advanced between
|
|
308
|
+
* local commit and push, e.g. a concurrent session or scheduled-tick already
|
|
309
|
+
* pushed), fetch+rebase and retry, bounded by BRAIN_PUSH_RETRY_MAX. A genuine
|
|
310
|
+
* rebase conflict (or an autostash pop that silently leaves changes stashed —
|
|
311
|
+
* git does not surface that as a rebase error) is treated as a hard failure
|
|
312
|
+
* needing manual sync, not retried further.
|
|
313
|
+
*
|
|
314
|
+
* `retryOptions` defaults to the BRAIN_PUSH_RETRY_* config; exposed as a
|
|
315
|
+
* parameter (rather than reading `config` directly inline) so tests can
|
|
316
|
+
* exercise the retry-exhaustion boundary deterministically without relying on
|
|
317
|
+
* process-wide env/module-load-order tricks.
|
|
179
318
|
*/
|
|
180
|
-
export async function pushBrainChanges(brainDir, brainId, workspaceId) {
|
|
319
|
+
export async function pushBrainChanges(brainDir, brainId, workspaceId, retryOptions) {
|
|
320
|
+
return withBrainLock(brainId, () => pushBrainChangesLocked(brainDir, brainId, workspaceId, retryOptions));
|
|
321
|
+
}
|
|
322
|
+
async function pushBrainChangesLocked(brainDir, brainId, workspaceId, retryOptions) {
|
|
323
|
+
await cleanupStaleLocks(brainDir);
|
|
181
324
|
try {
|
|
182
325
|
// Check if this is a git repo with a remote
|
|
183
326
|
try {
|
|
@@ -203,11 +346,121 @@ export async function pushBrainChanges(brainDir, brainId, workspaceId) {
|
|
|
203
346
|
logger.debug(`Brain ${brainId} has no unpushed commits`);
|
|
204
347
|
return;
|
|
205
348
|
}
|
|
349
|
+
// Guard: refuse to push if any blob about to be pushed exceeds the size limit.
|
|
350
|
+
// GitHub hard-rejects blobs over its limit, and once such a blob is committed
|
|
351
|
+
// the push can never succeed by retrying — it needs a history rewrite (tv#246).
|
|
352
|
+
// Skipping the push here keeps the brain retrying safely (and alerting) instead
|
|
353
|
+
// of hammering GitHub with a doomed push and losing the failure into poller.error.
|
|
354
|
+
try {
|
|
355
|
+
const oversized = await findOversizedBlobs(brainDir, config.BRAIN_MAX_BLOB_BYTES);
|
|
356
|
+
if (oversized.length > 0) {
|
|
357
|
+
const limitMb = (config.BRAIN_MAX_BLOB_BYTES / (1024 * 1024)).toFixed(0);
|
|
358
|
+
const details = oversized.map((o) => `${o.path} (${(o.size / (1024 * 1024)).toFixed(1)} MB)`).join(', ');
|
|
359
|
+
const message = `Brain ${brainId} push blocked — blob(s) over ${limitMb}MB limit in unpushed commits: ${details}`;
|
|
360
|
+
logger.error(message);
|
|
361
|
+
emitError('brain', message, { context: { brainId, oversizedPaths: oversized.map((o) => o.path) } });
|
|
362
|
+
return;
|
|
363
|
+
}
|
|
364
|
+
}
|
|
365
|
+
catch (error) {
|
|
366
|
+
// Guard itself failed (e.g. git plumbing hiccup) — don't block a legitimate
|
|
367
|
+
// push on that; fall through to the push attempt as before this guard existed.
|
|
368
|
+
// Still surface it as an error (not just a warn log): a silently-disabled
|
|
369
|
+
// guard defeats the whole point of tv#246, so it needs the same alerting
|
|
370
|
+
// visibility as the case it's meant to catch.
|
|
371
|
+
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
372
|
+
const message = `Brain ${brainId} oversized-blob guard failed, proceeding without it: ${errorMessage}`;
|
|
373
|
+
logger.error(message);
|
|
374
|
+
emitError('brain', message, { context: { brainId } });
|
|
375
|
+
}
|
|
206
376
|
const branch = (await gitExec('git branch --show-current', { cwd: brainDir })).stdout.trim() || 'main';
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
377
|
+
const maxAttempts = retryOptions?.maxAttempts ?? config.BRAIN_PUSH_RETRY_MAX;
|
|
378
|
+
const backoffMs = retryOptions?.backoffMs ?? config.BRAIN_PUSH_RETRY_BACKOFF_MS;
|
|
379
|
+
let attempt = 0;
|
|
380
|
+
let lastErrorMessage = '';
|
|
381
|
+
while (attempt < maxAttempts) {
|
|
382
|
+
attempt++;
|
|
383
|
+
try {
|
|
384
|
+
await gitExec(`git push -u origin ${branch}`, { cwd: brainDir });
|
|
385
|
+
if (attempt > 1) {
|
|
386
|
+
logger.info(`Brain ${brainId} push-race auto-resolved after ${attempt} attempts`);
|
|
387
|
+
emitError('brain', `Brain ${brainId} push-race auto-resolved after ${attempt} attempts`, {
|
|
388
|
+
severity: 'warn',
|
|
389
|
+
context: { brainId, errorCode: 'PUSH_RACE_CONVERGED', attempts: attempt },
|
|
390
|
+
});
|
|
391
|
+
}
|
|
392
|
+
else {
|
|
393
|
+
logger.info(`Brain ${brainId} changes pushed successfully`);
|
|
394
|
+
}
|
|
395
|
+
if (workspaceId)
|
|
396
|
+
reportBrainSync(brainId, workspaceId, 'pushed');
|
|
397
|
+
return;
|
|
398
|
+
}
|
|
399
|
+
catch (error) {
|
|
400
|
+
lastErrorMessage = error instanceof Error ? error.message : String(error);
|
|
401
|
+
if (!NON_FF_REJECTION_RE.test(lastErrorMessage) || attempt >= maxAttempts)
|
|
402
|
+
break;
|
|
403
|
+
logger.info(`Brain ${brainId} push rejected (non-fast-forward), rebasing and retrying (attempt ${attempt}/${maxAttempts})...`);
|
|
404
|
+
try {
|
|
405
|
+
await gitExec(`git fetch origin ${branch}`, { cwd: brainDir });
|
|
406
|
+
}
|
|
407
|
+
catch (fetchError) {
|
|
408
|
+
// No rebase has started yet at this point, so `git rebase --abort`
|
|
409
|
+
// below would itself fail ("no rebase in progress") and mislabel a
|
|
410
|
+
// plain network/fetch failure as PUSH_RACE_ABORT_FAILED. Keep fetch
|
|
411
|
+
// failures in their own errorCode instead.
|
|
412
|
+
const fetchMessage = fetchError instanceof Error ? fetchError.message : String(fetchError);
|
|
413
|
+
logger.error(`Brain ${brainId} push-race: git fetch failed: ${fetchMessage}`);
|
|
414
|
+
emitError('brain', `Brain ${brainId} push-race: fetch failed, cannot rebase: ${fetchMessage}`, {
|
|
415
|
+
severity: 'error',
|
|
416
|
+
context: { brainId, errorCode: 'PUSH_RACE_FETCH_FAILED', attempts: attempt },
|
|
417
|
+
});
|
|
418
|
+
return;
|
|
419
|
+
}
|
|
420
|
+
const stashBefore = await gitStashList(brainDir);
|
|
421
|
+
try {
|
|
422
|
+
await gitExec(`git rebase --autostash origin/${branch}`, { cwd: brainDir });
|
|
423
|
+
// `git rebase --autostash` does NOT surface a stash-pop conflict as a
|
|
424
|
+
// rebase error — it just leaves the change stashed and warns on
|
|
425
|
+
// stderr. Detect that silently-failed pop by diffing the stash list;
|
|
426
|
+
// otherwise the loop would proceed to push with local work missing.
|
|
427
|
+
const stashAfter = await gitStashList(brainDir);
|
|
428
|
+
if (stashAfter !== stashBefore) {
|
|
429
|
+
logger.error(`Brain ${brainId} push-race: autostash pop left changes stashed after rebase, treating as conflict`);
|
|
430
|
+
emitError('brain', `Brain ${brainId} push-race autostash pop conflict — manual sync needed (stash: ${stashAfter.split('\n')[0]})`, { severity: 'error', context: { brainId, errorCode: 'PUSH_RACE_CONFLICT', attempts: attempt, reason: 'autostash_pop_conflict' } });
|
|
431
|
+
return;
|
|
432
|
+
}
|
|
433
|
+
}
|
|
434
|
+
catch (rebaseError) {
|
|
435
|
+
try {
|
|
436
|
+
await gitExec('git rebase --abort', { cwd: brainDir });
|
|
437
|
+
}
|
|
438
|
+
catch (abortError) {
|
|
439
|
+
const abortMessage = abortError instanceof Error ? abortError.message : String(abortError);
|
|
440
|
+
logger.error(`Brain ${brainId} push-race: git rebase --abort itself failed, repo may be left mid-rebase: ${abortMessage}`);
|
|
441
|
+
emitError('brain', `Brain ${brainId} push-race: rebase abort failed, repo possibly stuck mid-rebase: ${abortMessage}`, {
|
|
442
|
+
severity: 'error',
|
|
443
|
+
context: { brainId, errorCode: 'PUSH_RACE_ABORT_FAILED', attempts: attempt },
|
|
444
|
+
});
|
|
445
|
+
return;
|
|
446
|
+
}
|
|
447
|
+
const rebaseMessage = rebaseError instanceof Error ? rebaseError.message : String(rebaseError);
|
|
448
|
+
logger.error(`Brain ${brainId} push-race rebase conflict: ${rebaseMessage}`);
|
|
449
|
+
emitError('brain', `Brain ${brainId} push-race rebase conflict — manual sync needed: ${rebaseMessage}`, {
|
|
450
|
+
severity: 'error',
|
|
451
|
+
context: { brainId, errorCode: 'PUSH_RACE_CONFLICT', attempts: attempt },
|
|
452
|
+
});
|
|
453
|
+
return;
|
|
454
|
+
}
|
|
455
|
+
await new Promise((resolve) => setTimeout(resolve, backoffMs * attempt));
|
|
456
|
+
}
|
|
457
|
+
}
|
|
458
|
+
const errorCode = NON_FF_REJECTION_RE.test(lastErrorMessage) ? 'PUSH_RACE_UNRESOLVED' : 'PUSH_FAILED';
|
|
459
|
+
logger.warn(`Failed to push brain ${brainId} changes: ${lastErrorMessage}`);
|
|
460
|
+
emitError('brain', `Failed to push brain ${brainId} changes: ${lastErrorMessage}`, {
|
|
461
|
+
severity: errorCode === 'PUSH_RACE_UNRESOLVED' ? 'error' : 'warn',
|
|
462
|
+
context: { brainId, errorCode, attempts: attempt },
|
|
463
|
+
});
|
|
211
464
|
}
|
|
212
465
|
catch (error) {
|
|
213
466
|
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// Decides how to surface a failed claude-spawner iteration to the user.
|
|
2
2
|
//
|
|
3
3
|
// Background (teamvibe.ai#193): poller-induced kills (timeout, idle watchdog,
|
|
4
|
-
// network watchdog, zombie cleanup) used to leak as generic
|
|
4
|
+
// network watchdog, zombie cleanup, poller shutdown) used to leak as generic
|
|
5
5
|
// "Error processing your request: Process exited with code 143" via
|
|
6
6
|
// sendSlackError. The explicit `killedByPoller` intent marker (set by the
|
|
7
7
|
// spawner before SIGTERM) lets us emit a neutral 1-sentence notice instead —
|
|
@@ -33,6 +33,38 @@ export function classifyFailedIteration(result) {
|
|
|
33
33
|
text: 'Network connectivity lost — send another message to retry.',
|
|
34
34
|
reaction: 'satellite_antenna',
|
|
35
35
|
};
|
|
36
|
+
case 'shutdown':
|
|
37
|
+
// PR#308 round-6 review: "will resume automatically" (round-5 wording)
|
|
38
|
+
// is still not reliably true even after the requeue-to-new/ fix in
|
|
39
|
+
// poller.ts. Requeuing correctly makes the message file findable in
|
|
40
|
+
// new/ again, but checkInboxForNewSessions() (inbox-watcher.ts) never
|
|
41
|
+
// scans the filesystem — it iterates getTrackedBrains(), an in-memory
|
|
42
|
+
// Map populated only by registerBrain() (maintenance-cron.ts), which
|
|
43
|
+
// only runs when the poller processes an SQS message for that brain
|
|
44
|
+
// (poller.ts ~line 578). On a fresh poller start, that map is empty
|
|
45
|
+
// and nothing hydrates it from disk, so a brain with no cron job and
|
|
46
|
+
// no incoming traffic can have its requeued message sit untouched
|
|
47
|
+
// until *some* SQS activity for that brain re-registers it — tracked
|
|
48
|
+
// separately as teamvibe.ai#310 (trackedBrains not hydrated from disk
|
|
49
|
+
// at startup). Deliberately not "fixed" here per DevGuru round-6: that
|
|
50
|
+
// hydration gap is a separate, deeper issue.
|
|
51
|
+
//
|
|
52
|
+
// "Some SQS activity" isn't only a human message — a user-created
|
|
53
|
+
// scheduled message (`source: 'cron'`, e.g. create_scheduled_message)
|
|
54
|
+
// also goes through the same registerBrain() call, so it re-registers
|
|
55
|
+
// the brain and would surface the requeued message just as well. This
|
|
56
|
+
// does NOT extend to the poller's own internal maintenance-cron jobs
|
|
57
|
+
// (`source: 'maintenance'`) — registerBrain() explicitly skips those
|
|
58
|
+
// (maintenance-cron.ts) — so a maintenance run alone will not unstick
|
|
59
|
+
// a requeued message. The text below names the two real paths (human
|
|
60
|
+
// message, user-scheduled run) instead of implying a human reply is
|
|
61
|
+
// the only way, while still not claiming guaranteed/automatic
|
|
62
|
+
// resumption for a brain with neither.
|
|
63
|
+
return {
|
|
64
|
+
kind: 'neutral',
|
|
65
|
+
text: "Poller is restarting — your message is saved and will resume with your next message, or this brain's next scheduled run.",
|
|
66
|
+
reaction: 'arrows_counterclockwise',
|
|
67
|
+
};
|
|
36
68
|
case 'timeout':
|
|
37
69
|
case undefined: {
|
|
38
70
|
// 'timeout' keeps the legacy ":warning: Session timed out after N
|
package/dist/claude-spawner.d.ts
CHANGED
|
@@ -1,8 +1,11 @@
|
|
|
1
|
+
import { type ChildProcess } from 'child_process';
|
|
1
2
|
import type { TeamVibeQueueMessage, SessionMetrics } from './types.js';
|
|
2
3
|
import { type SessionLogger } from './logger.js';
|
|
3
|
-
import type
|
|
4
|
+
import { type TypingIndicator } from './slack-client.js';
|
|
4
5
|
import { type ResolvedModelConfig } from './config-client.js';
|
|
5
6
|
import type { KilledByPoller } from './classify-failed-iteration.js';
|
|
7
|
+
export declare function progressNotificationMarks(estimateMinutes: number): number[];
|
|
8
|
+
export declare function computeCompactionKillDeadline(previousActivityTime: number, now: number, idleKillMs: number, graceMs: number): number;
|
|
6
9
|
export interface ContextHintParams {
|
|
7
10
|
isFirstMessage: boolean;
|
|
8
11
|
lastMessageTs?: string;
|
|
@@ -22,6 +25,8 @@ export interface SpawnResult {
|
|
|
22
25
|
error?: string;
|
|
23
26
|
metrics?: SessionMetrics;
|
|
24
27
|
killedByPoller?: KilledByPoller;
|
|
28
|
+
sawSendMessageCall?: boolean;
|
|
29
|
+
lastPlainTextContent?: string;
|
|
25
30
|
modelSource?: ModelSource;
|
|
26
31
|
}
|
|
27
32
|
/**
|
|
@@ -54,7 +59,175 @@ export declare function resolveModel(brainSettingsModel: string | undefined, ddb
|
|
|
54
59
|
model: string | undefined;
|
|
55
60
|
modelSource: ModelSource;
|
|
56
61
|
};
|
|
62
|
+
/**
|
|
63
|
+
* teamvibeai/teamvibe.ai#73: `claude` is spawned with `detached: true`, which
|
|
64
|
+
* (via `setsid`) makes it the leader of its own new process group, distinct
|
|
65
|
+
* from the poller's. `claude` itself spawns MCP server child processes under
|
|
66
|
+
* that same group. Signaling `proc` alone (`proc.kill()`) only reaches
|
|
67
|
+
* `claude` — its MCP children are never signaled and can survive as orphans,
|
|
68
|
+
* leaking RAM (the Railway staircase-memory pattern that motivated #73).
|
|
69
|
+
*
|
|
70
|
+
* This sends the signal to the whole process group instead (negative pid is
|
|
71
|
+
* the `kill(2)` convention for "signal the group", see `man 2 kill`). The
|
|
72
|
+
* group id equals `claude`'s own pid because `setsid` was used to create it.
|
|
73
|
+
*
|
|
74
|
+
* Group membership survives the leader's own exit — as long as at least one
|
|
75
|
+
* process (e.g. a lingering MCP server) is still alive in the group, `-pid`
|
|
76
|
+
* remains a valid target even after `claude` itself has already exited. This
|
|
77
|
+
* is what makes it safe to call from the `close` handler too (#73 gap 2),
|
|
78
|
+
* not just from the pre-exit timeout/idle/shutdown kill paths.
|
|
79
|
+
*
|
|
80
|
+
* Unlike `ChildProcess#kill()` (a safe no-op on an already-dead process),
|
|
81
|
+
* `process.kill(-pid, ...)` throws `ESRCH` once the whole group is gone —
|
|
82
|
+
* expected and common (the group is often already fully reaped by the time
|
|
83
|
+
* the follow-up SIGKILL sweep fires), so it's caught and swallowed here
|
|
84
|
+
* rather than propagating into an uncaught exception that would crash the
|
|
85
|
+
* poller.
|
|
86
|
+
*/
|
|
87
|
+
export declare function killProcessGroup(proc: ChildProcess, signal: NodeJS.Signals): void;
|
|
88
|
+
/**
|
|
89
|
+
* DevGuru PR#308 round-2 review (item 2): the group-SIGKILL sweeps below
|
|
90
|
+
* (`killAllActiveProcesses()` and the `close`-handler sweep) used to always
|
|
91
|
+
* pay their full fixed grace period (5s / `CLOSE_SWEEP_GRACE_MS`) before
|
|
92
|
+
* resolving, even in the overwhelmingly common case where `claude` exited
|
|
93
|
+
* cleanly and took its MCP children down with it — i.e. the group is already
|
|
94
|
+
* empty and there is nothing left to sweep. That matters specifically for
|
|
95
|
+
* `killAllActiveProcesses()`: `poller.ts`'s `shutdown()` awaits it BEFORE
|
|
96
|
+
* `releaseAllActiveLocks()`, and delaying that release costs the next
|
|
97
|
+
* message on the same thread a 15-minute stale-lock wait.
|
|
98
|
+
*
|
|
99
|
+
* Cheap fast-path: probe the group with signal `0` — a pure existence check
|
|
100
|
+
* per `man 2 kill`, it never actually signals anything. `ESRCH` means the
|
|
101
|
+
* group is provably already gone, so the caller can skip the grace-period
|
|
102
|
+
* wait entirely instead of always paying it. Any other outcome (including an
|
|
103
|
+
* unexpected errno) is treated as "still alive" — fail toward the existing,
|
|
104
|
+
* safe wait+SIGKILL behavior rather than skipping a real cleanup wait.
|
|
105
|
+
*
|
|
106
|
+
* Known limitation (DevGuru PR#308 round-4 review, deliberately deferred to
|
|
107
|
+
* teamvibeai/teamvibe.ai#309 rather than fixed here): a zombie (defunct,
|
|
108
|
+
* unreaped) process still answers this signal-0 probe without `ESRCH` — the
|
|
109
|
+
* process table entry is still present, it just has no live task behind it —
|
|
110
|
+
* so a group containing only zombies is still reported "alive." That makes
|
|
111
|
+
* the fast-path above silently miss and fall back to the fixed-cost
|
|
112
|
+
* wait+SIGKILL sequence instead of resolving immediately. This is a safe
|
|
113
|
+
* fallback (no hang, just no speedup), not a correctness bug — but it does
|
|
114
|
+
* mean the fast-path quietly stops paying off on a container that has
|
|
115
|
+
* accumulated zombies. What actually removes the limitation is giving the
|
|
116
|
+
* container a real init process (e.g. `tini`) as PID 1: unlike a bare `node`
|
|
117
|
+
* entrypoint, a real init reaps orphaned zombies as they appear, so they
|
|
118
|
+
* never accumulate and confuse this probe in the first place. See #309 for
|
|
119
|
+
* the tracked Docker-bake/tini follow-up.
|
|
120
|
+
*/
|
|
121
|
+
export declare function isProcessGroupAlive(pid: number): boolean;
|
|
122
|
+
export interface SweepProcessGroupOptions {
|
|
123
|
+
onGraceExpired?: () => void;
|
|
124
|
+
unref?: boolean;
|
|
125
|
+
}
|
|
126
|
+
/**
|
|
127
|
+
* Shared group-kill grace-period sweep: SIGTERM, then either resolve
|
|
128
|
+
* immediately via the `isProcessGroupAlive()` fast-path (group already
|
|
129
|
+
* provably empty — see its doc comment) or wait up to `graceMs` before
|
|
130
|
+
* escalating to SIGKILL and resolving. Extracted so both call sites
|
|
131
|
+
* (`killAllActiveProcesses()` and the `close`-handler sweep) share one
|
|
132
|
+
* tested implementation instead of duplicating the wait+escalate logic.
|
|
133
|
+
*
|
|
134
|
+
* PR#308 round-3 review item 2: the up-front `isProcessGroupAlive()` probe
|
|
135
|
+
* above only fast-paths the "already dead before we even started" case. The
|
|
136
|
+
* case that matters most — the group IS alive right this instant, which is
|
|
137
|
+
* always true during a real shutdown with an active session, exactly when
|
|
138
|
+
* `releaseAllActiveLocks()` is waiting on this — used to still unconditionally
|
|
139
|
+
* pay the *entire* fixed `graceMs` even if `claude` died 50ms after its
|
|
140
|
+
* SIGTERM. Fixed by polling `isProcessGroupAlive()` every
|
|
141
|
+
* `PROCESS_GROUP_POLL_INTERVAL_MS` after the SIGTERM instead of relying on a
|
|
142
|
+
* single fixed-delay timer to both wait AND escalate: whichever fires first —
|
|
143
|
+
* a poll observing ESRCH, or the grace period elapsing — resolves/escalates.
|
|
144
|
+
* `graceMs` remains the hard upper bound on how long this ever waits (a
|
|
145
|
+
* safety ceiling, unchanged from before); this only shortens the *common*
|
|
146
|
+
* case where the group dies well before that ceiling.
|
|
147
|
+
*/
|
|
148
|
+
export declare function sweepProcessGroup(proc: ChildProcess, graceMs: number, options?: SweepProcessGroupOptions): Promise<void>;
|
|
149
|
+
export interface BuildSpawnEnvParams {
|
|
150
|
+
spawnSecretsEnv: NodeJS.ProcessEnv;
|
|
151
|
+
processEnv: NodeJS.ProcessEnv;
|
|
152
|
+
envPath: string | undefined;
|
|
153
|
+
claudeConfigDir: string;
|
|
154
|
+
persistentStoragePath?: string;
|
|
155
|
+
slackContext?: {
|
|
156
|
+
channel: string;
|
|
157
|
+
thread_ts: string;
|
|
158
|
+
message_ts: string;
|
|
159
|
+
};
|
|
160
|
+
teamvibeApiUrl: string;
|
|
161
|
+
teamvibePollerToken: string;
|
|
162
|
+
teamvibeWorkspaceId: string;
|
|
163
|
+
teamvibeChannelId: string;
|
|
164
|
+
teamvibeBotId: string;
|
|
165
|
+
teamvibePollerId: string;
|
|
166
|
+
inboxThreadId: string;
|
|
167
|
+
brainPath: string;
|
|
168
|
+
}
|
|
169
|
+
/**
|
|
170
|
+
* Pure construction of the env object passed to the spawned `claude` child
|
|
171
|
+
* process. Extracted out of the `spawn()` call site so the "what does the
|
|
172
|
+
* child actually inherit" logic is unit-testable without spinning up a real
|
|
173
|
+
* process tree or mocking `fetchSecretsForSpawn`/DDB config.
|
|
174
|
+
*
|
|
175
|
+
* `NODE_OPTIONS` is explicitly stripped (same convention as `CLAUDECODE`
|
|
176
|
+
* below): the poller's own Railway start command sets
|
|
177
|
+
* `NODE_OPTIONS='--max-old-space-size=128 --max-semi-space-size=2'` to cap
|
|
178
|
+
* the *poller's* own idle V8 heap — a thin SQS-listener process is fine at
|
|
179
|
+
* 128MB. That var lives in `process.env` at the poller-process level, so
|
|
180
|
+
* without this override it would leak via the `...processEnv` spread below
|
|
181
|
+
* into the spawned `claude` CLI process. `claude` needs real headroom to do
|
|
182
|
+
* actual agentic work (parsing files, holding conversation context, etc.) —
|
|
183
|
+
* the poller's own tuned-for-idle heap cap is simply the wrong number for
|
|
184
|
+
* what that process does, independent of what any specific OOM failure
|
|
185
|
+
* would look like if left uncapped.
|
|
186
|
+
*/
|
|
187
|
+
export declare function buildSpawnEnv(params: BuildSpawnEnvParams): NodeJS.ProcessEnv;
|
|
57
188
|
export declare function buildPrompt(msg: TeamVibeQueueMessage, contextHint?: string): string;
|
|
189
|
+
export interface SpawnStdinPlan {
|
|
190
|
+
/** Extra CLI args to append. Empty for the plain-text path — this is the exact pre-tv293-bug2 behavior, unconditionally. */
|
|
191
|
+
extraArgs: string[];
|
|
192
|
+
/** Exact chunks written to the child process's stdin, in order, before `.end()`. */
|
|
193
|
+
stdinChunks: string[];
|
|
194
|
+
}
|
|
195
|
+
/**
|
|
196
|
+
* Decides how the built prompt reaches the `claude` CLI: the default
|
|
197
|
+
* plain-text stdin path (unchanged from before teamvibe.ai#293 bug 2 — this
|
|
198
|
+
* covers essentially all production traffic: every Slack message, every
|
|
199
|
+
* text-only email, cron, maintenance) or a `stream-json` structured-content
|
|
200
|
+
* path carrying inline image blocks, used only when `inlineImages` is
|
|
201
|
+
* present on the incoming message (populated by email-received.ts' Part A
|
|
202
|
+
* — see packages/modules-teamvibe/src/utils/email-inline-images.ts).
|
|
203
|
+
*
|
|
204
|
+
* Pure function — no child_process/spawn access — so the no-image path's
|
|
205
|
+
* exact output (and thus "no behavior change for non-image sessions") is
|
|
206
|
+
* directly unit-testable without mocking `spawn` at all. See
|
|
207
|
+
* claude-spawner.test.ts's "no-image path is unchanged" coverage.
|
|
208
|
+
*
|
|
209
|
+
* `now` is injectable (defaults to `Date.now()`) purely so tests can assert
|
|
210
|
+
* both sides of the `expiresAt` check deterministically — see below.
|
|
211
|
+
*/
|
|
212
|
+
export declare function buildSpawnStdinPlan(prompt: string, inlineImages: TeamVibeQueueMessage['inlineImages'], now?: number): SpawnStdinPlan;
|
|
213
|
+
export declare function extractAssistantText(event: any): string | undefined;
|
|
214
|
+
export declare function assistantCalledTool(event: any, toolName: string): boolean;
|
|
215
|
+
/**
|
|
216
|
+
* Decides whether a spawn attempt should report its session ID back to the
|
|
217
|
+
* caller (poller.ts) for persistence to DynamoDB via updateSessionId().
|
|
218
|
+
*
|
|
219
|
+
* First-message spawns always mint a fresh session ID locally (see
|
|
220
|
+
* spawnClaudeCode below, teamvibe.ai#77) instead of recycling whatever
|
|
221
|
+
* placeholder DynamoDB currently holds — so on ANY successful first-message
|
|
222
|
+
* spawn (not just the "already in use" collision-retry path), the ID actually
|
|
223
|
+
* used by Claude Code must be surfaced here. Otherwise DynamoDB keeps the
|
|
224
|
+
* createSession() placeholder forever and the thread's next message does
|
|
225
|
+
* `--resume <id-claude-code-never-used>`, which fails 100% of the time.
|
|
226
|
+
* (teamvibe.ai#77, DevGuru review round)
|
|
227
|
+
*/
|
|
228
|
+
export declare function finalizeSpawnResult(retryResult: SpawnResult, isFirstMessage: boolean, effectiveSessionId: string | undefined): SpawnResult & {
|
|
229
|
+
newSessionId?: string;
|
|
230
|
+
};
|
|
58
231
|
export declare function spawnClaudeCode(msg: TeamVibeQueueMessage, sessionLog: SessionLogger, cwd: string, sessionId?: string, isFirstMessage?: boolean, lastMessageTs?: string, onMessageSent?: () => void, typingIndicator?: TypingIndicator): Promise<SpawnResult & {
|
|
59
232
|
newSessionId?: string;
|
|
60
233
|
}>;
|
|
@@ -66,3 +239,7 @@ export declare function getActiveProcessCount(): number;
|
|
|
66
239
|
export declare function killZombieForThread(threadId: string): boolean;
|
|
67
240
|
export declare function isAtCapacity(): boolean;
|
|
68
241
|
export declare function killAllActiveProcesses(): Promise<void>;
|
|
242
|
+
export declare function __setActiveProcessForTest(id: string, proc: ChildProcess): void;
|
|
243
|
+
export declare function __clearActiveProcessesForTest(): void;
|
|
244
|
+
export declare function __setKillReasonForTest(proc: ChildProcess, reason: KilledByPoller): void;
|
|
245
|
+
export declare function __getKillReasonForTest(proc: ChildProcess): KilledByPoller | undefined;
|