@teamvibe/poller 0.6.2 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/brain-manager.d.ts +43 -1
- package/dist/brain-manager.js +259 -6
- package/dist/brains-bootstrap.d.ts +45 -0
- package/dist/brains-bootstrap.js +217 -0
- package/dist/classify-failed-iteration.d.ts +1 -1
- package/dist/classify-failed-iteration.js +33 -1
- package/dist/claude-spawner.d.ts +179 -1
- package/dist/claude-spawner.js +697 -92
- package/dist/config.d.ts +40 -0
- package/dist/config.js +32 -2
- package/dist/cron-telemetry.d.ts +3 -0
- package/dist/cron-telemetry.js +16 -0
- package/dist/event-queue.d.ts +30 -0
- package/dist/event-queue.js +4 -0
- package/dist/idle-restart.d.ts +12 -0
- package/dist/idle-restart.js +200 -0
- package/dist/inbox-manager.d.ts +25 -0
- package/dist/inbox-manager.js +47 -0
- package/dist/maintenance-activity.d.ts +18 -0
- package/dist/maintenance-activity.js +28 -0
- package/dist/maintenance-cron.d.ts +73 -0
- package/dist/maintenance-cron.js +85 -1
- package/dist/metrics/collector.d.ts +2 -0
- package/dist/metrics/collector.js +7 -0
- package/dist/poller.d.ts +12 -0
- package/dist/poller.js +345 -8
- package/dist/types.d.ts +6 -0
- package/package.json +1 -1
package/dist/claude-spawner.js
CHANGED
|
@@ -6,6 +6,7 @@ import { config } from './config.js';
|
|
|
6
6
|
import { logger } from './logger.js';
|
|
7
7
|
import { getBaseBrainPath } from './brain-manager.js';
|
|
8
8
|
import { getPollerId } from './auth-provider.js';
|
|
9
|
+
import { sendSlackMessage } from './slack-client.js';
|
|
9
10
|
import { emitError } from './event-queue.js';
|
|
10
11
|
import { createWorktree, removeWorktree, isWorktreeIsolationEnabled } from './worktree-manager.js';
|
|
11
12
|
import { fetchSecretsForSpawn, buildSecretsReminder } from './secrets-client.js';
|
|
@@ -119,6 +120,32 @@ async function countPriorThreadMessages(botToken, channel, threadTs, currentMess
|
|
|
119
120
|
return 0;
|
|
120
121
|
}
|
|
121
122
|
}
|
|
123
|
+
// tv#32 Layer B: given the agent's own estimate (via the set_expected_duration
|
|
124
|
+
// MCP tool) of how long a task will take, return which minute-marks should
|
|
125
|
+
// get a progress notification. Tiers per design discussion with Marosh
|
|
126
|
+
// (2026-07-30): a wrong estimate only costs a mistimed status ping, never a
|
|
127
|
+
// safety issue, so self-report is fine here (unlike the Layer A kill-deadline,
|
|
128
|
+
// which must stay driven by the objective compact_boundary signal only).
|
|
129
|
+
export function progressNotificationMarks(estimateMinutes) {
|
|
130
|
+
if (estimateMinutes <= 5)
|
|
131
|
+
return [];
|
|
132
|
+
if (estimateMinutes <= 10)
|
|
133
|
+
return [5];
|
|
134
|
+
if (estimateMinutes <= 15)
|
|
135
|
+
return [5, 10];
|
|
136
|
+
return [5, 10, 15];
|
|
137
|
+
}
|
|
138
|
+
// tv#32 Layer A: the idle-watchdog kill deadline on a compact_boundary event.
|
|
139
|
+
// max(original 10min-from-silence-start deadline, event-arrival + 5min grace) —
|
|
140
|
+
// an early compact_boundary (well before the original deadline) causes no
|
|
141
|
+
// extension; a late one gets a small fixed grace period. Never a large
|
|
142
|
+
// speculative ceiling, since compact_boundary only proves compaction STARTED,
|
|
143
|
+
// never that it finishes in bounded time (design revision per Marosh 2026-07-31,
|
|
144
|
+
// after anthropics/claude-code#19567 — a real 26+ hour hung compaction — showed
|
|
145
|
+
// a flat large ceiling would itself be an unfounded guess).
|
|
146
|
+
export function computeCompactionKillDeadline(previousActivityTime, now, idleKillMs, graceMs) {
|
|
147
|
+
return Math.max(previousActivityTime + idleKillMs, now + graceMs);
|
|
148
|
+
}
|
|
122
149
|
// Select and render the context hint that goes into the prompt. Pure aside
|
|
123
150
|
// from the injected counters — extracted from runClaudeCode so the branching
|
|
124
151
|
// (resume vs. thread-context vs. no-hint) can be unit tested without spinning
|
|
@@ -203,6 +230,240 @@ export function resolveModel(brainSettingsModel, ddbConfig, envDefaultModel) {
|
|
|
203
230
|
// WeakMap auto-cleans when proc is GC'd. Module-scope is required because
|
|
204
231
|
// killZombieForThread fires outside the runClaudeCode closure.
|
|
205
232
|
const killReasonByProc = new WeakMap();
|
|
233
|
+
// Grace period between a group SIGTERM and the follow-up group SIGKILL sweep
|
|
234
|
+
// on the *normal* (successful) completion path — see the `proc.on('close', ...)`
|
|
235
|
+
// handler below. Deliberately shorter than the 5s grace used on the
|
|
236
|
+
// timeout/idle/network/shutdown kill paths: this sweep runs on every single
|
|
237
|
+
// session close (the hot path), not just the relatively rare forced-kill
|
|
238
|
+
// paths, so we don't want to accumulate long-lived timers under load.
|
|
239
|
+
const CLOSE_SWEEP_GRACE_MS = 2000;
|
|
240
|
+
// PR#308 round-3 review item 2: poll interval for sweepProcessGroup()'s
|
|
241
|
+
// post-SIGTERM wait (see below). No existing sub-second polling-interval
|
|
242
|
+
// constant elsewhere in this package to match (checked config.ts /
|
|
243
|
+
// event-queue.ts / auth-provider.ts / inbox-watcher.ts — all *_INTERVAL_MS
|
|
244
|
+
// there are >=30s cron/heartbeat-style intervals, a different scale of
|
|
245
|
+
// problem). 200ms is within DevGuru's suggested 100-250ms range: fine-
|
|
246
|
+
// grained enough that the common "died quickly after SIGTERM" case (the one
|
|
247
|
+
// that matters most — see sweepProcessGroup's doc comment) resolves close to
|
|
248
|
+
// its real death time instead of the full grace period, without probing so
|
|
249
|
+
// often it adds meaningful overhead during shutdown, when this sweep can run
|
|
250
|
+
// for every still-active session concurrently.
|
|
251
|
+
const PROCESS_GROUP_POLL_INTERVAL_MS = 200;
|
|
252
|
+
/**
|
|
253
|
+
* teamvibeai/teamvibe.ai#73: `claude` is spawned with `detached: true`, which
|
|
254
|
+
* (via `setsid`) makes it the leader of its own new process group, distinct
|
|
255
|
+
* from the poller's. `claude` itself spawns MCP server child processes under
|
|
256
|
+
* that same group. Signaling `proc` alone (`proc.kill()`) only reaches
|
|
257
|
+
* `claude` — its MCP children are never signaled and can survive as orphans,
|
|
258
|
+
* leaking RAM (the Railway staircase-memory pattern that motivated #73).
|
|
259
|
+
*
|
|
260
|
+
* This sends the signal to the whole process group instead (negative pid is
|
|
261
|
+
* the `kill(2)` convention for "signal the group", see `man 2 kill`). The
|
|
262
|
+
* group id equals `claude`'s own pid because `setsid` was used to create it.
|
|
263
|
+
*
|
|
264
|
+
* Group membership survives the leader's own exit — as long as at least one
|
|
265
|
+
* process (e.g. a lingering MCP server) is still alive in the group, `-pid`
|
|
266
|
+
* remains a valid target even after `claude` itself has already exited. This
|
|
267
|
+
* is what makes it safe to call from the `close` handler too (#73 gap 2),
|
|
268
|
+
* not just from the pre-exit timeout/idle/shutdown kill paths.
|
|
269
|
+
*
|
|
270
|
+
* Unlike `ChildProcess#kill()` (a safe no-op on an already-dead process),
|
|
271
|
+
* `process.kill(-pid, ...)` throws `ESRCH` once the whole group is gone —
|
|
272
|
+
* expected and common (the group is often already fully reaped by the time
|
|
273
|
+
* the follow-up SIGKILL sweep fires), so it's caught and swallowed here
|
|
274
|
+
* rather than propagating into an uncaught exception that would crash the
|
|
275
|
+
* poller.
|
|
276
|
+
*/
|
|
277
|
+
export function killProcessGroup(proc, signal) {
|
|
278
|
+
if (proc.pid === undefined) {
|
|
279
|
+
// Spawn itself failed (or never produced a pid) — nothing to signal.
|
|
280
|
+
return;
|
|
281
|
+
}
|
|
282
|
+
try {
|
|
283
|
+
process.kill(-proc.pid, signal);
|
|
284
|
+
}
|
|
285
|
+
catch (err) {
|
|
286
|
+
const code = err?.code;
|
|
287
|
+
if (code !== 'ESRCH') {
|
|
288
|
+
logger.warn(`killProcessGroup: unexpected error sending ${signal} to group -${proc.pid}: ${err instanceof Error ? err.message : String(err)}`);
|
|
289
|
+
}
|
|
290
|
+
// ESRCH ("no such process") means the group is already gone — expected,
|
|
291
|
+
// not an error worth logging (this is the common case for the follow-up
|
|
292
|
+
// SIGKILL sweep after a clean exit).
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
/**
|
|
296
|
+
* DevGuru PR#308 round-2 review (item 2): the group-SIGKILL sweeps below
|
|
297
|
+
* (`killAllActiveProcesses()` and the `close`-handler sweep) used to always
|
|
298
|
+
* pay their full fixed grace period (5s / `CLOSE_SWEEP_GRACE_MS`) before
|
|
299
|
+
* resolving, even in the overwhelmingly common case where `claude` exited
|
|
300
|
+
* cleanly and took its MCP children down with it — i.e. the group is already
|
|
301
|
+
* empty and there is nothing left to sweep. That matters specifically for
|
|
302
|
+
* `killAllActiveProcesses()`: `poller.ts`'s `shutdown()` awaits it BEFORE
|
|
303
|
+
* `releaseAllActiveLocks()`, and delaying that release costs the next
|
|
304
|
+
* message on the same thread a 15-minute stale-lock wait.
|
|
305
|
+
*
|
|
306
|
+
* Cheap fast-path: probe the group with signal `0` — a pure existence check
|
|
307
|
+
* per `man 2 kill`, it never actually signals anything. `ESRCH` means the
|
|
308
|
+
* group is provably already gone, so the caller can skip the grace-period
|
|
309
|
+
* wait entirely instead of always paying it. Any other outcome (including an
|
|
310
|
+
* unexpected errno) is treated as "still alive" — fail toward the existing,
|
|
311
|
+
* safe wait+SIGKILL behavior rather than skipping a real cleanup wait.
|
|
312
|
+
*
|
|
313
|
+
* Known limitation (DevGuru PR#308 round-4 review, deliberately deferred to
|
|
314
|
+
* teamvibeai/teamvibe.ai#309 rather than fixed here): a zombie (defunct,
|
|
315
|
+
* unreaped) process still answers this signal-0 probe without `ESRCH` — the
|
|
316
|
+
* process table entry is still present, it just has no live task behind it —
|
|
317
|
+
* so a group containing only zombies is still reported "alive." That makes
|
|
318
|
+
* the fast-path above silently miss and fall back to the fixed-cost
|
|
319
|
+
* wait+SIGKILL sequence instead of resolving immediately. This is a safe
|
|
320
|
+
* fallback (no hang, just no speedup), not a correctness bug — but it does
|
|
321
|
+
* mean the fast-path quietly stops paying off on a container that has
|
|
322
|
+
* accumulated zombies. What actually removes the limitation is giving the
|
|
323
|
+
* container a real init process (e.g. `tini`) as PID 1: unlike a bare `node`
|
|
324
|
+
* entrypoint, a real init reaps orphaned zombies as they appear, so they
|
|
325
|
+
* never accumulate and confuse this probe in the first place. See #309 for
|
|
326
|
+
* the tracked Docker-bake/tini follow-up.
|
|
327
|
+
*/
|
|
328
|
+
export function isProcessGroupAlive(pid) {
|
|
329
|
+
try {
|
|
330
|
+
process.kill(-pid, 0);
|
|
331
|
+
return true;
|
|
332
|
+
}
|
|
333
|
+
catch (err) {
|
|
334
|
+
const code = err?.code;
|
|
335
|
+
if (code !== 'ESRCH') {
|
|
336
|
+
logger.warn(`isProcessGroupAlive: unexpected error probing group -${pid}: ${err instanceof Error ? err.message : String(err)}`);
|
|
337
|
+
return true;
|
|
338
|
+
}
|
|
339
|
+
return false;
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
/**
|
|
343
|
+
* Shared group-kill grace-period sweep: SIGTERM, then either resolve
|
|
344
|
+
* immediately via the `isProcessGroupAlive()` fast-path (group already
|
|
345
|
+
* provably empty — see its doc comment) or wait up to `graceMs` before
|
|
346
|
+
* escalating to SIGKILL and resolving. Extracted so both call sites
|
|
347
|
+
* (`killAllActiveProcesses()` and the `close`-handler sweep) share one
|
|
348
|
+
* tested implementation instead of duplicating the wait+escalate logic.
|
|
349
|
+
*
|
|
350
|
+
* PR#308 round-3 review item 2: the up-front `isProcessGroupAlive()` probe
|
|
351
|
+
* above only fast-paths the "already dead before we even started" case. The
|
|
352
|
+
* case that matters most — the group IS alive right this instant, which is
|
|
353
|
+
* always true during a real shutdown with an active session, exactly when
|
|
354
|
+
* `releaseAllActiveLocks()` is waiting on this — used to still unconditionally
|
|
355
|
+
* pay the *entire* fixed `graceMs` even if `claude` died 50ms after its
|
|
356
|
+
* SIGTERM. Fixed by polling `isProcessGroupAlive()` every
|
|
357
|
+
* `PROCESS_GROUP_POLL_INTERVAL_MS` after the SIGTERM instead of relying on a
|
|
358
|
+
* single fixed-delay timer to both wait AND escalate: whichever fires first —
|
|
359
|
+
* a poll observing ESRCH, or the grace period elapsing — resolves/escalates.
|
|
360
|
+
* `graceMs` remains the hard upper bound on how long this ever waits (a
|
|
361
|
+
* safety ceiling, unchanged from before); this only shortens the *common*
|
|
362
|
+
* case where the group dies well before that ceiling.
|
|
363
|
+
*/
|
|
364
|
+
export function sweepProcessGroup(proc, graceMs, options = {}) {
|
|
365
|
+
return new Promise((resolve) => {
|
|
366
|
+
if (proc.pid === undefined || !isProcessGroupAlive(proc.pid)) {
|
|
367
|
+
resolve();
|
|
368
|
+
return;
|
|
369
|
+
}
|
|
370
|
+
const pid = proc.pid;
|
|
371
|
+
killProcessGroup(proc, 'SIGTERM');
|
|
372
|
+
const deadline = Date.now() + graceMs;
|
|
373
|
+
let settled = false;
|
|
374
|
+
const poll = setInterval(() => {
|
|
375
|
+
if (settled)
|
|
376
|
+
return;
|
|
377
|
+
if (!isProcessGroupAlive(pid)) {
|
|
378
|
+
settled = true;
|
|
379
|
+
clearInterval(poll);
|
|
380
|
+
resolve();
|
|
381
|
+
return;
|
|
382
|
+
}
|
|
383
|
+
if (Date.now() >= deadline) {
|
|
384
|
+
settled = true;
|
|
385
|
+
clearInterval(poll);
|
|
386
|
+
options.onGraceExpired?.();
|
|
387
|
+
killProcessGroup(proc, 'SIGKILL');
|
|
388
|
+
resolve();
|
|
389
|
+
}
|
|
390
|
+
// Otherwise: still alive, deadline not yet reached — keep polling.
|
|
391
|
+
}, Math.min(PROCESS_GROUP_POLL_INTERVAL_MS, graceMs));
|
|
392
|
+
if (options.unref) {
|
|
393
|
+
poll.unref?.();
|
|
394
|
+
}
|
|
395
|
+
});
|
|
396
|
+
}
|
|
397
|
+
/**
|
|
398
|
+
* Pure construction of the env object passed to the spawned `claude` child
|
|
399
|
+
* process. Extracted out of the `spawn()` call site so the "what does the
|
|
400
|
+
* child actually inherit" logic is unit-testable without spinning up a real
|
|
401
|
+
* process tree or mocking `fetchSecretsForSpawn`/DDB config.
|
|
402
|
+
*
|
|
403
|
+
* `NODE_OPTIONS` is explicitly stripped (same convention as `CLAUDECODE`
|
|
404
|
+
* below): the poller's own Railway start command sets
|
|
405
|
+
* `NODE_OPTIONS='--max-old-space-size=128 --max-semi-space-size=2'` to cap
|
|
406
|
+
* the *poller's* own idle V8 heap — a thin SQS-listener process is fine at
|
|
407
|
+
* 128MB. That var lives in `process.env` at the poller-process level, so
|
|
408
|
+
* without this override it would leak via the `...processEnv` spread below
|
|
409
|
+
* into the spawned `claude` CLI process. `claude` needs real headroom to do
|
|
410
|
+
* actual agentic work (parsing files, holding conversation context, etc.) —
|
|
411
|
+
* the poller's own tuned-for-idle heap cap is simply the wrong number for
|
|
412
|
+
* what that process does, independent of what any specific OOM failure
|
|
413
|
+
* would look like if left uncapped.
|
|
414
|
+
*/
|
|
415
|
+
export function buildSpawnEnv(params) {
|
|
416
|
+
return {
|
|
417
|
+
// Secrets platform (#105) injection — SOLE call site of
|
|
418
|
+
// `SpawnSecretsEnvelope.unwrap()` in the entire poller codebase.
|
|
419
|
+
// Placed FIRST so subsequent spreads can overwrite. Belt-and-
|
|
420
|
+
// suspenders: even if the API's reserved-prefix validator fails
|
|
421
|
+
// (`AWS_`, `TEAMVIBE_`, `SLACK_`, `CLAUDE_CODE_`, `ANTHROPIC_`)
|
|
422
|
+
// a name collision is silently won by the platform variable
|
|
423
|
+
// explicit field below.
|
|
424
|
+
//
|
|
425
|
+
// Phase 2 of teamvibe.ai#215 (poller v0.5.1): the explicit
|
|
426
|
+
// `SLACK_BOT_TOKEN: msg.teamvibe.botToken` injection that used
|
|
427
|
+
// to live here was removed. `SLACK_BOT_TOKEN` now sources from
|
|
428
|
+
// the `/secrets/spawn` platform overlay (the spread above) —
|
|
429
|
+
// backend v0.2.2+ resolves it via the `Channel → SlackBot`
|
|
430
|
+
// relation. Requires backend on Phase 1; pollers on 0.5.0 still
|
|
431
|
+
// read the explicit field, so split rollout is safe.
|
|
432
|
+
...params.spawnSecretsEnv,
|
|
433
|
+
...params.processEnv,
|
|
434
|
+
CLAUDECODE: undefined,
|
|
435
|
+
// teamvibeai/teamvibe.ai#73 follow-up: strip the poller's own
|
|
436
|
+
// memory-tuning flags so they don't leak into `claude` — see docstring
|
|
437
|
+
// above.
|
|
438
|
+
NODE_OPTIONS: undefined,
|
|
439
|
+
PATH: params.envPath,
|
|
440
|
+
CI: 'true',
|
|
441
|
+
CLAUDE_CONFIG_DIR: params.claudeConfigDir,
|
|
442
|
+
...(params.persistentStoragePath && {
|
|
443
|
+
PERSISTENT_STORAGE_PATH: params.persistentStoragePath,
|
|
444
|
+
}),
|
|
445
|
+
...(params.slackContext && {
|
|
446
|
+
SLACK_CHANNEL: params.slackContext.channel,
|
|
447
|
+
SLACK_THREAD_TS: params.slackContext.thread_ts,
|
|
448
|
+
SLACK_MESSAGE_TS: params.slackContext.message_ts,
|
|
449
|
+
}),
|
|
450
|
+
// TeamVibe API env vars for scheduled messages MCP server
|
|
451
|
+
TEAMVIBE_API_URL: params.teamvibeApiUrl,
|
|
452
|
+
TEAMVIBE_POLLER_TOKEN: params.teamvibePollerToken,
|
|
453
|
+
TEAMVIBE_WORKSPACE_ID: params.teamvibeWorkspaceId,
|
|
454
|
+
TEAMVIBE_CHANNEL_ID: params.teamvibeChannelId,
|
|
455
|
+
TEAMVIBE_BOT_ID: params.teamvibeBotId,
|
|
456
|
+
TEAMVIBE_POLLER_ID: params.teamvibePollerId,
|
|
457
|
+
INBOX_THREAD_ID: params.inboxThreadId,
|
|
458
|
+
// Stable main-brain path (pre-worktree-rebind `cwd`), NOT `cwd` above —
|
|
459
|
+
// when worktree isolation is on, `cwd` here is the ephemeral
|
|
460
|
+
// `.worktrees/<thread_id>/` dir that's removed right after the
|
|
461
|
+
// session ends. bg-task-runner.mjs writes its .inbox/ checkpoint
|
|
462
|
+
// drops against this var so they land somewhere still being
|
|
463
|
+
// watched after the session (and worktree) is gone. See tv#265.
|
|
464
|
+
POLLER_BRAIN_PATH: params.brainPath,
|
|
465
|
+
};
|
|
466
|
+
}
|
|
206
467
|
export function buildPrompt(msg, contextHint) {
|
|
207
468
|
const isEmail = msg.source === 'email';
|
|
208
469
|
const isWebhook = msg.source === 'webhook';
|
|
@@ -276,6 +537,80 @@ export function buildPrompt(msg, contextHint) {
|
|
|
276
537
|
}
|
|
277
538
|
return prompt;
|
|
278
539
|
}
|
|
540
|
+
/**
|
|
541
|
+
* Decides how the built prompt reaches the `claude` CLI: the default
|
|
542
|
+
* plain-text stdin path (unchanged from before teamvibe.ai#293 bug 2 — this
|
|
543
|
+
* covers essentially all production traffic: every Slack message, every
|
|
544
|
+
* text-only email, cron, maintenance) or a `stream-json` structured-content
|
|
545
|
+
* path carrying inline image blocks, used only when `inlineImages` is
|
|
546
|
+
* present on the incoming message (populated by email-received.ts' Part A
|
|
547
|
+
* — see packages/modules-teamvibe/src/utils/email-inline-images.ts).
|
|
548
|
+
*
|
|
549
|
+
* Pure function — no child_process/spawn access — so the no-image path's
|
|
550
|
+
* exact output (and thus "no behavior change for non-image sessions") is
|
|
551
|
+
* directly unit-testable without mocking `spawn` at all. See
|
|
552
|
+
* claude-spawner.test.ts's "no-image path is unchanged" coverage.
|
|
553
|
+
*
|
|
554
|
+
* `now` is injectable (defaults to `Date.now()`) purely so tests can assert
|
|
555
|
+
* both sides of the `expiresAt` check deterministically — see below.
|
|
556
|
+
*/
|
|
557
|
+
export function buildSpawnStdinPlan(prompt, inlineImages, now = Date.now()) {
|
|
558
|
+
const images = (inlineImages ?? []).filter((img) => img.data || img.url);
|
|
559
|
+
if (images.length === 0) {
|
|
560
|
+
// Identical to the single `proc.stdin?.write(prompt)` call that existed
|
|
561
|
+
// before inline-image support — no extra args, one chunk, no trailing
|
|
562
|
+
// newline added.
|
|
563
|
+
return { extraArgs: [], stdinChunks: [prompt] };
|
|
564
|
+
}
|
|
565
|
+
const content = [{ type: 'text', text: prompt }];
|
|
566
|
+
for (const img of images) {
|
|
567
|
+
if (img.data) {
|
|
568
|
+
content.push({
|
|
569
|
+
type: 'image',
|
|
570
|
+
source: { type: 'base64', media_type: img.mediaType, data: img.data },
|
|
571
|
+
});
|
|
572
|
+
}
|
|
573
|
+
else if (img.url) {
|
|
574
|
+
// DevGuru review round 1 (non-blocking): the 15-minute presigned-URL
|
|
575
|
+
// TTL (PRESIGNED_IMAGE_URL_TTL_SECONDS) assumes this session consumes
|
|
576
|
+
// the URL within seconds of ingest, but SQS FIFO blocks an entire
|
|
577
|
+
// message group behind one in-flight message — a poller outage or
|
|
578
|
+
// backlog could leave this message queued long enough for the URL to
|
|
579
|
+
// expire before it's ever used. A dead URL request would likely just
|
|
580
|
+
// fail unhelpfully (the `url` source-type sub-path was never
|
|
581
|
+
// spike-tested end to end — see this function's docstring), so check
|
|
582
|
+
// expiry here and degrade to a text note instead of sending a URL
|
|
583
|
+
// that's already dead.
|
|
584
|
+
// DevGuru review round 2: `new Date(invalid).getTime()` is `NaN`, and
|
|
585
|
+
// `NaN <= now` is `false` — so a malformed (not missing) `expiresAt`
|
|
586
|
+
// used to pass through as "fresh" and send a URL whose real freshness
|
|
587
|
+
// is unknown. Only a genuinely absent `expiresAt` should be trusted
|
|
588
|
+
// as fresh (see the "no expiresAt at all" defensive case below); an
|
|
589
|
+
// unparseable one is treated the same as an expired one — fail
|
|
590
|
+
// toward the text-note degrade, not toward silently sending a URL
|
|
591
|
+
// that might already be dead.
|
|
592
|
+
const expiresAtMs = img.expiresAt ? new Date(img.expiresAt).getTime() : undefined;
|
|
593
|
+
if (expiresAtMs !== undefined && (Number.isNaN(expiresAtMs) || expiresAtMs <= now)) {
|
|
594
|
+
content.push({
|
|
595
|
+
type: 'text',
|
|
596
|
+
text: '[inline image expired before it could be sent]',
|
|
597
|
+
});
|
|
598
|
+
continue;
|
|
599
|
+
}
|
|
600
|
+
// Presigned-URL fallback (email-received.ts Part A step 5). This
|
|
601
|
+
// `url` source-type sub-path was NOT exercised by the earlier spike
|
|
602
|
+
// that verified the stream-json plumbing end to end (that spike only
|
|
603
|
+
// covered `type: 'base64'`) — implemented purely from the documented
|
|
604
|
+
// Anthropic API content-block shape. Flagged for reviewer attention.
|
|
605
|
+
content.push({
|
|
606
|
+
type: 'image',
|
|
607
|
+
source: { type: 'url', url: img.url },
|
|
608
|
+
});
|
|
609
|
+
}
|
|
610
|
+
}
|
|
611
|
+
const line = JSON.stringify({ type: 'user', message: { role: 'user', content } });
|
|
612
|
+
return { extraArgs: ['--input-format', 'stream-json'], stdinChunks: [line, '\n'] };
|
|
613
|
+
}
|
|
279
614
|
function truncateOutput(output, maxLen = 200) {
|
|
280
615
|
const outputStr = typeof output === 'string' ? output : JSON.stringify(output);
|
|
281
616
|
if (outputStr.includes('"type":"base64"') ||
|
|
@@ -332,7 +667,39 @@ function formatStreamEvent(event) {
|
|
|
332
667
|
return null;
|
|
333
668
|
}
|
|
334
669
|
}
|
|
335
|
-
|
|
670
|
+
// teamvibe.ai#284: pure extraction of the plain-text portion of an `assistant`
|
|
671
|
+
// stream-json event, so the end-of-turn guard's "what would we deliver as a
|
|
672
|
+
// fallback" logic is unit-testable without spinning up child_process. Mirrors
|
|
673
|
+
// the `case 'assistant':` branch in formatStreamEvent (same content-array
|
|
674
|
+
// shape), but keeps only `text` blocks — `tool_use` blocks are irrelevant
|
|
675
|
+
// here since tool calls are tracked separately via `sawSendMessageCall`.
|
|
676
|
+
// Returns undefined (not '') for "nothing to report" so callers can use a
|
|
677
|
+
// single truthiness check.
|
|
678
|
+
export function extractAssistantText(event) {
|
|
679
|
+
if (event?.type !== 'assistant' || !Array.isArray(event.message?.content)) {
|
|
680
|
+
return undefined;
|
|
681
|
+
}
|
|
682
|
+
const textParts = event.message.content
|
|
683
|
+
.filter((c) => c?.type === 'text' && typeof c.text === 'string')
|
|
684
|
+
.map((c) => c.text);
|
|
685
|
+
const combined = textParts.join('\n').trim();
|
|
686
|
+
return combined.length > 0 ? combined : undefined;
|
|
687
|
+
}
|
|
688
|
+
// teamvibe.ai#284 review fix: pure check for whether an `assistant`
|
|
689
|
+
// stream-json event contains a tool_use block calling `toolName`. Mirrors
|
|
690
|
+
// extractAssistantText's content-array parsing — `claude --output-format
|
|
691
|
+
// stream-json` never emits a standalone top-level `tool_use` event; tool_use
|
|
692
|
+
// blocks live nested in `event.message.content[]` (see formatStreamEvent's
|
|
693
|
+
// `case 'assistant':` branch for the same shape). A prior version of this
|
|
694
|
+
// check looked for `event.type === 'tool_use'` directly, which never matches
|
|
695
|
+
// against the real stream shape.
|
|
696
|
+
export function assistantCalledTool(event, toolName) {
|
|
697
|
+
if (event?.type !== 'assistant' || !Array.isArray(event.message?.content)) {
|
|
698
|
+
return false;
|
|
699
|
+
}
|
|
700
|
+
return event.message.content.some((c) => c?.type === 'tool_use' && c.name === toolName);
|
|
701
|
+
}
|
|
702
|
+
async function runClaudeCode(msg, sessionLog, cwd, brainPath, sessionId, isFirstMessage = true, lastMessageTs, onMessageSent, typingIndicator) {
|
|
336
703
|
const slackContext = msg.response_context.slack;
|
|
337
704
|
const isCronMessage = msg.source === 'cron';
|
|
338
705
|
const isMaintenance = msg.source === 'maintenance';
|
|
@@ -381,6 +748,10 @@ async function runClaudeCode(msg, sessionLog, cwd, sessionId, isFirstMessage = t
|
|
|
381
748
|
});
|
|
382
749
|
return new Promise((resolve) => {
|
|
383
750
|
const prompt = buildPrompt(msg, contextHint);
|
|
751
|
+
// teamvibe.ai#293 bug 2: decides plain-text vs. stream-json stdin. Empty
|
|
752
|
+
// `extraArgs`/single-chunk `stdinChunks` (the overwhelming majority of
|
|
753
|
+
// sessions — no inline images) reproduces the exact prior behavior.
|
|
754
|
+
const spawnStdinPlan = buildSpawnStdinPlan(prompt, msg.inlineImages);
|
|
384
755
|
// MCP server config passed via --mcp-config flag (no temp files needed)
|
|
385
756
|
const mcpServerPath = join(getBaseBrainPath(), 'mcp', 'slack.mjs');
|
|
386
757
|
const teamvibeApiServerPath = join(getBaseBrainPath(), 'mcp', 'teamvibe-api.mjs');
|
|
@@ -397,6 +768,7 @@ async function runClaudeCode(msg, sessionLog, cwd, sessionId, isFirstMessage = t
|
|
|
397
768
|
'--verbose',
|
|
398
769
|
'--output-format',
|
|
399
770
|
'stream-json',
|
|
771
|
+
...spawnStdinPlan.extraArgs,
|
|
400
772
|
'--max-turns',
|
|
401
773
|
'50',
|
|
402
774
|
'--mcp-config',
|
|
@@ -442,65 +814,99 @@ async function runClaudeCode(msg, sessionLog, cwd, sessionId, isFirstMessage = t
|
|
|
442
814
|
sessionLog.info(`Spawning Claude Code with args: ${JSON.stringify(args)}`);
|
|
443
815
|
sessionLog.info(`Working directory: ${cwd}`);
|
|
444
816
|
sessionLog.info(`Prompt length: ${prompt.length} chars`);
|
|
817
|
+
// Always log the human-readable text prompt only — never the raw
|
|
818
|
+
// stream-json stdin chunks, which can carry base64 image data that
|
|
819
|
+
// would otherwise bloat the session log file.
|
|
445
820
|
sessionLog.logPrompt(prompt);
|
|
821
|
+
if (spawnStdinPlan.extraArgs.length > 0) {
|
|
822
|
+
const imageCount = (msg.inlineImages ?? []).filter((img) => img.data || img.url).length;
|
|
823
|
+
sessionLog.info(`Inline image content: ${imageCount} image(s) sent via stream-json stdin (not logged)`);
|
|
824
|
+
}
|
|
446
825
|
// Prepend persistent bin dir to PATH if configured
|
|
447
826
|
const envPath = config.PERSISTENT_STORAGE_PATH
|
|
448
827
|
? `${config.PERSISTENT_STORAGE_PATH}/bin:${process.env['PATH'] || ''}`
|
|
449
828
|
: process.env['PATH'];
|
|
450
829
|
const proc = spawn(config.CLAUDE_CLI_PATH, args, {
|
|
451
830
|
cwd,
|
|
452
|
-
env: {
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
...spawnSecrets.env.unwrap(),
|
|
469
|
-
...process.env,
|
|
470
|
-
CLAUDECODE: undefined,
|
|
471
|
-
PATH: envPath,
|
|
472
|
-
CI: 'true',
|
|
473
|
-
CLAUDE_CONFIG_DIR: getBaseBrainPath(),
|
|
474
|
-
...(config.PERSISTENT_STORAGE_PATH && {
|
|
475
|
-
PERSISTENT_STORAGE_PATH: config.PERSISTENT_STORAGE_PATH,
|
|
476
|
-
}),
|
|
477
|
-
...(slackContext && {
|
|
478
|
-
SLACK_CHANNEL: slackContext.channel,
|
|
479
|
-
SLACK_THREAD_TS: slackContext.thread_ts,
|
|
480
|
-
SLACK_MESSAGE_TS: slackContext.message_ts,
|
|
481
|
-
}),
|
|
482
|
-
// TeamVibe API env vars for scheduled messages MCP server
|
|
483
|
-
TEAMVIBE_API_URL: config.TEAMVIBE_API_URL,
|
|
484
|
-
TEAMVIBE_POLLER_TOKEN: config.TEAMVIBE_POLLER_TOKEN || '',
|
|
485
|
-
TEAMVIBE_WORKSPACE_ID: msg.teamvibe.workspaceId,
|
|
486
|
-
TEAMVIBE_CHANNEL_ID: msg.teamvibe.channelId,
|
|
487
|
-
TEAMVIBE_BOT_ID: msg.teamvibe.botId,
|
|
488
|
-
TEAMVIBE_POLLER_ID: getPollerId() || '',
|
|
489
|
-
INBOX_THREAD_ID: msg.thread_id,
|
|
490
|
-
},
|
|
831
|
+
env: buildSpawnEnv({
|
|
832
|
+
spawnSecretsEnv: spawnSecrets.env.unwrap(),
|
|
833
|
+
processEnv: process.env,
|
|
834
|
+
envPath,
|
|
835
|
+
claudeConfigDir: getBaseBrainPath(),
|
|
836
|
+
persistentStoragePath: config.PERSISTENT_STORAGE_PATH,
|
|
837
|
+
slackContext,
|
|
838
|
+
teamvibeApiUrl: config.TEAMVIBE_API_URL,
|
|
839
|
+
teamvibePollerToken: config.TEAMVIBE_POLLER_TOKEN || '',
|
|
840
|
+
teamvibeWorkspaceId: msg.teamvibe.workspaceId,
|
|
841
|
+
teamvibeChannelId: msg.teamvibe.channelId,
|
|
842
|
+
teamvibeBotId: msg.teamvibe.botId,
|
|
843
|
+
teamvibePollerId: getPollerId() || '',
|
|
844
|
+
inboxThreadId: msg.thread_id,
|
|
845
|
+
brainPath,
|
|
846
|
+
}),
|
|
491
847
|
stdio: ['pipe', 'pipe', 'pipe'],
|
|
848
|
+
// teamvibeai/teamvibe.ai#73: run `claude` (via setsid) as the leader of
|
|
849
|
+
// its own new process group, separate from the poller's. This is what
|
|
850
|
+
// makes `killProcessGroup()` below able to reach the MCP server child
|
|
851
|
+
// processes `claude` itself spawns — signaling `proc` alone never did.
|
|
852
|
+
// Same trick poller-brain's bg-task.mjs background-task mechanism
|
|
853
|
+
// already relies on to escape session teardown; orthogonal here since
|
|
854
|
+
// that mechanism detaches *from* a `claude` session, not `claude`
|
|
855
|
+
// itself.
|
|
856
|
+
detached: true,
|
|
492
857
|
});
|
|
493
858
|
// Track active process by message ID and thread ID
|
|
494
859
|
activeProcesses.set(msg.id, proc);
|
|
495
860
|
activeProcessesByThread.set(msg.thread_id, proc);
|
|
496
|
-
|
|
861
|
+
for (const chunk of spawnStdinPlan.stdinChunks) {
|
|
862
|
+
proc.stdin?.write(chunk);
|
|
863
|
+
}
|
|
497
864
|
proc.stdin?.end();
|
|
498
865
|
let stdout = '';
|
|
499
866
|
let stderr = '';
|
|
500
867
|
let stdoutBuffer = '';
|
|
501
868
|
let metrics;
|
|
869
|
+
// teamvibe.ai#284 (poller-brain#235): end-of-turn guard state — tracked
|
|
870
|
+
// across the whole run so the `close` handler can tell poller.ts whether
|
|
871
|
+
// a successful turn actually delivered anything to Slack.
|
|
872
|
+
let sawSendMessageCall = false;
|
|
873
|
+
let lastPlainTextContent;
|
|
502
874
|
let lastActivityTime = Date.now();
|
|
503
875
|
let networkErrorStart = null; // timestamp when continuous network errors began
|
|
876
|
+
// Set on the first `compact_boundary` system event and held fixed until a
|
|
877
|
+
// subsequent non-compaction event confirms normal work resumed. While set,
|
|
878
|
+
// the idle watchdog's kill decision uses this absolute deadline instead of
|
|
879
|
+
// the default IDLE_WATCHDOG_KILL_MS window — legitimate context compaction
|
|
880
|
+
// (tv#32) produces no stdout for minutes and must not be killed as a hang.
|
|
881
|
+
// The deadline is max(original IDLE_WATCHDOG_KILL_MS-from-silence-start,
|
|
882
|
+
// compact_boundary-arrival-time + IDLE_WATCHDOG_COMPACTION_GRACE_MS) — an
|
|
883
|
+
// early compact_boundary (well before the original deadline) causes no
|
|
884
|
+
// extension at all; a late one gets a small fixed grace period instead of
|
|
885
|
+
// a large speculative ceiling, since compact_boundary only proves
|
|
886
|
+
// compaction STARTED, never that it will finish in bounded time (a stuck
|
|
887
|
+
// compaction can hang indefinitely — see anthropics/claude-code#19567).
|
|
888
|
+
// It never moves further on repeat compact_boundary events in the same run.
|
|
889
|
+
let compactionKillDeadline = null;
|
|
890
|
+
// tv#32 Layer B: pending progress-notification timers, and whether a
|
|
891
|
+
// schedule has already been installed (by the agent's own estimate, or by
|
|
892
|
+
// the compact_boundary floor-correction below). Cleared on reply/close so
|
|
893
|
+
// a "still working" ping never fires after the answer already landed.
|
|
894
|
+
const notificationTimers = [];
|
|
895
|
+
let notificationScheduleInstalled = false;
|
|
896
|
+
const clearNotificationTimers = () => {
|
|
897
|
+
for (const t of notificationTimers)
|
|
898
|
+
clearTimeout(t);
|
|
899
|
+
notificationTimers.length = 0;
|
|
900
|
+
};
|
|
901
|
+
const scheduleProgressNotifications = (marks) => {
|
|
902
|
+
for (const minutes of marks) {
|
|
903
|
+
notificationTimers.push(setTimeout(() => {
|
|
904
|
+
sendSlackMessage(msg, `Still working on this — about ${minutes} min in so far.`).catch((err) => {
|
|
905
|
+
sessionLog.warn(`[Watchdog] Failed to send progress notification: ${err instanceof Error ? err.message : err}`);
|
|
906
|
+
});
|
|
907
|
+
}, minutes * 60000));
|
|
908
|
+
}
|
|
909
|
+
};
|
|
504
910
|
proc.stdout.on('data', (data) => {
|
|
505
911
|
const chunk = data.toString();
|
|
506
912
|
stdout += chunk;
|
|
@@ -514,8 +920,41 @@ async function runClaudeCode(msg, sessionLog, cwd, sessionId, isFirstMessage = t
|
|
|
514
920
|
for (const line of lines) {
|
|
515
921
|
try {
|
|
516
922
|
const event = JSON.parse(line);
|
|
923
|
+
// Capture the pre-event activity timestamp BEFORE the unconditional
|
|
924
|
+
// overwrite below — needed to compute the "original 10min deadline
|
|
925
|
+
// from silence start" for a compact_boundary line, since that line
|
|
926
|
+
// is itself stdout activity and would otherwise clobber the very
|
|
927
|
+
// timestamp the deadline math depends on.
|
|
928
|
+
const previousActivityTime = lastActivityTime;
|
|
517
929
|
// Track activity for idle watchdog
|
|
518
930
|
lastActivityTime = Date.now();
|
|
931
|
+
// tv#32: recognize context compaction so the idle watchdog doesn't
|
|
932
|
+
// kill a legitimately-slow (not hung) session. First compact_boundary
|
|
933
|
+
// extends the kill deadline (see comment above compactionKillDeadline
|
|
934
|
+
// for the max() formula); further compact_boundary events in the same
|
|
935
|
+
// run do NOT push it out further. Any other event after one arrives
|
|
936
|
+
// means compaction ended — revert to normal idle-from-last-activity
|
|
937
|
+
// kill behavior.
|
|
938
|
+
if (event.type === 'system' && event.subtype === 'compact_boundary') {
|
|
939
|
+
if (compactionKillDeadline === null) {
|
|
940
|
+
compactionKillDeadline = computeCompactionKillDeadline(previousActivityTime, Date.now(), config.IDLE_WATCHDOG_KILL_MS, config.IDLE_WATCHDOG_COMPACTION_GRACE_MS);
|
|
941
|
+
sessionLog.info(`[Watchdog] Context compaction detected (trigger: ${event.trigger ?? 'unknown'}), ` +
|
|
942
|
+
`kill deadline set to ${new Date(compactionKillDeadline).toISOString()}`);
|
|
943
|
+
}
|
|
944
|
+
// Layer B floor-correction: compaction is objective proof this run
|
|
945
|
+
// is legitimately slow. If the agent never called
|
|
946
|
+
// set_expected_duration (or hasn't yet), make sure the user still
|
|
947
|
+
// gets at least one progress ping instead of relying solely on the
|
|
948
|
+
// agent's own — possibly too-optimistic or absent — estimate.
|
|
949
|
+
if (!notificationScheduleInstalled) {
|
|
950
|
+
notificationScheduleInstalled = true;
|
|
951
|
+
scheduleProgressNotifications([5]); // floor: at least one ping, matches the 5-10min tier
|
|
952
|
+
}
|
|
953
|
+
}
|
|
954
|
+
else if (compactionKillDeadline !== null) {
|
|
955
|
+
sessionLog.info('[Watchdog] Activity resumed after compaction, reverting to normal idle watchdog');
|
|
956
|
+
compactionKillDeadline = null;
|
|
957
|
+
}
|
|
519
958
|
// Extract metrics from result event
|
|
520
959
|
if (event.type === 'result') {
|
|
521
960
|
metrics = {
|
|
@@ -538,11 +977,32 @@ async function runClaudeCode(msg, sessionLog, cwd, sessionId, isFirstMessage = t
|
|
|
538
977
|
if (formatted) {
|
|
539
978
|
sessionLog.claude('stdout', formatted);
|
|
540
979
|
}
|
|
980
|
+
// teamvibe.ai#284: track the last non-empty plain-text assistant
|
|
981
|
+
// content seen, regardless of tool calls — candidate for the
|
|
982
|
+
// end-of-turn guard's fallback delivery.
|
|
983
|
+
const assistantText = extractAssistantText(event);
|
|
984
|
+
if (assistantText) {
|
|
985
|
+
lastPlainTextContent = assistantText;
|
|
986
|
+
}
|
|
541
987
|
// Detect when Claude sends a Slack message and clear typing indicator
|
|
542
|
-
if (
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
988
|
+
if (assistantCalledTool(event, 'mcp__slack__send_message')) {
|
|
989
|
+
sawSendMessageCall = true;
|
|
990
|
+
onMessageSent?.();
|
|
991
|
+
// tv#32 Layer B: the reply landed, no need for further "still
|
|
992
|
+
// working" pings for this run.
|
|
993
|
+
clearNotificationTimers();
|
|
994
|
+
}
|
|
995
|
+
// tv#32 Layer B: agent's own estimate of how long this task will
|
|
996
|
+
// take drives the progress-notification schedule (never the Layer A
|
|
997
|
+
// kill-deadline — see progressNotificationMarks doc comment).
|
|
998
|
+
if (event.type === 'tool_use' &&
|
|
999
|
+
event.name === 'mcp__slack__set_expected_duration' &&
|
|
1000
|
+
!notificationScheduleInstalled) {
|
|
1001
|
+
const minutes = Number(event.input?.minutes);
|
|
1002
|
+
if (Number.isFinite(minutes) && minutes > 0) {
|
|
1003
|
+
notificationScheduleInstalled = true;
|
|
1004
|
+
scheduleProgressNotifications(progressNotificationMarks(minutes));
|
|
1005
|
+
}
|
|
546
1006
|
}
|
|
547
1007
|
// Auto-update typing indicator status based on tool usage
|
|
548
1008
|
if (typingIndicator) {
|
|
@@ -589,8 +1049,10 @@ async function runClaudeCode(msg, sessionLog, cwd, sessionId, isFirstMessage = t
|
|
|
589
1049
|
channelId: msg.teamvibe.channelId,
|
|
590
1050
|
});
|
|
591
1051
|
killReasonByProc.set(proc, 'timeout');
|
|
592
|
-
|
|
593
|
-
|
|
1052
|
+
// #73: signal the whole process group (claude + its MCP children), not
|
|
1053
|
+
// just claude itself.
|
|
1054
|
+
killProcessGroup(proc, 'SIGTERM');
|
|
1055
|
+
setTimeout(() => killProcessGroup(proc, 'SIGKILL'), 5000);
|
|
594
1056
|
}, config.CLAUDE_TIMEOUT_MS);
|
|
595
1057
|
// Idle watchdog: kill if no stdout activity for too long
|
|
596
1058
|
let idleWarned = false;
|
|
@@ -607,21 +1069,32 @@ async function runClaudeCode(msg, sessionLog, cwd, sessionId, isFirstMessage = t
|
|
|
607
1069
|
channelId: msg.teamvibe.channelId,
|
|
608
1070
|
});
|
|
609
1071
|
killReasonByProc.set(proc, 'network');
|
|
610
|
-
|
|
611
|
-
|
|
1072
|
+
// #73: signal the whole process group, not just claude itself.
|
|
1073
|
+
killProcessGroup(proc, 'SIGTERM');
|
|
1074
|
+
setTimeout(() => killProcessGroup(proc, 'SIGKILL'), 5000);
|
|
612
1075
|
return;
|
|
613
1076
|
}
|
|
614
1077
|
}
|
|
615
|
-
|
|
1078
|
+
// tv#32: while a compaction deadline is set, kill on that fixed absolute
|
|
1079
|
+
// timestamp instead of the idle-from-last-activity window — compaction
|
|
1080
|
+
// legitimately produces no stdout for minutes.
|
|
1081
|
+
const isPastKillDeadline = compactionKillDeadline !== null
|
|
1082
|
+
? Date.now() >= compactionKillDeadline
|
|
1083
|
+
: idleMs > config.IDLE_WATCHDOG_KILL_MS;
|
|
1084
|
+
if (isPastKillDeadline) {
|
|
616
1085
|
const idleMin = Math.round(idleMs / 60000);
|
|
617
|
-
|
|
618
|
-
|
|
1086
|
+
const reason = compactionKillDeadline !== null
|
|
1087
|
+
? `idle for ${idleMin} minutes past the compaction ceiling`
|
|
1088
|
+
: `idle for ${idleMin} minutes`;
|
|
1089
|
+
logger.error(`Claude Code ${reason}, killing process`);
|
|
1090
|
+
emitError('session', `Claude Code ${reason} — killing hung process`, {
|
|
619
1091
|
context: { channelId: msg.teamvibe.channelId, threadId: msg.thread_id },
|
|
620
1092
|
channelId: msg.teamvibe.channelId,
|
|
621
1093
|
});
|
|
622
1094
|
killReasonByProc.set(proc, 'idle');
|
|
623
|
-
|
|
624
|
-
|
|
1095
|
+
// #73: signal the whole process group, not just claude itself.
|
|
1096
|
+
killProcessGroup(proc, 'SIGTERM');
|
|
1097
|
+
setTimeout(() => killProcessGroup(proc, 'SIGKILL'), 5000);
|
|
625
1098
|
}
|
|
626
1099
|
else if (!idleWarned && idleMs > config.IDLE_WATCHDOG_WARN_MS) {
|
|
627
1100
|
const idleMin = Math.round(idleMs / 60000);
|
|
@@ -637,11 +1110,33 @@ async function runClaudeCode(msg, sessionLog, cwd, sessionId, isFirstMessage = t
|
|
|
637
1110
|
idleWarned = true;
|
|
638
1111
|
}
|
|
639
1112
|
}, 30000); // Check every 30 seconds
|
|
640
|
-
proc.on('close', (code) => {
|
|
1113
|
+
proc.on('close', (code, signal) => {
|
|
641
1114
|
clearTimeout(timeout);
|
|
642
1115
|
clearInterval(idleWatchdog);
|
|
1116
|
+
clearNotificationTimers();
|
|
643
1117
|
activeProcesses.delete(msg.id);
|
|
644
1118
|
activeProcessesByThread.delete(msg.thread_id);
|
|
1119
|
+
lastSessionEndTs = Date.now();
|
|
1120
|
+
// #73 gap 2 (flagged independently by two reviews): `close` firing means
|
|
1121
|
+
// `claude` itself has exited, but its MCP server children — spawned
|
|
1122
|
+
// into the same detached process group, see `spawn()` above — are not
|
|
1123
|
+
// guaranteed to have exited with it (e.g. one that doesn't clean up on
|
|
1124
|
+
// stdin EOF). This is the *ordinary successful-completion* path, which
|
|
1125
|
+
// none of the timeout/idle/network/zombie kill paths above cover, and
|
|
1126
|
+
// production evidence (a RAM staircase that only resets on container
|
|
1127
|
+
// restart, i.e. grows on every session, not just forced-kill ones)
|
|
1128
|
+
// points at leaks happening here too. Always sweep, even when a kill
|
|
1129
|
+
// path above already ran for this proc — killProcessGroup() is a safe
|
|
1130
|
+
// ESRCH no-op once the group is actually already gone.
|
|
1131
|
+
// Round-2 review item 2: this is a hot path (every session close, not
|
|
1132
|
+
// just rare forced kills) — the ordinary case is `claude` exiting
|
|
1133
|
+
// cleanly and taking its MCP children with it, so the group is
|
|
1134
|
+
// already gone. sweepProcessGroup()'s isProcessGroupAlive() fast-path
|
|
1135
|
+
// skips both the (now-redundant) SIGTERM and the grace-period wait
|
|
1136
|
+
// entirely in that case instead of always paying CLOSE_SWEEP_GRACE_MS.
|
|
1137
|
+
// Fire-and-forget (unref'd) — this is the ordinary-completion path,
|
|
1138
|
+
// not something shutdown() awaits.
|
|
1139
|
+
void sweepProcessGroup(proc, CLOSE_SWEEP_GRACE_MS, { unref: true });
|
|
645
1140
|
const killReason = killReasonByProc.get(proc);
|
|
646
1141
|
const result = {
|
|
647
1142
|
success: code === 0,
|
|
@@ -649,6 +1144,8 @@ async function runClaudeCode(msg, sessionLog, cwd, sessionId, isFirstMessage = t
|
|
|
649
1144
|
exitCode: code,
|
|
650
1145
|
metrics,
|
|
651
1146
|
modelSource,
|
|
1147
|
+
sawSendMessageCall,
|
|
1148
|
+
lastPlainTextContent,
|
|
652
1149
|
...(killReason ? { killedByPoller: killReason } : {}),
|
|
653
1150
|
};
|
|
654
1151
|
if (killReason === 'timeout') {
|
|
@@ -664,7 +1161,16 @@ async function runClaudeCode(msg, sessionLog, cwd, sessionId, isFirstMessage = t
|
|
|
664
1161
|
result.error = 'Killed as zombie process (stale session lock)';
|
|
665
1162
|
}
|
|
666
1163
|
else if (code !== 0) {
|
|
667
|
-
|
|
1164
|
+
// `signal` matters here specifically: a process terminated by a
|
|
1165
|
+
// signal reports `code: null`, not a numeric exit code — e.g. a
|
|
1166
|
+
// cgroup OOM killer's SIGKILL. Before this PR that collapsed into
|
|
1167
|
+
// "Process exited with code null", indistinguishable in logs from
|
|
1168
|
+
// any other unsignaled null-code exit. Now that NODE_OPTIONS no
|
|
1169
|
+
// longer leaks the poller's tiny heap cap into `claude` (see
|
|
1170
|
+
// buildSpawnEnv above), a future OOM is more likely to surface this
|
|
1171
|
+
// way — a cgroup-level SIGKILL — rather than as a V8-level FATAL
|
|
1172
|
+
// ERROR visible in stderr, so this is the replacement visibility.
|
|
1173
|
+
result.error = stderr || (signal ? `Process was killed by signal ${signal}` : `Process exited with code ${code}`);
|
|
668
1174
|
}
|
|
669
1175
|
resolve(result);
|
|
670
1176
|
});
|
|
@@ -673,6 +1179,7 @@ async function runClaudeCode(msg, sessionLog, cwd, sessionId, isFirstMessage = t
|
|
|
673
1179
|
clearInterval(idleWatchdog);
|
|
674
1180
|
activeProcesses.delete(msg.id);
|
|
675
1181
|
activeProcessesByThread.delete(msg.thread_id);
|
|
1182
|
+
lastSessionEndTs = Date.now();
|
|
676
1183
|
emitError('session', `Claude Code process spawn error: ${error.message}`, {
|
|
677
1184
|
channelId: msg.teamvibe.channelId,
|
|
678
1185
|
context: { threadId: msg.thread_id },
|
|
@@ -682,6 +1189,8 @@ async function runClaudeCode(msg, sessionLog, cwd, sessionId, isFirstMessage = t
|
|
|
682
1189
|
output: stdout,
|
|
683
1190
|
exitCode: null,
|
|
684
1191
|
error: error.message,
|
|
1192
|
+
sawSendMessageCall,
|
|
1193
|
+
lastPlainTextContent,
|
|
685
1194
|
});
|
|
686
1195
|
});
|
|
687
1196
|
});
|
|
@@ -694,6 +1203,25 @@ const ISOLATED_SOURCES = new Set(['cron', 'maintenance']);
|
|
|
694
1203
|
function shouldIsolateInWorktree(msg) {
|
|
695
1204
|
return isWorktreeIsolationEnabled() && ISOLATED_SOURCES.has(msg.source);
|
|
696
1205
|
}
|
|
1206
|
+
/**
|
|
1207
|
+
* Decides whether a spawn attempt should report its session ID back to the
|
|
1208
|
+
* caller (poller.ts) for persistence to DynamoDB via updateSessionId().
|
|
1209
|
+
*
|
|
1210
|
+
* First-message spawns always mint a fresh session ID locally (see
|
|
1211
|
+
* spawnClaudeCode below, teamvibe.ai#77) instead of recycling whatever
|
|
1212
|
+
* placeholder DynamoDB currently holds — so on ANY successful first-message
|
|
1213
|
+
* spawn (not just the "already in use" collision-retry path), the ID actually
|
|
1214
|
+
* used by Claude Code must be surfaced here. Otherwise DynamoDB keeps the
|
|
1215
|
+
* createSession() placeholder forever and the thread's next message does
|
|
1216
|
+
* `--resume <id-claude-code-never-used>`, which fails 100% of the time.
|
|
1217
|
+
* (teamvibe.ai#77, DevGuru review round)
|
|
1218
|
+
*/
|
|
1219
|
+
export function finalizeSpawnResult(retryResult, isFirstMessage, effectiveSessionId) {
|
|
1220
|
+
if (isFirstMessage && retryResult.success && effectiveSessionId) {
|
|
1221
|
+
return { ...retryResult, newSessionId: effectiveSessionId };
|
|
1222
|
+
}
|
|
1223
|
+
return retryResult;
|
|
1224
|
+
}
|
|
697
1225
|
export async function spawnClaudeCode(msg, sessionLog, cwd, sessionId, isFirstMessage = true, lastMessageTs, onMessageSent, typingIndicator) {
|
|
698
1226
|
// Worktree isolation: spawned (non-foreground) sessions run in
|
|
699
1227
|
// <brain>/.worktrees/<thread_id>/ so the sync-watchdog's reset --hard on
|
|
@@ -712,29 +1240,32 @@ export async function spawnClaudeCode(msg, sessionLog, cwd, sessionId, isFirstMe
|
|
|
712
1240
|
}
|
|
713
1241
|
}
|
|
714
1242
|
try {
|
|
715
|
-
|
|
1243
|
+
// First-message spawns always mint a fresh session ID rather than
|
|
1244
|
+
// recycling whatever placeholder DynamoDB currently holds for this
|
|
1245
|
+
// thread (teamvibe.ai#77) — --resume on non-first messages is unchanged.
|
|
1246
|
+
let effectiveSessionId = sessionId;
|
|
1247
|
+
if (isFirstMessage) {
|
|
1248
|
+
const { randomUUID } = await import('crypto');
|
|
1249
|
+
effectiveSessionId = randomUUID();
|
|
1250
|
+
}
|
|
1251
|
+
const result = await runClaudeCode(msg, sessionLog, effectiveCwd, cwd, effectiveSessionId, isFirstMessage, lastMessageTs, onMessageSent, typingIndicator);
|
|
716
1252
|
// If --resume failed, retry as a fresh session (session files may have been lost on container restart)
|
|
717
1253
|
let retryResult = result;
|
|
718
|
-
if (!result.success &&
|
|
1254
|
+
if (!result.success && effectiveSessionId && !isFirstMessage) {
|
|
719
1255
|
sessionLog.info('Resume failed, retrying as fresh session (session files may have been lost)');
|
|
720
|
-
retryResult = await runClaudeCode(msg, sessionLog, effectiveCwd,
|
|
1256
|
+
retryResult = await runClaudeCode(msg, sessionLog, effectiveCwd, cwd, effectiveSessionId, true, lastMessageTs, onMessageSent, typingIndicator);
|
|
721
1257
|
}
|
|
722
|
-
// If session ID is "already in use" (stale lock file),
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
if (killed) {
|
|
727
|
-
sessionLog.info('Killed zombie process before fresh session retry');
|
|
728
|
-
// Give the process a moment to fully exit and release lock files
|
|
729
|
-
await new Promise((resolve) => setTimeout(resolve, 2000));
|
|
730
|
-
}
|
|
1258
|
+
// If session ID is "already in use" (stale lock file), retry immediately with a
|
|
1259
|
+
// fresh ID — no zombie kill first. Zombie cleanup is #73's concern, not spawn
|
|
1260
|
+
// retry's (teamvibe.ai#77 acceptance criteria: "no zombie kill needed first").
|
|
1261
|
+
if (!retryResult.success && effectiveSessionId && retryResult.error?.includes('already in use')) {
|
|
731
1262
|
sessionLog.info('Session ID already in use (stale lock), retrying with fresh session ID');
|
|
732
1263
|
const { randomUUID } = await import('crypto');
|
|
733
1264
|
const freshId = randomUUID();
|
|
734
|
-
const freshResult = await runClaudeCode(msg, sessionLog, effectiveCwd, freshId, true, lastMessageTs, onMessageSent, typingIndicator);
|
|
735
|
-
return
|
|
1265
|
+
const freshResult = await runClaudeCode(msg, sessionLog, effectiveCwd, cwd, freshId, true, lastMessageTs, onMessageSent, typingIndicator);
|
|
1266
|
+
return finalizeSpawnResult(freshResult, true, freshId);
|
|
736
1267
|
}
|
|
737
|
-
return retryResult;
|
|
1268
|
+
return finalizeSpawnResult(retryResult, isFirstMessage, effectiveSessionId);
|
|
738
1269
|
}
|
|
739
1270
|
finally {
|
|
740
1271
|
if (worktreeOwner) {
|
|
@@ -746,9 +1277,17 @@ export async function spawnClaudeCode(msg, sessionLog, cwd, sessionId, isFirstMe
|
|
|
746
1277
|
}
|
|
747
1278
|
const activeProcesses = new Map();
|
|
748
1279
|
const activeProcessesByThread = new Map();
|
|
1280
|
+
// tv#317: when the last `claude` process ended, so the idle-restart gate can require
|
|
1281
|
+
// a quiet period since real work last happened. Starts at module load (poller boot) so
|
|
1282
|
+
// a poller that never runs a session still becomes eligible once IDLE_RESTART_MIN_UPTIME_MS
|
|
1283
|
+
// passes, rather than being stuck at 0/undefined forever.
|
|
1284
|
+
let lastSessionEndTs = Date.now();
|
|
749
1285
|
export function getActiveProcessCount() {
|
|
750
1286
|
return activeProcesses.size;
|
|
751
1287
|
}
|
|
1288
|
+
export function getLastSessionEndTs() {
|
|
1289
|
+
return lastSessionEndTs;
|
|
1290
|
+
}
|
|
752
1291
|
/**
|
|
753
1292
|
* Kill a zombie process for a given thread ID.
|
|
754
1293
|
* Returns true if a process was found and killed.
|
|
@@ -762,12 +1301,14 @@ export function killZombieForThread(threadId) {
|
|
|
762
1301
|
logger.warn(`Killing zombie process for thread ${threadId} (pid: ${proc.pid})`);
|
|
763
1302
|
emitError('session', `Killing zombie process for thread ${threadId}`, { severity: 'warn' });
|
|
764
1303
|
killReasonByProc.set(proc, 'zombie');
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
1304
|
+
// #73: signal the whole process group, not just claude itself. The
|
|
1305
|
+
// follow-up SIGKILL sweep always fires after the grace period — it is
|
|
1306
|
+
// NOT gated on `proc.exitCode` here, because `claude` itself can look
|
|
1307
|
+
// exited while its MCP server children are still alive in the same
|
|
1308
|
+
// group (killProcessGroup() is a safe ESRCH no-op if the group is
|
|
1309
|
+
// actually already fully gone by then).
|
|
1310
|
+
killProcessGroup(proc, 'SIGTERM');
|
|
1311
|
+
setTimeout(() => killProcessGroup(proc, 'SIGKILL'), 5000);
|
|
771
1312
|
return true;
|
|
772
1313
|
}
|
|
773
1314
|
export function isAtCapacity() {
|
|
@@ -777,23 +1318,87 @@ export function killAllActiveProcesses() {
|
|
|
777
1318
|
if (activeProcesses.size === 0)
|
|
778
1319
|
return Promise.resolve();
|
|
779
1320
|
logger.info(`Killing ${activeProcesses.size} active Claude process(es)...`);
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
1321
|
+
// DevGuru PR#308 round-1 review (non-blocking item 2): both branches below
|
|
1322
|
+
// used to `resolve()` as soon as `claude` itself was observed dead
|
|
1323
|
+
// (immediately, or on `close`), WITHOUT waiting for the scheduled
|
|
1324
|
+
// group-SIGKILL sweep — so the comments here promised a cleanup guarantee
|
|
1325
|
+
// ("leaving any of them running is exactly the leak #73 tracks") that the
|
|
1326
|
+
// actual control flow didn't provide: poller.ts's `shutdown()` typically
|
|
1327
|
+
// reaches `releaseAllActiveLocks()` → drain loop → `process.exit()` in
|
|
1328
|
+
// well under the 5s sweep grace window, so the sweep's own timer would
|
|
1329
|
+
// often die with the process before it ever fired. Fixed by actually
|
|
1330
|
+
// AWAITING the sweep (resolving only once it has run) instead of loosening
|
|
1331
|
+
// the comment to "best-effort" — `shutdown()` already has slack for this:
|
|
1332
|
+
// a 30s drain-loop timeout on the normal path, and a 15s hard cap even on
|
|
1333
|
+
// the circuit-breaker path (poller.ts's `Promise.race([shutdown(...),
|
|
1334
|
+
// sleep(15000)])`), both comfortably above this ~5s, run-once-in-parallel
|
|
1335
|
+
// (not per-process) addition.
|
|
1336
|
+
const killPromises = Array.from(activeProcesses.entries()).map(([id, proc]) => {
|
|
1337
|
+
// The retry-classification fix in poller.ts (round-2 review item 1)
|
|
1338
|
+
// reads this to distinguish an in-flight iteration killed by poller
|
|
1339
|
+
// shutdown from a genuine application failure — without it, a message
|
|
1340
|
+
// caught mid-flight by shutdown could get counted toward MAX_RETRIES
|
|
1341
|
+
// and eventually dropped instead of retried.
|
|
1342
|
+
//
|
|
1343
|
+
// Round-3 review item 3: only set 'shutdown' if no reason is already
|
|
1344
|
+
// recorded for this process. A process can already be marked with a
|
|
1345
|
+
// more specific reason (e.g. 'timeout') whose `close` event just
|
|
1346
|
+
// hasn't fired yet when shutdown begins — unconditionally overwriting
|
|
1347
|
+
// it here would replace the specific reason with the generic one,
|
|
1348
|
+
// surfacing a "Poller is restarting" notice instead of the more
|
|
1349
|
+
// accurate (and deliberately legacy-shaped) timeout message.
|
|
1350
|
+
if (!killReasonByProc.has(proc))
|
|
1351
|
+
killReasonByProc.set(proc, 'shutdown');
|
|
1352
|
+
// Round-2 review item 2: sweepProcessGroup()'s isProcessGroupAlive()
|
|
1353
|
+
// fast-path resolves immediately when the group is already provably
|
|
1354
|
+
// empty, instead of always paying the full 5s grace-period wait — this
|
|
1355
|
+
// matters here specifically because shutdown() (poller.ts) awaits this
|
|
1356
|
+
// whole function BEFORE releaseAllActiveLocks(), and delaying that
|
|
1357
|
+
// costs the next message on the same thread a 15-minute stale-lock
|
|
1358
|
+
// wait. This promise still resolves only once a real sweep has
|
|
1359
|
+
// actually run for a genuinely-still-alive group — not merely once
|
|
1360
|
+
// `claude` itself closes — so the caller genuinely waits out the grace
|
|
1361
|
+
// window instead of racing it. `killProcessGroup()` is a safe ESRCH
|
|
1362
|
+
// no-op if the group is actually already fully gone by the time a
|
|
1363
|
+
// scheduled SIGKILL fires.
|
|
1364
|
+
//
|
|
1365
|
+
// `proc.exitCode !== null` (claude itself already exited before
|
|
1366
|
+
// shutdown started) only suppresses the "didn't fully exit" warning
|
|
1367
|
+
// below — per #73, that alone does NOT mean its MCP server children
|
|
1368
|
+
// (same detached process group) are gone too, so the sweep (and its
|
|
1369
|
+
// fast-path probe) still runs unconditionally either way.
|
|
1370
|
+
const claudeAlreadyExited = proc.exitCode !== null;
|
|
1371
|
+
return sweepProcessGroup(proc, 5000, {
|
|
1372
|
+
onGraceExpired: claudeAlreadyExited
|
|
1373
|
+
? undefined
|
|
1374
|
+
: () => {
|
|
1375
|
+
logger.warn(`Process ${id} group did not fully exit after SIGTERM, sending group SIGKILL`);
|
|
1376
|
+
emitError('session', `Process ${id} group did not fully exit after SIGTERM, sending group SIGKILL`, { severity: 'warn' });
|
|
1377
|
+
},
|
|
793
1378
|
});
|
|
794
|
-
|
|
795
|
-
}));
|
|
1379
|
+
});
|
|
796
1380
|
return Promise.all(killPromises).then(() => {
|
|
797
1381
|
logger.info('All active processes terminated');
|
|
798
1382
|
});
|
|
799
1383
|
}
|
|
1384
|
+
// Exported for testing only (PR#308 round-3 review item 3). `activeProcesses`
|
|
1385
|
+
// and `killReasonByProc` have no other external mutators/readers — production
|
|
1386
|
+
// code only ever populates them via the real spawnClaudeCode()/kill-path
|
|
1387
|
+
// call sites above. These let a test put a fake process into
|
|
1388
|
+
// killAllActiveProcesses()'s input set with a reason already recorded
|
|
1389
|
+
// (mirroring the exact race DevGuru described: a process killed for a more
|
|
1390
|
+
// specific reason whose `close` handler hasn't run yet when shutdown starts)
|
|
1391
|
+
// and then read the reason back afterward, without invoking any real spawn.
|
|
1392
|
+
// Same pattern as poller.ts's __setShuttingDownForTest.
|
|
1393
|
+
export function __setActiveProcessForTest(id, proc) {
|
|
1394
|
+
activeProcesses.set(id, proc);
|
|
1395
|
+
}
|
|
1396
|
+
export function __clearActiveProcessesForTest() {
|
|
1397
|
+
activeProcesses.clear();
|
|
1398
|
+
}
|
|
1399
|
+
export function __setKillReasonForTest(proc, reason) {
|
|
1400
|
+
killReasonByProc.set(proc, reason);
|
|
1401
|
+
}
|
|
1402
|
+
export function __getKillReasonForTest(proc) {
|
|
1403
|
+
return killReasonByProc.get(proc);
|
|
1404
|
+
}
|