@teamvibe/poller 0.7.0 → 0.7.2

package/dist/idle-restart.js

@@ -0,0 +1,200 @@
+import { appendFileSync, readdirSync, readFileSync } from 'fs';
+import { join } from 'path';
+import { config } from './config.js';
+import { logger } from './logger.js';
+import { getActiveProcessCount, getLastSessionEndTs } from './claude-spawner.js';
+import { emitEvent } from './event-queue.js';
+// tv#317: self-restart once genuinely idle, to let Railway reclaim page-cache RAM that
+// Linux never proactively frees under our large memory limit. See config.ts for the
+// IDLE_RESTART_* thresholds this checks against.
+const processStartTime = Date.now();
+// cgroup v2 only. Railway containers run cgroup v2; on anything else (local dev, a
+// future cgroup v1 host) this just returns null and the gate never fires — the memory
+// pressure this exists to relieve is Railway-specific, so failing closed is correct
+// rather than guessing at a v1 path/format.
+const MEMORY_STAT_PATH = '/sys/fs/cgroup/memory.stat';
+export function readReclaimableFileBytes(path = MEMORY_STAT_PATH) {
+    let text;
+    try {
+        text = readFileSync(path, 'utf-8');
+    }
+    catch {
+        return null;
+    }
+    for (const line of text.split('\n')) {
+        const spaceIdx = line.indexOf(' ');
+        if (spaceIdx > 0 && line.slice(0, spaceIdx) === 'file') {
+            const value = Number(line.slice(spaceIdx + 1));
+            return Number.isFinite(value) ? value : null;
+        }
+    }
+    return null;
+}
+// --- bg-task liveness ---------------------------------------------------------------
+//
+// Reimplemented (not imported): bg-task.mjs lives in poller-brain and runs inside a
+// Claude session, not the poller process — there's no shared module to import here.
+// This mirrors its parseStatus/lifecycleOf classification (poller-brain
+// skills/background-task/scripts/bg-task.mjs) rather than a raw /proc scan, which
+// false-negatives on interpreter/path variations and can't distinguish "stuck past TTL"
+// from "healthy" the way the status file's own started/ttl fields can.
+const ABANDON_GRACE_SEC = 60; // mirrors bg-task.mjs's ABANDON_GRACE_SEC
+function parseStatus(text) {
+    const kv = {};
+    for (const line of text.split('\n')) {
+        const i = line.indexOf('=');
+        if (i > 0)
+            kv[line.slice(0, i)] = line.slice(i + 1);
+    }
+    return kv;
+}
+function pidAlive(pid) {
+    try {
+        process.kill(pid, 0);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+// Same four-way classification as bg-task.mjs's lifecycleOf, minus the tests-only
+// `alive` injection point (this module has its own dedicated tests instead).
+function lifecycleOf(kv, now) {
+    // Trust the runner's own claim, whatever it says (bg-task.mjs:139) — not just the two
+    // values the runner writes today (`finished`/`timed-out`, bg-task-runner.mjs:403).
+    // Mirroring the check itself, not just its current output, means the two
+    // implementations of this format can't quietly diverge if the runner ever starts
+    // writing a state value this list doesn't happen to enumerate — hardcoding a closed
+    // set here would silently fall through to the pid/TTL heuristics below instead.
+    if (kv.state)
+        return kv.state;
+    if (kv.runner_crashed)
+        return 'crashed';
+    if (Object.keys(kv).length === 0)
+        return 'unknown';
+    const pid = Number(kv.pid);
+    if (!Number.isInteger(pid) || pid <= 0 || !pidAlive(pid))
+        return 'abandoned';
+    const started = kv.started ? Date.parse(kv.started) : NaN;
+    const ttl = Number(String(kv.ttl ?? '').replace(/s$/, ''));
+    if (Number.isFinite(started) && Number.isFinite(ttl) && now - started > (ttl + ABANDON_GRACE_SEC) * 1000) {
+        return 'abandoned';
+    }
+    return 'running';
+}
+function bgTaskRoot() {
+    // Mirrors bg-task.mjs's taskRoot() default exactly, minus the per-channel
+    // namespace — the poller checks liveness across every channel it serves, not just
+    // one, so it walks the parent directory instead of joining a single channel ID.
+    return process.env['BG_TASK_ROOT'] || join(config.PERSISTENT_STORAGE_PATH || '/tmp', 'bg-tasks');
+}
+function listDirs(path) {
+    try {
+        return readdirSync(path, { withFileTypes: true })
+            .filter((entry) => entry.isDirectory())
+            .map((entry) => entry.name);
+    }
+    catch {
+        return [];
+    }
+}
+// Returns a "channel/taskId" label for the first live (running or unknown-state) task
+// found, or null if none. `unknown` (no status written yet) is treated as live —
+// deliberately conservative, per tv#317: a task a few milliseconds from its first
+// status line looks identical to an orphaned dir, and "skip this idle cycle" is cheap
+// while restarting mid-launch is not recoverable.
+export function findLiveBgTask(now = Date.now()) {
+    const root = bgTaskRoot();
+    for (const channel of listDirs(root)) {
+        const channelPath = join(root, channel);
+        for (const taskDir of listDirs(channelPath)) {
+            let statusText = '';
+            try {
+                statusText = readFileSync(join(channelPath, taskDir, 'status'), 'utf-8');
+            }
+            catch {
+                // No status file at all — same "unknown" treatment as an empty one below.
+            }
+            const state = lifecycleOf(parseStatus(statusText), now);
+            if (state === 'running' || state === 'unknown') {
+                return `${channel}/${taskDir}`;
+            }
+        }
+    }
+    return null;
+}
+// tv#317 "Startup reconciliation": bg-tasks don't survive a poller restart (their
+// runner is a child of the container that just went away), but a status file on a
+// persistent volume has no way to know that on its own. Without this, a task dir
+// whose runner never wrote a terminal line looks identical to a live one — worse,
+// after a restart the fresh container's own low PIDs can coincidentally collide with
+// a stale recorded `pid=`, so pidAlive() alone isn't reliable evidence either. Called
+// once at boot so every pre-restart dir is unambiguously stamped before the idle-restart
+// gate (or `--list`) ever reads it again.
+export function reconcileBgTasksOnBoot() {
+    const root = bgTaskRoot();
+    let stamped = 0;
+    for (const channel of listDirs(root)) {
+        const channelPath = join(root, channel);
+        for (const taskDir of listDirs(channelPath)) {
+            const statusPath = join(channelPath, taskDir, 'status');
+            let statusText;
+            try {
+                statusText = readFileSync(statusPath, 'utf-8');
+            }
+            catch {
+                continue; // no status file yet — nothing to stamp
+            }
+            const kv = parseStatus(statusText);
+            if (kv.state || kv.runner_crashed)
+                continue; // already terminal
+            try {
+                appendFileSync(statusPath, 'runner_crashed=container_restart\n');
+                stamped++;
+            }
+            catch (err) {
+                logger.warn(`[IdleRestart] Failed to stamp stale bg-task ${channel}/${taskDir}: ${err instanceof Error ? err.message : err}`);
+            }
+        }
+    }
+    if (stamped > 0) {
+        logger.info(`[IdleRestart] Reconciled ${stamped} bg-task dir(s) left over from before this boot`);
+    }
+}
+export function checkIdleRestart(deps = {}) {
+    const now = deps.now ?? Date.now();
+    const startTime = deps.startTime ?? processStartTime;
+    const activeProcessCount = deps.activeProcessCount ?? getActiveProcessCount;
+    const lastSessionEndTs = deps.lastSessionEndTs ?? getLastSessionEndTs;
+    const reclaimableBytesFn = deps.reclaimableBytes ?? readReclaimableFileBytes;
+    const liveBgTask = deps.liveBgTask ?? findLiveBgTask;
+    if (!config.IDLE_RESTART_ENABLED)
+        return false;
+    const uptimeMs = now - startTime;
+    if (uptimeMs < config.IDLE_RESTART_MIN_UPTIME_MS)
+        return false;
+    if (activeProcessCount() > 0)
+        return false;
+    const idleMs = now - lastSessionEndTs();
+    if (idleMs < config.IDLE_RESTART_IDLE_MS)
+        return false;
+    const reclaimableBytes = reclaimableBytesFn();
+    if (reclaimableBytes === null || reclaimableBytes < config.IDLE_RESTART_MEMORY_FILE_THRESHOLD_BYTES)
+        return false;
+    if (liveBgTask())
+        return false; // skip this cycle, recheck next time — never cut a bg-task short just to free memory sooner
+    // Re-verify the two liveness conditions right before signaling restart — a
+    // session or bg-task could have started in the time it took to reach this point.
+    if (activeProcessCount() > 0)
+        return false;
+    if (liveBgTask())
+        return false;
+    logger.info(`[IdleRestart] Idle ${Math.round(idleMs / 60000)}min, reclaimable page cache ` +
+        `${Math.round(reclaimableBytes / 1024 / 1024)}MB — restarting to let Railway reclaim it`);
+    emitEvent('poller.idle_restart', 'system', {
+        idleMs,
+        uptimeMs,
+        reclaimableBytes,
+    });
+    return true;
+}

package/dist/maintenance-cron.d.ts

@@ -21,6 +21,72 @@ export declare function getTrackedBrains(): ReadonlyMap<string, TrackedBrainInfo
  * @internal
  */
 export declare function __resetTrackedBrainsForTest(): void;
+/**
+ * Shape returned by GET /poller/brains (teamvibe.ai#127 Option B) — the
+ * fields registerBrain() needs to build a synthetic maintenance message,
+ * minus `lastRealMessageAt` (see seedTrackedBrains below for why that's
+ * deliberately not part of the bootstrap payload).
+ */
+export interface BootstrapBrainInfo {
+    brainId: string;
+    gitRepoUrl: string;
+    branch: string;
+    workspaceId: string;
+    botId: string;
+    botToken: string;
+    channelId: string;
+}
+/**
+ * Seed trackedBrains at startup from the backend's authoritative brain list
+ * (teamvibe.ai#127) so a "quiet" brain — one that hasn't received a real
+ * message since this poller process started — still gets included in the
+ * maintenance/heartbeat cron, instead of being silently invisible until a
+ * human happens to message it.
+ *
+ * Deliberately does NOT set `lastRealMessageAt` (left undefined). That
+ * field specifically means "we've seen a real message since our own last
+ * restart" — shouldSkipMaintenance (maintenance-activity.ts, #131) treats
+ * `undefined` as fail-open (never skip) for exactly this reason. Bootstrap
+ * seeding isn't a real message; claiming one occurred here would risk the
+ * opposite bug — a genuinely-quiet brain wrongly evaluated as "nothing new,
+ * skip" on its very first post-restart cron tick.
+ *
+ * Skips (rather than overwrites) any brainId already present in
+ * trackedBrains. This guard is load-bearing, not just defensive (DevGuru
+ * review, round 4): since round 3's critical-path detachment
+ * (brains-bootstrap.ts's `void (async () => {...})()` retry), the retry
+ * branch can finish AFTER startPoller() has already returned and real user
+ * messages have started arriving. That means a real registerBrain() call
+ * for a brain (which sets `lastRealMessageAt`) can race ahead of — and
+ * complete before — the detached bootstrap retry's seedTrackedBrains()
+ * call for that same brain. Without this skip, the late-arriving bootstrap
+ * seed would silently overwrite the real registration back to a fresh
+ * entry with `lastRealMessageAt: undefined`, losing real activity data
+ * (and, per the fail-open contract above, artificially reopening the
+ * skip-maintenance window for a brain that's actually been active).
+ *
+ * DOES also register each seeded brain for hourly metrics (see
+ * `registerBrainForMetrics` below) — DevGuru review caught that this
+ * module's original "metrics has its own disk-walk path" reasoning was
+ * wrong: `metrics/scheduler.ts`'s `registered` Map has exactly one
+ * production caller (`registerBrain()` below), with no disk-walk discovery
+ * of its own. Disk reads only happen in `collector.ts`, and only for a
+ * brain already present in that Map. Skipping the metrics seed here would
+ * reproduce the exact same restart-amnesia bug for metrics that this whole
+ * change exists to fix for maintenance: a quiet brain would get maintenance
+ * triggers again, but zero metrics until someone messages it.
+ *
+ * Deliberately does NOT replicate `registerBrain()`'s *immediate*
+ * collect+send side effect (the `if (isNew) { void (async () => ... })()`
+ * block below) — only the Map insert. Firing an immediate collect+send for
+ * every brain this poller owns, all at once, on every restart would be a
+ * fleet-wide burst; the existing hourly cron with its per-brain jitter
+ * (`collectAndSendAll`, `jitterMsFor`) already picks up newly-registered
+ * brains on its own schedule without a startup spike.
+ *
+ * Returns the number of brains actually seeded (for startup logging).
+ */
+export declare function seedTrackedBrains(brains: readonly BootstrapBrainInfo[]): number;
 /**
  * Register a brain from a real message so the maintenance cron knows about it.
  * Called from processMessage() after brain path is resolved.

package/dist/maintenance-cron.js

@@ -27,6 +27,75 @@ export function __resetTrackedBrainsForTest() {
     trackedBrains.clear();
 }
 let maintenanceJob = null;
+/**
+ * Seed trackedBrains at startup from the backend's authoritative brain list
+ * (teamvibe.ai#127) so a "quiet" brain — one that hasn't received a real
+ * message since this poller process started — still gets included in the
+ * maintenance/heartbeat cron, instead of being silently invisible until a
+ * human happens to message it.
+ *
+ * Deliberately does NOT set `lastRealMessageAt` (left undefined). That
+ * field specifically means "we've seen a real message since our own last
+ * restart" — shouldSkipMaintenance (maintenance-activity.ts, #131) treats
+ * `undefined` as fail-open (never skip) for exactly this reason. Bootstrap
+ * seeding isn't a real message; claiming one occurred here would risk the
+ * opposite bug — a genuinely-quiet brain wrongly evaluated as "nothing new,
+ * skip" on its very first post-restart cron tick.
+ *
+ * Skips (rather than overwrites) any brainId already present in
+ * trackedBrains. This guard is load-bearing, not just defensive (DevGuru
+ * review, round 4): since round 3's critical-path detachment
+ * (brains-bootstrap.ts's `void (async () => {...})()` retry), the retry
+ * branch can finish AFTER startPoller() has already returned and real user
+ * messages have started arriving. That means a real registerBrain() call
+ * for a brain (which sets `lastRealMessageAt`) can race ahead of — and
+ * complete before — the detached bootstrap retry's seedTrackedBrains()
+ * call for that same brain. Without this skip, the late-arriving bootstrap
+ * seed would silently overwrite the real registration back to a fresh
+ * entry with `lastRealMessageAt: undefined`, losing real activity data
+ * (and, per the fail-open contract above, artificially reopening the
+ * skip-maintenance window for a brain that's actually been active).
+ *
+ * DOES also register each seeded brain for hourly metrics (see
+ * `registerBrainForMetrics` below) — DevGuru review caught that this
+ * module's original "metrics has its own disk-walk path" reasoning was
+ * wrong: `metrics/scheduler.ts`'s `registered` Map has exactly one
+ * production caller (`registerBrain()` below), with no disk-walk discovery
+ * of its own. Disk reads only happen in `collector.ts`, and only for a
+ * brain already present in that Map. Skipping the metrics seed here would
+ * reproduce the exact same restart-amnesia bug for metrics that this whole
+ * change exists to fix for maintenance: a quiet brain would get maintenance
+ * triggers again, but zero metrics until someone messages it.
+ *
+ * Deliberately does NOT replicate `registerBrain()`'s *immediate*
+ * collect+send side effect (the `if (isNew) { void (async () => ... })()`
+ * block below) — only the Map insert. Firing an immediate collect+send for
+ * every brain this poller owns, all at once, on every restart would be a
+ * fleet-wide burst; the existing hourly cron with its per-brain jitter
+ * (`collectAndSendAll`, `jitterMsFor`) already picks up newly-registered
+ * brains on its own schedule without a startup spike.
+ *
+ * Returns the number of brains actually seeded (for startup logging).
+ */
+export function seedTrackedBrains(brains) {
+    let seeded = 0;
+    for (const b of brains) {
+        if (trackedBrains.has(b.brainId))
+            continue;
+        trackedBrains.set(b.brainId, {
+            brainId: b.brainId,
+            gitRepoUrl: b.gitRepoUrl,
+            branch: b.branch,
+            workspaceId: b.workspaceId,
+            botId: b.botId,
+            botToken: b.botToken,
+            channelId: b.channelId,
+        });
+        registerBrainForMetrics(b.brainId);
+        seeded++;
+    }
+    return seeded;
+}
 /**
  * Register a brain from a real message so the maintenance cron knows about it.
  * Called from processMessage() after brain path is resolved.

package/dist/poller.d.ts

@@ -29,8 +29,11 @@ export declare function isPollerInitiatedFailure(result: {
     exitCode: number | null;
     killedByPoller?: KilledByPoller;
 }): boolean;
+export declare function describeFailureReason(killedByPoller: KilledByPoller | undefined, exitCode: number | null): string;
+export declare const END_OF_TURN_NO_FURTHER_RESPONSE_SENTINEL = "NO_FURTHER_RESPONSE_NEEDED";
 export declare function endOfTurnFallbackText(hasSlackContext: boolean, result: Pick<SpawnResult, 'success' | 'sawSendMessageCall' | 'lastPlainTextContent'>): string | undefined;
 export declare function runInboxLoop(threadId: string, kbPath: string, queueMessage: TeamVibeQueueMessage, sessionLog: ReturnType<typeof logger.createSession>): Promise<void>;
+export declare function __setSleepForTest(impl: (ms: number) => Promise<void>): void;
 export declare function __setShuttingDownForTest(value: boolean): void;
 export declare function startPoller(): Promise<void>;
 export {};