@sleep2agi/commhub-server 0.8.8 → 0.9.0-preview.1

package/src/retention.ts

@@ -0,0 +1,269 @@
+// Round-2/4 review ② — retention sweep + incremental VACUUM.
+//
+// Background maintenance for the commhub database. Multi-tenant
+// public-facing hubs (e.g. <hub-domain>) write to high-frequency tables
+// on every heartbeat (agent_telemetry, task_events, completions) — left
+// unswept these grow unbounded, slow down indexed scans, and bloat the
+// on-disk footprint. SQLite never reclaims freed pages without VACUUM,
+// so a row delete by itself doesn't shrink the file.
+//
+// This module exposes:
+//   - sweepRetention(cfg): one-shot sweep across high-growth tables
+//   - startRetentionSweeper(): wires a 1-hour setInterval, fires once
+//     at startup with an immediate (non-blocking) sweep
+//
+// Defaults are conservative — long-tail debugging traces survive a
+// week. Operators can override per env (COMMHUB_RETENTION_*_DAYS) for
+// hot hubs.
+//
+// Run isolated tests against retention.test.ts:
+//   COMMHUB_DB=/tmp/test-retention.db bun test src/retention.test.ts
+
+import { db } from "./db.js";
+
+export type RetentionConfig = {
+  // High-frequency telemetry; default = 7 days.
+  // Heartbeat is ~30s → 50 agents × 86400 s/day ÷ 30 s = 144000 rows/day.
+  // At 7d that's ~1M rows. Indexed scans stay fast; raw disk = a few MB.
+  telemetryDays: number;
+  // Task state-change audit; default = 30 days.
+  // One row per status transition. Useful for incident forensics.
+  taskEventsDays: number;
+  // Acked inbox; default = 7 days post-ack.
+  // Once acked the row is no longer hot — keep a week for debugging.
+  // Unacked rows are NEVER swept (could be in-flight delivery).
+  ackedInboxDays: number;
+  // Completed/cancelled/failed tasks; default = 30 days.
+  // Active tasks (created/delivered/acked/running/replied less than
+  // 30 days old) are NEVER swept.
+  terminalTasksDays: number;
+  // Audit log; default = 90 days.
+  // Longer retention because audit_log is a security artifact.
+  auditLogDays: number;
+};
+
+export type SweepResult = {
+  startedAt: string;
+  durationMs: number;
+  deletes: {
+    agent_telemetry: number;
+    task_events: number;
+    inbox: number;
+    tasks: number;
+    audit_log: number;
+  };
+  vacuum: {
+    walCheckpointPagesMoved: number | null;
+    incrementalFreedPages: number | null;
+    errored: boolean;
+  };
+};
+
+const DEFAULTS: RetentionConfig = {
+  telemetryDays: 7,
+  taskEventsDays: 30,
+  ackedInboxDays: 7,
+  terminalTasksDays: 30,
+  auditLogDays: 90,
+};
+
+function readEnvDays(name: string, fallback: number): number {
+  const raw = process.env[name];
+  if (raw === undefined || raw === "") return fallback;
+  const n = Number(raw);
+  // Negative = disable that table's sweep entirely (operator opt-out).
+  // NaN / non-finite falls back to default — guard against fat-fingered env.
+  if (!Number.isFinite(n)) return fallback;
+  return n;
+}
+
+export function readRetentionConfig(): RetentionConfig {
+  return {
+    telemetryDays: readEnvDays("COMMHUB_RETENTION_TELEMETRY_DAYS", DEFAULTS.telemetryDays),
+    taskEventsDays: readEnvDays("COMMHUB_RETENTION_TASK_EVENTS_DAYS", DEFAULTS.taskEventsDays),
+    ackedInboxDays: readEnvDays("COMMHUB_RETENTION_INBOX_ACKED_DAYS", DEFAULTS.ackedInboxDays),
+    terminalTasksDays: readEnvDays("COMMHUB_RETENTION_TASKS_TERMINAL_DAYS", DEFAULTS.terminalTasksDays),
+    auditLogDays: readEnvDays("COMMHUB_RETENTION_AUDIT_DAYS", DEFAULTS.auditLogDays),
+  };
+}
+
+function sweepOne(
+  sql: string,
+  params: (string | number)[],
+  days: number,
+): number {
+  // days < 0 → operator opt-out for this table; skip the DELETE.
+  // days === 0 is a legitimate "sweep everything older than now"
+  // (effectively a TRUNCATE); we honour it.
+  if (days < 0) return 0;
+  try {
+    const res = db.run(sql, params);
+    return res.changes ?? 0;
+  } catch (e: any) {
+    console.log(`[commhub retention] sweep failed: ${e?.message ?? e}`);
+    return 0;
+  }
+}
+
+export function sweepRetention(cfg: RetentionConfig = readRetentionConfig()): SweepResult {
+  const startedAt = new Date().toISOString();
+  const t0 = Date.now();
+
+  const telemetry = sweepOne(
+    `DELETE FROM agent_telemetry WHERE created_at < datetime('now', ?1)`,
+    [`-${cfg.telemetryDays} days`],
+    cfg.telemetryDays,
+  );
+  const taskEvents = sweepOne(
+    `DELETE FROM task_events WHERE created_at < datetime('now', ?1)`,
+    [`-${cfg.taskEventsDays} days`],
+    cfg.taskEventsDays,
+  );
+  // Only sweep ACKED inbox rows — unacked may be in-flight delivery
+  // for an agent that's offline but coming back. Acked rows have
+  // served their purpose and exist only for debugging.
+  const inbox = sweepOne(
+    `DELETE FROM inbox WHERE acked = 1 AND created_at < datetime('now', ?1)`,
+    [`-${cfg.ackedInboxDays} days`],
+    cfg.ackedInboxDays,
+  );
+  // Terminal-task sweep — split into two DELETEs:
+  //
+  // (a) "safe terminals": cancelled / failed / expired / (theoretical)
+  //     completed have no chain-ancestor risk and are safe to reap
+  //     unconditionally past the horizon. NOTE: as of this writing
+  //     'completed' is dead code — no code path sets it (every reply
+  //     uses 'replied'/'failed'/'cancelled' via send_reply or
+  //     chainReplyToParent). Kept in the IN-list as a forward guard:
+  //     if someone introduces a 'completed' write later, the sweep
+  //     picks it up without code changes.
+  //
+  // (b) "replied" — was excluded entirely in the original PR, which
+  //     missed the goal: every fulfilled task lives at 'replied'
+  //     status, so the tasks table's MAIN GROWTH SOURCE would never
+  //     be reclaimed. Reviewer caught this. Fix: child-ref-aware
+  //     delete — reap an old replied row ONLY IF no other task
+  //     references it as parent_task_id. A row with no children is
+  //     not a chain ancestor and is safe to drop.
+  //
+  // **Time field** (both DELETEs): use COALESCE(completed_at, created_at).
+  // Terminal status means the server set `completed_at` at the
+  // reply/fail/cancel moment. Sweeping on `created_at` alone would
+  // reap a task created 60d ago but completed today the instant it
+  // entered terminal state — zero post-terminal retention. The
+  // COALESCE falls back to created_at for legacy rows that pre-date
+  // the completed_at column (back-compat).
+  const tasksSafeTerminal = sweepOne(
+    `DELETE FROM tasks
+     WHERE status IN ('completed', 'cancelled', 'failed', 'expired')
+       AND COALESCE(completed_at, created_at) < datetime('now', ?1)`,
+    [`-${cfg.terminalTasksDays} days`],
+    cfg.terminalTasksDays,
+  );
+  const tasksRepliedChildSafe = sweepOne(
+    `DELETE FROM tasks
+     WHERE status = 'replied'
+       AND COALESCE(completed_at, created_at) < datetime('now', ?1)
+       AND NOT EXISTS (
+         SELECT 1 FROM tasks AS child
+         WHERE child.parent_task_id = tasks.task_id
+       )`,
+    [`-${cfg.terminalTasksDays} days`],
+    cfg.terminalTasksDays,
+  );
+  const tasks = tasksSafeTerminal + tasksRepliedChildSafe;
+  const auditLog = sweepOne(
+    `DELETE FROM audit_log WHERE created_at < datetime('now', ?1)`,
+    [`-${cfg.auditLogDays} days`],
+    cfg.auditLogDays,
+  );
+
+  // VACUUM:
+  //   - wal_checkpoint(TRUNCATE): trims the WAL file. Works on every
+  //     SQLite DB regardless of auto_vacuum setting. Useful after
+  //     big DELETEs so the WAL doesn't stay bloated until the next
+  //     natural checkpoint.
+  //   - incremental_vacuum: ONLY reclaims main-DB pages on databases
+  //     created with `PRAGMA auto_vacuum = INCREMENTAL`. On legacy
+  //     DBs (the default mode 0) this is a no-op and the main file
+  //     does NOT shrink — freed pages stay in the freelist for
+  //     future inserts. Operators who want disk reclamation on old
+  //     DBs must run a one-time blocking `VACUUM` themselves (out of
+  //     scope for the sweeper, which deliberately stays non-
+  //     blocking).
+  let walCheckpointPagesMoved: number | null = null;
+  let incrementalFreedPages: number | null = null;
+  let errored = false;
+  try {
+    // wal_checkpoint(TRUNCATE) returns a row { busy, log, checkpointed }.
+    // We expose `log` (page count moved into the main DB) for ops visibility.
+    const checkpoint = db.get<{ busy: number; log: number; checkpointed: number }>(
+      "PRAGMA wal_checkpoint(TRUNCATE)",
+    );
+    walCheckpointPagesMoved = checkpoint?.log ?? null;
+  } catch (e: any) {
+    console.log(`[commhub retention] wal_checkpoint failed: ${e?.message ?? e}`);
+    errored = true;
+  }
+  try {
+    // incremental_vacuum is a no-op on databases not created with
+    // PRAGMA auto_vacuum = INCREMENTAL. For existing prod DBs that
+    // weren't, the operator would have to run a full one-time VACUUM
+    // to migrate — out of scope for the sweeper.
+    const freelistBefore = db.get<{ freelist_count: number }>("PRAGMA freelist_count");
+    db.exec("PRAGMA incremental_vacuum");
+    const freelistAfter = db.get<{ freelist_count: number }>("PRAGMA freelist_count");
+    incrementalFreedPages =
+      (freelistBefore?.freelist_count ?? 0) - (freelistAfter?.freelist_count ?? 0);
+  } catch (e: any) {
+    console.log(`[commhub retention] incremental_vacuum failed: ${e?.message ?? e}`);
+    errored = true;
+  }
+
+  return {
+    startedAt,
+    durationMs: Date.now() - t0,
+    deletes: {
+      agent_telemetry: telemetry,
+      task_events: taskEvents,
+      inbox,
+      tasks,
+      audit_log: auditLog,
+    },
+    vacuum: { walCheckpointPagesMoved, incrementalFreedPages, errored },
+  };
+}
+
+// Used by index.ts to wire the periodic sweeper. Returns the timer so
+// callers can stop it in tests / shutdown.
+export function startRetentionSweeper(
+  intervalMs = 60 * 60 * 1000, // 1 hour
+  cfg?: RetentionConfig,
+): ReturnType<typeof setInterval> {
+  const sweep = () => {
+    try {
+      const r = sweepRetention(cfg);
+      const totalDeletes =
+        r.deletes.agent_telemetry +
+        r.deletes.task_events +
+        r.deletes.inbox +
+        r.deletes.tasks +
+        r.deletes.audit_log;
+      if (totalDeletes > 0 || r.vacuum.errored) {
+        console.log(
+          `[commhub retention] swept in ${r.durationMs}ms — ` +
+            `telemetry=${r.deletes.agent_telemetry} events=${r.deletes.task_events} ` +
+            `inbox=${r.deletes.inbox} tasks=${r.deletes.tasks} audit=${r.deletes.audit_log} ` +
+            `walPages=${r.vacuum.walCheckpointPagesMoved} freedPages=${r.vacuum.incrementalFreedPages}`,
+        );
+      }
+    } catch (e: any) {
+      console.log(`[commhub retention] sweep top-level failed: ${e?.message ?? e}`);
+    }
+  };
+  // Fire once on startup so a long-running hub eventually catches up
+  // even if it's restarted just after the previous sweep. setImmediate
+  // (vs synchronous) keeps server startup from blocking on the sweep.
+  setImmediate(sweep);
+  return setInterval(sweep, intervalMs);
+}

package/src/stale-sweeper.test.ts

@@ -0,0 +1,181 @@
+// Round-2/4 review ③ — stale session sweeper tests.
+//
+// Drives the real sweepStaleSessions against isolated in-process
+// SQLite (COMMHUB_DB=/tmp/...). Pins behaviour the read-path
+// regression (per-request UPDATE) would re-introduce if anyone
+// undoes the fix:
+//
+//   1. fresh sessions stay non-offline regardless of how many
+//      times the sweeper runs;
+//   2. stale sessions get marked offline exactly once (idempotent);
+//   3. global sweep crosses tenant boundaries — a stale agent in
+//      network B is marked offline even when no one in net B is
+//      polling /api/status;
+//   4. env-var cutoff is respected and bad values fall back silently.
+//
+// Run with:
+//   COMMHUB_DB=/tmp/stale-test.db bun test src/stale-sweeper.test.ts
+
+import { describe, expect, test, beforeEach } from "bun:test";
+import { db } from "./db.js";
+import { sweepStaleSessions, staleCutoffMinutes } from "./stale-sweeper.js";
+
+function insertSession(opts: {
+  alias: string;
+  status?: string;
+  minutesAgo: number;
+  network_id?: string;
+}): void {
+  const id = `r_${opts.alias}`;
+  db.run(
+    `INSERT OR REPLACE INTO sessions
+       (resume_id, alias, status, network_id, updated_at, registered_at)
+     VALUES (?1, ?2, ?3, ?4, datetime('now', ?5), datetime('now', ?5))`,
+    [id, opts.alias, opts.status ?? "idle", opts.network_id ?? "default", `-${opts.minutesAgo} minutes`]
+  );
+}
+
+function getStatus(alias: string): string | null {
+  const row = db.get<{ status: string }>(
+    "SELECT status FROM sessions WHERE alias = ?1",
+    alias
+  );
+  return row?.status ?? null;
+}
+
+beforeEach(() => {
+  db.run("DELETE FROM sessions");
+});
+
+describe("sweepStaleSessions — fresh agents not touched", () => {
+  test("agent updated < cutoff stays at its current status", () => {
+    insertSession({ alias: "live-1", status: "idle", minutesAgo: 1 });
+    insertSession({ alias: "live-2", status: "working", minutesAgo: 5 });
+
+    sweepStaleSessions(10);
+
+    expect(getStatus("live-1")).toBe("idle");
+    expect(getStatus("live-2")).toBe("working");
+  });
+
+  test("multiple sweeps on quiet hub are pure no-ops (no write fired)", () => {
+    insertSession({ alias: "live", status: "idle", minutesAgo: 1 });
+
+    const r1 = sweepStaleSessions(10);
+    const r2 = sweepStaleSessions(10);
+    const r3 = sweepStaleSessions(10);
+
+    expect(r1.markedOffline).toBe(0);
+    expect(r2.markedOffline).toBe(0);
+    expect(r3.markedOffline).toBe(0);
+    expect(getStatus("live")).toBe("idle");
+  });
+});
+
+describe("sweepStaleSessions — stale agents marked offline", () => {
+  test("agent updated > cutoff gets marked offline exactly once", () => {
+    insertSession({ alias: "stale", status: "idle", minutesAgo: 30 });
+
+    const r1 = sweepStaleSessions(10);
+    expect(r1.markedOffline).toBe(1);
+    expect(getStatus("stale")).toBe("offline");
+
+    // Second sweep does nothing because the row is already offline
+    // (the predicate is `status != 'offline'`).
+    const r2 = sweepStaleSessions(10);
+    expect(r2.markedOffline).toBe(0);
+    expect(getStatus("stale")).toBe("offline");
+  });
+
+  test("already-offline stale row not re-marked (write avoided)", () => {
+    insertSession({ alias: "old-offline", status: "offline", minutesAgo: 60 });
+
+    const r = sweepStaleSessions(10);
+
+    expect(r.markedOffline).toBe(0);
+    expect(getStatus("old-offline")).toBe("offline");
+  });
+});
+
+describe("sweepStaleSessions — global across tenants (the read-path fix)", () => {
+  test("stale row in network B is marked offline even with no net-B polling", () => {
+    // Pre-fix: GET /api/status from net-A would only UPDATE rows in
+    // net-A's scope. A stale agent in net-B would stay non-offline
+    // until someone polled net-B. After the fix, the global sweeper
+    // is tenant-blind on the maintenance write.
+    insertSession({ alias: "stale-a", status: "idle", minutesAgo: 30, network_id: "net-A" });
+    insertSession({ alias: "stale-b", status: "idle", minutesAgo: 30, network_id: "net-B" });
+    insertSession({ alias: "stale-c", status: "idle", minutesAgo: 30, network_id: null as any });
+
+    const r = sweepStaleSessions(10);
+
+    expect(r.markedOffline).toBe(3);
+    expect(getStatus("stale-a")).toBe("offline");
+    expect(getStatus("stale-b")).toBe("offline");
+    expect(getStatus("stale-c")).toBe("offline");
+  });
+});
+
+describe("sweepStaleSessions — cutoff boundary + env", () => {
+  test("exactly-at-cutoff agent treated as stale (< strict, not <=)", () => {
+    // Just past the cutoff window.
+    insertSession({ alias: "just-stale", status: "idle", minutesAgo: 11 });
+    sweepStaleSessions(10);
+    expect(getStatus("just-stale")).toBe("offline");
+  });
+
+  test("just-under-cutoff agent treated as live", () => {
+    insertSession({ alias: "just-live", status: "working", minutesAgo: 9 });
+    sweepStaleSessions(10);
+    expect(getStatus("just-live")).toBe("working");
+  });
+
+  test("env COMMHUB_STALE_CUTOFF_MINUTES overrides default", () => {
+    process.env.COMMHUB_STALE_CUTOFF_MINUTES = "3";
+    expect(staleCutoffMinutes()).toBe(3);
+    delete process.env.COMMHUB_STALE_CUTOFF_MINUTES;
+  });
+
+  test("env invalid value falls back to default 10 (no operator footgun)", () => {
+    process.env.COMMHUB_STALE_CUTOFF_MINUTES = "not-a-number";
+    expect(staleCutoffMinutes()).toBe(10);
+    delete process.env.COMMHUB_STALE_CUTOFF_MINUTES;
+
+    process.env.COMMHUB_STALE_CUTOFF_MINUTES = "0";
+    expect(staleCutoffMinutes()).toBe(10);
+    delete process.env.COMMHUB_STALE_CUTOFF_MINUTES;
+
+    process.env.COMMHUB_STALE_CUTOFF_MINUTES = "-5";
+    expect(staleCutoffMinutes()).toBe(10);
+    delete process.env.COMMHUB_STALE_CUTOFF_MINUTES;
+  });
+});
+
+describe("sweepStaleSessions — read-path regression guard (pin the fix)", () => {
+  // This is the load-bearing pin: if anyone adds back a per-request
+  // UPDATE in /api/status / /api/servers / /api/server-detail-* /
+  // get_all_status, this test plus the absence of those UPDATEs in
+  // the source should make the regression visible. Since this is a
+  // unit-level test of the sweeper itself, the regression-prevention
+  // role is mostly social — the comments in this file + the comments
+  // in index.ts and tools.ts at the former UPDATE sites tell future
+  // editors "do not put the UPDATE back; use the sweeper."
+  test("after sweep, sessions row has the offline status persisted (single source of truth)", () => {
+    insertSession({ alias: "x", status: "working", minutesAgo: 30 });
+    const before = sweepStaleSessions(10);
+    expect(before.markedOffline).toBe(1);
+
+    // Re-read directly — no extra UPDATE needed by a caller.
+    const status = getStatus("x");
+    expect(status).toBe("offline");
+
+    // 100 simulated read-path lookups don't fire any maintenance writes.
+    for (let i = 0; i < 100; i++) {
+      const row = db.get<{ status: string }>(
+        "SELECT status FROM sessions WHERE alias = ?1",
+        "x"
+      );
+      expect(row?.status).toBe("offline");
+    }
+  });
+});

package/src/stale-sweeper.ts

@@ -0,0 +1,114 @@
+// Round-2/4 review ③ — read-path write amplification fix.
+//
+// Before this module: GET /api/status, GET /api/servers, GET
+// /api/server-health/:host, and the MCP `get_all_status` tool each
+// fired:
+//
+//   UPDATE sessions SET status = 'offline'
+//   WHERE updated_at < ?1 AND status != 'offline'
+//
+// on every call. With the dashboard polling /api/status every few
+// seconds × N tabs × M users, that's a high-frequency WRITE on a
+// heavily-read table just to maintain a derived field. Wait events
+// stack up (WAL fsync, page locks) and the writes are 99% no-ops.
+//
+// Fix: move the stale-marking off the read path into a single
+// background timer. Reads now SELECT only; the timer runs every 60s
+// and applies the same UPDATE GLOBALLY (no per-network scope — stale
+// is stale regardless of which tenant is asking). Worst-case delay
+// between an agent crashing and going `offline` in the dashboard:
+//
+//   10min stale cutoff + 60s sweep = up to ~11min
+//
+// (unchanged from before the fix for any reader that wasn't actively
+// polling).
+//
+// Same pattern as `src/retention.ts` for shutdown (clearInterval in
+// the graceful shutdown handler).
+
+import { db } from "./db.js";
+
+export type StaleSweepResult = {
+  startedAt: string;
+  durationMs: number;
+  markedOffline: number;
+  staleCutoffMinutes: number;
+};
+
+// Stale window: an agent that hasn't reported in this many minutes is
+// considered offline. Matches the pre-fix per-request cutoff so the
+// observable behaviour from the dashboard's perspective is unchanged.
+const DEFAULT_STALE_MINUTES = 10;
+const DEFAULT_SWEEP_MS = 60 * 1000;
+
+export function staleCutoffMinutes(): number {
+  const raw = process.env.COMMHUB_STALE_CUTOFF_MINUTES;
+  if (raw === undefined || raw === "") return DEFAULT_STALE_MINUTES;
+  const n = Number(raw);
+  if (!Number.isFinite(n) || n <= 0) return DEFAULT_STALE_MINUTES;
+  return n;
+}
+
+function sqliteCutoff(minutes: number): string {
+  return new Date(Date.now() - minutes * 60 * 1000)
+    .toISOString()
+    .replace("T", " ")
+    .slice(0, 19);
+}
+
+// Single GLOBAL UPDATE — no per-network filtering. The pre-fix per-
+// request stale marker WAS scoped (each REST call only saw its own
+// network's stale rows), but the semantics of "this row hasn't been
+// heard from in 10min" is global. A network that no one's looking at
+// still wants its agents marked offline so an admin scanning all
+// networks gets consistent state.
+export function sweepStaleSessions(cutoffMinutes = staleCutoffMinutes()): StaleSweepResult {
+  const startedAt = new Date().toISOString();
+  const t0 = Date.now();
+  let markedOffline = 0;
+  try {
+    const cutoff = sqliteCutoff(cutoffMinutes);
+    const res = db.run(
+      "UPDATE sessions SET status = 'offline' WHERE updated_at < ?1 AND status != 'offline'",
+      [cutoff],
+    );
+    markedOffline = res.changes ?? 0;
+  } catch (e: any) {
+    console.log(`[commhub stale-sweep] failed: ${e?.message ?? e}`);
+  }
+  return {
+    startedAt,
+    durationMs: Date.now() - t0,
+    markedOffline,
+    staleCutoffMinutes: cutoffMinutes,
+  };
+}
+
+export function startStaleSessionSweeper(
+  intervalMs?: number,
+): ReturnType<typeof setInterval> {
+  const env = process.env.COMMHUB_STALE_SWEEP_SECONDS;
+  const envMs = env && Number.isFinite(Number(env)) && Number(env) > 0
+    ? Number(env) * 1000
+    : undefined;
+  const ms = intervalMs ?? envMs ?? DEFAULT_SWEEP_MS;
+  const sweep = () => {
+    try {
+      const r = sweepStaleSessions();
+      // Log only when something actually changed — quiet hubs stay quiet.
+      if (r.markedOffline > 0) {
+        console.log(
+          `[commhub stale-sweep] marked ${r.markedOffline} session(s) offline in ${r.durationMs}ms ` +
+            `(cutoff=${r.staleCutoffMinutes}min)`,
+        );
+      }
+    } catch (e: any) {
+      console.log(`[commhub stale-sweep] top-level failed: ${e?.message ?? e}`);
+    }
+  };
+  // Fire once at boot via setImmediate so the very-first /api/status
+  // after server start gets fresh stale state without blocking
+  // startup.
+  setImmediate(sweep);
+  return setInterval(sweep, ms);
+}