@checkstack/satellite-backend 0.4.0 → 0.5.0

package/src/heartbeat-monitor.it.test.ts

@@ -0,0 +1,232 @@
+/**
+ * Integration test (real Postgres): cross-pod heartbeat-lost detection.
+ *
+ * This is the DETERMINISTIC backstop for `.agent/rules/state-and-scale.md` that
+ * the single-process unit suite structurally cannot provide for the heartbeat
+ * monitor. The prior fix made connection STATUS durable but left the
+ * online→offline transition DETECTION pod-local (an in-memory `previousStatuses`
+ * map): under N pods, the heartbeat-check job runs on a VARYING pod, so a pod
+ * with an empty map never observed the satellite online and therefore never
+ * fired the `heartbeat_lost` edge — leaving `connectionStatus` stuck `online`
+ * forever after a pod crash. This test proves that bug cannot recur.
+ *
+ * ## Two-pod model (faithful proxy)
+ *
+ * We model TWO independent "pods" as two independent `SatelliteService` +
+ * `HeartbeatMonitor` instances, EACH over its OWN `pg.Pool`, BOTH pointed at the
+ * SAME Postgres database + schema (mirroring the automation-backend cross-pod
+ * read-consistency IT). Separate pools = separate processes for the property
+ * under test (no shared JS heap, so no shared in-memory baseline), one DB =
+ * the shared durable substrate N pods share in production.
+ *
+ *   - pod A — the pod that handled the satellite's WS connection.
+ *   - pod B — a DIFFERENT pod that later claims the heartbeat-check job and has
+ *     NEVER seen this satellite online in memory.
+ *
+ * ## What it asserts
+ *
+ *   1. Pod A connects the satellite (durable write: lastHeartbeatAt=now,
+ *      lastConnectionEvent="connected"). Pod B's entity read sees it ONLINE.
+ *   2. The satellite's heartbeat ages out (we backdate lastHeartbeatAt past the
+ *      offline threshold — a crashed pod that stopped heartbeating). Pod B's
+ *      read now self-heals to OFFLINE purely from compute-on-read.
+ *   3. Pod B's heartbeat monitor — fresh heap, no prior in-memory knowledge —
+ *      DETECTS the heartbeat_lost edge from durable state and drives the entity
+ *      mutate (flips lastConnectionEvent="heartbeat_lost").
+ *   4. Re-running the monitor on EITHER pod is a no-op (idempotent across pods +
+ *      redelivery).
+ *
+ * Gated behind `CHECKSTACK_IT=1`; connection from `CHECKSTACK_IT_PG_URL`. Each
+ * run isolates itself in a freshly created Postgres schema and cleans up.
+ */
+import { afterAll, beforeAll, describe, expect, it } from "bun:test";
+import { drizzle } from "drizzle-orm/node-postgres";
+import { Pool } from "pg";
+
+import type { SafeDatabase } from "@checkstack/backend-api";
+import {
+  createMockLogger,
+  createMockSignalService,
+} from "@checkstack/test-utils-backend";
+
+import * as schema from "./schema";
+import { SatelliteService } from "./service";
+import { HeartbeatMonitor } from "./heartbeat-monitor";
+import { OFFLINE_THRESHOLD_MS } from "@checkstack/satellite-common";
+
+const PG_URL =
+  process.env.CHECKSTACK_IT_PG_URL ??
+  "postgres://postgres:postgres@localhost:5432/postgres";
+
+const SCHEMA = `it_sat_heartbeat_${crypto.randomUUID().replace(/-/g, "")}`;
+
+/**
+ * One simulated pod: an independent `SatelliteService` + `HeartbeatMonitor` over
+ * its OWN pool to the shared DB. The monitor's entity sink performs the REAL
+ * durable write via this pod's service (no framework handle needed — the
+ * property under test is whether DETECTION reads durable state, not the
+ * transition-log append).
+ */
+interface Pod {
+  readonly pool: Pool;
+  readonly service: SatelliteService;
+  readonly monitor: HeartbeatMonitor;
+  end(): Promise<void>;
+}
+
+describe.skipIf(!process.env.CHECKSTACK_IT)(
+  "cross-pod heartbeat-lost detection (real Postgres)",
+  () => {
+    const pods: Pod[] = [];
+
+    function makePod(): Pod {
+      const pool = new Pool({
+        connectionString: PG_URL,
+        options: `-c search_path=${SCHEMA}`,
+      });
+      const db = drizzle({
+        client: pool,
+        schema,
+      }) as unknown as SafeDatabase<typeof schema>;
+      const service = new SatelliteService(db);
+      const logger = createMockLogger();
+      const signalService = createMockSignalService();
+
+      // The monitor's entity sink: the REAL durable heartbeat_lost write through
+      // this pod's service. This is exactly what `index.ts` wires (minus the
+      // framework `handle.mutate` wrapper, which is out of scope for the
+      // cross-pod DETECTION property).
+      const monitor = new HeartbeatMonitor(service, signalService, logger, {
+        mirror: async (satelliteId) => {
+          await service.applyConnectionState({
+            satelliteId,
+            lastEvent: "heartbeat_lost",
+          });
+        },
+      });
+
+      return { pool, service, monitor, end: () => pool.end() };
+    }
+
+    let podA: Pod;
+    let podB: Pod;
+
+    beforeAll(async () => {
+      const setupPool = new Pool({ connectionString: PG_URL });
+      try {
+        await setupPool.query(`CREATE SCHEMA IF NOT EXISTS "${SCHEMA}"`);
+        // The satellites table the service reads/writes. Only the columns the
+        // service touches are needed.
+        await setupPool.query(`
+          CREATE TABLE "${SCHEMA}".satellites (
+            id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
+            name text NOT NULL,
+            region text NOT NULL,
+            tags jsonb NOT NULL DEFAULT '{}',
+            token_hash text NOT NULL,
+            last_heartbeat_at timestamp,
+            version text,
+            last_connection_event text,
+            created_at timestamp NOT NULL DEFAULT now()
+          )
+        `);
+      } finally {
+        await setupPool.end();
+      }
+
+      podA = makePod();
+      podB = makePod();
+      pods.push(podA, podB);
+    });
+
+    afterAll(async () => {
+      await Promise.all(pods.map((p) => p.end()));
+      const cleanupPool = new Pool({ connectionString: PG_URL });
+      try {
+        await cleanupPool.query(`DROP SCHEMA IF EXISTS "${SCHEMA}" CASCADE`);
+      } finally {
+        await cleanupPool.end();
+      }
+    });
+
+    /** Insert a satellite row; returns its id. */
+    async function insertSatellite(): Promise<string> {
+      const { rows } = await podA.pool.query<{ id: string }>(
+        `INSERT INTO "${SCHEMA}".satellites (name, region, token_hash)
+         VALUES ('eu-west', 'eu-west-1', 'hash') RETURNING id`,
+      );
+      return rows[0]!.id;
+    }
+
+    /** Read the durable connection columns of one satellite (any pod). */
+    async function readRow(
+      pod: Pod,
+      id: string,
+    ): Promise<{ lastHeartbeatAt: Date | null; lastConnectionEvent: string | null }> {
+      const { rows } = await pod.pool.query<{
+        last_heartbeat_at: Date | null;
+        last_connection_event: string | null;
+      }>(
+        `SELECT last_heartbeat_at, last_connection_event
+         FROM "${SCHEMA}".satellites WHERE id = $1`,
+        [id],
+      );
+      return {
+        lastHeartbeatAt: rows[0]!.last_heartbeat_at,
+        lastConnectionEvent: rows[0]!.last_connection_event,
+      };
+    }
+
+    it("pod B detects heartbeat_lost for a satellite it never saw online, idempotently", async () => {
+      const id = await insertSatellite();
+
+      // 1. Pod A connects the satellite: durable write of the connected edge.
+      await podA.service.applyConnectionState({
+        satelliteId: id,
+        lastEvent: "connected",
+        lastHeartbeatAt: new Date(),
+      });
+
+      // Pod B (different pool/heap) reads it ONLINE via compute-on-read.
+      const onlineB = await podB.service.getManyConnectionStates([id]);
+      expect(onlineB[id]).toBeDefined();
+      expect(onlineB[id]!.status).toBe("online");
+      expect(onlineB[id]!.lastEvent).toBe("connected");
+
+      // 2. The owning pod crashes / stops heartbeating: backdate the heartbeat
+      //    past the offline threshold (simulating elapsed wall-clock time).
+      const aged = new Date(Date.now() - OFFLINE_THRESHOLD_MS - 60_000);
+      await podA.pool.query(
+        `UPDATE "${SCHEMA}".satellites SET last_heartbeat_at = $1 WHERE id = $2`,
+        [aged, id],
+      );
+
+      // Pod B's read self-heals to OFFLINE from durable state alone — no pod
+      // ever wrote "offline"; the status is computed.
+      const offlineB = await podB.service.getManyConnectionStates([id]);
+      expect(offlineB[id]!.status).toBe("offline");
+      // ...but the last recorded edge is still "connected" — the heartbeat_lost
+      // edge has NOT been fired yet (this is the bug surface: under the old
+      // pod-local design it would NEVER fire on a fresh pod).
+      expect(offlineB[id]!.lastEvent).toBe("connected");
+
+      // 3. Pod B's monitor — fresh heap, never saw this satellite online —
+      //    detects the edge from DURABLE state and drives the mutate.
+      await podB.monitor.checkHeartbeats();
+
+      const afterDetect = await readRow(podB, id);
+      expect(afterDetect.lastConnectionEvent).toBe("heartbeat_lost");
+      // The entity read now reports the heartbeat_lost edge globally (pod A too).
+      const lostA = await podA.service.getManyConnectionStates([id]);
+      expect(lostA[id]!.lastEvent).toBe("heartbeat_lost");
+      expect(lostA[id]!.status).toBe("offline");
+
+      // 4. Idempotent: re-running on pod B AND on pod A is a no-op (the durable
+      //    lastConnectionEvent="heartbeat_lost" makes the predicate false).
+      await podB.monitor.checkHeartbeats();
+      await podA.monitor.checkHeartbeats();
+      const afterReRun = await readRow(podA, id);
+      expect(afterReRun.lastConnectionEvent).toBe("heartbeat_lost");
+    });
+  },
+);

package/src/heartbeat-monitor.test.ts

@@ -1,21 +1,56 @@
 import { describe, it, expect, mock, beforeEach } from "bun:test";
-import { HeartbeatMonitor } from "./heartbeat-monitor";
+import {
+  HeartbeatMonitor,
+  type SatelliteHeartbeatEntitySink,
+} from "./heartbeat-monitor";
 import {
   createMockLogger,
   createMockSignalService,
   type MockSignalService,
 } from "@checkstack/test-utils-backend";
-import { SATELLITE_STATUS_CHANGED } from "@checkstack/satellite-common";
+import {
+  SATELLITE_STATUS_CHANGED,
+  OFFLINE_THRESHOLD_MS,
+} from "@checkstack/satellite-common";
 import type { SatelliteService } from "./service";
-import type { SatelliteWithStatus } from "@checkstack/satellite-common";
-
-const createMockSatelliteService = (
-  satellites: SatelliteWithStatus[],
-): SatelliteService =>
+import type { SatelliteConnectionEvent } from "./entity";
+
+type LivenessRow = {
+  id: string;
+  name: string;
+  region: string;
+  lastHeartbeatAt: Date | null;
+  lastConnectionEvent: SatelliteConnectionEvent | null;
+};
+
+function makeEntitySink(): {
+  sink: SatelliteHeartbeatEntitySink;
+  mirrors: string[];
+} {
+  const mirrors: string[] = [];
+  return {
+    sink: {
+      mirror: mock(async (id: string) => {
+        mirrors.push(id);
+      }),
+    },
+    mirrors,
+  };
+}
+
+/**
+ * Mock the service's durable liveness read. The array is captured by reference
+ * so a test can mutate it (e.g. flip `lastConnectionEvent` to "heartbeat_lost"
+ * to simulate the idempotent durable write) between `checkHeartbeats` calls.
+ */
+const createMockSatelliteService = (rows: LivenessRow[]): SatelliteService =>
   ({
-    listSatellites: mock(async () => satellites),
+    listConnectionLiveness: mock(async () => rows),
   }) as unknown as SatelliteService;
 
+const recentHeartbeat = () => new Date(Date.now() - 5_000);
+const agedHeartbeat = () => new Date(Date.now() - OFFLINE_THRESHOLD_MS - 10_000);
+
 describe("HeartbeatMonitor", () => {
   let signalService: MockSignalService;
   let logger: ReturnType<typeof createMockLogger>;
@@ -25,49 +60,45 @@ describe("HeartbeatMonitor", () => {
     logger = createMockLogger();
   });
 
-  it("should not broadcast signals on first check (initialization)", async () => {
+  it("does NOT detect a lost edge for an online (recent heartbeat) satellite", async () => {
     const service = createMockSatelliteService([
       {
         id: "sat-1",
         name: "eu-west",
         region: "eu-west-1",
-        tags: {},
-        status: "online",
-        createdAt: new Date(),
+        lastHeartbeatAt: recentHeartbeat(),
+        lastConnectionEvent: "connected",
       },
     ]);
+    const { sink, mirrors } = makeEntitySink();
+    const monitor = new HeartbeatMonitor(service, signalService, logger, sink);
 
-    const monitor = new HeartbeatMonitor(service, signalService, logger);
     await monitor.checkHeartbeats();
 
-    // First check should NOT broadcast — it only initializes state
+    expect(mirrors).toHaveLength(0);
     expect(signalService.getRecordedSignals()).toHaveLength(0);
   });
 
-  it("should broadcast when a satellite goes offline", async () => {
-    const satellites: SatelliteWithStatus[] = [
+  it("detects heartbeat_lost from DURABLE state with NO prior in-memory knowledge", async () => {
+    // The core horizontal-scale property: a fresh monitor (empty heap, as on a
+    // pod that never saw this satellite online) still detects the online→offline
+    // edge purely from the durable (agedHeartbeat + lastConnectionEvent:
+    // "connected") row, mirrors heartbeat_lost, and broadcasts.
+    const service = createMockSatelliteService([
       {
         id: "sat-1",
         name: "eu-west",
         region: "eu-west-1",
-        tags: {},
-        status: "online",
-        createdAt: new Date(),
+        lastHeartbeatAt: agedHeartbeat(),
+        lastConnectionEvent: "connected",
       },
-    ];
-    const service = createMockSatelliteService(satellites);
-    const monitor = new HeartbeatMonitor(service, signalService, logger);
-
-    // First check: initialize state
-    await monitor.checkHeartbeats();
+    ]);
+    const { sink, mirrors } = makeEntitySink();
+    const monitor = new HeartbeatMonitor(service, signalService, logger, sink);
 
-    // Simulate satellite going offline
-    satellites[0] = { ...satellites[0], status: "offline" };
     await monitor.checkHeartbeats();
 
-    expect(
-      signalService.wasSignalEmitted(SATELLITE_STATUS_CHANGED.id),
-    ).toBe(true);
+    expect(mirrors).toEqual(["sat-1"]);
 
     const recorded = signalService.getRecordedSignalsById(
       SATELLITE_STATUS_CHANGED.id,
@@ -81,95 +112,137 @@ describe("HeartbeatMonitor", () => {
     });
   });
 
-  it("should broadcast when a satellite comes back online", async () => {
-    const satellites: SatelliteWithStatus[] = [
+  it("is idempotent: re-running after the durable flip is a no-op (any pod, redelivery)", async () => {
+    const rows: LivenessRow[] = [
       {
         id: "sat-1",
         name: "eu-west",
         region: "eu-west-1",
-        tags: {},
-        status: "offline",
-        createdAt: new Date(),
+        lastHeartbeatAt: agedHeartbeat(),
+        lastConnectionEvent: "connected",
       },
     ];
-    const service = createMockSatelliteService(satellites);
-    const monitor = new HeartbeatMonitor(service, signalService, logger);
+    const service = createMockSatelliteService(rows);
+    const { sink, mirrors } = makeEntitySink();
+    const monitor = new HeartbeatMonitor(service, signalService, logger, sink);
 
-    // First check: initialize state
+    // First check detects + mirrors the edge.
     await monitor.checkHeartbeats();
+    expect(mirrors).toEqual(["sat-1"]);
 
-    // Simulate satellite coming online
-    satellites[0] = { ...satellites[0], status: "online" };
+    // Simulate the durable write the mirror performed: lastConnectionEvent is
+    // now "heartbeat_lost". A re-run (same pod OR another pod, since the
+    // predicate is over durable state) detects nothing more.
+    rows[0] = { ...rows[0], lastConnectionEvent: "heartbeat_lost" };
     await monitor.checkHeartbeats();
+    expect(mirrors).toEqual(["sat-1"]); // still just one
 
-    const recorded = signalService.getRecordedSignalsById(
-      SATELLITE_STATUS_CHANGED.id,
-    );
-    expect(recorded).toHaveLength(1);
-    expect(recorded[0].payload).toEqual({
-      satelliteId: "sat-1",
-      status: "online",
-      name: "eu-west",
-      region: "eu-west-1",
-    });
+    // A SECOND pod with a fresh (empty) monitor likewise sees a no-op.
+    const { sink: sink2, mirrors: mirrors2 } = makeEntitySink();
+    const monitor2 = new HeartbeatMonitor(service, signalService, logger, sink2);
+    await monitor2.checkHeartbeats();
+    expect(mirrors2).toHaveLength(0);
   });
 
-  it("should not broadcast if status stays the same", async () => {
-    const satellites: SatelliteWithStatus[] = [
+  it("does NOT mirror the offline→online edge (the WS handler owns reconnect)", async () => {
+    // A satellite that reconnected: recent heartbeat, lastConnectionEvent already
+    // "connected". Nothing for the monitor to do — and crucially it must not
+    // treat a heartbeat_lost→connected transition as a lost edge.
+    const service = createMockSatelliteService([
       {
         id: "sat-1",
         name: "eu-west",
         region: "eu-west-1",
-        tags: {},
-        status: "online",
-        createdAt: new Date(),
+        lastHeartbeatAt: recentHeartbeat(),
+        lastConnectionEvent: "connected",
       },
-    ];
-    const service = createMockSatelliteService(satellites);
-    const monitor = new HeartbeatMonitor(service, signalService, logger);
+    ]);
+    const { sink, mirrors } = makeEntitySink();
+    const monitor = new HeartbeatMonitor(service, signalService, logger, sink);
 
-    // First check: initialize
     await monitor.checkHeartbeats();
-    // Second check: same status
+    expect(mirrors).toHaveLength(0);
+  });
+
+  it("does NOT detect a lost edge for a cleanly-disconnected satellite", async () => {
+    // Clean disconnect already set lastConnectionEvent="disconnected" and nulled
+    // the heartbeat: status is offline, but the edge was already recorded, so the
+    // monitor must not re-fire heartbeat_lost.
+    const service = createMockSatelliteService([
+      {
+        id: "sat-1",
+        name: "eu-west",
+        region: "eu-west-1",
+        lastHeartbeatAt: null,
+        lastConnectionEvent: "disconnected",
+      },
+    ]);
+    const { sink, mirrors } = makeEntitySink();
+    const monitor = new HeartbeatMonitor(service, signalService, logger, sink);
+
     await monitor.checkHeartbeats();
+    expect(mirrors).toHaveLength(0);
+    expect(signalService.getRecordedSignals()).toHaveLength(0);
+  });
+
+  it("does NOT detect a lost edge for a never-connected satellite", async () => {
+    const service = createMockSatelliteService([
+      {
+        id: "sat-1",
+        name: "eu-west",
+        region: "eu-west-1",
+        lastHeartbeatAt: null,
+        lastConnectionEvent: null,
+      },
+    ]);
+    const { sink, mirrors } = makeEntitySink();
+    const monitor = new HeartbeatMonitor(service, signalService, logger, sink);
 
+    await monitor.checkHeartbeats();
+    expect(mirrors).toHaveLength(0);
     expect(signalService.getRecordedSignals()).toHaveLength(0);
   });
 
-  it("should clean up tracked state for deleted satellites", async () => {
-    const satellites: SatelliteWithStatus[] = [
+  it("broadcasts the offline edge only once per pod across back-to-back checks", async () => {
+    const rows: LivenessRow[] = [
       {
         id: "sat-1",
         name: "eu-west",
         region: "eu-west-1",
-        tags: {},
-        status: "online",
-        createdAt: new Date(),
+        lastHeartbeatAt: agedHeartbeat(),
+        lastConnectionEvent: "connected",
       },
     ];
-    const service = createMockSatelliteService(satellites);
-    const monitor = new HeartbeatMonitor(service, signalService, logger);
+    const service = createMockSatelliteService(rows);
+    const { sink } = makeEntitySink();
+    const monitor = new HeartbeatMonitor(service, signalService, logger, sink);
 
-    // First check
+    // Two checks BEFORE the durable flip lands (e.g. mirror failed once). The
+    // broadcast-dedup set suppresses a second broadcast from THIS pod.
     await monitor.checkHeartbeats();
-
-    // Satellite removed (empty list returned)
-    satellites.length = 0;
     await monitor.checkHeartbeats();
 
-    // Adding it back should not trigger a transition
-    // (it's treated as a new satellite on first check)
-    satellites.push({
-      id: "sat-1",
-      name: "eu-west",
-      region: "eu-west-1",
-      tags: {},
-      status: "online",
-      createdAt: new Date(),
-    });
+    expect(
+      signalService.getRecordedSignalsById(SATELLITE_STATUS_CHANGED.id),
+    ).toHaveLength(1);
+  });
+
+  it("works without an entity sink (broadcast-only)", async () => {
+    const service = createMockSatelliteService([
+      {
+        id: "sat-1",
+        name: "eu-west",
+        region: "eu-west-1",
+        lastHeartbeatAt: agedHeartbeat(),
+        lastConnectionEvent: "connected",
+      },
+    ]);
+    const monitor = new HeartbeatMonitor(service, signalService, logger);
+
     await monitor.checkHeartbeats();
 
-    // No transition signal should have been emitted for the re-add
-    expect(signalService.getRecordedSignals()).toHaveLength(0);
+    expect(
+      signalService.wasSignalEmitted(SATELLITE_STATUS_CHANGED.id),
+    ).toBe(true);
   });
 });