@checkstack/satellite-backend 0.3.6 → 0.5.0

package/src/satellite-ws-handler.test.ts

@@ -2,10 +2,12 @@ import { describe, it, expect, mock, beforeEach } from "bun:test";
 import {
   SatelliteWsHandler,
   type SatelliteResultHandler,
+  type SatelliteScriptPackageSink,
 } from "./satellite-ws-handler";
 import { createMockLogger } from "@checkstack/test-utils-backend";
 import type { SatelliteService } from "./service";
 import type { ConfigRelay } from "./config-relay";
+import type { SatelliteConnectionEvent } from "./entity";
 import type { SatelliteWithStatus } from "@checkstack/satellite-common";
 
 const MOCK_SATELLITE: SatelliteWithStatus = {
@@ -262,4 +264,269 @@ describe("SatelliteWsHandler", () => {
       await handler.pushConfigUpdate("non-existent");
     });
   });
+
+  describe("script-package distribution", () => {
+    function makeSink(
+      lockfileHash: string | null,
+    ): {
+      sink: SatelliteScriptPackageSink;
+      reports: Parameters<SatelliteScriptPackageSink["reportSyncState"]>[0][];
+    } {
+      const reports: Parameters<
+        SatelliteScriptPackageSink["reportSyncState"]
+      >[0][] = [];
+      return {
+        reports,
+        sink: {
+          getDesiredLockfileHash: mock(async () => lockfileHash),
+          reportSyncState: mock(async (input) => {
+            reports.push(input);
+          }),
+          getManifest: mock(async () => [
+            { name: "leftpad", version: "0.0.1", integrity: "sha-1" },
+          ]),
+          getBlobBase64: mock(async () => "YmxvYg=="),
+        },
+      };
+    }
+
+    async function authedHandlerWithSink(lockfileHash: string | null) {
+      const { sink, reports } = makeSink(lockfileHash);
+      const h = new SatelliteWsHandler(
+        service,
+        configRelay,
+        resultHandler,
+        logger,
+        undefined,
+        sink,
+      );
+      const ws = createMockWs();
+      const { onMessage } = h.onConnection(ws);
+      await onMessage(
+        JSON.stringify({
+          type: "authenticate",
+          clientId: "sat-1",
+          token: "csat_valid-token",
+        }),
+      );
+      return { h, ws, onMessage, reports };
+    }
+
+    it("carries the desired lockfile hash in the authenticated payload", async () => {
+      const { ws } = await authedHandlerWithSink("hash-123");
+      const auth = JSON.parse(ws.messages[0]);
+      expect(auth.type).toBe("authenticated");
+      expect(auth.scriptPackagesLockfileHash).toBe("hash-123");
+    });
+
+    it("omits the hash entirely when no sink is wired (version-skew safe)", async () => {
+      // Default `handler` has no script-package sink.
+      const ws = createMockWs();
+      const { onMessage } = handler.onConnection(ws);
+      await onMessage(
+        JSON.stringify({
+          type: "authenticate",
+          clientId: "sat-1",
+          token: "csat_valid-token",
+        }),
+      );
+      const auth = JSON.parse(ws.messages[0]);
+      expect("scriptPackagesLockfileHash" in auth).toBe(false);
+    });
+
+    it("fans refresh_script_packages out to every connected satellite", async () => {
+      const { h, ws } = await authedHandlerWithSink("hash-123");
+      ws.messages.length = 0;
+
+      h.pushRefreshScriptPackagesToAll("hash-456");
+
+      expect(ws.messages).toHaveLength(1);
+      const msg = JSON.parse(ws.messages[0]);
+      expect(msg.type).toBe("refresh_script_packages");
+      expect(msg.lockfileHash).toBe("hash-456");
+    });
+
+    it("persists a satellite's reported sync state", async () => {
+      const { onMessage, reports } = await authedHandlerWithSink("hash-123");
+      await onMessage(
+        JSON.stringify({
+          type: "script_package_sync_state",
+          lockfileHash: "hash-123",
+          status: "ready",
+        }),
+      );
+      expect(reports).toHaveLength(1);
+      expect(reports[0]).toMatchObject({
+        satelliteId: "sat-1",
+        lockfileHash: "hash-123",
+        status: "ready",
+      });
+    });
+
+    it("answers a manifest request over the WS channel", async () => {
+      const { ws, onMessage } = await authedHandlerWithSink("hash-123");
+      ws.messages.length = 0;
+      await onMessage(
+        JSON.stringify({
+          type: "request_script_package_manifest",
+          lockfileHash: "hash-123",
+        }),
+      );
+      const reply = JSON.parse(ws.messages[0]);
+      expect(reply.type).toBe("script_package_manifest");
+      expect(reply.entries[0].name).toBe("leftpad");
+    });
+
+    it("answers a blob request over the WS channel", async () => {
+      const { ws, onMessage } = await authedHandlerWithSink("hash-123");
+      ws.messages.length = 0;
+      await onMessage(
+        JSON.stringify({
+          type: "request_script_package_blob",
+          integrity: "sha-1",
+        }),
+      );
+      const reply = JSON.parse(ws.messages[0]);
+      expect(reply.type).toBe("script_package_blob");
+      expect(reply.data).toBe("YmxvYg==");
+    });
+  });
+
+  describe("connection-state entity mirror", () => {
+    function makeEntitySink() {
+      const mirrors: Array<{
+        satelliteId: string;
+        lastEvent: SatelliteConnectionEvent;
+        lastHeartbeatAt: Date | null;
+      }> = [];
+      return {
+        sink: {
+          mirror: mock(
+            async (input: {
+              satelliteId: string;
+              lastEvent: SatelliteConnectionEvent;
+              lastHeartbeatAt: Date | null;
+            }) => {
+              mirrors.push(input);
+            },
+          ),
+        },
+        mirrors,
+      };
+    }
+
+    it("drives the connected edge with lastHeartbeatAt=now on successful authentication", async () => {
+      const { sink, mirrors } = makeEntitySink();
+      const h = new SatelliteWsHandler(
+        service,
+        configRelay,
+        resultHandler,
+        logger,
+        sink,
+      );
+      const ws = createMockWs();
+      const { onMessage } = h.onConnection(ws);
+      await onMessage(
+        JSON.stringify({
+          type: "authenticate",
+          clientId: "sat-1",
+          token: "csat_valid-token",
+        }),
+      );
+
+      expect(mirrors).toHaveLength(1);
+      expect(mirrors[0]!.satelliteId).toBe("sat-1");
+      expect(mirrors[0]!.lastEvent).toBe("connected");
+      // lastHeartbeatAt = now (non-null) so the computed status reads online.
+      expect(mirrors[0]!.lastHeartbeatAt).toBeInstanceOf(Date);
+    });
+
+    it("does NOT also call updateHeartbeat on connect when a sink is wired (the mirror writes the heartbeat)", async () => {
+      const { sink } = makeEntitySink();
+      const h = new SatelliteWsHandler(
+        service,
+        configRelay,
+        resultHandler,
+        logger,
+        sink,
+      );
+      const ws = createMockWs();
+      const { onMessage } = h.onConnection(ws);
+      await onMessage(
+        JSON.stringify({
+          type: "authenticate",
+          clientId: "sat-1",
+          token: "csat_valid-token",
+        }),
+      );
+      // The connect-time heartbeat is written by the mirror's apply, not by a
+      // separate updateHeartbeat call.
+      expect(service.updateHeartbeat).not.toHaveBeenCalled();
+    });
+
+    it("writes the connect heartbeat directly when NO sink is wired", async () => {
+      // The default `handler` has no sink: it must still record the heartbeat.
+      const ws = createMockWs();
+      const { onMessage } = handler.onConnection(ws);
+      await onMessage(
+        JSON.stringify({
+          type: "authenticate",
+          clientId: "sat-1",
+          token: "csat_valid-token",
+        }),
+      );
+      expect(service.updateHeartbeat).toHaveBeenCalledWith("sat-1", {});
+    });
+
+    it("drives the disconnected edge with lastHeartbeatAt=null (immediate offline) when the socket closes", async () => {
+      const { sink, mirrors } = makeEntitySink();
+      const h = new SatelliteWsHandler(
+        service,
+        configRelay,
+        resultHandler,
+        logger,
+        sink,
+      );
+      const ws = createMockWs();
+      const { onMessage, onClose } = h.onConnection(ws);
+      await onMessage(
+        JSON.stringify({
+          type: "authenticate",
+          clientId: "sat-1",
+          token: "csat_valid-token",
+        }),
+      );
+      onClose?.();
+      // onClose fires the mirror fire-and-forget; flush the microtask queue.
+      await Promise.resolve();
+      await Promise.resolve();
+
+      const disconnected = mirrors.find((m) => m.lastEvent === "disconnected");
+      expect(disconnected).toBeDefined();
+      // Clearing lastHeartbeatAt makes the computed status flip offline at once.
+      expect(disconnected!.lastHeartbeatAt).toBeNull();
+      expect(disconnected!.satelliteId).toBe("sat-1");
+    });
+
+    it("does not mirror on a failed authentication", async () => {
+      const { sink, mirrors } = makeEntitySink();
+      const h = new SatelliteWsHandler(
+        service,
+        configRelay,
+        resultHandler,
+        logger,
+        sink,
+      );
+      const ws = createMockWs();
+      const { onMessage } = h.onConnection(ws);
+      await onMessage(
+        JSON.stringify({
+          type: "authenticate",
+          clientId: "sat-1",
+          token: "csat_invalid-token",
+        }),
+      );
+      expect(mirrors).toHaveLength(0);
+    });
+  });
 });

package/src/satellite-ws-handler.ts

@@ -4,8 +4,10 @@ import type {
   WsConnection,
   WsConnectionHandlers,
 } from "@checkstack/backend-api";
+import { extractErrorMessage } from "@checkstack/common";
 import type { SatelliteService } from "./service";
 import type { ConfigRelay } from "./config-relay";
+import type { SatelliteConnectionEvent } from "./entity";
 import {
   SatelliteToCoreMessageSchema,
   type CoreToSatelliteMessage,
@@ -13,6 +15,27 @@ import {
   type SatelliteWithStatus,
 } from "@checkstack/satellite-common";
 
+/**
+ * Optional plug-point for driving a satellite connection lifecycle edge into
+ * the reactive `satellite-connection` entity (reactive automation engine
+ * §10.6). Bound from `afterPluginsReady` where the entity handle is available —
+ * when not provided, no entity state is mirrored (graceful no-op in unit tests).
+ *
+ * The WS handler calls `mirror` at the same connect / disconnect lifecycle
+ * points it previously emitted the `satellite.connected` / `.disconnected`
+ * hooks; the change-deriver re-fires the equivalent trigger events. The status
+ * is COMPUTED on read from `lastHeartbeatAt`, so the sink carries the new
+ * heartbeat value for the edge rather than a status: `now` on connect (online),
+ * `null` on clean disconnect (offline immediately).
+ */
+export interface SatelliteConnectionEntitySink {
+  mirror: (input: {
+    satelliteId: string;
+    lastEvent: SatelliteConnectionEvent;
+    lastHeartbeatAt: Date | null;
+  }) => Promise<void>;
+}
+
 /**
  * Callback for handling health check results received from satellites.
  */
@@ -24,6 +47,49 @@ export interface SatelliteResultHandler {
   }): Promise<void>;
 }
 
+/**
+ * Optional plug-point for script-package distribution to satellites. Wired
+ * from `afterPluginsReady` against the script-packages RPC. When absent,
+ * satellites simply never receive a `scriptPackagesLockfileHash` or refresh
+ * push (graceful no-op on installs without the plugin).
+ */
+export interface SatelliteScriptPackageSink {
+  /** The desired lockfile hash to carry in assignment payloads, or null. */
+  getDesiredLockfileHash(): Promise<string | null>;
+  /** Persist a satellite's reconcile state for the admin UI. */
+  reportSyncState(input: {
+    satelliteId: string;
+    lockfileHash: string | null;
+    status: "pending" | "syncing" | "ready" | "error";
+    errorMessage?: string;
+  }): Promise<void>;
+  /** Manifest entries for a lockfile hash (for satellite delta diffing). */
+  getManifest(input: {
+    lockfileHash: string;
+  }): Promise<{ name: string; version: string; integrity: string }[]>;
+  /** One content-addressed blob as base64, or null if not found. */
+  getBlobBase64(input: { integrity: string }): Promise<string | null>;
+}
+
+/**
+ * Optional plug-point for just-in-time secret delivery to satellites.
+ * Wired from `afterPluginsReady` against `secretResolverRef`. When absent,
+ * a `request_run_secrets` is answered with an error (no secrets available),
+ * so a collector that declares `secretEnv` fails clearly rather than
+ * running without it.
+ *
+ * The resolver reads the declared `secretEnv` from the satellite's persisted
+ * assignment (the satellite does not choose which secrets), resolves ONLY
+ * those refs, and returns the env map. Resolved values are never persisted.
+ */
+export interface SatelliteSecretSink {
+  resolveRunSecrets(input: {
+    satelliteId: string;
+    configId: string;
+    collectorId: string;
+  }): Promise<Record<string, string>>;
+}
+
 /**
  * Active satellite connection tracking.
  */
@@ -37,7 +103,15 @@ interface SatelliteConnection {
  * Manages authentication, heartbeats, result ingestion, and config pushes.
  */
 export class SatelliteWsHandler implements WebSocketRouteHandler {
-  /** Map of satelliteId → active WebSocket connection */
+  /**
+   * Pod-local live-socket registry: satelliteId → the WebSocket connection
+   * physically held by THIS pod. This is NOT the reactive entity's source of
+   * truth (that is the durable `satellites` connection columns, globally
+   * readable from any pod). It exists ONLY to route messages — config pushes,
+   * script-package refreshes, shutdowns — to a socket this pod actually owns;
+   * a satellite connected to another pod is simply absent here. Treat it as
+   * transport infrastructure, not state.
+   */
   private connections = new Map<string, SatelliteConnection>();
 
   constructor(
@@ -45,6 +119,25 @@ export class SatelliteWsHandler implements WebSocketRouteHandler {
     private configRelay: ConfigRelay,
     private resultHandler: SatelliteResultHandler,
     private logger: Logger,
+    /**
+     * Optional. When set, the handler mirrors `online` / `offline`
+     * connection state into the reactive `satellite-connection` entity at
+     * the same lifecycle points it logs. Wired by `afterPluginsReady` so the
+     * action graph stays decoupled from entity-handle availability.
+     */
+    private connectionEntitySink?: SatelliteConnectionEntitySink,
+    /**
+     * Optional. When set, assignment payloads carry the desired script-package
+     * lockfile hash and the handler can push `refresh_script_packages` +
+     * persist per-satellite sync state.
+     */
+    private scriptPackageSink?: SatelliteScriptPackageSink,
+    /**
+     * Optional. When set, the handler answers `request_run_secrets` by
+     * resolving the collector's declared secretEnv just-in-time. When
+     * unset, such a request is answered with an error.
+     */
+    private secretSink?: SatelliteSecretSink,
   ) {}
 
   /**
@@ -94,17 +187,47 @@ export class SatelliteWsHandler implements WebSocketRouteHandler {
         // Track connection
         this.connections.set(satellite.id, { satellite, ws });
 
-        // Update heartbeat on connect
-        await this.service.updateHeartbeat(satellite.id, {});
+        // Drive the `connected` edge into the reactive entity (best-effort —
+        // never block the auth handshake on a mirror failure). `apply` sets
+        // `lastHeartbeatAt = now` so the computed status reads `online`, and
+        // `lastConnectionEvent = "connected"`; the change-deriver re-fires the
+        // `satellite.connected` trigger event. This is also the connect-time
+        // heartbeat write (no separate `updateHeartbeat` needed), and it runs
+        // through `handle.mutate` so `prev` is snapshotted BEFORE the write.
+        if (this.connectionEntitySink) {
+          try {
+            await this.connectionEntitySink.mirror({
+              satelliteId: satellite.id,
+              lastEvent: "connected",
+              lastHeartbeatAt: new Date(),
+            });
+          } catch (error) {
+            this.logger.error(
+              `Failed to mirror satellite-connection (connected) for ${satellite.name}:`,
+              error,
+            );
+          }
+        } else {
+          // No entity sink wired (e.g. unit tests): still record the
+          // connect-time heartbeat directly so liveness is correct.
+          await this.service.updateHeartbeat(satellite.id, {});
+        }
 
-        // Send authenticated response with full config
+        // Send authenticated response with full config. Carry the desired
+        // script-package lockfile hash as the durable convergence backstop:
+        // a satellite that missed a refresh push reconciles on connect.
         const assignments =
           await this.configRelay.getAssignmentsForSatellite(satellite.id);
+        const scriptPackagesLockfileHash =
+          await this.resolveDesiredLockfileHash();
 
         this.sendMessage(ws, {
           type: "authenticated",
           satelliteId: satellite.id,
           assignments,
+          ...(scriptPackagesLockfileHash === undefined
+            ? {}
+            : { scriptPackagesLockfileHash }),
         });
 
         this.logger.info(
@@ -135,6 +258,80 @@ export class SatelliteWsHandler implements WebSocketRouteHandler {
           );
           break;
         }
+        case "script_package_sync_state": {
+          // Persist the satellite's reconcile state for the admin UI.
+          try {
+            await this.scriptPackageSink?.reportSyncState({
+              satelliteId: authenticatedSatellite.id,
+              lockfileHash: parsed.lockfileHash,
+              status: parsed.status,
+              errorMessage: parsed.errorMessage,
+            });
+          } catch (error) {
+            this.logger.error(
+              `Failed to persist script-package sync state for ${authenticatedSatellite.name}:`,
+              error,
+            );
+          }
+          break;
+        }
+        case "request_script_package_manifest": {
+          const entries =
+            (await this.scriptPackageSink?.getManifest({
+              lockfileHash: parsed.lockfileHash,
+            })) ?? [];
+          this.sendMessage(ws, {
+            type: "script_package_manifest",
+            lockfileHash: parsed.lockfileHash,
+            entries,
+          });
+          break;
+        }
+        case "request_script_package_blob": {
+          const data =
+            (await this.scriptPackageSink?.getBlobBase64({
+              integrity: parsed.integrity,
+            })) ?? null;
+          this.sendMessage(ws, {
+            type: "script_package_blob",
+            integrity: parsed.integrity,
+            data,
+          });
+          break;
+        }
+        case "request_run_secrets": {
+          // JIT secret delivery: resolve ONLY the collector's declared
+          // secretEnv (read from the persisted assignment, not chosen by
+          // the satellite) and reply with the env map. On any failure,
+          // reply with an error so the satellite fails the run clearly.
+          if (!this.secretSink) {
+            this.sendMessage(ws, {
+              type: "run_secrets",
+              requestId: parsed.requestId,
+              error: "Secret delivery is not available on this core instance.",
+            });
+            break;
+          }
+          try {
+            const env = await this.secretSink.resolveRunSecrets({
+              satelliteId: authenticatedSatellite.id,
+              configId: parsed.configId,
+              collectorId: parsed.collectorId,
+            });
+            this.sendMessage(ws, {
+              type: "run_secrets",
+              requestId: parsed.requestId,
+              env,
+            });
+          } catch (error) {
+            this.sendMessage(ws, {
+              type: "run_secrets",
+              requestId: parsed.requestId,
+              error: extractErrorMessage(error),
+            });
+          }
+          break;
+        }
         case "authenticate": {
           // Already authenticated, ignore duplicate auth attempts
           this.logger.debug(
@@ -147,10 +344,34 @@ export class SatelliteWsHandler implements WebSocketRouteHandler {
 
     const onClose = () => {
       if (authenticatedSatellite) {
-        this.connections.delete(authenticatedSatellite.id);
+        const closedSatellite = authenticatedSatellite;
+        this.connections.delete(closedSatellite.id);
         this.logger.info(
-          `Satellite disconnected: ${authenticatedSatellite.name} (${authenticatedSatellite.region})`,
+          `Satellite disconnected: ${closedSatellite.name} (${closedSatellite.region})`,
         );
+        if (this.connectionEntitySink) {
+          // Fire-and-forget — `onClose` is sync, so don't await; we don't have
+          // a place to surface a rejection anyway. Clear `lastHeartbeatAt`
+          // (`null`) so the computed status flips `offline` IMMEDIATELY on a
+          // clean disconnect (no waiting for the heartbeat to age out), and set
+          // `lastConnectionEvent = "disconnected"` so the deriver re-fires
+          // `satellite.disconnected`. Nulling the heartbeat coincides with the
+          // "never connected" representation, but `lastConnectionEvent` stays
+          // `"disconnected"` (non-null), so the entity still HAS state — the
+          // read only omits a satellite whose `lastConnectionEvent` is null.
+          void this.connectionEntitySink
+            .mirror({
+              satelliteId: closedSatellite.id,
+              lastEvent: "disconnected",
+              lastHeartbeatAt: null,
+            })
+            .catch((error: unknown) => {
+              this.logger.error(
+                `Failed to mirror satellite-connection (disconnected) for ${closedSatellite.name}:`,
+                error,
+              );
+            });
+        }
       }
     };
 
@@ -166,10 +387,14 @@ export class SatelliteWsHandler implements WebSocketRouteHandler {
 
     const assignments =
       await this.configRelay.getAssignmentsForSatellite(satelliteId);
+    const scriptPackagesLockfileHash = await this.resolveDesiredLockfileHash();
 
     this.sendMessage(conn.ws, {
       type: "config_updated",
       assignments,
+      ...(scriptPackagesLockfileHash === undefined
+        ? {}
+        : { scriptPackagesLockfileHash }),
     });
 
     this.logger.debug(
@@ -177,6 +402,41 @@ export class SatelliteWsHandler implements WebSocketRouteHandler {
     );
   }
 
+  /**
+   * Push a `refresh_script_packages` to every connected satellite. Called by
+   * the `script-packages.changed` broadcast handler so each core instance
+   * fans the refresh out to its own satellites. Best-effort liveness; the
+   * assignment-carried hash is the durable backstop.
+   */
+  pushRefreshScriptPackagesToAll(lockfileHash: string): void {
+    for (const conn of this.connections.values()) {
+      this.sendMessage(conn.ws, {
+        type: "refresh_script_packages",
+        lockfileHash,
+      });
+    }
+    this.logger.debug(
+      `Pushed refresh_script_packages (${lockfileHash}) to ${this.connections.size} satellite(s)`,
+    );
+  }
+
+  /**
+   * Resolve the desired lockfile hash for assignment payloads. Returns
+   * `undefined` when the sink isn't wired (so the field is omitted entirely
+   * for version-skew safety), or `string | null` from the sink.
+   */
+  private async resolveDesiredLockfileHash(): Promise<
+    string | null | undefined
+  > {
+    if (!this.scriptPackageSink) return undefined;
+    try {
+      return await this.scriptPackageSink.getDesiredLockfileHash();
+    } catch (error) {
+      this.logger.error("Failed to resolve desired lockfile hash:", error);
+      return undefined;
+    }
+  }
+
   /**
    * Send a shutdown message to a specific satellite (e.g., on token revocation).
    */

package/src/schema.ts

@@ -19,9 +19,30 @@ export const satellites = pgTable("satellites", {
   tags: jsonb("tags").$type<Record<string, string>>().default({}).notNull(),
   /** Bcrypt hash of the satellite's API token */
   tokenHash: text("token_hash").notNull(),
-  /** Last heartbeat timestamp — null means never connected */
+  /**
+   * Last heartbeat timestamp — null means never connected (or cleanly
+   * disconnected). This is the SINGLE durable liveness source of truth: the
+   * reactive `satellite-connection` entity's `status` and `lastSeenAt` are
+   * COMPUTED on read from it (via `computeStatus` / `OFFLINE_THRESHOLD_MS`), so
+   * the entity is globally consistent from any pod and self-heals — a stale row
+   * reads `offline` once this timestamp ages past the offline threshold, even
+   * if the pod that owned the socket crashed without writing offline.
+   */
   lastHeartbeatAt: timestamp("last_heartbeat_at"),
   /** Satellite version reported on connect/heartbeat */
   version: text("version"),
+  /**
+   * Which lifecycle edge produced the latest connection-status change. Preserves
+   * the distinction between a socket drop (`disconnected`) and the heartbeat-lost
+   * offline edge (`heartbeat_lost`) that a bare status diff cannot encode. This
+   * is the ONLY durable connection column the reactive `satellite-connection`
+   * entity needs beyond `lastHeartbeatAt`: the deriver reads it as `lastEvent`,
+   * and the heartbeat monitor uses it to make heartbeat-lost detection
+   * idempotent (once it is `"heartbeat_lost"`, re-runs are no-ops). Nullable: a
+   * satellite that never connected has no last event.
+   */
+  lastConnectionEvent: text("last_connection_event", {
+    enum: ["connected", "disconnected", "heartbeat_lost"],
+  }),
   createdAt: timestamp("created_at").defaultNow().notNull(),
 });