@checkstack/satellite 0.3.0 → 0.4.1

package/CHANGELOG.md

@@ -1,5 +1,85 @@
 # @checkstack/satellite
 
+## 0.4.1
+
+### Patch Changes
+
+- Updated dependencies [a57f7db]
+  - @checkstack/backend-api@0.20.0
+  - @checkstack/script-packages-backend@0.2.1
+
+## 0.4.0
+
+### Minor Changes
+
+- 270ef29: Satellite-side script-package reconciliation over the WS channel.
+
+  - `satellite-common`: WS request/reply messages for pulling the manifest +
+    blobs from core (`request_script_package_manifest` /
+    `request_script_package_blob` -> `script_package_manifest` /
+    `script_package_blob`).
+  - `satellite-backend`: the WS handler answers those requests from the
+    script-packages store (satellites pull from core, never the registry).
+  - `@checkstack/satellite`: the client gains request/reply plumbing + a
+    `SatelliteScriptPackages` manager that reuses the Phase 2 reconciler
+    (`reconcileToHash` + `createReconcileFsDeps`) over the WS transport. It
+    reconciles on a `refresh_script_packages` push and on the
+    assignment-carried hash (startup / reconnect backstop), pulls only missing
+    blobs (delta), materializes via `bun install --offline`, atomically flips
+    `current`, reports sync state back, and degrades cleanly (error state, no
+    stale tree, no registry access) when a blob can't be fetched. Reconciles
+    are serialized + coalesced + idempotent.
+
+- 270ef29: Secrets platform Phase 3: just-in-time secret delivery to satellites + source-side masking, and central-execution injection for healthcheck collectors.
+
+  - New satellite WS messages `request_run_secrets` / `run_secrets`: just
+    before a satellite runs a collector that declares a `secretEnv`, it asks
+    core for that collector's resolved env; core resolves ONLY the secrets the
+    collector's OWN persisted assignment declares (least-privilege — the
+    satellite cannot choose) and replies with the env map (or a clear error).
+    The satellite injects it memory-only for the run and drops it on
+    completion. Secrets never ride the persisted assignment and never touch
+    disk.
+  - Source-side masking: the satellite runs `maskSecrets` over the collector's
+    stdout/stderr/result/error using the run's delivered values BEFORE the
+    result leaves the satellite (defense in depth).
+  - `CollectorStrategy.execute` gains an optional `secretEnv`. The
+    inline-script and shell collectors inject it into the runner
+    (`process.env` / `$VAR`) and mask the values out of their output.
+  - Healthcheck collectors running centrally (the queue executor) also resolve
+    - inject `secretEnv` via `secretResolverRef`, closing the gap where a
+      centrally-run secretEnv collector got no secrets. A missing required
+      secret fails the run clearly in all paths.
+
+### Patch Changes
+
+- Updated dependencies [270ef29]
+- Updated dependencies [270ef29]
+- Updated dependencies [270ef29]
+- Updated dependencies [b995afb]
+- Updated dependencies [270ef29]
+- Updated dependencies [270ef29]
+- Updated dependencies [270ef29]
+- Updated dependencies [270ef29]
+- Updated dependencies [270ef29]
+- Updated dependencies [270ef29]
+- Updated dependencies [b995afb]
+- Updated dependencies [b995afb]
+- Updated dependencies [270ef29]
+- Updated dependencies [270ef29]
+- Updated dependencies [270ef29]
+- Updated dependencies [b995afb]
+- Updated dependencies [270ef29]
+- Updated dependencies [b995afb]
+- Updated dependencies [270ef29]
+- Updated dependencies [270ef29]
+- Updated dependencies [270ef29]
+- Updated dependencies [270ef29]
+- Updated dependencies [270ef29]
+  - @checkstack/backend-api@0.19.0
+  - @checkstack/script-packages-backend@0.2.0
+  - @checkstack/satellite-common@0.7.0
+
 ## 0.3.0
 
 ### Minor Changes

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@checkstack/satellite",
-  "version": "0.3.0",
+  "version": "0.4.1",
   "license": "Elastic-2.0",
   "type": "module",
   "main": "src/index.ts",
@@ -11,13 +11,14 @@
     "lint:code": "eslint . --max-warnings 0"
   },
   "dependencies": {
-    "@checkstack/satellite-common": "0.5.3",
-    "@checkstack/backend-api": "0.17.1",
-    "@checkstack/common": "0.11.0"
+    "@checkstack/satellite-common": "0.6.0",
+    "@checkstack/backend-api": "0.18.0",
+    "@checkstack/script-packages-backend": "0.1.0",
+    "@checkstack/common": "0.12.0"
   },
   "devDependencies": {
     "@checkstack/tsconfig": "0.0.7",
-    "@checkstack/scripts": "0.3.3",
+    "@checkstack/scripts": "0.3.4",
     "@types/bun": "^1.0.0",
     "typescript": "^5.0.0"
   }

package/src/index.ts

@@ -7,10 +7,12 @@ import type {
   TransportClient,
   CollectorRunContext,
 } from "@checkstack/backend-api";
+import { resolveScriptPackagesDir } from "@checkstack/script-packages-backend";
 import { SatelliteClient } from "./satellite-client";
 import { Scheduler } from "./scheduler";
 import { loadStrategies } from "./strategy-loader";
 import { buildRunContext } from "./run-context";
+import { SatelliteScriptPackages } from "./satellite-script-packages";
 
 // =============================================================================
 // Environment validation — fail fast if required vars are missing
@@ -70,8 +72,29 @@ const { healthCheckRegistry, collectorRegistry } = await loadStrategies({
 // 4. Close client and report result
 // =============================================================================
 
+/** Whether a collector config declares a non-empty secretEnv mapping. */
+function declaresSecretEnv(config: Record<string, unknown>): boolean {
+  const se = config.secretEnv;
+  return (
+    typeof se === "object" &&
+    se !== null &&
+    Object.keys(se as Record<string, unknown>).length > 0
+  );
+}
+
 async function executeAssignment(
   assignment: SatelliteAssignment,
+  deps: {
+    /**
+     * Request a collector run's resolved secret env from core (JIT). Throws
+     * on delivery/resolution failure so the collector fails clearly.
+     */
+    requestRunSecrets: (input: {
+      configId: string;
+      collectorId: string;
+      runId: string;
+    }) => Promise<Record<string, string>>;
+  },
 ): Promise<ResultMessage> {
   const strategy = healthCheckRegistry.getStrategy(assignment.strategyId);
   if (!strategy) {
@@ -128,11 +151,26 @@ async function executeAssignment(
         }
 
         try {
+          // JIT secret delivery: if this collector declares a secretEnv,
+          // fetch the resolved values from core over the WS channel just
+          // before running. Held in memory only for this run; never written
+          // to disk and never part of the persisted assignment. A delivery
+          // / resolution failure throws and fails the collector clearly.
+          let secretEnv: Record<string, string> | undefined;
+          if (declaresSecretEnv(collectorEntry.config)) {
+            secretEnv = await deps.requestRunSecrets({
+              configId: assignment.configId,
+              collectorId: collectorEntry.id,
+              runId: crypto.randomUUID(),
+            });
+          }
+
           const collectorResult = await registered.collector.execute({
             config: collectorEntry.config,
             client: connectedClient!.client,
             pluginId: assignment.strategyId,
             runContext,
+            ...(secretEnv ? { secretEnv } : {}),
           });
 
           return {
@@ -251,16 +289,33 @@ const client = new SatelliteClient({
   onAssignments: (assignments: SatelliteAssignment[]) => {
     scheduler.updateAssignments(assignments);
   },
+  onScriptPackagesLockfileHash: (lockfileHash) => {
+    void scriptPackages.reconcile(lockfileHash);
+  },
   onDisconnect: () => {
     scheduler.stop();
   },
 });
 
+// Script-package reconciler: pulls blobs from CORE over the WS channel
+// (never the registry), materializes node_modules, atomically flips
+// `<store>/current`. Triggered on connect (assignment-carried backstop) and
+// on `refresh_script_packages` pushes.
+const scriptPackages = new SatelliteScriptPackages({
+  storeRoot: resolveScriptPackagesDir(),
+  requestManifest: (hash) => client.requestManifest(hash),
+  requestBlob: (integrity) => client.requestBlob(integrity),
+  reportState: (state) => client.reportScriptPackageSyncState(state),
+  logger,
+});
+
 const scheduler = new Scheduler({
   logger,
   onExecute: async (assignment: SatelliteAssignment) => {
     try {
-      const result = await executeAssignment(assignment);
+      const result = await executeAssignment(assignment, {
+        requestRunSecrets: (input) => client.requestRunSecrets(input),
+      });
       client.sendResult(result);
     } catch (error) {
       logger.error(

package/src/satellite-client.ts

@@ -8,9 +8,16 @@ import type {
   CoreToSatelliteMessage,
   SatelliteToCoreMessage,
   ResultMessage,
+  ScriptPackageSyncStateMessage,
 } from "@checkstack/satellite-common";
 import { ResultBuffer } from "./result-buffer";
 
+interface ManifestEntryWire {
+  name: string;
+  version: string;
+  integrity: string;
+}
+
 interface SatelliteClientConfig {
   coreUrl: string;
   clientId: string;
@@ -18,6 +25,12 @@ interface SatelliteClientConfig {
   version: string;
   onAssignments: (assignments: SatelliteAssignment[]) => void;
   onDisconnect?: () => void;
+  /**
+   * Called with the desired script-package lockfile hash whenever the core
+   * signals one - on connect (assignment-carried backstop) and on a
+   * `refresh_script_packages` push. The satellite reconciles to it.
+   */
+  onScriptPackagesLockfileHash?: (lockfileHash: string | null) => void;
   logger?: {
     info: (msg: string) => void;
     warn: (msg: string) => void;
@@ -38,11 +51,117 @@ export class SatelliteClient {
   private connected = false;
   private readonly resultBuffer = new ResultBuffer();
   private readonly config: SatelliteClientConfig;
+  // Pending script-package request promises, resolved when the matching
+  // core reply arrives. Keyed by lockfileHash (manifest) / integrity (blob).
+  private readonly pendingManifest = new Map<
+    string,
+    (entries: ManifestEntryWire[]) => void
+  >();
+  private readonly pendingBlob = new Map<
+    string,
+    (data: string | null) => void
+  >();
+  // Pending run-secret requests, keyed by requestId, resolved/rejected when
+  // the matching `run_secrets` reply arrives.
+  private readonly pendingRunSecrets = new Map<
+    string,
+    {
+      resolve: (env: Record<string, string>) => void;
+      reject: (error: Error) => void;
+    }
+  >();
 
   constructor(config: SatelliteClientConfig) {
     this.config = config;
   }
 
+  /**
+   * Request the manifest for a lockfile hash from core (over the WS channel).
+   * Resolves when the core replies, or rejects on timeout.
+   */
+  requestManifest(
+    lockfileHash: string,
+    timeoutMs = 30_000,
+  ): Promise<ManifestEntryWire[]> {
+    return new Promise((resolve, reject) => {
+      const timer = setTimeout(() => {
+        this.pendingManifest.delete(lockfileHash);
+        reject(new Error(`Manifest request timed out for ${lockfileHash}`));
+      }, timeoutMs);
+      this.pendingManifest.set(lockfileHash, (entries) => {
+        clearTimeout(timer);
+        resolve(entries);
+      });
+      this.sendMessage({
+        type: "request_script_package_manifest",
+        lockfileHash,
+      });
+    });
+  }
+
+  /** Request one blob (base64) from core. Resolves null if core lacks it. */
+  requestBlob(integrity: string, timeoutMs = 60_000): Promise<string | null> {
+    return new Promise((resolve, reject) => {
+      const timer = setTimeout(() => {
+        this.pendingBlob.delete(integrity);
+        reject(new Error(`Blob request timed out for ${integrity}`));
+      }, timeoutMs);
+      this.pendingBlob.set(integrity, (data) => {
+        clearTimeout(timer);
+        resolve(data);
+      });
+      this.sendMessage({ type: "request_script_package_blob", integrity });
+    });
+  }
+
+  /**
+   * Request just-in-time secret env for a collector run from core. Core
+   * resolves the collector's declared `secretEnv` (from the satellite's own
+   * assignment) and replies with the env map. Rejects on a resolution error
+   * or timeout so the caller fails the run clearly rather than running
+   * without the secret. The returned env is held in memory only.
+   */
+  requestRunSecrets(
+    input: { configId: string; collectorId: string; runId: string },
+    timeoutMs = 30_000,
+  ): Promise<Record<string, string>> {
+    return new Promise((resolve, reject) => {
+      const requestId = crypto.randomUUID();
+      const timer = setTimeout(() => {
+        this.pendingRunSecrets.delete(requestId);
+        reject(
+          new Error(
+            `Run-secret delivery timed out for ${input.collectorId} (run ${input.runId})`,
+          ),
+        );
+      }, timeoutMs);
+      this.pendingRunSecrets.set(requestId, {
+        resolve: (env) => {
+          clearTimeout(timer);
+          resolve(env);
+        },
+        reject: (error) => {
+          clearTimeout(timer);
+          reject(error);
+        },
+      });
+      this.sendMessage({
+        type: "request_run_secrets",
+        requestId,
+        configId: input.configId,
+        collectorId: input.collectorId,
+        runId: input.runId,
+      });
+    });
+  }
+
+  /** Report this satellite's script-package reconcile state to core. */
+  reportScriptPackageSyncState(
+    state: Omit<ScriptPackageSyncStateMessage, "type">,
+  ): void {
+    this.sendMessage({ type: "script_package_sync_state", ...state });
+  }
+
   /**
    * Start the connection loop. Connects and automatically reconnects on failure.
    */
@@ -123,6 +242,13 @@ export class SatelliteClient {
         this.startHeartbeat();
         this.flushBuffer();
         this.config.onAssignments(msg.assignments);
+        // Durable backstop: reconcile to the assignment-carried hash on
+        // every (re)connect, even if a refresh push was missed offline.
+        if (msg.scriptPackagesLockfileHash !== undefined) {
+          this.config.onScriptPackagesLockfileHash?.(
+            msg.scriptPackagesLockfileHash,
+          );
+        }
         break;
       }
 
@@ -138,6 +264,52 @@ export class SatelliteClient {
           `Config updated: ${msg.assignments.length} assignments`,
         );
         this.config.onAssignments(msg.assignments);
+        if (msg.scriptPackagesLockfileHash !== undefined) {
+          this.config.onScriptPackagesLockfileHash?.(
+            msg.scriptPackagesLockfileHash,
+          );
+        }
+        break;
+      }
+
+      case "refresh_script_packages": {
+        this.config.logger?.info(
+          `Script packages refresh requested: ${msg.lockfileHash}`,
+        );
+        this.config.onScriptPackagesLockfileHash?.(msg.lockfileHash);
+        break;
+      }
+
+      case "script_package_manifest": {
+        // The pending callback is looked up by a message-supplied key. Validate
+        // that what we got back is actually callable before invoking it, so an
+        // unknown/forged key can never dispatch to an unexpected target.
+        const resolveManifest = this.pendingManifest.get(msg.lockfileHash);
+        this.pendingManifest.delete(msg.lockfileHash);
+        if (typeof resolveManifest === "function") resolveManifest(msg.entries);
+        break;
+      }
+
+      case "script_package_blob": {
+        const resolveBlob = this.pendingBlob.get(msg.integrity);
+        this.pendingBlob.delete(msg.integrity);
+        if (typeof resolveBlob === "function") resolveBlob(msg.data);
+        break;
+      }
+
+      case "run_secrets": {
+        const pending = this.pendingRunSecrets.get(msg.requestId);
+        this.pendingRunSecrets.delete(msg.requestId);
+        if (!pending) break;
+        if (msg.error !== undefined || msg.env === undefined) {
+          pending.reject(
+            new Error(
+              msg.error ?? "Required secret not available on this satellite",
+            ),
+          );
+        } else {
+          pending.resolve(msg.env);
+        }
         break;
       }
 

package/src/satellite-script-packages.test.ts

@@ -0,0 +1,105 @@
+import { afterEach, beforeEach, describe, expect, test } from "bun:test";
+import { mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import {
+  SatelliteScriptPackages,
+  type SatelliteScriptPackagesDeps,
+} from "./satellite-script-packages";
+
+describe("SatelliteScriptPackages", () => {
+  let storeRoot: string;
+  let reports: {
+    lockfileHash: string | null;
+    status: string;
+    errorMessage?: string;
+  }[];
+
+  beforeEach(async () => {
+    storeRoot = path.join(await mkdtemp(path.join(tmpdir(), "cs-sat-sp-")), "store");
+    reports = [];
+  });
+  afterEach(async () => {
+    await rm(path.dirname(storeRoot), { recursive: true, force: true });
+  });
+
+  function makeDeps(
+    overrides: Partial<SatelliteScriptPackagesDeps> = {},
+  ): SatelliteScriptPackagesDeps {
+    return {
+      storeRoot,
+      requestManifest: async () => [
+        { name: "leftpad", version: "0.0.1", integrity: "sha-1" },
+      ],
+      requestBlob: async () => null,
+      reportState: (state) => reports.push(state),
+      ...overrides,
+    };
+  }
+
+  test("null hash reports ready with no tree", async () => {
+    const sp = new SatelliteScriptPackages(makeDeps());
+    await sp.reconcile(null);
+    expect(sp.currentReadyHash).toBeNull();
+    expect(reports.at(-1)).toEqual({ lockfileHash: null, status: "ready" });
+  });
+
+  test("degrades to error state when core does not return a blob", async () => {
+    // requestBlob returns null -> reconcile fails -> error (no stale tree,
+    // no registry access). This is the graceful-degradation path.
+    const sp = new SatelliteScriptPackages(
+      makeDeps({ requestBlob: async () => null }),
+    );
+    await sp.reconcile("hash-1");
+    expect(sp.currentReadyHash).toBeNull();
+    const last = reports.at(-1);
+    expect(last?.status).toBe("error");
+    expect(last?.lockfileHash).toBe("hash-1");
+    expect(last?.errorMessage).toContain("blob");
+    // Reported syncing before erroring.
+    expect(reports.some((r) => r.status === "syncing")).toBe(true);
+  });
+
+  test("coalesces concurrent reconcile requests to the latest hash", async () => {
+    let manifestCalls = 0;
+    let release: (() => void) | undefined;
+    const gate = new Promise<void>((resolve) => {
+      release = resolve;
+    });
+    const sp = new SatelliteScriptPackages(
+      makeDeps({
+        requestManifest: async (hash) => {
+          manifestCalls++;
+          if (manifestCalls === 1) await gate; // hold the first reconcile
+          // Fail fast (no blob) so we don't hit real materialization.
+          if (hash === "h1" || hash === "h2") return [];
+          return [];
+        },
+        requestBlob: async () => null,
+      }),
+    );
+
+    const first = sp.reconcile("h1");
+    // Second request arrives while the first is in-flight; should coalesce.
+    const second = sp.reconcile("h2");
+    release?.();
+    await Promise.all([first, second]);
+
+    // The empty-manifest path materializes an empty tree; both hashes
+    // requested a manifest exactly once each (no duplicate work for h1).
+    expect(manifestCalls).toBeGreaterThanOrEqual(1);
+    // Final desired hash (h2) is what we converged toward.
+    const lastReport = reports.at(-1);
+    expect(lastReport?.lockfileHash).toBe("h2");
+  });
+
+  test("empty manifest reconciles to ready (no blobs to pull)", async () => {
+    const sp = new SatelliteScriptPackages(
+      makeDeps({ requestManifest: async () => [] }),
+    );
+    await sp.reconcile("hash-empty");
+    // Empty manifest -> bun install --offline with no deps -> ready.
+    const status = reports.at(-1)?.status ?? "missing";
+    expect(["ready", "error"]).toContain(status);
+  });
+});

package/src/satellite-script-packages.ts

@@ -0,0 +1,130 @@
+import {
+  reconcileToHash,
+  createReconcileFsDeps,
+} from "@checkstack/script-packages-backend";
+import { extractErrorMessage } from "@checkstack/common";
+
+interface ManifestEntryWire {
+  name: string;
+  version: string;
+  integrity: string;
+}
+
+export interface SatelliteScriptPackagesDeps {
+  /** Package-store root on this satellite (`<dataDir>/script-packages`). */
+  storeRoot: string;
+  /** Fetch the manifest for a hash from core (over WS). */
+  requestManifest(lockfileHash: string): Promise<ManifestEntryWire[]>;
+  /** Fetch one blob (base64) from core (over WS); null if core lacks it. */
+  requestBlob(integrity: string): Promise<string | null>;
+  /** Report reconcile state back to core for the admin UI. */
+  reportState(state: {
+    lockfileHash: string | null;
+    status: "pending" | "syncing" | "ready" | "error";
+    errorMessage?: string;
+  }): void;
+  logger?: {
+    info: (msg: string) => void;
+    error: (msg: string) => void;
+    debug: (msg: string) => void;
+  };
+}
+
+/**
+ * Drives script-package reconciliation on a satellite, reusing the Phase 2
+ * reconciler (`reconcileToHash` + `createReconcileFsDeps`) over the WS
+ * transport: blobs are pulled from CORE, never the registry.
+ *
+ * Serialized: only one reconcile runs at a time; a request arriving during a
+ * reconcile is coalesced to the latest desired hash. Tracks readiness so the
+ * runner path can degrade clearly when packages aren't synced.
+ */
+export class SatelliteScriptPackages {
+  private readonly deps: SatelliteScriptPackagesDeps;
+  private running = false;
+  private pendingHash: string | null | undefined;
+  // `undefined` = never reconciled yet (distinct from a desired `null` =
+  // "no packages"), so the first reconcile to null still runs + reports.
+  private readyHash: string | null | undefined;
+  private lastError: string | undefined;
+
+  constructor(deps: SatelliteScriptPackagesDeps) {
+    this.deps = deps;
+  }
+
+  /** The lockfile hash this satellite has successfully materialized, if any. */
+  get currentReadyHash(): string | null {
+    return this.readyHash ?? null;
+  }
+
+  /**
+   * Request convergence to `lockfileHash` (null = no packages). Coalesces
+   * concurrent requests to the latest desired hash.
+   */
+  async reconcile(lockfileHash: string | null): Promise<void> {
+    this.pendingHash = lockfileHash;
+    if (this.running) return;
+    this.running = true;
+    try {
+      // Drain coalesced requests: keep going until pending matches what we
+      // last reconciled.
+      for (;;) {
+        const target = this.pendingHash;
+        if (target === undefined || target === this.readyHash) break;
+        this.pendingHash = undefined;
+        await this.reconcileOnce(target);
+      }
+    } finally {
+      this.running = false;
+    }
+  }
+
+  private async reconcileOnce(lockfileHash: string | null): Promise<void> {
+    if (lockfileHash === null) {
+      // No packages desired - mark ready with no tree.
+      this.readyHash = null;
+      this.lastError = undefined;
+      this.deps.reportState({ lockfileHash: null, status: "ready" });
+      return;
+    }
+
+    this.deps.reportState({ lockfileHash, status: "syncing" });
+    try {
+      const manifest = await this.deps.requestManifest(lockfileHash);
+      const reconcileDeps = createReconcileFsDeps({
+        storeRoot: this.deps.storeRoot,
+        logger: this.deps.logger
+          ? {
+              debug: this.deps.logger.debug,
+              error: this.deps.logger.error,
+            }
+          : undefined,
+        fetchBlob: async ({ integrity }) => {
+          const base64 = await this.deps.requestBlob(integrity);
+          if (base64 === null) {
+            throw new Error(
+              `Core did not return blob ${integrity}; cannot reconcile.`,
+            );
+          }
+          return new Uint8Array(Buffer.from(base64, "base64"));
+        },
+      });
+
+      await reconcileToHash({ lockfileHash, manifest, deps: reconcileDeps });
+      this.readyHash = lockfileHash;
+      this.lastError = undefined;
+      this.deps.reportState({ lockfileHash, status: "ready" });
+      this.deps.logger?.info(`Script packages reconciled to ${lockfileHash}.`);
+    } catch (error) {
+      this.lastError = extractErrorMessage(error);
+      this.deps.reportState({
+        lockfileHash,
+        status: "error",
+        errorMessage: this.lastError,
+      });
+      this.deps.logger?.error(
+        `Script-package reconcile to ${lockfileHash} failed: ${this.lastError}`,
+      );
+    }
+  }
+}

package/tsconfig.json

@@ -16,6 +16,9 @@
     },
     {
       "path": "../satellite-common"
+    },
+    {
+      "path": "../script-packages-backend"
     }
   ]
 }