@checkstack/script-packages-backend 0.2.0

package/src/reconcile-fs.ts

@@ -0,0 +1,199 @@
+import { spawn } from "bun";
+import { mkdir, writeFile, readdir, stat } from "node:fs/promises";
+import path from "node:path";
+import type { ManifestEntry } from "@checkstack/script-packages-common";
+import type { ReconcileDeps } from "./reconciler";
+import { unpackInto } from "./cache-archive";
+import { verifyBlobSha256 } from "./blob-hash";
+import { atomicSymlinkSwap, readCurrentTarget } from "./atomic-symlink";
+import { storePaths } from "./data-dir";
+import { sweepTreeGc } from "./tree-gc";
+import { extractErrorMessage } from "@checkstack/common";
+
+/**
+ * Concrete filesystem/Bun adapter for the host {@link ReconcileDeps}.
+ *
+ * Layout under the store root:
+ *   - `cache/`                 Bun cache (blobs extracted here)
+ *   - `cache/.cs-integrity/`   marker files (hex integrity) tracking which
+ *                              blobs are seeded - the cache dir names are
+ *                              `name@version@@@n`, not integrities, so we
+ *                              track presence out-of-band for delta diffing
+ *   - `trees/<lockfileHash>/`  a materialized install (package.json + the
+ *                              `bun install --offline` node_modules)
+ *   - `current -> trees/<hash>`atomically-flipped symlink the runner's
+ *                              `resolutionRoot` points at
+ *
+ * Materialize seeds nothing itself - the reconciler has already extracted
+ * the needed blobs into `cache/`. It writes a package.json whose deps are
+ * the manifest's top-level set, runs `bun install --offline` (zero network,
+ * reconstructs node_modules from the warm cache), then atomically flips
+ * `current`.
+ */
+
+const INTEGRITY_MARKER_DIR = ".cs-integrity";
+
+function integrityMarkerName(integrity: string): string {
+  return Buffer.from(integrity, "utf8").toString("hex");
+}
+
+function bunfigDisableAutoInstall(): string {
+  return '[install]\nauto = "disable"\n';
+}
+
+export function createReconcileFsDeps(input: {
+  storeRoot: string;
+  /** Fetch a blob (shared store on core; HTTP on satellite). */
+  fetchBlob: (input: { integrity: string }) => Promise<Uint8Array>;
+  /**
+   * Tree-GC grace window (ms). Defaults inside {@link sweepTreeGc} to a value
+   * comfortably exceeding the max run timeout so a live run pinned to an
+   * old tree is never deleted. Injectable for tests.
+   */
+  treeGcGraceMs?: number;
+  logger?: { debug(msg: string): void; error(msg: string): void };
+}): ReconcileDeps {
+  const paths = storePaths(input.storeRoot);
+  const markerDir = path.join(paths.cache, INTEGRITY_MARKER_DIR);
+
+  return {
+    logger: input.logger,
+
+    async localCacheIntegrities() {
+      try {
+        const hexes = await readdir(markerDir);
+        return hexes.map((hex) =>
+          Buffer.from(hex, "hex").toString("utf8"),
+        );
+      } catch {
+        return [];
+      }
+    },
+
+    fetchBlob: input.fetchBlob,
+
+    async seedBlob({ entry, bytes }) {
+      // Verify the transported bytes against the manifest's recorded blob
+      // hash BEFORE extracting. On a mismatch, discard + error clearly so a
+      // corrupt/tampered blob is never materialized. (Entries published
+      // before `blobSha256` skip verification — backward-safe.)
+      const verdict = verifyBlobSha256({ entry, bytes });
+      if (!verdict.ok) {
+        throw new Error(
+          `Blob integrity check failed for ${entry.name}@${entry.version} ` +
+            `(${entry.integrity}): expected sha256 ${verdict.expected}, got ` +
+            `${verdict.actual}. Refusing to seed.`,
+        );
+      }
+      await mkdir(paths.cache, { recursive: true });
+      await unpackInto({ targetDir: paths.cache, bytes });
+      await mkdir(markerDir, { recursive: true });
+      await writeFile(
+        path.join(markerDir, integrityMarkerName(entry.integrity)),
+        entry.integrity,
+      );
+    },
+
+    async currentLockfileHash() {
+      const target = await readCurrentTarget(paths.current);
+      if (!target) return;
+      // target is `trees/<hash>`; the hash is the basename.
+      return path.basename(target);
+    },
+
+    async materializeAndFlip({ lockfileHash, manifest }) {
+      const treeDir = path.join(paths.trees, lockfileHash);
+      // If the tree already exists with a node_modules, just (re)flip.
+      const alreadyBuilt = await dirExists(path.join(treeDir, "node_modules"));
+      if (!alreadyBuilt) {
+        await mkdir(treeDir, { recursive: true });
+        await writeFile(
+          path.join(treeDir, "package.json"),
+          renderManifestPackageJson(manifest),
+        );
+        await writeFile(
+          path.join(treeDir, "bunfig.toml"),
+          bunfigDisableAutoInstall(),
+        );
+        const proc = spawn({
+          cmd: [
+            process.execPath,
+            "install",
+            "--offline",
+            "--no-save",
+            "--ignore-scripts",
+          ],
+          cwd: treeDir,
+          env: { ...process.env, BUN_INSTALL_CACHE_DIR: paths.cache },
+          stdout: "pipe",
+          stderr: "pipe",
+        });
+        const [stderr, exitCode] = await Promise.all([
+          new Response(proc.stderr).text(),
+          proc.exited,
+        ]);
+        if (exitCode !== 0) {
+          throw new Error(
+            `bun install --offline failed for ${lockfileHash} (exit ${exitCode}): ${stderr.slice(0, 800)}`,
+          );
+        }
+      }
+      await atomicSymlinkSwap({
+        linkPath: paths.current,
+        target: path.join("trees", lockfileHash),
+      });
+
+      // Host-local tree GC after a successful flip: prune old non-current
+      // trees past the grace window. Best-effort — a sweep failure must never
+      // fail a reconcile (the new tree is already live). The grace window
+      // guards live runs still pinned to a just-superseded tree.
+      try {
+        await sweepTreeGc({
+          storeRoot: input.storeRoot,
+          graceMs: input.treeGcGraceMs,
+          logger: input.logger,
+        });
+      } catch (error) {
+        input.logger?.error(
+          `Tree GC after flip to ${lockfileHash} failed: ${extractErrorMessage(error)}`,
+        );
+      }
+    },
+  };
+}
+
+/**
+ * Build the package.json for a materialized tree. Pins every manifest
+ * package by exact version so `bun install --offline` resolves the same
+ * tree the installer produced (transitive deps are already in the cache).
+ */
+function renderManifestPackageJson(manifest: ManifestEntry[]): string {
+  const dependencies: Record<string, string> = {};
+  for (const entry of manifest) {
+    dependencies[entry.name] = entry.version;
+  }
+  return (
+    JSON.stringify(
+      {
+        name: "checkstack-script-packages-tree",
+        private: true,
+        dependencies: Object.fromEntries(
+          Object.entries(dependencies).toSorted(([a], [b]) =>
+            a.localeCompare(b),
+          ),
+        ),
+      },
+      null,
+      2,
+    ) + "\n"
+  );
+}
+
+async function dirExists(dir: string): Promise<boolean> {
+  try {
+    const info = await stat(dir);
+    return info.isDirectory();
+  } catch {
+    return false;
+  }
+}

package/src/reconciler.test.ts

@@ -0,0 +1,289 @@
+import { afterEach, beforeEach, describe, expect, test } from "bun:test";
+import { mkdtemp, mkdir, writeFile, readdir, readlink, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import type { ManifestEntry } from "@checkstack/script-packages-common";
+import { reconcileToHash, type ReconcileDeps } from "./reconciler";
+import { createReconcileFsDeps } from "./reconcile-fs";
+import { packDir } from "./cache-archive";
+import { blobSha256 } from "./blob-hash";
+import { storePaths } from "./data-dir";
+
+const a: ManifestEntry = { name: "a", version: "1.0.0", integrity: "sha-a" };
+const b: ManifestEntry = { name: "b", version: "2.0.0", integrity: "sha-b" };
+
+function makeDeps(overrides: Partial<ReconcileDeps> = {}): {
+  deps: ReconcileDeps;
+  fetched: string[];
+  seeded: string[];
+  materialized: string[];
+} {
+  const fetched: string[] = [];
+  const seeded: string[] = [];
+  const materialized: string[] = [];
+  const deps: ReconcileDeps = {
+    currentLockfileHash: async () => undefined,
+    localCacheIntegrities: async () => [],
+    fetchBlob: async ({ integrity }) => {
+      fetched.push(integrity);
+      return new Uint8Array([1]);
+    },
+    seedBlob: async ({ entry }) => {
+      seeded.push(entry.integrity);
+    },
+    materializeAndFlip: async ({ lockfileHash }) => {
+      materialized.push(lockfileHash);
+    },
+    ...overrides,
+  };
+  return { deps, fetched, seeded, materialized };
+}
+
+describe("reconcileToHash", () => {
+  test("pulls only missing blobs, then materializes + flips", async () => {
+    const { deps, fetched, seeded, materialized } = makeDeps({
+      localCacheIntegrities: async () => ["sha-a"],
+    });
+    const res = await reconcileToHash({
+      lockfileHash: "HASH1",
+      manifest: [a, b],
+      deps,
+    });
+    expect(fetched).toEqual(["sha-b"]); // delta
+    expect(seeded).toEqual(["sha-b"]);
+    expect(materialized).toEqual(["HASH1"]);
+    expect(res.alreadyConverged).toBe(false);
+    expect(res.pulledIntegrities).toEqual(["sha-b"]);
+  });
+
+  test("is a no-op when already at the desired hash", async () => {
+    const { deps, fetched, materialized } = makeDeps({
+      currentLockfileHash: async () => "HASH1",
+    });
+    const res = await reconcileToHash({
+      lockfileHash: "HASH1",
+      manifest: [a, b],
+      deps,
+    });
+    expect(res.alreadyConverged).toBe(true);
+    expect(fetched).toEqual([]);
+    expect(materialized).toEqual([]);
+  });
+
+  test("cold cache pulls the full blob set", async () => {
+    const { deps, fetched } = makeDeps();
+    await reconcileToHash({ lockfileHash: "H", manifest: [a, b], deps });
+    expect(fetched.sort()).toEqual(["sha-a", "sha-b"]);
+  });
+});
+
+describe("seedBlob blob-integrity verification (M1)", () => {
+  let work: string;
+  let blob: Uint8Array;
+
+  beforeEach(async () => {
+    work = await mkdtemp(path.join(tmpdir(), "cs-seed-"));
+    // Build a real gzip-tar blob of a fake cache entry.
+    const src = path.join(work, "src");
+    const entryDir = path.join(src, "pkg@1.0.0");
+    await mkdir(entryDir, { recursive: true });
+    await writeFile(path.join(entryDir, "index.js"), "module.exports = 1;\n");
+    blob = await packDir({ parentDir: src, entryName: "pkg@1.0.0" });
+  });
+  afterEach(async () => {
+    await rm(work, { recursive: true, force: true });
+  });
+
+  const entryWith = (over: Partial<ManifestEntry> = {}): ManifestEntry => ({
+    name: "pkg",
+    version: "1.0.0",
+    integrity: "sha512-npm-tarball-hash",
+    ...over,
+  });
+
+  test("seeds a blob whose recorded blobSha256 matches", async () => {
+    const storeRoot = path.join(work, "store");
+    const deps = createReconcileFsDeps({
+      storeRoot,
+      fetchBlob: async () => blob,
+    });
+    await deps.seedBlob({
+      entry: entryWith({ blobSha256: blobSha256(blob) }),
+      bytes: blob,
+    });
+    // Extracted into the cache.
+    const cacheEntries = await readdir(storePaths(storeRoot).cache);
+    expect(cacheEntries).toContain("pkg@1.0.0");
+  });
+
+  test("rejects a tampered blob and does NOT unpack it", async () => {
+    const storeRoot = path.join(work, "store2");
+    const deps = createReconcileFsDeps({
+      storeRoot,
+      fetchBlob: async () => blob,
+    });
+    const tampered = new Uint8Array(blob);
+    tampered[tampered.length - 1] ^= 0xff; // flip a byte
+
+    await expect(
+      deps.seedBlob({
+        // Recorded hash is of the ORIGINAL blob; tampered bytes won't match.
+        entry: entryWith({ blobSha256: blobSha256(blob) }),
+        bytes: tampered,
+      }),
+    ).rejects.toThrow(/integrity check failed/i);
+
+    // Nothing was extracted (cache dir absent or empty).
+    const cacheDir = storePaths(storeRoot).cache;
+    const entries = await readdir(cacheDir).catch(() => []);
+    expect(entries).not.toContain("pkg@1.0.0");
+  });
+
+  test("backward-safe: an entry without blobSha256 still seeds", async () => {
+    const storeRoot = path.join(work, "store3");
+    const deps = createReconcileFsDeps({
+      storeRoot,
+      fetchBlob: async () => blob,
+    });
+    await deps.seedBlob({ entry: entryWith(), bytes: blob });
+    const cacheEntries = await readdir(storePaths(storeRoot).cache);
+    expect(cacheEntries).toContain("pkg@1.0.0");
+  });
+});
+
+/**
+ * N-host (multi-pod) convergence: two hosts over SEPARATE store roots
+ * sharing ONE in-memory blob map (models two pods backed by one blob
+ * store). Both reconcile to the same lockfileHash and must converge to the
+ * same materialized tree; a second pass on each is idempotent.
+ *
+ * No network: the materialize uses an EMPTY manifest, so `bun install
+ * --offline` builds an empty tree without touching a registry. The shared
+ * blob map is still exercised by the delta-pull scenario below (a blob
+ * seeded into the shared store is pulled identically by both hosts).
+ */
+describe("N-host reconcile convergence (shared blob store, two pods)", () => {
+  let work: string;
+
+  beforeEach(async () => {
+    work = await mkdtemp(path.join(tmpdir(), "cs-nhost-"));
+  });
+  afterEach(async () => {
+    await rm(work, { recursive: true, force: true });
+  });
+
+  /** Build a host's deps over its own store root, reading from a shared map. */
+  function hostDeps(storeRoot: string, sharedBlobs: Map<string, Uint8Array>) {
+    return createReconcileFsDeps({
+      storeRoot,
+      fetchBlob: async ({ integrity }) => {
+        const bytes = sharedBlobs.get(integrity);
+        if (!bytes) throw new Error(`blob ${integrity} not in shared store`);
+        return bytes;
+      },
+    });
+  }
+
+  test("two hosts converge to the same tree at one hash; second pass is idempotent", async () => {
+    // One shared blob store; two pods with independent local store roots.
+    const sharedBlobs = new Map<string, Uint8Array>();
+    const rootA = path.join(work, "podA");
+    const rootB = path.join(work, "podB");
+    const depsA = hostDeps(rootA, sharedBlobs);
+    const depsB = hostDeps(rootB, sharedBlobs);
+
+    const lockfileHash = "convergehash";
+    const manifest: ManifestEntry[] = []; // empty → offline materialize, no net
+
+    // First pass on each pod: both converge (not already-converged).
+    const a1 = await reconcileToHash({ lockfileHash, manifest, deps: depsA });
+    const b1 = await reconcileToHash({ lockfileHash, manifest, deps: depsB });
+    expect(a1.alreadyConverged).toBe(false);
+    expect(b1.alreadyConverged).toBe(false);
+
+    // Both `current` symlinks point at the SAME tree (trees/<hash>).
+    const targetA = await readlink(storePaths(rootA).current);
+    const targetB = await readlink(storePaths(rootB).current);
+    expect(path.basename(targetA)).toBe(lockfileHash);
+    expect(path.basename(targetB)).toBe(lockfileHash);
+    expect(path.basename(targetA)).toBe(path.basename(targetB));
+
+    // Each pod materialized the tree (node_modules present under trees/<hash>).
+    for (const root of [rootA, rootB]) {
+      const tree = path.join(storePaths(root).trees, lockfileHash);
+      const entries = await readdir(tree);
+      expect(entries).toContain("node_modules");
+      expect(entries).toContain("package.json");
+    }
+
+    // Second pass on each pod is a no-op (idempotent convergence).
+    const a2 = await reconcileToHash({ lockfileHash, manifest, deps: depsA });
+    const b2 = await reconcileToHash({ lockfileHash, manifest, deps: depsB });
+    expect(a2.alreadyConverged).toBe(true);
+    expect(b2.alreadyConverged).toBe(true);
+    expect(a2.pulledIntegrities).toEqual([]);
+    expect(b2.pulledIntegrities).toEqual([]);
+  }, 60_000);
+
+  test("both pods pull the SAME blob from the one shared store (delta sync)", async () => {
+    // Seed one real packed blob into the shared store (no network).
+    const src = path.join(work, "src");
+    const entryDir = path.join(src, "pkg@1.0.0");
+    await mkdir(entryDir, { recursive: true });
+    await writeFile(path.join(entryDir, "index.js"), "module.exports = 1;\n");
+    const bytes = await packDir({ parentDir: src, entryName: "pkg@1.0.0" });
+
+    const entry: ManifestEntry = {
+      name: "pkg",
+      version: "1.0.0",
+      integrity: "sha512-shared-blob",
+      blobSha256: blobSha256(bytes),
+    };
+    const sharedBlobs = new Map<string, Uint8Array>([[entry.integrity, bytes]]);
+
+    const rootA = path.join(work, "podA");
+    const rootB = path.join(work, "podB");
+
+    // Each pod, cold cache, seeds the missing blob — pulling the SAME bytes
+    // from the one shared store into its own local cache.
+    const fetchedA: string[] = [];
+    const fetchedB: string[] = [];
+    const depsA = createReconcileFsDeps({
+      storeRoot: rootA,
+      fetchBlob: async ({ integrity }) => {
+        fetchedA.push(integrity);
+        const b = sharedBlobs.get(integrity);
+        if (!b) throw new Error("missing");
+        return b;
+      },
+    });
+    const depsB = createReconcileFsDeps({
+      storeRoot: rootB,
+      fetchBlob: async ({ integrity }) => {
+        fetchedB.push(integrity);
+        const b = sharedBlobs.get(integrity);
+        if (!b) throw new Error("missing");
+        return b;
+      },
+    });
+
+    // Drive only the seeding side (delta diff + seedBlob) — both pods start
+    // cold, so both pull the one shared blob.
+    expect(await depsA.localCacheIntegrities()).toEqual([]);
+    expect(await depsB.localCacheIntegrities()).toEqual([]);
+    await depsA.seedBlob({ entry, bytes: await depsA.fetchBlob({ integrity: entry.integrity }) });
+    await depsB.seedBlob({ entry, bytes: await depsB.fetchBlob({ integrity: entry.integrity }) });
+
+    expect(fetchedA).toEqual([entry.integrity]);
+    expect(fetchedB).toEqual([entry.integrity]);
+
+    // Both pods now have the blob locally, materialized into the same cache
+    // entry layout from the identical shared bytes.
+    for (const root of [rootA, rootB]) {
+      const cacheEntries = await readdir(storePaths(root).cache);
+      expect(cacheEntries).toContain("pkg@1.0.0");
+    }
+    expect(await depsA.localCacheIntegrities()).toEqual([entry.integrity]);
+    expect(await depsB.localCacheIntegrities()).toEqual([entry.integrity]);
+  });
+});

package/src/reconciler.ts

@@ -0,0 +1,81 @@
+import type { ManifestEntry } from "@checkstack/script-packages-common";
+import { computeMissingBlobs } from "./reconcile-diff";
+
+/**
+ * Host reconciler: converge this host's local materialized `node_modules`
+ * to the desired `lockfileHash`. Identical logic on every core instance and
+ * each satellite - only the blob transport differs (shared blob store on
+ * core; HTTP-via-core on satellites), injected as `fetchBlob`.
+ *
+ * Idempotent: a host already at `lockfileHash` is a no-op. Reconcile is
+ * triggered on startup, on the `script-packages.changed` hook (core) /
+ * `RefreshScriptPackages` push (satellite), and on assignment-sync
+ * (backstop), so a host that missed the broadcast still converges.
+ *
+ * The heavy I/O (cache seeding, `bun install --offline`, symlink flip) is
+ * injected so the orchestration is unit-testable; the concrete adapter
+ * lives in `reconcile-fs.ts`.
+ */
+
+export interface ReconcileDeps {
+  /** Integrities already present in this host's local cache. */
+  localCacheIntegrities(): Promise<string[]>;
+  /** Fetch one blob by integrity (shared store on core; HTTP on satellite). */
+  fetchBlob(input: { integrity: string }): Promise<Uint8Array>;
+  /** Seed a fetched blob into the local cache. */
+  seedBlob(input: { entry: ManifestEntry; bytes: Uint8Array }): Promise<void>;
+  /**
+   * Materialize the tree for `lockfileHash` from the (now-complete) local
+   * cache and atomically flip `current` at it. Returns when `current`
+   * points at the new tree. No-op-safe if already materialized.
+   */
+  materializeAndFlip(input: {
+    lockfileHash: string;
+    manifest: ManifestEntry[];
+  }): Promise<void>;
+  /** The lockfileHash `current` already points at, if any. */
+  currentLockfileHash(): Promise<string | undefined>;
+  logger?: { debug(msg: string): void; error(msg: string): void };
+}
+
+export interface ReconcileResult {
+  /** True when nothing needed doing (already at the desired hash). */
+  alreadyConverged: boolean;
+  pulledIntegrities: string[];
+}
+
+export async function reconcileToHash({
+  lockfileHash,
+  manifest,
+  deps,
+}: {
+  lockfileHash: string;
+  manifest: ManifestEntry[];
+  deps: ReconcileDeps;
+}): Promise<ReconcileResult> {
+  const current = await deps.currentLockfileHash();
+  if (current === lockfileHash) {
+    deps.logger?.debug(
+      `Script packages already at ${lockfileHash}; reconcile is a no-op.`,
+    );
+    return { alreadyConverged: true, pulledIntegrities: [] };
+  }
+
+  const localIntegrities = await deps.localCacheIntegrities();
+  const missing = computeMissingBlobs({ manifest, localIntegrities });
+
+  for (const entry of missing) {
+    const bytes = await deps.fetchBlob({ integrity: entry.integrity });
+    await deps.seedBlob({ entry, bytes });
+  }
+
+  await deps.materializeAndFlip({ lockfileHash, manifest });
+
+  deps.logger?.debug(
+    `Reconciled script packages to ${lockfileHash} (pulled ${missing.length} blob(s)).`,
+  );
+  return {
+    alreadyConverged: false,
+    pulledIntegrities: missing.map((e) => e.integrity),
+  };
+}