@checkstack/script-packages-backend 0.2.0

package/src/data-dir.test.ts

@@ -0,0 +1,41 @@
+import { describe, expect, test } from "bun:test";
+import path from "node:path";
+import {
+  resolveDataDir,
+  resolveScriptPackagesDir,
+  storePaths,
+} from "./data-dir";
+
+describe("resolveDataDir", () => {
+  test("uses CHECKSTACK_DATA_DIR when set", () => {
+    expect(resolveDataDir({ CHECKSTACK_DATA_DIR: "/srv/cs" })).toBe("/srv/cs");
+  });
+
+  test("defaults to .data under cwd when unset", () => {
+    expect(resolveDataDir({})).toBe(path.resolve(process.cwd(), ".data"));
+  });
+
+  test("ignores a blank value", () => {
+    expect(resolveDataDir({ CHECKSTACK_DATA_DIR: "   " })).toBe(
+      path.resolve(process.cwd(), ".data"),
+    );
+  });
+});
+
+describe("resolveScriptPackagesDir", () => {
+  test("nests script-packages under the data dir", () => {
+    expect(resolveScriptPackagesDir({ CHECKSTACK_DATA_DIR: "/srv/cs" })).toBe(
+      "/srv/cs/script-packages",
+    );
+  });
+});
+
+describe("storePaths", () => {
+  test("derives the cache / trees / current / runs subpaths", () => {
+    const p = storePaths("/srv/cs/script-packages");
+    expect(p.cache).toBe("/srv/cs/script-packages/cache");
+    expect(p.trees).toBe("/srv/cs/script-packages/trees");
+    expect(p.current).toBe("/srv/cs/script-packages/current");
+    expect(p.runs).toBe("/srv/cs/script-packages/.runs");
+  });
+});

package/src/data-dir.ts

@@ -0,0 +1,42 @@
+import path from "node:path";
+
+/**
+ * Resolve the on-disk root for the managed package store on this host.
+ *
+ * No general backend data-dir convention exists in the repo yet, so we
+ * introduce `CHECKSTACK_DATA_DIR` (env), defaulting to a `.data/` dir under
+ * the backend's CWD. Script packages live at `<dataDir>/script-packages/`.
+ *
+ * The store contains a generated `package.json`, `bun.lock`, the
+ * content-addressed cache, versioned `trees/<lockfileHash>/`, the `current`
+ * symlink, and `.runs/<uuid>/` scratch dirs for runner resolution.
+ */
+export function resolveDataDir(env: NodeJS.ProcessEnv = process.env): string {
+  const configured = env.CHECKSTACK_DATA_DIR;
+  if (configured && configured.trim().length > 0) {
+    return path.resolve(configured);
+  }
+  return path.resolve(process.cwd(), ".data");
+}
+
+/** `<dataDir>/script-packages` - the managed package store root. */
+export function resolveScriptPackagesDir(
+  env: NodeJS.ProcessEnv = process.env,
+): string {
+  return path.join(resolveDataDir(env), "script-packages");
+}
+
+/** Subpaths inside the package store. */
+export function storePaths(storeRoot: string) {
+  return {
+    root: storeRoot,
+    /** Content-addressed blob cache (compressed blobs keyed by integrity). */
+    cache: path.join(storeRoot, "cache"),
+    /** Versioned materialized trees: `trees/<lockfileHash>/node_modules`. */
+    trees: path.join(storeRoot, "trees"),
+    /** Atomically-flipped symlink to the active tree. */
+    current: path.join(storeRoot, "current"),
+    /** Per-run scratch dirs for runner resolution. */
+    runs: path.join(storeRoot, ".runs"),
+  };
+}

package/src/e2e-install-reconcile.test.ts

@@ -0,0 +1,121 @@
+import { afterAll, beforeAll, describe, expect, test } from "bun:test";
+import { mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { defaultEsmScriptRunner } from "@checkstack/backend-api";
+import { performInstall, type BlobIndex } from "./install-service";
+import { createCentralResolver } from "./resolver";
+import { reconcileToHash } from "./reconciler";
+import { createReconcileFsDeps } from "./reconcile-fs";
+import { storePaths } from "./data-dir";
+
+/**
+ * End-to-end proof of the central-store + core-reconciliation pipeline
+ * against a real (tiny, pure-JS) package:
+ *
+ *   resolve+install -> publish per-package blobs -> reconcile (delta pull
+ *   from the "blob store") -> bun install --offline materialize -> atomic
+ *   `current` flip -> a user script imports the package via resolutionRoot.
+ *
+ * Network is only touched once (the central resolve); the reconcile side
+ * pulls from the in-memory blob store, never the registry.
+ */
+// This test reaches the public npm registry once (the central resolve), so
+// it's opt-in to avoid flakiness / failures in offline CI. Run with
+// `CHECKSTACK_E2E_NETWORK=1 bun test` to exercise the full pipeline.
+const E2E_ENABLED = process.env.CHECKSTACK_E2E_NETWORK === "1";
+
+describe.skipIf(!E2E_ENABLED)("e2e: install + reconcile + import", () => {
+  let work: string;
+  // In-memory blob store standing in for the active BlobStore.
+  const blobs = new Map<string, Uint8Array>();
+
+  beforeAll(async () => {
+    work = await mkdtemp(path.join(tmpdir(), "cs-e2e-"));
+  });
+  afterAll(async () => {
+    await rm(work, { recursive: true, force: true });
+  });
+
+  test(
+    "resolves leftpad centrally, reconciles on a fresh host, and runs an import",
+    async () => {
+      const resolver = createCentralResolver({
+        scratchDir: path.join(work, "install-scratch"),
+        cacheDir: path.join(work, "install-cache"),
+        registry: {
+          registryUrl: "https://registry.npmjs.org/",
+          scopedRegistries: [],
+        },
+      });
+
+      const index: BlobIndex = { record: async () => {} };
+      const blobStore = {
+        id: "memory",
+        has: async ({ integrity }: { integrity: string }) =>
+          blobs.has(integrity),
+        put: async ({
+          integrity,
+          bytes,
+        }: {
+          integrity: string;
+          bytes: Uint8Array;
+        }) => {
+          blobs.set(integrity, bytes);
+        },
+      };
+
+      const installResult = await performInstall({
+        packages: [{ name: "leftpad", version: "0.0.1", enabled: true }],
+        ignoreScripts: true,
+        resolver,
+        blobStore,
+        blobIndex: index,
+      });
+
+      expect(installResult.manifest.length).toBeGreaterThanOrEqual(1);
+      expect(installResult.lockfileHash.length).toBe(64);
+      expect(blobs.size).toBeGreaterThanOrEqual(1);
+
+      // ── Fresh host reconcile (cold cache, pulls all blobs from the store) ──
+      const storeRoot = path.join(work, "host-store");
+      const deps = createReconcileFsDeps({
+        storeRoot,
+        fetchBlob: async ({ integrity }) => {
+          const bytes = blobs.get(integrity);
+          if (!bytes) throw new Error(`blob ${integrity} not in store`);
+          return bytes;
+        },
+      });
+
+      const recRes = await reconcileToHash({
+        lockfileHash: installResult.lockfileHash,
+        manifest: installResult.manifest,
+        deps,
+      });
+      expect(recRes.alreadyConverged).toBe(false);
+      expect(recRes.pulledIntegrities.length).toBe(blobs.size);
+
+      // A second reconcile is a no-op (idempotent convergence).
+      const again = await reconcileToHash({
+        lockfileHash: installResult.lockfileHash,
+        manifest: installResult.manifest,
+        deps,
+      });
+      expect(again.alreadyConverged).toBe(true);
+
+      // ── Run a user script that imports the materialized package ──
+      const current = storePaths(storeRoot).current;
+      const run = await defaultEsmScriptRunner.run({
+        script: `import leftpad from "leftpad";\nexport default leftpad("7", 3);`,
+        context: {},
+        timeoutMs: 20_000,
+        resolutionRoot: current,
+      });
+      expect(run.error).toBeUndefined();
+      // leftpad pads with "0" by default.
+      expect(run.result).toBe("007");
+    },
+    60_000,
+  );
+});

package/src/hooks.ts

@@ -0,0 +1,20 @@
+import { createHook } from "@checkstack/backend-api";
+import {
+  SCRIPT_PACKAGES_CHANGED_HOOK_ID,
+  type ScriptPackagesChangedPayload,
+} from "@checkstack/script-packages-common";
+
+/**
+ * Backend hook fired by the elected installer after a successful install.
+ *
+ * Core instances subscribe in `broadcast` mode (every instance receives it,
+ * the deliberate inverse of installer-election) and kick their reconciler
+ * to delta-sync to the new `lockfileHash`. Each instance's broadcast
+ * handler also pushes a `RefreshScriptPackages` control message to its
+ * connected satellites (Phase 3). The hook is best-effort liveness; the
+ * durable desired state (`install_state.lockfileHash`) drives idempotent
+ * convergence on the next startup / assignment-sync regardless.
+ */
+export const scriptPackagesChangedHook = createHook<ScriptPackagesChangedPayload>(
+  SCRIPT_PACKAGES_CHANGED_HOOK_ID,
+);