@checkstack/script-packages-backend 0.2.0

package/src/resolution-root.ts

@@ -0,0 +1,127 @@
+import { stat } from "node:fs/promises";
+import path from "node:path";
+import type { SafeDatabase } from "@checkstack/backend-api";
+import { storePaths, resolveScriptPackagesDir } from "./data-dir";
+import { readCurrentTarget } from "./atomic-symlink";
+import { createInstallStateStore } from "./install-state-store";
+import { scriptPackageInstallState } from "./schema";
+
+/**
+ * What a script-execution call site should do about package resolution.
+ *
+ *  - `none`     — no packages are configured (desired hash is null). The
+ *                 caller leaves `resolutionRoot` UNSET so the runner uses
+ *                 `os.tmpdir()` exactly as before: non-package scripts are
+ *                 completely unaffected.
+ *  - `ready`    — packages are configured AND this host has materialized
+ *                 the desired tree. The caller passes `root` as the runner's
+ *                 `resolutionRoot` so `import "<pkg>"` resolves.
+ *  - `notReady` — packages ARE configured but this host hasn't materialized
+ *                 the desired hash yet (cold start, mid-reconcile, or a
+ *                 failed reconcile). The caller MUST surface a clear error
+ *                 on any package import rather than running against a stale
+ *                 or empty tree. `reason` is a ready-to-surface message.
+ */
+export type ResolutionRootStatus =
+  | { mode: "none" }
+  | { mode: "ready"; root: string }
+  | { mode: "notReady"; reason: string };
+
+async function symlinkResolvesToTree(currentPath: string): Promise<boolean> {
+  // `current` is a symlink to `trees/<hash>`; require it to resolve to a
+  // real dir with a node_modules so we never point the runner at a
+  // half-built / missing tree.
+  const target = await readCurrentTarget(currentPath);
+  if (!target) return false;
+  const resolved = path.isAbsolute(target)
+    ? target
+    : path.join(path.dirname(currentPath), target);
+  try {
+    const info = await stat(path.join(resolved, "node_modules"));
+    return info.isDirectory();
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Decide the resolution-root status for a script execution on this host.
+ *
+ * @param desiredLockfileHash the install state's desired hash (null = no
+ *   packages configured).
+ * @param storeRoot the package-store root (`<dataDir>/script-packages`),
+ *   identical shape on core and satellites.
+ * @param hostLabel "central backend" | "satellite" — woven into the
+ *   not-ready error so operators know which host to look at.
+ */
+export async function resolveResolutionRoot({
+  desiredLockfileHash,
+  storeRoot,
+  hostLabel = "central backend",
+}: {
+  desiredLockfileHash: string | null;
+  storeRoot: string;
+  hostLabel?: string;
+}): Promise<ResolutionRootStatus> {
+  if (desiredLockfileHash === null) {
+    return { mode: "none" };
+  }
+  const paths = storePaths(storeRoot);
+  const currentHash = await readCurrentTarget(paths.current).then((target) =>
+    target ? path.basename(target) : undefined,
+  );
+  if (
+    currentHash === desiredLockfileHash &&
+    (await symlinkResolvesToTree(paths.current))
+  ) {
+    return { mode: "ready", root: paths.current };
+  }
+  return {
+    mode: "notReady",
+    reason: `npm packages not ready on this ${hostLabel} (expected ${desiredLockfileHash}${
+      currentHash ? `, have ${currentHash}` : ", none materialized"
+    }). The package set is still syncing; retry shortly.`,
+  };
+}
+
+/**
+ * Filesystem-only resolution: returns `ready` when `<store>/current`
+ * resolves to a materialized tree, else `none`. Host-agnostic and needs no
+ * DB / RPC / desired-hash, so it works on satellites (which have no script-
+ * packages RPC) as well as core. Execution safety does NOT depend on the
+ * `notReady` distinction: the runner always disables Bun auto-install, so a
+ * missing package errors regardless. This resolver simply points the runner
+ * at the synced tree when one exists.
+ */
+export async function resolveResolutionRootFromStore(
+  storeRoot: string,
+): Promise<ResolutionRootStatus> {
+  const paths = storePaths(storeRoot);
+  if (await symlinkResolvesToTree(paths.current)) {
+    return { mode: "ready", root: paths.current };
+  }
+  return { mode: "none" };
+}
+
+/**
+ * Convenience for central-backend call sites: read the desired hash from the
+ * install state and resolve the local resolution-root status against the
+ * default `<dataDir>/script-packages` store. Plugins that execute user
+ * scripts call this per run.
+ */
+export async function resolveResolutionRootForHost({
+  db,
+  hostLabel = "central backend",
+}: {
+  db: SafeDatabase<{
+    scriptPackageInstallState: typeof scriptPackageInstallState;
+  }>;
+  hostLabel?: string;
+}): Promise<ResolutionRootStatus> {
+  const state = await createInstallStateStore(db).load();
+  return resolveResolutionRoot({
+    desiredLockfileHash: state.lockfileHash,
+    storeRoot: resolveScriptPackagesDir(),
+    hostLabel,
+  });
+}

package/src/resolver.test.ts

@@ -0,0 +1,133 @@
+import { afterEach, beforeEach, describe, expect, test } from "bun:test";
+import { mkdtemp, rm, stat } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { createCentralResolver } from "./resolver";
+
+/**
+ * Hermetic resolver tests. No internet: we point the registry at an
+ * unreachable loopback address so `bun install` fails fast (connection
+ * refused), exercising the REAL resolve path through to its error.
+ *
+ * The security-relevant property under test: the scratch dir holds a
+ * plaintext `.npmrc` with the registry auth token, so it MUST be removed on
+ * EVERY exit path - including failures - not only on the success path.
+ */
+describe("createCentralResolver scratch cleanup", () => {
+  let work: string;
+
+  beforeEach(async () => {
+    work = await mkdtemp(path.join(tmpdir(), "cs-resolver-"));
+  });
+  afterEach(async () => {
+    await rm(work, { recursive: true, force: true });
+  });
+
+  async function exists(p: string): Promise<boolean> {
+    try {
+      await stat(p);
+      return true;
+    } catch {
+      return false;
+    }
+  }
+
+  test("removes the token-bearing scratch dir when bun install fails", async () => {
+    const scratchDir = path.join(work, "scratch");
+    const resolver = createCentralResolver({
+      scratchDir,
+      cacheDir: path.join(work, "cache"),
+      registry: {
+        // Unreachable loopback → fast ConnectionRefused, no real network.
+        registryUrl: "http://127.0.0.1:1/",
+        scopedRegistries: [],
+        authToken: "super-secret-token",
+      },
+    });
+
+    await expect(
+      resolver.resolve({
+        packages: [
+          {
+            name: "definitely-not-a-real-pkg-xyz-123",
+            version: "1.0.0",
+            enabled: true,
+          },
+        ],
+        ignoreScripts: true,
+      }),
+    ).rejects.toThrow();
+
+    // The scratch dir (and its .npmrc token) must be gone after the failure.
+    expect(await exists(scratchDir)).toBe(false);
+    expect(await exists(path.join(scratchDir, ".npmrc"))).toBe(false);
+  }, 30_000);
+});
+
+describe("createCentralResolver empty allowlist", () => {
+  let work: string;
+
+  beforeEach(async () => {
+    work = await mkdtemp(path.join(tmpdir(), "cs-resolver-empty-"));
+  });
+  afterEach(async () => {
+    await rm(work, { recursive: true, force: true });
+  });
+
+  async function exists(p: string): Promise<boolean> {
+    try {
+      await stat(p);
+      return true;
+    } catch {
+      return false;
+    }
+  }
+
+  // Regression: with no enabled packages `bun install` writes no `bun.lock`,
+  // so unconditionally reading it threw ENOENT and failed "Install now". The
+  // resolver must short-circuit to an empty resolved set without running a
+  // subprocess or touching the (unreachable) registry.
+  test("resolves to an empty set without running bun install", async () => {
+    const scratchDir = path.join(work, "scratch");
+    const resolver = createCentralResolver({
+      scratchDir,
+      cacheDir: path.join(work, "cache"),
+      registry: {
+        // Unreachable: proves no install subprocess / network is attempted.
+        registryUrl: "http://127.0.0.1:1/",
+        scopedRegistries: [],
+        authToken: "super-secret-token",
+      },
+    });
+
+    const result = await resolver.resolve({
+      packages: [],
+      ignoreScripts: true,
+    });
+
+    expect(result).toEqual([]);
+    // No scratch dir was created (short-circuited before any disk write).
+    expect(await exists(scratchDir)).toBe(false);
+  });
+
+  test("treats an all-disabled allowlist as empty", async () => {
+    const scratchDir = path.join(work, "scratch-disabled");
+    const resolver = createCentralResolver({
+      scratchDir,
+      cacheDir: path.join(work, "cache-disabled"),
+      registry: {
+        registryUrl: "http://127.0.0.1:1/",
+        scopedRegistries: [],
+        authToken: "super-secret-token",
+      },
+    });
+
+    const result = await resolver.resolve({
+      packages: [{ name: "left-pad", version: "1.0.0", enabled: false }],
+      ignoreScripts: true,
+    });
+
+    expect(result).toEqual([]);
+    expect(await exists(scratchDir)).toBe(false);
+  });
+});

package/src/resolver.ts

@@ -0,0 +1,132 @@
+import { spawn } from "bun";
+import { mkdir, writeFile, rm } from "node:fs/promises";
+import path from "node:path";
+import type { PackageSpec } from "@checkstack/script-packages-common";
+import type { Resolver, ResolvedPackage } from "./install-service";
+import { buildDependencies, buildStorePackageJson } from "./lockfile";
+import { renderNpmrc, type NpmrcInput } from "./npmrc";
+import { parseBunLock } from "./parse-bun-lock";
+import { packDir } from "./cache-archive";
+import { findCacheEntry } from "./cache-layout";
+
+/**
+ * Concrete central-install {@link Resolver}.
+ *
+ * In an isolated scratch dir it writes the generated `package.json` +
+ * `.npmrc` + a `bunfig.toml` that disables auto-install, runs `bun install`
+ * against the (possibly internal) registry into a dedicated cache dir,
+ * parses the resulting `bun.lock` for the manifest, then packs each
+ * package's Bun cache entry into a content-addressed blob.
+ *
+ * Runs only on the elected installer. The auth token is written to the
+ * scratch `.npmrc` and the whole scratch dir is removed afterwards; the
+ * token is never logged.
+ */
+
+export interface CentralResolverOptions {
+  /** Scratch dir for the resolve (created + removed per resolve). */
+  scratchDir: string;
+  /** Dedicated Bun cache dir to install into (the publish source). */
+  cacheDir: string;
+  /** Registry config (token already resolved from the secret store). */
+  registry: NpmrcInput;
+}
+
+function bunfigDisableAutoInstall(): string {
+  // Disable Bun auto-install so a missing package fails fast instead of
+  // silently fetching at import time on hosts (matches reconcile behavior).
+  return '[install]\nauto = "disable"\n';
+}
+
+export function createCentralResolver(
+  options: CentralResolverOptions,
+): Resolver {
+  return {
+    async resolve({
+      packages,
+      ignoreScripts,
+    }: {
+      packages: PackageSpec[];
+      ignoreScripts: boolean;
+    }): Promise<ResolvedPackage[]> {
+      const { scratchDir, cacheDir, registry } = options;
+
+      // No enabled packages → `bun install` writes no `bun.lock`, so reading
+      // it would throw ENOENT and fail the install. Short-circuit to an empty
+      // resolved set: an empty manifest yields totalSizeBytes 0 and a
+      // deterministic empty-lockfile hash, ending the install in `ready`/0 MB.
+      if (Object.keys(buildDependencies(packages)).length === 0) {
+        return [];
+      }
+
+      await rm(scratchDir, { recursive: true, force: true });
+      await mkdir(scratchDir, { recursive: true });
+      await mkdir(cacheDir, { recursive: true });
+
+      // From the moment the token-bearing `.npmrc` is written, the scratch
+      // dir MUST be removed on every exit path - success OR throw - so the
+      // plaintext registry token never lingers on disk (a throw between
+      // install and pack would otherwise leak it until the next resolve).
+      await writeFile(
+        path.join(scratchDir, "package.json"),
+        buildStorePackageJson(packages),
+      );
+      await writeFile(path.join(scratchDir, ".npmrc"), renderNpmrc(registry));
+      await writeFile(
+        path.join(scratchDir, "bunfig.toml"),
+        bunfigDisableAutoInstall(),
+      );
+
+      try {
+        // Plain `bun install` (no --no-save) so `bun.lock` is written - it's
+        // the manifest source. The scratch dir is throwaway, so saving is fine.
+        const installArgs = ["install"];
+        if (ignoreScripts) installArgs.push("--ignore-scripts");
+
+        const proc = spawn({
+          cmd: [process.execPath, ...installArgs],
+          cwd: scratchDir,
+          env: { ...process.env, BUN_INSTALL_CACHE_DIR: cacheDir },
+          stdout: "pipe",
+          stderr: "pipe",
+        });
+        const [stderr, exitCode] = await Promise.all([
+          new Response(proc.stderr).text(),
+          proc.exited,
+        ]);
+        if (exitCode !== 0) {
+          // Never include the .npmrc / token; only Bun's own stderr.
+          throw new Error(
+            `bun install failed (exit ${exitCode}): ${stderr.slice(0, 800)}`,
+          );
+        }
+
+        const lockText = await Bun.file(
+          path.join(scratchDir, "bun.lock"),
+        ).text();
+        const manifest = parseBunLock(lockText);
+
+        const resolved: ResolvedPackage[] = [];
+        for (const entry of manifest) {
+          const loc = await findCacheEntry({
+            cacheDir,
+            name: entry.name,
+            version: entry.version,
+          });
+          if (!loc) {
+            throw new Error(
+              `Resolved ${entry.name}@${entry.version} but its cache entry was not found; cannot publish blob.`,
+            );
+          }
+          const blob = await packDir(loc);
+          resolved.push({ entry, blob });
+        }
+
+        return resolved;
+      } finally {
+        // Best-effort: a cleanup failure must not mask the original outcome.
+        await rm(scratchDir, { recursive: true, force: true }).catch(() => {});
+      }
+    },
+  };
+}

package/src/router.ts

@@ -0,0 +1,273 @@
+import { implement, ORPCError } from "@orpc/server";
+import {
+  autoAuthMiddleware,
+  correlationMiddleware,
+  resolveActor,
+  type Logger,
+  type RpcContext,
+  type SafeDatabase,
+} from "@checkstack/backend-api";
+import type { RegistryTokenStore } from "./registry-token";
+import {
+  scriptPackagesContract,
+  type BlobGcSummary,
+} from "@checkstack/script-packages-common";
+import type { BlobStoreRegistry } from "./blob-store-registry";
+import {
+  createPackageStore,
+  createRegistryConfigStore,
+  createSizeCapStore,
+  createStorageConfigStore,
+  createSatelliteStateStore,
+  createBlobGcStateStore,
+} from "./stores";
+import { createInstallStateStore } from "./install-state-store";
+import { resolveRegistryRequestConfig } from "./registry-request-config";
+import {
+  searchPackages as registrySearchPackages,
+  getPackageVersions as registryGetPackageVersions,
+  RegistryClientError,
+} from "./registry-client";
+import * as schema from "./schema";
+
+export interface ScriptPackagesRouterDeps {
+  db: SafeDatabase<typeof schema>;
+  blobStores: BlobStoreRegistry;
+  logger: Logger;
+  /** Trigger an install (elected). Provided by the plugin (wires the resolver). */
+  triggerInstall(): Promise<{ started: boolean; reason?: string }>;
+  /**
+   * Kick a storage migration to `target` in the background. Provided by the
+   * plugin (wires the blob stores). Returns immediately; progress is polled
+   * via `getStorageMigrationState`.
+   */
+  triggerMigration(input: {
+    target: string;
+  }): Promise<{ started: boolean; reason?: string }>;
+  /**
+   * Run a blob-GC pass (elected via the installer advisory lock; refuses
+   * while an install / migration is in flight). Provided by the plugin
+   * (wires the blob stores + retention/grace). Returns a summary.
+   */
+  triggerBlobGc(): Promise<BlobGcSummary>;
+  /**
+   * Registry auth-token store, backed by the secrets platform's internal
+   * secrets. Provided by the plugin (which injects `internalSecretsRef`).
+   */
+  registryToken: RegistryTokenStore;
+}
+
+export function createScriptPackagesRouter({
+  db,
+  blobStores,
+  logger,
+  triggerInstall,
+  triggerMigration,
+  triggerBlobGc,
+  registryToken,
+}: ScriptPackagesRouterDeps) {
+  const packages = createPackageStore(db);
+  const registry = createRegistryConfigStore(db);
+  const storage = createStorageConfigStore(db);
+  const sizeCap = createSizeCapStore(db);
+  const satellites = createSatelliteStateStore(db);
+  const installState = createInstallStateStore(db);
+  const blobGcState = createBlobGcStateStore(db);
+
+  const os = implement(scriptPackagesContract)
+    .$context<RpcContext>()
+    .use(correlationMiddleware)
+    .use(autoAuthMiddleware);
+
+  return os.router({
+    // ─── Allowlist ────────────────────────────────────────────────────────
+    listPackages: os.listPackages.handler(async () => ({
+      items: await packages.list(),
+    })),
+
+    addPackage: os.addPackage.handler(async ({ input, context }) =>
+      packages.upsert({
+        name: input.name,
+        version: input.version,
+        addedBy: resolveActor(context.user).id ?? null,
+      }),
+    ),
+
+    removePackage: os.removePackage.handler(async ({ input }) => {
+      await packages.remove(input.name);
+      return { success: true };
+    }),
+
+    setPackageEnabled: os.setPackageEnabled.handler(async ({ input }) =>
+      packages.setEnabled({ name: input.name, enabled: input.enabled }),
+    ),
+
+    // ─── Registry autocomplete ────────────────────────────────────────────
+    searchPackages: os.searchPackages.handler(async ({ input }) => {
+      const reqConfig = await resolveRegistryRequestConfig({
+        registry,
+        registryToken,
+        logger,
+      });
+      try {
+        const items = await registrySearchPackages({
+          registry: reqConfig,
+          text: input.text,
+        });
+        return { items };
+      } catch (error) {
+        if (error instanceof RegistryClientError) {
+          throw new ORPCError("BAD_GATEWAY", { message: error.message });
+        }
+        throw error;
+      }
+    }),
+
+    getPackageVersions: os.getPackageVersions.handler(async ({ input }) => {
+      const reqConfig = await resolveRegistryRequestConfig({
+        registry,
+        registryToken,
+        logger,
+      });
+      try {
+        return await registryGetPackageVersions({
+          registry: reqConfig,
+          name: input.name,
+        });
+      } catch (error) {
+        if (error instanceof RegistryClientError) {
+          throw new ORPCError("BAD_GATEWAY", { message: error.message });
+        }
+        throw error;
+      }
+    }),
+
+    // ─── Registry config ──────────────────────────────────────────────────
+    getRegistryConfig: os.getRegistryConfig.handler(async () => registry.get()),
+
+    setRegistryConfig: os.setRegistryConfig.handler(async ({ input }) => {
+      // Store the token (if provided) in the secrets platform's internal
+      // secrets and persist the stable marker as the "secret ref".
+      // `undefined` leaves the existing token; "" clears it.
+      let authSecretRef: string | null | undefined;
+      if (input.authToken !== undefined) {
+        if (input.authToken.length > 0) {
+          authSecretRef = await registryToken.store(input.authToken);
+        } else {
+          await registryToken.clear();
+          authSecretRef = null;
+        }
+      }
+      await registry.set({
+        registryUrl: input.registryUrl,
+        scopedRegistries: input.scopedRegistries,
+        ignoreScripts: input.ignoreScripts,
+        authSecretRef,
+      });
+      return registry.get();
+    }),
+
+    // ─── Install ──────────────────────────────────────────────────────────
+    installNow: os.installNow.handler(async () => triggerInstall()),
+
+    getSizeCapConfig: os.getSizeCapConfig.handler(async () => sizeCap.get()),
+    setSizeCapConfig: os.setSizeCapConfig.handler(async ({ input }) => {
+      await sizeCap.set(input);
+      return sizeCap.get();
+    }),
+
+    // ─── Storage ──────────────────────────────────────────────────────────
+    getStorageConfig: os.getStorageConfig.handler(async () => storage.get()),
+
+    listStorageBackends: os.listStorageBackends.handler(async () => ({
+      backends: blobStores.ids(),
+    })),
+
+    setStorageBackend: os.setStorageBackend.handler(async ({ input }) => {
+      if (!blobStores.has(input.backend)) {
+        throw new ORPCError("BAD_REQUEST", {
+          message: `Blob store backend "${input.backend}" is not available.`,
+        });
+      }
+      const current = await storage.get();
+      if (current.migrationStatus === "migrating") {
+        throw new ORPCError("CONFLICT", {
+          message:
+            "A storage migration is in progress; cannot change the active backend until it completes.",
+        });
+      }
+      // Directly setting the active backend does NOT move existing blobs. Use
+      // `migrateStorage` to copy blobs to the new backend. This is for an
+      // initial selection / when both backends already hold the blobs.
+      await storage.setActiveBackend(input.backend);
+      return storage.get();
+    }),
+
+    migrateStorage: os.migrateStorage.handler(async ({ input }) => {
+      if (!blobStores.has(input.target)) {
+        throw new ORPCError("BAD_REQUEST", {
+          message: `Target blob store backend "${input.target}" is not available.`,
+        });
+      }
+      return triggerMigration({ target: input.target });
+    }),
+
+    getStorageMigrationState: os.getStorageMigrationState.handler(async () =>
+      storage.get(),
+    ),
+
+    // ─── Garbage collection ───────────────────────────────────────────────
+    gcBlobs: os.gcBlobs.handler(async () => triggerBlobGc()),
+
+    getBlobGcState: os.getBlobGcState.handler(async () => blobGcState.get()),
+
+    // ─── Per-host status ──────────────────────────────────────────────────
+    listSatelliteSyncState: os.listSatelliteSyncState.handler(async () => ({
+      items: await satellites.list(),
+    })),
+
+    reportSatelliteSyncState: os.reportSatelliteSyncState.handler(
+      async ({ input }) => {
+        await satellites.report(input);
+        return { success: true };
+      },
+    ),
+
+    // ─── Authoring / runtime ──────────────────────────────────────────────
+    getInstallState: os.getInstallState.handler(async () => installState.load()),
+
+    getManifest: os.getManifest.handler(async ({ input }) => {
+      const state = await installState.load();
+      if (state.lockfileHash !== input.lockfileHash) {
+        // We only retain the current desired manifest; an older hash isn't
+        // reconstructable. Return empty so the caller falls back to a full
+        // pull against the current manifest.
+        return { entries: [] };
+      }
+      return { entries: state.manifest };
+    }),
+
+    downloadBlob: os.downloadBlob.handler(async ({ input }) => {
+      const storageConfig = await storage.get();
+      const active = storageConfig.activeBackend;
+      const result = await blobStores.readWithFallback({
+        integrity: input.integrity,
+        activeBackendId: active,
+      });
+      if (!result) {
+        throw new ORPCError("NOT_FOUND", {
+          message: `Blob ${input.integrity} not found in any backend.`,
+        });
+      }
+      return {
+        integrity: input.integrity,
+        data: Buffer.from(result.bytes).toString("base64"),
+        sizeBytes: result.bytes.byteLength,
+      };
+    }),
+  });
+}
+
+export type ScriptPackagesRouter = ReturnType<
+  typeof createScriptPackagesRouter
+>;