@checkstack/script-packages-backend 0.2.0

package/src/install-state-store.ts

@@ -0,0 +1,131 @@
+import { eq } from "drizzle-orm";
+import type {
+  AdvisoryLockHandle,
+  AdvisoryLockService,
+  SafeDatabase,
+} from "@checkstack/backend-api";
+import type {
+  InstallState,
+  ManifestEntry,
+} from "@checkstack/script-packages-common";
+import { scriptPackageInstallState } from "./schema";
+
+/**
+ * Singleton install-state persistence (data) and the installer-election
+ * advisory lock (coordination), kept as two separate factories so read-only
+ * call sites (per-run resolution-root, router reads) depend only on the data
+ * store and never need the pool-backed lock service.
+ *
+ * Exactly one core instance performs the registry-facing `bun install` at a
+ * time, guarded by a Postgres advisory lock. The lock is held across a
+ * MINUTES-long `bun install`, so it MUST use a dedicated pooled connection
+ * (not a long-open transaction): {@link createInstallerLock} uses the shared
+ * {@link AdvisoryLockService}, which checks out one client, acquires the
+ * session lock on it, and releases on the SAME client. The lock auto-releases
+ * when that connection dies, so a crashed installer never wedges the
+ * election.
+ */
+
+const INSTALLER_LOCK_KEY = "script-packages.installer";
+const SINGLETON_ID = "singleton";
+
+type Schema = { scriptPackageInstallState: typeof scriptPackageInstallState };
+
+export interface InstallStateStore {
+  load(): Promise<InstallState>;
+  setInstalling(): Promise<void>;
+  setReady(input: {
+    lockfileHash: string;
+    manifest: ManifestEntry[];
+    totalSizeBytes: number;
+  }): Promise<void>;
+  setError(message: string): Promise<void>;
+}
+
+export interface InstallerLock {
+  /**
+   * Acquire the installer-election lock on a dedicated client. Returns a
+   * handle (call `release()` in a `finally`) on success, or `null` if
+   * another instance already holds it.
+   */
+  tryInstallerLock(): Promise<AdvisoryLockHandle | null>;
+}
+
+/**
+ * The installer-election lock, backed by the pool-backed advisory-lock
+ * service so the session lock keeps connection affinity across the long
+ * `bun install`.
+ */
+export function createInstallerLock(
+  advisoryLock: AdvisoryLockService,
+): InstallerLock {
+  return {
+    async tryInstallerLock() {
+      return advisoryLock.tryAcquire(INSTALLER_LOCK_KEY);
+    },
+  };
+}
+
+const DEFAULT_STATE: InstallState = {
+  status: "idle",
+  lockfileHash: null,
+  manifest: [],
+  totalSizeBytes: 0,
+  lastInstalledAt: null,
+  errorMessage: null,
+};
+
+export function createInstallStateStore(
+  db: SafeDatabase<Schema>,
+): InstallStateStore {
+  async function upsert(
+    set: Partial<typeof scriptPackageInstallState.$inferInsert>,
+  ): Promise<void> {
+    await db
+      .insert(scriptPackageInstallState)
+      .values({ id: SINGLETON_ID, ...set })
+      .onConflictDoUpdate({
+        target: scriptPackageInstallState.id,
+        set,
+      });
+  }
+
+  return {
+    async load() {
+      const rows = await db
+        .select()
+        .from(scriptPackageInstallState)
+        .where(eq(scriptPackageInstallState.id, SINGLETON_ID))
+        .limit(1);
+      const row = rows[0];
+      if (!row) return DEFAULT_STATE;
+      return {
+        status: row.status as InstallState["status"],
+        lockfileHash: row.lockfileHash,
+        manifest: row.manifest,
+        totalSizeBytes: row.totalSizeBytes,
+        lastInstalledAt: row.lastInstalledAt,
+        errorMessage: row.errorMessage,
+      };
+    },
+
+    async setInstalling() {
+      await upsert({ status: "installing", errorMessage: null });
+    },
+
+    async setReady({ lockfileHash, manifest, totalSizeBytes }) {
+      await upsert({
+        status: "ready",
+        lockfileHash,
+        manifest,
+        totalSizeBytes,
+        lastInstalledAt: new Date(),
+        errorMessage: null,
+      });
+    },
+
+    async setError(message) {
+      await upsert({ status: "error", errorMessage: message });
+    },
+  };
+}

package/src/lockfile.test.ts

@@ -0,0 +1,60 @@
+import { describe, expect, test } from "bun:test";
+import type { ManifestEntry } from "@checkstack/script-packages-common";
+import {
+  buildDependencies,
+  buildStorePackageJson,
+  computeLockfileHash,
+  sortManifest,
+} from "./lockfile";
+
+const a: ManifestEntry = { name: "a", version: "1.0.0", integrity: "sha-a" };
+const b: ManifestEntry = { name: "b", version: "2.0.0", integrity: "sha-b" };
+
+describe("sortManifest", () => {
+  test("orders by name then version", () => {
+    expect(sortManifest([b, a]).map((e) => e.name)).toEqual(["a", "b"]);
+  });
+});
+
+describe("computeLockfileHash", () => {
+  test("is order-independent", () => {
+    expect(computeLockfileHash([a, b])).toBe(computeLockfileHash([b, a]));
+  });
+
+  test("changes when an integrity changes (content-addressed)", () => {
+    const a2: ManifestEntry = { ...a, integrity: "sha-a-v2" };
+    expect(computeLockfileHash([a, b])).not.toBe(computeLockfileHash([a2, b]));
+  });
+
+  test("changes when a version changes", () => {
+    const a2: ManifestEntry = { ...a, version: "1.0.1" };
+    expect(computeLockfileHash([a])).not.toBe(computeLockfileHash([a2]));
+  });
+
+  test("empty manifest hashes deterministically", () => {
+    expect(computeLockfileHash([])).toBe(computeLockfileHash([]));
+  });
+});
+
+describe("buildDependencies", () => {
+  test("excludes disabled packages and sorts keys", () => {
+    const deps = buildDependencies([
+      { name: "zeta", version: "1.0.0", enabled: true },
+      { name: "alpha", version: "2.0.0", enabled: true },
+      { name: "off", version: "3.0.0", enabled: false },
+    ]);
+    expect(Object.keys(deps)).toEqual(["alpha", "zeta"]);
+    expect(deps.alpha).toBe("2.0.0");
+    expect(deps.off).toBeUndefined();
+  });
+});
+
+describe("buildStorePackageJson", () => {
+  test("renders a private package with pinned deps", () => {
+    const json = JSON.parse(
+      buildStorePackageJson([{ name: "lodash", version: "4.17.21", enabled: true }]),
+    );
+    expect(json.private).toBe(true);
+    expect(json.dependencies.lodash).toBe("4.17.21");
+  });
+});

package/src/npmrc.test.ts

@@ -0,0 +1,48 @@
+import { describe, expect, test } from "bun:test";
+import { renderNpmrc } from "./npmrc";
+
+describe("renderNpmrc", () => {
+  test("renders the default registry without auth when no token", () => {
+    const out = renderNpmrc({
+      registryUrl: "https://registry.npmjs.org/",
+      scopedRegistries: [],
+    });
+    expect(out).toContain("registry=https://registry.npmjs.org/");
+    expect(out).not.toContain("_authToken");
+  });
+
+  test("renders scoped registries", () => {
+    const out = renderNpmrc({
+      registryUrl: "https://artifactory.acme.com/npm/",
+      scopedRegistries: [
+        { scope: "@acme", registryUrl: "https://artifactory.acme.com/npm-acme/" },
+      ],
+    });
+    expect(out).toContain(
+      "@acme:registry=https://artifactory.acme.com/npm-acme/",
+    );
+  });
+
+  test("emits protocol-relative _authToken lines for the registry and scoped registries", () => {
+    const out = renderNpmrc({
+      registryUrl: "https://artifactory.acme.com/npm/",
+      scopedRegistries: [
+        { scope: "@acme", registryUrl: "https://artifactory.acme.com/npm-acme/" },
+      ],
+      authToken: "TOKEN123",
+    });
+    expect(out).toContain("//artifactory.acme.com/npm/:_authToken=TOKEN123");
+    expect(out).toContain(
+      "//artifactory.acme.com/npm-acme/:_authToken=TOKEN123",
+    );
+  });
+
+  test("omits auth lines for an empty token", () => {
+    const out = renderNpmrc({
+      registryUrl: "https://registry.npmjs.org/",
+      scopedRegistries: [],
+      authToken: "",
+    });
+    expect(out).not.toContain("_authToken");
+  });
+});

package/src/npmrc.ts

@@ -0,0 +1,42 @@
+/**
+ * Render an `.npmrc` for the managed store from registry config.
+ *
+ * The auth token is written to the on-disk `.npmrc` at install time only;
+ * it is NEVER logged or returned to the client. Callers pass the resolved
+ * plaintext token (fetched from the connection-store secret) directly to
+ * this pure function and write the result to disk.
+ */
+
+export interface NpmrcInput {
+  registryUrl: string;
+  scopedRegistries: { scope: string; registryUrl: string }[];
+  /** Resolved plaintext token, or undefined for an anonymous registry. */
+  authToken?: string;
+}
+
+/** Strip protocol from a registry URL for the `//host/:_authToken` key. */
+function registryAuthKey(registryUrl: string): string {
+  // `//registry.example.com/:_authToken=...` - npm keys auth by the
+  // protocol-relative registry path.
+  return registryUrl.replace(/^https?:/, "");
+}
+
+export function renderNpmrc(input: NpmrcInput): string {
+  const lines: string[] = [`registry=${input.registryUrl}`];
+
+  for (const scoped of input.scopedRegistries) {
+    lines.push(`${scoped.scope}:registry=${scoped.registryUrl}`);
+  }
+
+  if (input.authToken && input.authToken.length > 0) {
+    const key = registryAuthKey(input.registryUrl);
+    lines.push(`${key}:_authToken=${input.authToken}`);
+    // Scoped registries on the same host commonly share the token; emit a
+    // per-scoped-registry auth line too so private scopes authenticate.
+    for (const scoped of input.scopedRegistries) {
+      lines.push(`${registryAuthKey(scoped.registryUrl)}:_authToken=${input.authToken}`);
+    }
+  }
+
+  return lines.join("\n") + "\n";
+}

package/src/package-types.test.ts

@@ -0,0 +1,293 @@
+import { afterEach, beforeEach, describe, expect, test } from "bun:test";
+import { mkdtemp, mkdir, writeFile, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import {
+  extractReferences,
+  resolvePackageTypeClosure,
+  typesPackageDirName,
+} from "./package-types";
+
+describe("typesPackageDirName", () => {
+  test("unscoped name maps to itself", () => {
+    expect(typesPackageDirName("lodash")).toBe("lodash");
+  });
+  test("scoped name mangles @scope/name -> scope__name", () => {
+    expect(typesPackageDirName("@babel/core")).toBe("babel__core");
+  });
+  test("only the first slash is mangled", () => {
+    expect(typesPackageDirName("@scope/name")).toBe("scope__name");
+  });
+});
+
+describe("extractReferences", () => {
+  test("captures triple-slash path + types references", () => {
+    const refs = extractReferences(
+      `/// <reference path="./common/array.d.ts" />\n/// <reference types="node" />`,
+    );
+    expect(refs).toContainEqual({
+      kind: "path",
+      target: "./common/array.d.ts",
+    });
+    expect(refs).toContainEqual({ kind: "types", target: "node" });
+  });
+  test("captures relative import/export/require, ignores bare", () => {
+    const refs = extractReferences(
+      [
+        `import x from "./sibling";`,
+        `export { y } from "../up";`,
+        `import _ = require("../index");`,
+        `import bare from "lodash";`,
+      ].join("\n"),
+    );
+    const targets = refs.map((r) => r.target);
+    expect(targets).toContain("./sibling");
+    expect(targets).toContain("../up");
+    expect(targets).toContain("../index");
+    expect(targets).not.toContain("lodash");
+  });
+});
+
+describe("resolvePackageTypeClosure", () => {
+  let nm: string;
+  beforeEach(async () => {
+    nm = path.join(
+      await mkdtemp(path.join(tmpdir(), "cs-types-")),
+      "node_modules",
+    );
+    await mkdir(nm, { recursive: true });
+  });
+  afterEach(async () => {
+    await rm(path.dirname(nm), { recursive: true, force: true });
+  });
+
+  /** Write a file under node_modules at the given relative path. */
+  async function file(rel: string, content: string): Promise<void> {
+    const full = path.join(nm, rel);
+    await mkdir(path.dirname(full), { recursive: true });
+    await writeFile(full, content);
+  }
+
+  test("lodash-like: own types absent, @types present, multi-file /// <reference> closure, unwrapped", async () => {
+    // lodash itself ships NO own types.
+    await file(
+      "lodash/package.json",
+      JSON.stringify({ name: "lodash", main: "index.js" }),
+    );
+    await file("lodash/index.js", "module.exports = {};");
+    // @types/lodash: index fans out via /// <reference path>.
+    await file(
+      "@types/lodash/package.json",
+      JSON.stringify({ name: "@types/lodash", types: "index.d.ts" }),
+    );
+    await file(
+      "@types/lodash/index.d.ts",
+      [
+        `/// <reference path="./common/common.d.ts" />`,
+        `export = _;`,
+        `export as namespace _;`,
+        `declare const _: _.LoDashStatic;`,
+        `declare namespace _ { interface LoDashStatic {} }`,
+      ].join("\n"),
+    );
+    await file(
+      "@types/lodash/common/common.d.ts",
+      `import _ = require("../index");\ndeclare module "../index" { interface LoDashStatic { debounce(): void; } }`,
+    );
+
+    const closure = await resolvePackageTypeClosure({
+      nodeModulesDir: nm,
+      specifier: "lodash",
+    });
+
+    expect(closure.notFound).toBe(false);
+    expect(closure.hasOwnTypes).toBe(false);
+    expect(closure.hasAtTypes).toBe(true);
+    const paths = closure.files.map((f) => f.path);
+    expect(paths).toContain("node_modules/@types/lodash/index.d.ts");
+    // The referenced common/common.d.ts must be followed + included.
+    expect(paths).toContain("node_modules/@types/lodash/common/common.d.ts");
+    // The @types package.json is included so resolution finds the entry.
+    expect(paths).toContain("node_modules/@types/lodash/package.json");
+    // UNWRAPPED: no `declare module "lodash"` envelope was added.
+    const indexFile = closure.files.find(
+      (f) => f.path === "node_modules/@types/lodash/index.d.ts",
+    );
+    expect(indexFile?.content).toContain("export = _;");
+    expect(indexFile?.content).not.toContain('declare module "lodash"');
+  });
+
+  test("bundled-types (zod/dayjs-like): own index.d.ts present, no @types, returns bundled files (not a 404)", async () => {
+    await file(
+      "zod/package.json",
+      JSON.stringify({ name: "zod", types: "index.d.ts", main: "index.js" }),
+    );
+    await file("zod/index.d.ts", `export declare function parse(): unknown;`);
+    await file("zod/index.js", "module.exports = {};");
+    // No @types/zod on purpose.
+
+    const closure = await resolvePackageTypeClosure({
+      nodeModulesDir: nm,
+      specifier: "zod",
+    });
+
+    expect(closure.notFound).toBe(false);
+    expect(closure.hasOwnTypes).toBe(true);
+    expect(closure.hasAtTypes).toBe(false);
+    const paths = closure.files.map((f) => f.path);
+    expect(paths).toContain("node_modules/zod/index.d.ts");
+    expect(paths).toContain("node_modules/zod/package.json");
+  });
+
+  test("bundled-types via exports map `types` condition", async () => {
+    await file(
+      "dayjs/package.json",
+      JSON.stringify({
+        name: "dayjs",
+        exports: {
+          ".": { types: "./esm/index.d.ts", import: "./esm/index.js" },
+        },
+      }),
+    );
+    await file(
+      "dayjs/esm/index.d.ts",
+      `export declare function dayjs(): unknown;`,
+    );
+
+    const closure = await resolvePackageTypeClosure({
+      nodeModulesDir: nm,
+      specifier: "dayjs",
+    });
+
+    expect(closure.hasOwnTypes).toBe(true);
+    expect(closure.files.map((f) => f.path)).toContain(
+      "node_modules/dayjs/esm/index.d.ts",
+    );
+  });
+
+  test("scoped: @scope/name resolves to @types/scope__name", async () => {
+    await file(
+      "@babel/core/package.json",
+      JSON.stringify({ name: "@babel/core", main: "lib/index.js" }),
+    );
+    await file("@babel/core/lib/index.js", "module.exports = {};");
+    await file(
+      "@types/babel__core/package.json",
+      JSON.stringify({ name: "@types/babel__core", types: "index.d.ts" }),
+    );
+    await file(
+      "@types/babel__core/index.d.ts",
+      `export declare function transform(): unknown;`,
+    );
+
+    const closure = await resolvePackageTypeClosure({
+      nodeModulesDir: nm,
+      specifier: "@babel/core",
+    });
+
+    expect(closure.hasAtTypes).toBe(true);
+    expect(closure.files.map((f) => f.path)).toContain(
+      "node_modules/@types/babel__core/index.d.ts",
+    );
+  });
+
+  test("both own types AND @types present -> both included", async () => {
+    await file(
+      "thing/package.json",
+      JSON.stringify({ name: "thing", types: "index.d.ts" }),
+    );
+    await file("thing/index.d.ts", `export const own: number;`);
+    await file(
+      "@types/thing/package.json",
+      JSON.stringify({ name: "@types/thing", types: "index.d.ts" }),
+    );
+    await file("@types/thing/index.d.ts", `export const fromTypes: string;`);
+
+    const closure = await resolvePackageTypeClosure({
+      nodeModulesDir: nm,
+      specifier: "thing",
+    });
+
+    expect(closure.hasOwnTypes).toBe(true);
+    expect(closure.hasAtTypes).toBe(true);
+    const paths = closure.files.map((f) => f.path);
+    expect(paths).toContain("node_modules/thing/index.d.ts");
+    expect(paths).toContain("node_modules/@types/thing/index.d.ts");
+  });
+
+  test("neither own types nor @types -> graceful empty closure, no throw", async () => {
+    await file(
+      "notyped/package.json",
+      JSON.stringify({ name: "notyped", main: "index.js" }),
+    );
+    await file("notyped/index.js", "module.exports = 1;");
+
+    const closure = await resolvePackageTypeClosure({
+      nodeModulesDir: nm,
+      specifier: "notyped",
+    });
+
+    expect(closure.notFound).toBe(true);
+    expect(closure.hasOwnTypes).toBe(false);
+    expect(closure.hasAtTypes).toBe(false);
+    // package.json is still included (harmless), but no .d.ts.
+    expect(closure.files.every((f) => !f.path.endsWith(".d.ts"))).toBe(true);
+  });
+
+  test("missing package entirely -> notFound, never throws", async () => {
+    const closure = await resolvePackageTypeClosure({
+      nodeModulesDir: nm,
+      specifier: "ghost",
+    });
+    expect(closure.notFound).toBe(true);
+    expect(closure.files).toHaveLength(0);
+  });
+
+  test("follows cross-package /// <reference types> into another @types", async () => {
+    await file(
+      "@types/needsnode/package.json",
+      JSON.stringify({ name: "@types/needsnode", types: "index.d.ts" }),
+    );
+    await file(
+      "@types/needsnode/index.d.ts",
+      `/// <reference types="node" />\nexport declare const x: number;`,
+    );
+    await file(
+      "@types/node/package.json",
+      JSON.stringify({ name: "@types/node", types: "index.d.ts" }),
+    );
+    await file("@types/node/index.d.ts", `declare const process: unknown;`);
+    await file("needsnode/package.json", JSON.stringify({ name: "needsnode" }));
+
+    const closure = await resolvePackageTypeClosure({
+      nodeModulesDir: nm,
+      specifier: "needsnode",
+    });
+
+    expect(closure.files.map((f) => f.path)).toContain(
+      "node_modules/@types/node/index.d.ts",
+    );
+  });
+
+  test("truncated flag set (not silent) when the closure exceeds the ceiling", async () => {
+    const big = "x".repeat(2000);
+    await file(
+      "@types/huge/package.json",
+      JSON.stringify({ name: "@types/huge", types: "index.d.ts" }),
+    );
+    await file(
+      "@types/huge/index.d.ts",
+      `/// <reference path="./a.d.ts" />\nexport const a: 1;`,
+    );
+    await file("@types/huge/a.d.ts", `export const big = "${big}";`);
+    await file("huge/package.json", JSON.stringify({ name: "huge" }));
+
+    const closure = await resolvePackageTypeClosure({
+      nodeModulesDir: nm,
+      specifier: "huge",
+      maxTotalBytes: 100, // tiny ceiling forces a drop
+    });
+
+    expect(closure.truncated).toBe(true);
+  });
+});