@checkstack/script-packages-backend 0.2.0

package/src/tree-gc.test.ts

@@ -0,0 +1,184 @@
+import { afterEach, beforeEach, describe, expect, test } from "bun:test";
+import { mkdtemp, mkdir, writeFile, rm, stat, utimes } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { sweepTreeGc } from "./tree-gc";
+import { atomicSymlinkSwap } from "./atomic-symlink";
+import { storePaths } from "./data-dir";
+import { RETIRED_AT_MARKER } from "./tree-retirement";
+
+const HOUR = 60 * 60 * 1000;
+
+describe("sweepTreeGc", () => {
+  let storeRoot: string;
+
+  beforeEach(async () => {
+    storeRoot = await mkdtemp(path.join(tmpdir(), "cs-treegc-"));
+  });
+  afterEach(async () => {
+    await rm(storeRoot, { recursive: true, force: true });
+  });
+
+  /** Create `trees/<hash>/node_modules` and set its mtime `ageMs` in the past. */
+  async function makeTree(hash: string, ageMs: number) {
+    const paths = storePaths(storeRoot);
+    const dir = path.join(paths.trees, hash);
+    await mkdir(path.join(dir, "node_modules"), { recursive: true });
+    await writeFile(path.join(dir, "package.json"), "{}");
+    const when = new Date(Date.now() - ageMs);
+    await utimes(dir, when, when);
+    return dir;
+  }
+
+  /** Stamp a `.retired-at` marker `ageMs` in the past onto a tree dir. */
+  async function retireTree(hash: string, ageMs: number) {
+    const dir = path.join(storePaths(storeRoot).trees, hash);
+    await writeFile(
+      path.join(dir, RETIRED_AT_MARKER),
+      `${Date.now() - ageMs}\n`,
+    );
+  }
+
+  async function exists(dir: string): Promise<boolean> {
+    try {
+      await stat(dir);
+      return true;
+    } catch {
+      return false;
+    }
+  }
+
+  async function pointCurrentAt(hash: string) {
+    const paths = storePaths(storeRoot);
+    await atomicSymlinkSwap({
+      linkPath: paths.current,
+      target: path.join("trees", hash),
+    });
+  }
+
+  test("never deletes the current tree, regardless of age", async () => {
+    const cur = await makeTree("hashCur", 100 * HOUR); // very old
+    await pointCurrentAt("hashCur");
+
+    const res = await sweepTreeGc({ storeRoot, graceMs: HOUR });
+    expect(res.currentHash).toBe("hashCur");
+    expect(res.deleted).toEqual([]);
+    expect(await exists(cur)).toBe(true);
+  });
+
+  test("prunes a non-current tree retired past the grace window", async () => {
+    const cur = await makeTree("hashCur", 0);
+    const old = await makeTree("hashOld", 100 * HOUR); // ancient mtime
+    await retireTree("hashOld", 5 * HOUR); // but retired 5h ago > grace
+    await pointCurrentAt("hashCur");
+
+    const res = await sweepTreeGc({ storeRoot, graceMs: HOUR });
+    expect(res.deleted).toEqual(["hashOld"]);
+    expect(await exists(old)).toBe(false);
+    expect(await exists(cur)).toBe(true);
+  });
+
+  test("keeps a non-current tree still within the grace window (live run may be pinned)", async () => {
+    const recent = await makeTree("hashRecent", 100 * HOUR); // ancient mtime
+    await retireTree("hashRecent", 0.25 * HOUR); // retired 15 min ago < grace
+    await makeTree("hashCur", 0);
+    await pointCurrentAt("hashCur");
+
+    const res = await sweepTreeGc({ storeRoot, graceMs: HOUR });
+    expect(res.deleted).toEqual([]);
+    expect(res.keptWithinGrace).toContain("hashRecent");
+    expect(await exists(recent)).toBe(true);
+  });
+
+  test("respects the grace boundary using the injected clock", async () => {
+    await makeTree("hashCur", 0);
+    const old = path.join(storePaths(storeRoot).trees, "hashEdge");
+    await mkdir(path.join(old, "node_modules"), { recursive: true });
+    await retireTree("hashEdge", 2 * HOUR);
+    await pointCurrentAt("hashCur");
+
+    // now far in the future → well past grace-since-retirement → deleted.
+    const res = await sweepTreeGc({
+      storeRoot,
+      graceMs: HOUR,
+      now: Date.now() + 10 * HOUR,
+    });
+    expect(res.deleted).toContain("hashEdge");
+  });
+
+  test("back-fills a marker for an unmarked non-current tree and retains it that pass", async () => {
+    await makeTree("hashCur", 0);
+    const orphan = await makeTree("hashOrphan", 100 * HOUR); // ancient, no marker
+    await pointCurrentAt("hashCur");
+
+    // First pass: no retirement marker → retained + back-filled.
+    const first = await sweepTreeGc({ storeRoot, graceMs: HOUR });
+    expect(first.deleted).toEqual([]);
+    expect(first.keptWithinGrace).toContain("hashOrphan");
+    expect(await exists(orphan)).toBe(true);
+    expect(await exists(path.join(orphan, RETIRED_AT_MARKER))).toBe(true);
+
+    // A later pass past grace (from the back-filled time) deletes it.
+    const later = await sweepTreeGc({
+      storeRoot,
+      graceMs: HOUR,
+      now: Date.now() + 10 * HOUR,
+    });
+    expect(later.deleted).toContain("hashOrphan");
+    expect(await exists(orphan)).toBe(false);
+  });
+
+  test("no trees dir → no-op", async () => {
+    const res = await sweepTreeGc({ storeRoot, graceMs: HOUR });
+    expect(res.deleted).toEqual([]);
+    expect(res.keptWithinGrace).toEqual([]);
+  });
+
+  // ── H3 regression: grace must key on RETIREMENT time, not materialize mtime ──
+  //
+  // A tree that was `current` for days has an old mtime. The instant it is
+  // superseded by a flip it would (under the old mtime-keyed logic) be
+  // INSTANTLY eligible — and sweepTreeGc runs right after a flip, so it would
+  // delete a tree an in-flight run is still pinned to. The grace window must
+  // start ticking from when the tree BECAME non-current, not from its mtime.
+
+  test("retains a long-current tree the instant it is superseded by a flip", async () => {
+    // Tree A has been current for ~days (very old mtime).
+    const treeA = await makeTree("hashA", 100 * HOUR);
+    // Tree B has just been materialized.
+    await makeTree("hashB", 0);
+
+    // Flip current A -> ... -> B (A becomes non-current right now).
+    await pointCurrentAt("hashA");
+    await pointCurrentAt("hashB");
+
+    // Immediately sweep. A's mtime is ancient, but it only just retired, so it
+    // MUST be retained (a run started before the flip is still pinned to it).
+    const res = await sweepTreeGc({ storeRoot, graceMs: HOUR });
+    expect(res.currentHash).toBe("hashB");
+    expect(res.deleted).toEqual([]);
+    expect(res.keptWithinGrace).toContain("hashA");
+    expect(await exists(treeA)).toBe(true);
+  });
+
+  test("deletes a superseded tree only once grace-since-RETIREMENT elapses", async () => {
+    const treeA = await makeTree("hashA", 100 * HOUR);
+    await makeTree("hashB", 0);
+    await pointCurrentAt("hashA");
+    await pointCurrentAt("hashB"); // A retires now
+
+    // Within grace from retirement → retained.
+    const within = await sweepTreeGc({ storeRoot, graceMs: HOUR });
+    expect(within.deleted).toEqual([]);
+    expect(await exists(treeA)).toBe(true);
+
+    // Well past grace from retirement (injected clock) → deleted.
+    const after = await sweepTreeGc({
+      storeRoot,
+      graceMs: HOUR,
+      now: Date.now() + 10 * HOUR,
+    });
+    expect(after.deleted).toContain("hashA");
+    expect(await exists(treeA)).toBe(false);
+  });
+});

package/src/tree-gc.ts

@@ -0,0 +1,160 @@
+import { readdir, stat, rm } from "node:fs/promises";
+import path from "node:path";
+import { DEFAULT_TREE_GC_GRACE_MS } from "@checkstack/script-packages-common";
+import { extractErrorMessage } from "@checkstack/common";
+import { storePaths } from "./data-dir";
+import { readCurrentTarget } from "./atomic-symlink";
+import { markTreeRetired, readTreeRetiredAt } from "./tree-retirement";
+
+/**
+ * Host-local tree garbage collection: prune materialized
+ * `trees/<lockfileHash>/` dirs whose hash is NOT the one `current` points at,
+ * after a grace period. Applies identically to core pods and satellites (the
+ * materialize/flip code is shared, so this sweep is too).
+ *
+ * ## Why the conservative grace window (active-run safety)
+ *
+ * The runner pins in-flight runs to the tree they STARTED on by snapshotting
+ * `resolutionRoot` (which dereferences `<store>/current` at run start). After
+ * an atomic flip, a live run keeps resolving against the old tree's
+ * `node_modules`; deleting that tree mid-run would break the run.
+ *
+ * There is no robust cross-process refcount available here: runs execute in
+ * throwaway Bun subprocesses under `.runs/<uuid>/` whose `resolutionRoot`
+ * points at `current` (a symlink), not at a specific tree, and a crashed
+ * run leaves no reliable "release" signal. A naive refcount would leak on
+ * crash and could under-count, risking deletion of a live tree. So we take
+ * the **robust** option: a grace window chosen to comfortably exceed the
+ * longest possible script-run timeout. A tree only becomes eligible once it
+ * has been non-current for longer than any run could still be using it. When
+ * in doubt, we retain.
+ *
+ * ## Why grace is keyed on RETIREMENT time, not dir mtime
+ *
+ * A tree's mtime reflects when it was MATERIALIZED, not when it stopped being
+ * current. A tree that served as `current` for days has an ancient mtime, so
+ * keying the grace window on mtime makes it eligible for deletion the instant
+ * it is superseded — and this sweep runs right after a flip. That would
+ * delete a tree an in-flight run (which snapshotted `resolutionRoot` before
+ * the flip) is still pinned to. Instead the flip stamps a `.retired-at`
+ * marker into the superseded tree (see `tree-retirement.ts`), and the grace
+ * window is measured from THAT timestamp.
+ *
+ * A non-current tree with NO marker (e.g. one superseded before this scheme
+ * existed, or where the flip-time marker write failed) is RETAINED and the
+ * marker is lazily back-filled with `now`, so it ages out on subsequent
+ * sweeps instead of leaking forever — but is never deleted on a missing
+ * signal.
+ *
+ * `current`'s target is NEVER a candidate, regardless of age.
+ */
+
+export interface TreeGcResult {
+  /** Tree hashes deleted this pass. */
+  deleted: string[];
+  /** Non-current tree hashes kept because still within the grace window. */
+  keptWithinGrace: string[];
+  /** The hash `current` points at (never a candidate), if any. */
+  currentHash?: string;
+}
+
+/**
+ * Sweep the `trees/` dir of one host store, deleting non-current trees older
+ * than the grace window. Idempotent and crash-safe (a partially-deleted tree
+ * is simply re-swept next pass).
+ *
+ * @param storeRoot the package-store root (`<dataDir>/script-packages`).
+ * @param graceMs grace window in ms (default 1h, > the max run timeout).
+ * @param now injectable clock for deterministic tests.
+ */
+export async function sweepTreeGc({
+  storeRoot,
+  graceMs = DEFAULT_TREE_GC_GRACE_MS,
+  now = Date.now(),
+  logger,
+}: {
+  storeRoot: string;
+  graceMs?: number;
+  now?: number;
+  logger?: { debug(msg: string): void; error(msg: string): void };
+}): Promise<TreeGcResult> {
+  const paths = storePaths(storeRoot);
+
+  // The hash `current` points at (basename of `trees/<hash>`). NEVER deleted.
+  const currentTarget = await readCurrentTarget(paths.current);
+  const currentHash = currentTarget
+    ? path.basename(currentTarget)
+    : undefined;
+
+  let entries: string[];
+  try {
+    entries = await readdir(paths.trees);
+  } catch {
+    // No trees dir yet → nothing to GC.
+    return { deleted: [], keptWithinGrace: [], currentHash };
+  }
+
+  const deleted: string[] = [];
+  const keptWithinGrace: string[] = [];
+
+  for (const hash of entries) {
+    if (hash === currentHash) continue; // never delete the live tree
+
+    const treeDir = path.join(paths.trees, hash);
+    let info: Awaited<ReturnType<typeof stat>>;
+    try {
+      info = await stat(treeDir);
+    } catch {
+      continue; // vanished between readdir and stat; nothing to do
+    }
+    if (!info.isDirectory()) continue;
+
+    // Grace keyed on RETIREMENT time (the flip stamps `.retired-at` into the
+    // superseded tree), NOT the dir mtime. A long-current tree has an ancient
+    // mtime but only just retired, so mtime would make it instantly eligible
+    // and this sweep (run right after a flip) would delete a tree a live run
+    // is still pinned to.
+    const retiredAt = await readTreeRetiredAt({ treeDir });
+    if (retiredAt === undefined) {
+      // No marker yet: never delete on a missing signal. Back-fill `now` so
+      // the tree ages out on later sweeps instead of leaking forever, and
+      // retain it this pass.
+      await markTreeRetired({ treeDir, at: now });
+      keptWithinGrace.push(hash);
+      logger?.debug(
+        `Tree GC: non-current tree ${hash} has no retirement marker; ` +
+          `back-filling now and retaining this pass.`,
+      );
+      continue;
+    }
+
+    const ageMs = now - retiredAt;
+    if (ageMs < graceMs) {
+      keptWithinGrace.push(hash);
+      logger?.debug(
+        `Tree GC: keeping non-current tree ${hash} (retired ${Math.round(
+          ageMs / 1000,
+        )}s ago < grace ${Math.round(graceMs / 1000)}s; a run may still be pinned).`,
+      );
+      continue;
+    }
+
+    try {
+      await rm(treeDir, { recursive: true, force: true });
+      deleted.push(hash);
+      logger?.debug(`Tree GC: deleted non-current tree ${hash}.`);
+    } catch (error) {
+      logger?.error(
+        `Tree GC: failed to delete tree ${hash}: ${extractErrorMessage(
+          error,
+        )}. Retaining for the next pass.`,
+      );
+    }
+  }
+
+  logger?.debug(
+    `Tree GC complete: current=${currentHash ?? "(none)"}, ${deleted.length} deleted, ${keptWithinGrace.length} kept within grace.`,
+  );
+
+  return { deleted, keptWithinGrace, currentHash };
+}

package/src/tree-retirement.ts

@@ -0,0 +1,81 @@
+import { readFile, writeFile, stat } from "node:fs/promises";
+import path from "node:path";
+
+/**
+ * Per-tree "became-non-current-at" sentinel.
+ *
+ * Tree GC must key its grace window on the moment a tree RETIRED (stopped
+ * being `current`), not on the tree dir's mtime. A tree that was `current`
+ * for days has an ancient mtime, so keying grace on mtime makes it eligible
+ * for deletion the instant it is superseded — while an in-flight run is still
+ * pinned to it (the runner snapshots `resolutionRoot` at run start). Keying on
+ * retirement time guarantees a tree is only collected once it has been
+ * non-current for longer than any run could still be using it.
+ *
+ * The marker is a tiny file `<treeDir>/.retired-at` holding the epoch-ms
+ * timestamp of retirement. It is written at flip time (in
+ * {@link import("./atomic-symlink").atomicSymlinkSwap}) for the tree being
+ * superseded. Tree GC reads it; a tree with no marker yet is RETAINED (and
+ * lazily back-filled — see `tree-gc.ts`) so we never delete on a missing
+ * signal.
+ */
+
+/** Sentinel file name written into a tree dir when it stops being current. */
+export const RETIRED_AT_MARKER = ".retired-at";
+
+/**
+ * Record that the tree at `treeDir` became non-current at `at` (epoch ms).
+ * Best-effort and idempotent-by-overwrite; never throws (the caller is on a
+ * flip hot path where a marker write must not fail the flip).
+ */
+export async function markTreeRetired({
+  treeDir,
+  at,
+}: {
+  treeDir: string;
+  at: number;
+}): Promise<void> {
+  try {
+    await writeFile(path.join(treeDir, RETIRED_AT_MARKER), `${at}\n`, "utf8");
+  } catch {
+    // A failed marker write degrades to the "no marker" path in tree-gc,
+    // which RETAINS the tree — safe by construction.
+  }
+}
+
+/**
+ * Read the retirement timestamp (epoch ms) for the tree at `treeDir`, or
+ * `undefined` when no marker exists (or it is unreadable / malformed).
+ */
+export async function readTreeRetiredAt({
+  treeDir,
+}: {
+  treeDir: string;
+}): Promise<number | undefined> {
+  let raw: string;
+  try {
+    raw = await readFile(path.join(treeDir, RETIRED_AT_MARKER), "utf8");
+  } catch {
+    return undefined;
+  }
+  const parsed = Number.parseInt(raw.trim(), 10);
+  return Number.isFinite(parsed) ? parsed : undefined;
+}
+
+/**
+ * True when `treeDir` already carries a retirement marker. Used by tree-GC to
+ * decide whether to back-fill one for a tree that is non-current but has no
+ * marker yet (e.g. a tree superseded before this scheme existed).
+ */
+export async function hasRetirementMarker({
+  treeDir,
+}: {
+  treeDir: string;
+}): Promise<boolean> {
+  try {
+    await stat(path.join(treeDir, RETIRED_AT_MARKER));
+    return true;
+  } catch {
+    return false;
+  }
+}

package/src/type-acquisition-route.ts

@@ -0,0 +1,178 @@
+import path from "node:path";
+import type { AuthService, Logger } from "@checkstack/backend-api";
+import {
+  parseTypeAcquisitionPath,
+  scriptPackagesAccess,
+  TYPE_ACQUISITION_PATH_PREFIX,
+  type PackageTypeClosure,
+} from "@checkstack/script-packages-common";
+import { extractErrorMessage } from "@checkstack/common";
+import { resolvePackageTypeClosure } from "./package-types";
+import { storePaths } from "./data-dir";
+
+/**
+ * Raw, HTTP-cacheable route serving one package's `.d.ts` closure for editor
+ * lazy ATA. Mounted (via `rpc.registerHttpHandler`) at
+ * `/api/script-packages/types/:lockfileHash/:encodedSpecifier`.
+ *
+ * Why a raw route and not an oRPC procedure: oRPC responses in this codebase
+ * cannot set custom HTTP response headers (the Hono `/api` handler builds the
+ * RpcContext WITHOUT a `responseHeaders` bag, so a procedure can't emit
+ * `Cache-Control`). The user explicitly wants HTTP-level caching keyed to the
+ * install, so we serve a raw handler and set the headers directly.
+ *
+ * Caching: the path carries the current `lockfileHash`, and the response sets
+ * `Cache-Control: private, max-age=<1y>, immutable`. A given (hash, specifier)
+ * closure is immutable for the life of that install; a new install changes the
+ * hash, so the browser simply requests a fresh URL and never reuses the old
+ * cache entry. `private` because the registry (and thus the types) may be
+ * private.
+ *
+ * Auth: this is a raw route, so it bypasses the oRPC auto-auth middleware. We
+ * authenticate the request and enforce the same global `script-packages.read`
+ * access the (removed) oRPC procedure used. The resource has no instance-level
+ * access, so a global-access check is exactly equivalent.
+ */
+
+/** One year. The (hash, specifier) closure is immutable for the install. */
+const CACHE_MAX_AGE_SECONDS = 60 * 60 * 24 * 365;
+
+interface CreateTypeClosureHttpHandlerDeps {
+  auth: AuthService;
+  /** Returns the current desired lockfile hash (null = no packages). */
+  getLockfileHash: () => Promise<string | null>;
+  storeRoot: string;
+  logger: Logger;
+}
+
+/**
+ * Whether the authenticated user holds global `script-packages.read`.
+ * Services are trusted; users/applications must carry the rule (or `*`).
+ */
+function hasReadAccess(
+  user: Awaited<ReturnType<AuthService["authenticate"]>>,
+): boolean {
+  if (!user) return false;
+  if (user.type === "service") return true;
+  const rules = user.accessRules ?? [];
+  return (
+    rules.includes("*") || rules.includes(scriptPackagesAccess.read.id)
+  );
+}
+
+function jsonResponse(body: unknown, status: number): Response {
+  return Response.json(body, { status });
+}
+
+export function createTypeClosureHttpHandler({
+  auth,
+  getLockfileHash,
+  storeRoot,
+  logger,
+}: CreateTypeClosureHttpHandlerDeps): (req: Request) => Promise<Response> {
+  return async (req: Request): Promise<Response> => {
+    if (req.method !== "GET") {
+      return jsonResponse({ error: "Method not allowed" }, 405);
+    }
+
+    // ─── Auth ─────────────────────────────────────────────────────────────
+    let user: Awaited<ReturnType<AuthService["authenticate"]>>;
+    try {
+      user = await auth.authenticate(req);
+    } catch (error) {
+      logger.error(
+        `Type-closure auth failed: ${extractErrorMessage(error)}`,
+      );
+      return jsonResponse({ error: "Unauthorized" }, 401);
+    }
+    if (!user) return jsonResponse({ error: "Unauthorized" }, 401);
+    if (!hasReadAccess(user)) {
+      return jsonResponse({ error: "Forbidden" }, 403);
+    }
+
+    // ─── Parse path ───────────────────────────────────────────────────────
+    const pathname = new URL(req.url).pathname;
+    const prefixIndex = pathname.indexOf(TYPE_ACQUISITION_PATH_PREFIX);
+    if (prefixIndex === -1) {
+      return jsonResponse({ error: "Not found" }, 404);
+    }
+    const afterPrefix = pathname.slice(
+      prefixIndex + TYPE_ACQUISITION_PATH_PREFIX.length,
+    );
+    const parsed = parseTypeAcquisitionPath(afterPrefix);
+    if (!parsed) {
+      return jsonResponse({ error: "Bad request" }, 400);
+    }
+    const { lockfileHash, specifier } = parsed;
+
+    // ─── Hash must match the current install ──────────────────────────────
+    // We only retain the current materialized tree; an old hash's tree may be
+    // gone. A mismatch means the client's install state is stale — tell it so
+    // it refetches install state and retries against the new hash.
+    let currentHash: string | null;
+    try {
+      currentHash = await getLockfileHash();
+    } catch (error) {
+      logger.error(
+        `Type-closure install-state load failed: ${extractErrorMessage(error)}`,
+      );
+      return jsonResponse({ error: "Internal error" }, 500);
+    }
+    if (currentHash === null) {
+      // No packages configured: nothing to acquire. Cacheable empty result.
+      return cacheableClosure({
+        specifier,
+        files: [],
+        hasOwnTypes: false,
+        hasAtTypes: false,
+        notFound: true,
+        truncated: false,
+      });
+    }
+    if (currentHash !== lockfileHash) {
+      // Don't cache a stale-hash miss (no immutable guarantee).
+      return jsonResponse(
+        { error: "Stale lockfile hash; refetch install state." },
+        409,
+      );
+    }
+
+    // ─── Resolve the closure off the materialized tree ────────────────────
+    const nodeModulesDir = path.join(
+      storePaths(storeRoot).trees,
+      lockfileHash,
+      "node_modules",
+    );
+    let closure: PackageTypeClosure;
+    try {
+      closure = await resolvePackageTypeClosure({
+        nodeModulesDir,
+        specifier,
+      });
+    } catch (error) {
+      logger.error(
+        `Type-closure resolution failed for "${specifier}": ${extractErrorMessage(
+          error,
+        )}`,
+      );
+      return jsonResponse({ error: "Internal error" }, 500);
+    }
+    if (closure.truncated) {
+      logger.warn(
+        `Type closure for "${specifier}" (${lockfileHash}) hit the size ceiling; some declarations were dropped.`,
+      );
+    }
+    return cacheableClosure(closure);
+  };
+}
+
+function cacheableClosure(closure: PackageTypeClosure): Response {
+  return Response.json(closure, {
+    status: 200,
+    headers: {
+      // Per-install immutable: the (hash, specifier) closure never changes
+      // for the life of the install. `private` because types may be private.
+      "Cache-Control": `private, max-age=${CACHE_MAX_AGE_SECONDS}, immutable`,
+    },
+  });
+}

package/tsconfig.json

@@ -0,0 +1,23 @@
+{
+  "extends": "@checkstack/tsconfig/backend.json",
+  "include": [
+    "src"
+  ],
+  "references": [
+    {
+      "path": "../backend-api"
+    },
+    {
+      "path": "../common"
+    },
+    {
+      "path": "../script-packages-common"
+    },
+    {
+      "path": "../secrets-backend"
+    },
+    {
+      "path": "../secrets-common"
+    }
+  ]
+}