@checkstack/script-packages-backend 0.2.0

package/src/schema.ts

@@ -0,0 +1,166 @@
+import {
+  pgTable,
+  text,
+  jsonb,
+  integer,
+  bigint,
+  boolean,
+  timestamp,
+  index,
+} from "drizzle-orm/pg-core";
+import type { ManifestEntry } from "@checkstack/script-packages-common";
+
+/**
+ * Drizzle schema for the script-packages plugin.
+ *
+ * Tables (per the feature plan §3.7). Singleton tables use a fixed text PK
+ * (`"singleton"`) so an upsert always targets the one row. Core instances
+ * are ephemeral and not individually addressable - they reconcile to the
+ * desired `lockfile_hash` without per-pod rows.
+ */
+
+/** The admin-curated allowlist of pinned packages. */
+export const scriptPackages = pgTable("script_packages", {
+  name: text("name").primaryKey(),
+  version: text("version").notNull(),
+  enabled: boolean("enabled").notNull().default(true),
+  addedBy: text("added_by"),
+  addedAt: timestamp("added_at").defaultNow().notNull(),
+  updatedAt: timestamp("updated_at").defaultNow().notNull(),
+});
+
+/** Singleton registry config. Auth token is a connection-store secret ref. */
+export const scriptPackageRegistryConfig = pgTable(
+  "script_package_registry_config",
+  {
+    id: text("id").primaryKey().default("singleton"),
+    registryUrl: text("registry_url")
+      .notNull()
+      .default("https://registry.npmjs.org/"),
+    scopedRegistries: jsonb("scoped_registries")
+      .$type<{ scope: string; registryUrl: string }[]>()
+      .notNull()
+      .default([]),
+    /** Secret ref into the connection-store; never the plaintext token. */
+    authSecretRef: text("auth_secret_ref"),
+    ignoreScripts: boolean("ignore_scripts").notNull().default(true),
+    updatedAt: timestamp("updated_at").defaultNow().notNull(),
+  },
+);
+
+/** Singleton desired install state + lockfile manifest. */
+export const scriptPackageInstallState = pgTable(
+  "script_package_install_state",
+  {
+    id: text("id").primaryKey().default("singleton"),
+    /** "idle" | "installing" | "ready" | "error" */
+    status: text("status").notNull().default("idle"),
+    /** Desired lockfile hash every host reconciles to. */
+    lockfileHash: text("lockfile_hash"),
+    manifest: jsonb("manifest").$type<ManifestEntry[]>().notNull().default([]),
+    totalSizeBytes: bigint("total_size_bytes", { mode: "number" })
+      .notNull()
+      .default(0),
+    lastInstalledAt: timestamp("last_installed_at"),
+    errorMessage: text("error_message"),
+  },
+);
+
+/** Singleton size-cap config (warn / block thresholds). */
+export const scriptPackageSizeCap = pgTable("script_package_size_cap", {
+  id: text("id").primaryKey().default("singleton"),
+  warnBytes: bigint("warn_bytes", { mode: "number" })
+    .notNull()
+    .default(150 * 1024 * 1024),
+  blockBytes: bigint("block_bytes", { mode: "number" })
+    .notNull()
+    .default(300 * 1024 * 1024),
+  updatedAt: timestamp("updated_at").defaultNow().notNull(),
+});
+
+/**
+ * Content-addressed blob index. `integrity` is the stable identity across
+ * blob-store backends; `backend` records which `BlobStore` currently holds
+ * it (well-defined during migration). Powers delta sync + blob GC.
+ */
+export const scriptPackageBlob = pgTable(
+  "script_package_blob",
+  {
+    integrity: text("integrity").primaryKey(),
+    name: text("name").notNull(),
+    version: text("version").notNull(),
+    /** "postgres" | "s3" | ... */
+    backend: text("backend").notNull(),
+    sizeBytes: bigint("size_bytes", { mode: "number" }).notNull(),
+    createdAt: timestamp("created_at").defaultNow().notNull(),
+  },
+  (t) => ({
+    backendIdx: index("script_package_blob_backend_idx").on(t.backend),
+  }),
+);
+
+/** Singleton storage config + in-flight migration state. */
+export const scriptPackageStorageConfig = pgTable(
+  "script_package_storage_config",
+  {
+    id: text("id").primaryKey().default("singleton"),
+    activeBackend: text("active_backend").notNull().default("postgres"),
+    /** "idle" | "migrating" | "completed" | "error" */
+    migrationStatus: text("migration_status").notNull().default("idle"),
+    migrationTarget: text("migration_target"),
+    migratedCount: integer("migrated_count").notNull().default(0),
+    migrationError: text("migration_error"),
+    updatedAt: timestamp("updated_at").defaultNow().notNull(),
+  },
+);
+
+/**
+ * Recent lockfile-manifest history. The installer records each successful
+ * `lockfileHash` + its manifest here so the blob GC can compute the
+ * "retained set" (current + the previous N hashes) without that history
+ * being reconstructable from the singleton install state (which only holds
+ * the CURRENT desired manifest). Old rows beyond the retention window are
+ * pruned by the GC itself.
+ */
+export const scriptPackageLockfileHistory = pgTable(
+  "script_package_lockfile_history",
+  {
+    lockfileHash: text("lockfile_hash").primaryKey(),
+    manifest: jsonb("manifest").$type<ManifestEntry[]>().notNull().default([]),
+    recordedAt: timestamp("recorded_at").defaultNow().notNull(),
+  },
+  (t) => ({
+    recordedIdx: index("script_package_lockfile_history_recorded_idx").on(
+      t.recordedAt,
+    ),
+  }),
+);
+
+/** Singleton blob-GC last-run state (for the admin UI; never gates safety). */
+export const scriptPackageBlobGcState = pgTable(
+  "script_package_blob_gc_state",
+  {
+    id: text("id").primaryKey().default("singleton"),
+    lastRunAt: timestamp("last_run_at"),
+    lastDeleted: integer("last_deleted").notNull().default(0),
+    lastBytesReclaimed: bigint("last_bytes_reclaimed", { mode: "number" })
+      .notNull()
+      .default(0),
+    totalBytesReclaimed: bigint("total_bytes_reclaimed", { mode: "number" })
+      .notNull()
+      .default(0),
+  },
+);
+
+/** Per-satellite reconcile state (satellites are individually addressable). */
+export const scriptPackageSatelliteState = pgTable(
+  "script_package_satellite_state",
+  {
+    satelliteId: text("satellite_id").primaryKey(),
+    lockfileHash: text("lockfile_hash"),
+    /** "pending" | "syncing" | "ready" | "error" */
+    status: text("status").notNull().default("pending"),
+    errorMessage: text("error_message"),
+    syncedAt: timestamp("synced_at"),
+  },
+);

package/src/size-cap.test.ts

@@ -0,0 +1,32 @@
+import { describe, expect, test } from "bun:test";
+import { evaluateSizeCap } from "./size-cap";
+
+const cap = { warnBytes: 150 * 1024 * 1024, blockBytes: 300 * 1024 * 1024 };
+
+describe("evaluateSizeCap", () => {
+  test("ok below the warn threshold", () => {
+    expect(evaluateSizeCap({ totalSizeBytes: 50 * 1024 * 1024, cap }).level).toBe(
+      "ok",
+    );
+  });
+
+  test("warns between warn and block", () => {
+    const v = evaluateSizeCap({ totalSizeBytes: 200 * 1024 * 1024, cap });
+    expect(v.level).toBe("warn");
+    if (v.level !== "ok") expect(v.message).toContain("200.0MB");
+  });
+
+  test("blocks above the block threshold", () => {
+    const v = evaluateSizeCap({ totalSizeBytes: 400 * 1024 * 1024, cap });
+    expect(v.level).toBe("block");
+  });
+
+  test("exactly at a threshold is not over it", () => {
+    expect(
+      evaluateSizeCap({ totalSizeBytes: cap.warnBytes, cap }).level,
+    ).toBe("ok");
+    expect(
+      evaluateSizeCap({ totalSizeBytes: cap.blockBytes, cap }).level,
+    ).toBe("warn");
+  });
+});

package/src/size-cap.ts

@@ -0,0 +1,40 @@
+import type { SizeCapConfig } from "@checkstack/script-packages-common";
+
+export type SizeCapVerdict =
+  | { level: "ok" }
+  | { level: "warn"; message: string }
+  | { level: "block"; message: string };
+
+function mb(bytes: number): string {
+  return `${(bytes / (1024 * 1024)).toFixed(1)}MB`;
+}
+
+/**
+ * Evaluate a resolved total size against the configured cap. `block` should
+ * refuse the install; `warn` surfaces a non-fatal notice in the admin UI.
+ */
+export function evaluateSizeCap({
+  totalSizeBytes,
+  cap,
+}: {
+  totalSizeBytes: number;
+  cap: SizeCapConfig;
+}): SizeCapVerdict {
+  if (totalSizeBytes > cap.blockBytes) {
+    return {
+      level: "block",
+      message: `Resolved package size ${mb(totalSizeBytes)} exceeds the ${mb(
+        cap.blockBytes,
+      )} block threshold. Remove packages or raise the cap.`,
+    };
+  }
+  if (totalSizeBytes > cap.warnBytes) {
+    return {
+      level: "warn",
+      message: `Resolved package size ${mb(totalSizeBytes)} exceeds the ${mb(
+        cap.warnBytes,
+      )} warning threshold.`,
+    };
+  }
+  return { level: "ok" };
+}

package/src/storage-migration.test.ts

@@ -0,0 +1,318 @@
+import { describe, expect, test } from "bun:test";
+import type { BlobStore } from "./blob-store";
+import {
+  runStorageMigration,
+  resumeCrashedMigration,
+  type MigrationStateSnapshot,
+  type StorageMigrationConfig,
+  type StorageMigrationStores,
+} from "./storage-migration";
+import type { AdvisoryLockHandle } from "@checkstack/backend-api";
+
+function memStore(id: string, seed: Record<string, string> = {}): BlobStore {
+  const map = new Map<string, Uint8Array>(
+    Object.entries(seed).map(([k, v]) => [k, new TextEncoder().encode(v)]),
+  );
+  return {
+    id,
+    async put({ integrity, bytes }) {
+      map.set(integrity, bytes);
+    },
+    async get({ integrity }) {
+      return map.get(integrity);
+    },
+    async has({ integrity }) {
+      return map.has(integrity);
+    },
+    async delete({ integrity }) {
+      map.delete(integrity);
+    },
+    async list() {
+      return [...map.keys()];
+    },
+  };
+}
+
+/** In-memory blob index + storage-config doubles. */
+function harness(initial: { integrity: string; backend: string }[]) {
+  const index = new Map(initial.map((b) => [b.integrity, b.backend]));
+  const config = {
+    status: "idle" as string,
+    activeBackend: "postgres",
+    target: null as string | null,
+    migratedCount: 0,
+    error: null as string | null,
+  };
+  const blobIndex: StorageMigrationStores = {
+    listNotOnBackend: async (target) =>
+      [...index.entries()]
+        .filter(([, backend]) => backend !== target)
+        .map(([integrity, backend]) => ({ integrity, backend })),
+    setBackend: async (integrity, backend) => {
+      index.set(integrity, backend);
+    },
+  };
+  const storage: StorageMigrationConfig = {
+    beginMigration: async (target) => {
+      config.status = "migrating";
+      config.target = target;
+      config.migratedCount = 0;
+      config.error = null;
+    },
+    setMigratedCount: async (n) => {
+      config.migratedCount = n;
+    },
+    completeMigration: async (target) => {
+      config.status = "completed";
+      config.activeBackend = target;
+    },
+    failMigration: async (message) => {
+      config.status = "error";
+      config.error = message;
+    },
+  };
+  return { index, config, blobIndex, storage };
+}
+
+describe("runStorageMigration", () => {
+  test("copies + verifies every blob, then atomically flips active backend", async () => {
+    const pg = memStore("postgres", { "sha-a": "AAA", "sha-b": "BBB" });
+    const s3 = memStore("s3");
+    const { index, config, blobIndex, storage } = harness([
+      { integrity: "sha-a", backend: "postgres" },
+      { integrity: "sha-b", backend: "postgres" },
+    ]);
+
+    const res = await runStorageMigration({
+      blobIndex,
+      storage,
+      getStore: (id) => (id === "postgres" ? pg : s3),
+      activeBackend: "postgres",
+      target: "s3",
+    });
+
+    expect(res.completed).toBe(true);
+    expect(res.migrated).toBe(2);
+    // Bytes copied to target + index flipped.
+    expect(new TextDecoder().decode(await s3.get({ integrity: "sha-a" }))).toBe("AAA");
+    expect(index.get("sha-a")).toBe("s3");
+    expect(index.get("sha-b")).toBe("s3");
+    // Active backend flipped only at completion.
+    expect(config.activeBackend).toBe("s3");
+    expect(config.status).toBe("completed");
+    expect(config.migratedCount).toBe(2);
+  });
+
+  test("resumes from a partial state (blobs already on target are skipped)", async () => {
+    // sha-a already migrated; only sha-b remains.
+    const pg = memStore("postgres", { "sha-b": "BBB" });
+    const s3 = memStore("s3", { "sha-a": "AAA" });
+    const { index, config, blobIndex, storage } = harness([
+      { integrity: "sha-a", backend: "s3" }, // already flipped
+      { integrity: "sha-b", backend: "postgres" },
+    ]);
+
+    const res = await runStorageMigration({
+      blobIndex,
+      storage,
+      getStore: (id) => (id === "postgres" ? pg : s3),
+      activeBackend: "postgres",
+      target: "s3",
+    });
+
+    expect(res.migrated).toBe(1); // only sha-b
+    expect(index.get("sha-b")).toBe("s3");
+    expect(config.activeBackend).toBe("s3");
+  });
+
+  test("aborts cleanly on an integrity mismatch (active backend unchanged)", async () => {
+    const pg = memStore("postgres", { "sha-a": "AAA" });
+    // Target silently corrupts on write (put stores different bytes).
+    const corrupt: BlobStore = {
+      id: "s3",
+      async put() {
+        /* drop the write */
+      },
+      async get() {
+        return new TextEncoder().encode("CORRUPT");
+      },
+      async has() {
+        return false;
+      },
+      async delete() {},
+      async list() {
+        return [];
+      },
+    };
+    const { index, config, blobIndex, storage } = harness([
+      { integrity: "sha-a", backend: "postgres" },
+    ]);
+
+    const res = await runStorageMigration({
+      blobIndex,
+      storage,
+      getStore: (id) => (id === "postgres" ? pg : corrupt),
+      activeBackend: "postgres",
+      target: "s3",
+    });
+
+    expect(res.completed).toBe(false);
+    expect(res.error).toMatch(/integrity verification failed/i);
+    expect(config.status).toBe("error");
+    expect(config.activeBackend).toBe("postgres"); // NOT flipped
+    expect(index.get("sha-a")).toBe("postgres"); // NOT flipped
+  });
+
+  test("GCs the source after a verified copy when gcSource is set", async () => {
+    const pg = memStore("postgres", { "sha-a": "AAA" });
+    const s3 = memStore("s3");
+    const { blobIndex, storage } = harness([
+      { integrity: "sha-a", backend: "postgres" },
+    ]);
+
+    await runStorageMigration({
+      blobIndex,
+      storage,
+      getStore: (id) => (id === "postgres" ? pg : s3),
+      activeBackend: "postgres",
+      target: "s3",
+      gcSource: true,
+    });
+
+    expect(await pg.has({ integrity: "sha-a" })).toBe(false); // GC'd
+    expect(await s3.has({ integrity: "sha-a" })).toBe(true);
+  });
+
+  test("no-op when target equals the active backend", async () => {
+    const { config, blobIndex, storage } = harness([]);
+    const res = await runStorageMigration({
+      blobIndex,
+      storage,
+      getStore: () => memStore("postgres"),
+      activeBackend: "postgres",
+      target: "postgres",
+    });
+    expect(res.completed).toBe(true);
+    expect(res.migrated).toBe(0);
+    expect(config.status).toBe("idle"); // beginMigration not called
+  });
+
+  test("aborts when a source blob is missing", async () => {
+    const pg = memStore("postgres"); // empty — sha-a missing
+    const s3 = memStore("s3");
+    const { config, blobIndex, storage } = harness([
+      { integrity: "sha-a", backend: "postgres" },
+    ]);
+    const res = await runStorageMigration({
+      blobIndex,
+      storage,
+      getStore: (id) => (id === "postgres" ? pg : s3),
+      activeBackend: "postgres",
+      target: "s3",
+    });
+    expect(res.completed).toBe(false);
+    expect(res.error).toMatch(/missing from its source backend/i);
+    expect(config.activeBackend).toBe("postgres");
+  });
+});
+
+describe("resumeCrashedMigration", () => {
+  function fakeLock(): { handle: AdvisoryLockHandle; released: () => boolean } {
+    let released = false;
+    return {
+      handle: {
+        async release() {
+          released = true;
+        },
+      },
+      released: () => released,
+    };
+  }
+
+  test("resumes a store left in 'migrating' on init and completes it", async () => {
+    // H3 regression: a migration that crashed mid-flight leaves status
+    // "migrating", which wedges installs AND blocks triggerMigration. Init
+    // must relaunch the resumable migration toward the recorded target.
+    const pg = memStore("postgres", { "sha-a": "AAA", "sha-b": "BBB" });
+    const s3 = memStore("s3");
+    const { index, config, blobIndex, storage } = harness([
+      { integrity: "sha-a", backend: "postgres" },
+      { integrity: "sha-b", backend: "postgres" },
+    ]);
+    // Simulate the crash: status stuck at "migrating" toward "s3".
+    config.status = "migrating";
+    config.target = "s3";
+
+    const lock = fakeLock();
+    const result = await resumeCrashedMigration({
+      loadState: async (): Promise<MigrationStateSnapshot> => ({
+        migrationStatus: config.status,
+        migrationTarget: config.target,
+        activeBackend: config.activeBackend,
+      }),
+      tryLock: async () => lock.handle,
+      runMigration: ({ target, activeBackend }) =>
+        runStorageMigration({
+          blobIndex,
+          storage,
+          getStore: (id) => (id === "postgres" ? pg : s3),
+          activeBackend,
+          target,
+        }),
+    });
+
+    expect(result.resumed).toBe(true);
+    await result.done; // wait for the relaunched migration to finish
+
+    expect(config.status).toBe("completed");
+    expect(config.activeBackend).toBe("s3");
+    expect(index.get("sha-a")).toBe("s3");
+    expect(index.get("sha-b")).toBe("s3");
+    expect(lock.released()).toBe(true); // lock freed when done
+  });
+
+  test("does nothing when no migration is in flight", async () => {
+    const { blobIndex, storage } = harness([]);
+    let ran = false;
+    const result = await resumeCrashedMigration({
+      loadState: async () => ({
+        migrationStatus: "idle",
+        migrationTarget: null,
+        activeBackend: "postgres",
+      }),
+      tryLock: async () => {
+        throw new Error("should not try to lock when nothing to resume");
+      },
+      runMigration: async () => {
+        ran = true;
+        return runStorageMigration({
+          blobIndex,
+          storage,
+          getStore: () => memStore("x"),
+          activeBackend: "postgres",
+          target: "s3",
+        });
+      },
+    });
+    expect(result.resumed).toBe(false);
+    expect(ran).toBe(false);
+  });
+
+  test("defers to another pod that already holds the lock", async () => {
+    let ran = false;
+    const result = await resumeCrashedMigration({
+      loadState: async () => ({
+        migrationStatus: "migrating",
+        migrationTarget: "s3",
+        activeBackend: "postgres",
+      }),
+      tryLock: async () => null, // held elsewhere
+      runMigration: async () => {
+        ran = true;
+      },
+    });
+    expect(result.resumed).toBe(false);
+    expect(result.reason).toMatch(/another instance/i);
+    expect(ran).toBe(false);
+  });
+});