@checkstack/backend 0.12.0 → 0.13.0

package/CHANGELOG.md

@@ -1,5 +1,43 @@
 # @checkstack/backend
 
+## 0.13.0
+
+### Minor Changes
+
+- af6bda7: Fix plugin migrations failing on upgrade with `type "..." does not exist`.
+
+  Plugin migrations are schema-agnostic and rely on `search_path` to resolve
+  unqualified names into the plugin's schema (e.g. `plugin_healthcheck`). The
+  loader set `search_path` at the session level on the shared admin pool and
+  then called Drizzle's `migrate()`. Because `migrate()` runs all pending
+  migrations inside its own transaction, a `pg.Pool` could service that
+  transaction on a different physical connection than the one the `SET` ran on,
+  so the migration SQL executed against `public` instead.
+
+  This was invisible on a fresh database (every object is created within that
+  one transaction, so unqualified references still resolve), but broke upgrades:
+  the healthcheck plugin's new `health_check_state_transitions` migration
+  references the pre-existing `health_check_status` enum, which an earlier
+  migration created in the plugin schema. On a different pooled connection that
+  enum is not on the `public` `search_path`, so startup failed with
+  `type "health_check_status" does not exist` and the pod crash-looped.
+
+  Migrations now run on a single pinned pool connection: the loader checks out
+  one dedicated client, sets `search_path` on it, and binds the migrator to that
+  same client, mirroring the connection-affinity pattern already used by the
+  advisory-lock service. Every migration statement now runs under the intended
+  schema.
+
+  Boot was also restructured into two passes over the topologically-sorted
+  plugins: pass 1 runs every plugin's migrations, pass 2 runs every plugin's
+  `init()`. Previously the two were interleaved per plugin, so an
+  already-initialized plugin's background work (queue consumers, sweepers,
+  reactive-entity/event wiring) could compete for pool connections while a later
+  plugin was still migrating. Running all migrations first keeps the pool quiet
+  during migrations and removes that race entirely. The pinned connection and the
+  two-pass ordering are each independently sufficient for the fix above; together
+  they make boot robust regardless of what else touches the pool.
+
 ## 0.12.0
 
 ### Minor Changes

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@checkstack/backend",
-  "version": "0.12.0",
+  "version": "0.13.0",
   "license": "Elastic-2.0",
   "checkstack": {
     "type": "backend"

package/src/plugin-manager/plugin-loader.ts

@@ -1,9 +1,7 @@
-import { migrate } from "drizzle-orm/node-postgres/migrator";
-import { NodePgDatabase } from "drizzle-orm/node-postgres";
 import path from "node:path";
 import fs from "node:fs";
 import type { Hono } from "hono";
-import { eq, and, sql } from "drizzle-orm";
+import { eq, and } from "drizzle-orm";
 import type { SafeDatabase } from "@checkstack/backend-api";
 import {
   coreServices,
@@ -23,6 +21,8 @@ import { rootLogger } from "../logger";
 import type { ServiceRegistry } from "../services/service-registry";
 import { plugins } from "../schema";
 import { stripPublicSchemaFromMigrations } from "../utils/strip-public-schema";
+import { runPluginMigrations } from "../utils/run-plugin-migrations";
+import { adminPool } from "../db";
 import {
   discoverLocalPlugins,
   syncPluginsToDatabase,
@@ -309,6 +309,17 @@ export async function loadPlugins({
   }
 
   // Phase 2: Initialize Plugins (Topological Sort)
+  //
+  // Done in two passes over the topologically-sorted plugins:
+  //   Pass 1 - run EVERY plugin's migrations.
+  //   Pass 2 - resolve deps + run EVERY plugin's init().
+  // Splitting the passes guarantees no plugin's init() (which may start
+  // background DB work - queue consumers, sweepers, reactive-entity/event
+  // wiring) is running while another plugin is still migrating. That
+  // interleaving is what could divert a migration onto a pooled connection
+  // without the plugin's search_path. `runPluginMigrations` pins a connection
+  // too, so each measure is independently sufficient; together they make boot
+  // robust regardless of what touches the pool.
   const logger = await deps.registry.get(coreServices.logger, {
     pluginId: "core",
   });
@@ -347,9 +358,9 @@ export async function loadPlugins({
   // pre-existing behavior and is preferable to a multi-second hang.
   deps.onApiRouteRegistered?.();
 
+  // Phase 2, pass 1: run every plugin's migrations BEFORE any plugin init.
   for (const id of sortedIds) {
     const p = pendingInits.find((x) => x.metadata.pluginId === id)!;
-    rootLogger.info(`🚀 Initializing ${p.metadata.pluginId}...`);
 
     try {
       /**
@@ -372,34 +383,32 @@ export async function loadPlugins({
        * causing "relation does not exist" errors since the tables are actually in
        * the plugin's schema (e.g., `plugin_maintenance.maintenances`).
        *
-       * ## Session-Level vs Transaction-Level search_path
+       * ## Why a pinned connection (not a session-level SET on the pool)
+       *
+       * The migration `search_path` MUST be set on the exact connection the
+       * migration statements run on. Setting it at the session level on the
+       * shared `adminPool` does not achieve that: `migrate()` runs all pending
+       * migrations inside its own transaction, which a `pg.Pool` may service on
+       * a *different* physical connection than the `SET` ran on. The migration
+       * SQL would then execute against `public`.
        *
-       * We use **session-level** `SET search_path` (not `SET LOCAL`) here because:
-       * - `migrate()` runs multiple statements and may manage its own transactions
-       * - `SET LOCAL` only persists within a single transaction
-       * - Session-level SET persists until explicitly changed or session ends
+       * `runPluginMigrations()` therefore checks out ONE dedicated client from
+       * the pool, sets `search_path` on it, and binds the migrator to that same
+       * client - the same connection-affinity pattern the advisory-lock service
+       * uses (see `advisory-lock.ts`). The bug this prevents is invisible on a
+       * fresh database (every object is created in one transaction, so
+       * unqualified references still resolve) but breaks UPGRADES: a new
+       * migration that references an enum an earlier migration created in the
+       * plugin schema fails with `type "..." does not exist`.
        *
        * ## Why This Doesn't Affect Runtime Queries
        *
        * After migrations complete, plugins receive their database via
        * `createScopedDb()` which wraps every query in a transaction with
        * `SET LOCAL search_path`. This ensures runtime queries always use the
-       * correct schema, regardless of the session-level search_path.
-       *
-       * ## Potential Hazards
-       *
-       * 1. **Error During Migration**: If a migration fails, the search_path may
-       *    remain set to that plugin's schema. The next plugin's migration would
-       *    fail visibly (wrong schema), which is better than silent data corruption.
-       *
-       * 2. **Parallel Migration Execution**: This code assumes sequential plugin
-       *    initialization (which is enforced by the topologically-sorted loop).
-       *    If migrations ever run in parallel, search_path conflicts would occur.
-       *
-       * 3. **Connection Pool Pollution**: `SET` without `LOCAL` affects the entire
-       *    session. However, we reset to `public` after each plugin's migrations,
-       *    and runtime queries use `SET LOCAL` anyway, so this is safe.
+       * correct schema.
        *
+       * @see runPluginMigrations in ../utils/run-plugin-migrations.ts
        * @see createScopedDb in ../utils/scoped-db.ts for runtime query isolation
        * @see getPluginSchemaName in @checkstack/drizzle-helper for schema naming
        * =======================================================================
@@ -419,29 +428,13 @@ export async function loadPlugins({
             `   -> Running migrations for ${p.metadata.pluginId} from ${migrationsFolder}`,
           );
 
-          // Create schema if it doesn't exist BEFORE running migrations.
-          // Without this, SET search_path to a non-existent schema causes
-          // PostgreSQL to fall back to 'public', creating tables in the wrong schema.
-          await deps.db.execute(
-            sql.raw(`CREATE SCHEMA IF NOT EXISTS "${migrationsSchema}"`),
-          );
-
-          // Set search_path to plugin schema before running migrations.
-          // Uses session-level SET (not SET LOCAL) because migrate() may run
-          // multiple statements across transaction boundaries.
-          // No 'public' fallback: schema is guaranteed to exist from CREATE above.
-          await deps.db.execute(
-            sql.raw(`SET search_path = "${migrationsSchema}"`),
-          );
-          // Drizzle migrate() requires NodePgDatabase, cast from SafeDatabase
-          await migrate(deps.db as NodePgDatabase<Record<string, unknown>>, {
+          // Run on a single pinned connection so the search_path we set is the
+          // one the migration statements actually execute under.
+          await runPluginMigrations({
+            pool: adminPool,
             migrationsFolder,
             migrationsSchema,
           });
-
-          // Reset search_path to public after migrations complete.
-          // This prevents search_path leaking into subsequent plugin migrations.
-          await deps.db.execute(sql.raw(`SET search_path = public`));
         } catch (error) {
           rootLogger.error(
             `❌ Failed migration of plugin ${p.metadata.pluginId}:`,
@@ -456,7 +449,25 @@ export async function loadPlugins({
           `   -> No migrations found for ${p.metadata.pluginId} (skipping)`,
         );
       }
+    } catch (error) {
+      rootLogger.error(
+        `❌ Critical error loading plugin ${p.metadata.pluginId}:`,
+        error,
+      );
+      throw new Error(`Critical error loading plugin ${p.metadata.pluginId}`, {
+        cause: error,
+      });
+    }
+  }
+
+  // Phase 2, pass 2: initialize plugins in topological order. Every plugin -
+  // and therefore every dependency - is fully migrated by now, so an init()
+  // can assume all plugin schemas exist.
+  for (const id of sortedIds) {
+    const p = pendingInits.find((x) => x.metadata.pluginId === id)!;
+    rootLogger.info(`🚀 Initializing ${p.metadata.pluginId}...`);
 
+    try {
       // Resolve Dependencies
       const resolvedDeps: Record<string, unknown> = {};
       for (const [key, ref] of Object.entries(p.deps)) {

package/src/plugin-manager.test.ts

@@ -486,5 +486,49 @@ describe("PluginManager", () => {
 
       expect(testBackendInit).toHaveBeenCalled();
     });
+
+    it("initializes every plugin across the two-pass (migrate-all, then init-all) loop", async () => {
+      const mockRouter = {
+        route: mock(),
+        all: mock(),
+        newResponse: mock(),
+      } as never;
+
+      // Boot runs migrations for all plugins in pass 1, then inits in pass 2.
+      // Manual test plugins have no plugin path so pass 1 is a no-op for them;
+      // this guards that pass 2 still initializes EVERY plugin (the split loop
+      // doesn't drop any) and follows topological order.
+      const initOrder: string[] = [];
+      const makePlugin = (pluginId: string) =>
+        createBackendPlugin({
+          metadata: { pluginId },
+          register(env) {
+            env.registerInit({
+              deps: {},
+              init: async () => {
+                initOrder.push(pluginId);
+              },
+            });
+          },
+        });
+
+      pluginManager.registerService(
+        coreServices.queueManager,
+        createMockQueueManager(),
+      );
+      pluginManager.registerService(coreServices.logger, createMockLogger());
+      pluginManager.registerService(
+        coreServices.database,
+        createMockDb() as never,
+      );
+
+      await pluginManager.loadPlugins(
+        mockRouter,
+        [makePlugin("plugin-a"), makePlugin("plugin-b"), makePlugin("plugin-c")],
+        { skipDiscovery: true },
+      );
+
+      expect(initOrder).toEqual(["plugin-a", "plugin-b", "plugin-c"]);
+    });
   });
 });

package/src/utils/run-plugin-migrations.test.ts

@@ -0,0 +1,124 @@
+import { describe, it, expect } from "bun:test";
+import type { PoolClient } from "pg";
+import type { NodePgDatabase } from "drizzle-orm/node-postgres";
+import { runPluginMigrations } from "./run-plugin-migrations";
+
+type MigrationDb = NodePgDatabase<Record<string, unknown>>;
+
+/**
+ * A fake pooled client that records the SQL it runs and whether it was
+ * released, modelling the bits of `pg.PoolClient` the helper touches.
+ */
+function makeFakeClient() {
+  const queries: string[] = [];
+  let released = false;
+  const client = {
+    query: async (text: string) => {
+      queries.push(text);
+      return { rows: [] };
+    },
+    release: () => {
+      released = true;
+    },
+  };
+  return {
+    client: client as unknown as PoolClient,
+    queries,
+    isReleased: () => released,
+  };
+}
+
+describe("runPluginMigrations", () => {
+  it("runs the migrator on a single pinned connection with search_path set first", async () => {
+    const { client, queries, isReleased } = makeFakeClient();
+
+    let connectCount = 0;
+    const pool = {
+      connect: async () => {
+        connectCount++;
+        return client;
+      },
+    };
+
+    const fakeDb = { __fake: true } as unknown as MigrationDb;
+    let dbPassedToMigrate: unknown;
+    let clientPassedToFactory: PoolClient | undefined;
+    let queriesBeforeMigrate: string[] = [];
+
+    await runPluginMigrations({
+      pool,
+      migrationsFolder: "/plugins/healthcheck/drizzle",
+      migrationsSchema: "plugin_healthcheck",
+      createMigrationDb: (c) => {
+        clientPassedToFactory = c;
+        return fakeDb;
+      },
+      migrate: async (db, config) => {
+        dbPassedToMigrate = db;
+        queriesBeforeMigrate = [...queries];
+        expect(config.migrationsFolder).toBe("/plugins/healthcheck/drizzle");
+        expect(config.migrationsSchema).toBe("plugin_healthcheck");
+      },
+    });
+
+    // Exactly ONE connection is checked out: the SET and the migration must
+    // share a physical connection, which was the whole bug.
+    expect(connectCount).toBe(1);
+
+    // The migrator runs against a Drizzle instance bound to that same pinned
+    // client.
+    expect(clientPassedToFactory).toBe(client);
+    expect(dbPassedToMigrate).toBe(fakeDb);
+
+    // search_path is pointed at the plugin schema (after creating it) BEFORE
+    // the migrator runs.
+    expect(queriesBeforeMigrate).toEqual([
+      'CREATE SCHEMA IF NOT EXISTS "plugin_healthcheck"',
+      'SET search_path = "plugin_healthcheck"',
+    ]);
+
+    // Afterwards the connection is reset and returned to the pool.
+    expect(queries.at(-1)).toBe("SET search_path = public");
+    expect(isReleased()).toBe(true);
+  });
+
+  it("resets search_path and releases the connection even when the migrator throws", async () => {
+    const { client, queries, isReleased } = makeFakeClient();
+    const pool = { connect: async () => client };
+    const boom = new Error("migration failed");
+
+    await expect(
+      runPluginMigrations({
+        pool,
+        migrationsFolder: "/x",
+        migrationsSchema: "plugin_x",
+        createMigrationDb: () => ({}) as unknown as MigrationDb,
+        migrate: async () => {
+          throw boom;
+        },
+      }),
+    ).rejects.toThrow("migration failed");
+
+    expect(queries.at(-1)).toBe("SET search_path = public");
+    expect(isReleased()).toBe(true);
+  });
+
+  it("never touches anything but connect() on the pool (no session SET on the shared pool)", async () => {
+    const { client } = makeFakeClient();
+    const pool = { connect: async () => client };
+
+    await runPluginMigrations({
+      pool,
+      migrationsFolder: "/x",
+      migrationsSchema: "plugin_x",
+      createMigrationDb: () => ({}) as unknown as MigrationDb,
+      migrate: async () => {},
+    });
+
+    // The pool surface the helper depends on is exactly `connect`; everything
+    // else (CREATE SCHEMA, SET search_path, the migration itself) happens on
+    // the checked-out client. If this contract ever widens, the regression
+    // that motivated the pinned connection could creep back in.
+    expect(Object.keys(pool)).toEqual(["connect"]);
+  });
+});

package/src/utils/run-plugin-migrations.ts

@@ -0,0 +1,88 @@
+import { drizzle, type NodePgDatabase } from "drizzle-orm/node-postgres";
+import { migrate as defaultMigrate } from "drizzle-orm/node-postgres/migrator";
+import type { Pool, PoolClient } from "pg";
+
+type MigrationDb = NodePgDatabase<Record<string, unknown>>;
+
+export interface RunPluginMigrationsArgs {
+  /** Shared admin pool; the helper checks out ONE dedicated client from it. */
+  pool: Pick<Pool, "connect">;
+  /** Absolute path to the plugin's Drizzle migrations folder. */
+  migrationsFolder: string;
+  /**
+   * Postgres schema the plugin's objects live in (e.g. `plugin_healthcheck`).
+   * Also used by Drizzle for the per-plugin `__drizzle_migrations` table.
+   */
+  migrationsSchema: string;
+  /**
+   * Builds the Drizzle instance the migrator runs against. Defaults to one
+   * bound to the pinned `client`. Injectable so tests can run without a real
+   * connection.
+   */
+  createMigrationDb?: (client: PoolClient) => MigrationDb;
+  /** Drizzle's migrator. Injectable for tests. */
+  migrate?: (
+    db: MigrationDb,
+    config: { migrationsFolder: string; migrationsSchema: string },
+  ) => Promise<void>;
+}
+
+/**
+ * Run a plugin's Drizzle migrations on a SINGLE pinned pool connection.
+ *
+ * ## Why a pinned connection is required
+ *
+ * Plugin migrations are schema-agnostic: they reference the plugin's tables,
+ * types, and enums *unqualified* and rely on `search_path` to resolve them
+ * into the plugin's schema (e.g. `plugin_healthcheck`). So `search_path` must
+ * be set before the migration SQL runs.
+ *
+ * Setting it at the *session* level on the shared pool does NOT work, for the
+ * same reason session-level advisory locks don't (see `advisory-lock.ts`):
+ * Drizzle's `migrate()` wraps all pending migrations in one transaction, and
+ * with a `pg.Pool` that transaction checks out a *different* physical
+ * connection than the one the `SET` ran on. The migration statements then
+ * execute with the default `public` search_path.
+ *
+ * This stays invisible on a fresh database - every object (including each
+ * enum) is created within that one transaction, so unqualified references
+ * still resolve against whatever schema that connection happens to use. But on
+ * an UPGRADE, where earlier migrations already created an enum in the plugin
+ * schema and only newer migrations run, a new migration that references the
+ * pre-existing enum fails with `type "..." does not exist`.
+ *
+ * Binding the migrator to ONE pinned client, on which we set `search_path`
+ * first, guarantees every migration statement runs under the intended schema.
+ */
+export async function runPluginMigrations({
+  pool,
+  migrationsFolder,
+  migrationsSchema,
+  createMigrationDb = (client) => drizzle(client),
+  migrate = defaultMigrate,
+}: RunPluginMigrationsArgs): Promise<void> {
+  const client = await pool.connect();
+  try {
+    // Ensure the schema exists before pointing search_path at it. SET to a
+    // missing schema silently falls back to `public` at resolution time, which
+    // would recreate the very bug this helper exists to prevent.
+    await client.query(`CREATE SCHEMA IF NOT EXISTS "${migrationsSchema}"`);
+    await client.query(`SET search_path = "${migrationsSchema}"`);
+
+    await migrate(createMigrationDb(client), {
+      migrationsFolder,
+      migrationsSchema,
+    });
+  } finally {
+    // Reset before the client returns to the pool so the setting never leaks
+    // onto an unrelated query that later reuses this physical connection.
+    try {
+      await client.query("SET search_path = public");
+    } catch (resetError) {
+      // Best-effort: the release below still returns the connection. Reference
+      // the binding so lint doesn't flag an empty catch.
+      void resetError;
+    }
+    client.release();
+  }
+}