@checkstack/backend 0.13.0 → 0.15.0

package/CHANGELOG.md

@@ -1,5 +1,92 @@
 # @checkstack/backend
 
+## 0.15.0
+
+### Minor Changes
+
+- a57f7db: fix(backend): give advisory locks a dedicated connection pool to prevent pool-starvation deadlock
+
+  Both the session-lock service and `withXactLock` HOLD a Postgres connection for
+  the lock's whole lifetime while the gated work runs on a _different_ connection.
+  Both lock and work were drawing from the single shared `adminPool` (which, with
+  no explicit config, defaulted to `max: 10` and `connectionTimeoutMillis: 0` -
+  wait forever). Under concurrency >= pool size, every slot became a lock-holding
+  connection waiting for a work connection that could never free up: a permanent
+  deadlock. It surfaced as all connections stuck `idle in transaction` on
+  `pg_advisory_xact_lock` and every API request hanging into an upstream 502,
+  only after the server had been running long enough to hit that concurrency
+  (e.g. a burst of health-check evaluations or incident dedups).
+
+  Advisory locks now run on a dedicated `lockPool`, separate from `adminPool`, so
+  the acquire graph is acyclic (`lockPool -> adminPool`, never back) and the
+  deadlock class is impossible. `AdvisoryLockService` gains a pooled
+  `withXactLock({ key, fn })` method (lock on the lock pool, work on the admin
+  pool); healthcheck's per-system serializer, incident's dedup-create, and the
+  automation single-mode concurrency lock now use it. The deadlock-prone
+  standalone `withXactLock({ db, ... })` helper is REMOVED.
+
+  Both pools are explicitly configured with `connectionTimeoutMillis` so any
+  future exhaustion fails fast and self-heals instead of hanging, and both get a
+  pool-level `error` handler (an idle pooled client whose backend dies otherwise
+  crashes the pod). The lock pool additionally sets
+  `idle_in_transaction_session_timeout` and `lock_timeout` so a stalled critical
+  section is reaped server-side (auto-releasing the lock) rather than stranding a
+  key forever. The advisory-lock service also now removes its per-client error
+  listener on release (it previously leaked one listener per acquisition on each
+  reused pooled connection - an unbounded `MaxListenersExceeded` leak).
+
+  New env vars (all optional): `DATABASE_POOL_MAX` (default 20),
+  `DATABASE_LOCK_POOL_MAX` (default 10), `DATABASE_POOL_CONNECTION_TIMEOUT_MS`
+  (default 10000), `DATABASE_POOL_IDLE_TIMEOUT_MS` (default 30000),
+  `DATABASE_LOCK_IDLE_TX_TIMEOUT_MS` (default 30000), `DATABASE_LOCK_TIMEOUT_MS`
+  (default 30000). Size pools off
+  `N_pods * (DATABASE_POOL_MAX + DATABASE_LOCK_POOL_MAX) <= max_connections`.
+
+  BREAKING CHANGE: the standalone `withXactLock({ db, key, fn })` export is
+  removed - use `coreServices.advisoryLock.withXactLock({ key, fn })` instead.
+  `IncidentService`'s constructor now requires an `AdvisoryLockService` as its
+  second argument, and the healthcheck `createHealthEntitySerializer` /
+  `executeHealthCheckJob` / `setupHealthCheckWorker` helpers take `advisoryLock`
+  instead of `db` for the serializer.
+
+### Patch Changes
+
+- Updated dependencies [a57f7db]
+  - @checkstack/backend-api@0.20.0
+  - @checkstack/cache-api@0.3.8
+  - @checkstack/queue-api@0.3.8
+  - @checkstack/signal-backend@0.2.12
+
+## 0.14.0
+
+### Minor Changes
+
+- 79b3487: Relocate plugin objects stranded in `public` into their plugin schema, and run
+  migrations under a strict plugin-only `search_path`.
+
+  Some databases predate per-plugin schema isolation and have a plugin's tables
+  and enums sitting in `public` while the `__drizzle_migrations` ledger lives in
+  the plugin schema. Runtime kept working because the scoped-db `search_path`
+  falls back to `public`, but migrations did not: a new migration referencing a
+  pre-existing object (e.g. the `health_check_status` enum) failed at startup with
+  `type "health_check_status" does not exist`, crash-looping the pod. The previous
+  pinned-connection fix made this deterministic by reliably targeting the
+  (empty-of-that-object) plugin schema.
+
+  The loader now, before running a plugin's migrations, MOVES any of that plugin's
+  objects still in `public` into `plugin_<id>` with fully-qualified
+  `ALTER ... SET SCHEMA` statements (by-OID, so columns, foreign keys, enum
+  references, and owned sequences keep working). The relocation is idempotent
+  (only moves objects that are in `public` and not already in the plugin schema)
+  and is driven by the union of every Drizzle snapshot the plugin ships, so a
+  table an early migration created and a later one drops is moved first and its
+  unqualified `DROP TABLE` still resolves.
+
+  With the stragglers relocated, migrations run under a strict
+  `search_path = "plugin_<id>"` (no `public` fallback). Combined with creating the
+  schema before the `SET`, unqualified `CREATE TABLE` / `CREATE TYPE` can only ever
+  land in the plugin schema, never silently in `public`.
+
 ## 0.13.0
 
 ### Minor Changes

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@checkstack/backend",
-  "version": "0.13.0",
+  "version": "0.15.0",
   "license": "Elastic-2.0",
   "checkstack": {
     "type": "backend"

package/src/db.ts

@@ -1,6 +1,7 @@
 import { drizzle } from "drizzle-orm/node-postgres";
-import { Pool } from "pg";
+import { Pool, type PoolConfig } from "pg";
 import * as schema from "./schema";
+import { rootLogger } from "./logger";
 
 // Basic connection string sometimes fails with Bun + pg + docker SASL
 // parsing manually or relying on pg to pick up ENV variables if we don't pass anything
@@ -13,8 +14,114 @@ if (!connectionString) {
   throw new Error("DATABASE_URL is not defined");
 }
 
-export const adminPool = new Pool({
+/** Parse a positive-integer env var, falling back to `fallback` when unset/invalid. */
+function intFromEnv(name: string, fallback: number): number {
+  const raw = process.env[name];
+  if (raw === undefined || raw === "") return fallback;
+  const parsed = Number.parseInt(raw, 10);
+  return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
+}
+
+/**
+ * ## Connection budget (read this before bumping the defaults)
+ *
+ * The platform runs as N horizontally-scaled pods sharing ONE Postgres. The
+ * server-wide ceiling is `max_connections`, so the real budget is:
+ *
+ *   N_pods * (adminPool.max + lockPool.max) <= max_connections - headroom
+ *
+ * Size the pools off that budget (via the env vars below), NOT off the number
+ * of plugins: connections are never pinned per-plugin. The scoped-db proxy sets
+ * `SET LOCAL search_path` per transaction on a borrowed-then-returned
+ * connection, so a connection is occupied only for one transaction (or one
+ * held advisory lock), never for a plugin's lifetime.
+ *
+ * ## Why two pools
+ *
+ * Session/transaction advisory locks (see `advisory-lock.ts`) HOLD a connection
+ * for the lock's whole lifetime while the locked work runs on a *different*
+ * connection. If both came from one pool, then under concurrency >= pool size
+ * every slot becomes a lock-holding connection waiting for a work connection
+ * that can never free up - a self-inflicted deadlock (observed: 10 connections
+ * all `idle in transaction` on `pg_advisory_xact_lock`). Giving advisory locks
+ * their own `lockPool` makes the acquire graph acyclic (lockPool -> adminPool,
+ * never back), so that deadlock class is impossible.
+ *
+ * `connectionTimeoutMillis` is the universal safety net: a pg Pool defaults to
+ * waiting FOREVER for a free connection, which turns any future exhaustion into
+ * a permanent hang (and an upstream 502). With a finite timeout, an exhausted
+ * acquire throws, its holder unwinds and releases, and the pool self-heals.
+ */
+const COMMON_POOL_CONFIG = {
   connectionString,
+  connectionTimeoutMillis: intFromEnv(
+    "DATABASE_POOL_CONNECTION_TIMEOUT_MS",
+    10_000,
+  ),
+  idleTimeoutMillis: intFromEnv("DATABASE_POOL_IDLE_TIMEOUT_MS", 30_000),
+} satisfies PoolConfig;
+
+/**
+ * The main pool: serves every plugin query and all API request work. Never used
+ * to hold an advisory lock open (that is `lockPool`'s job).
+ */
+export const adminPool = new Pool({
+  ...COMMON_POOL_CONFIG,
+  max: intFromEnv("DATABASE_POOL_MAX", 20),
+});
+
+/**
+ * Dedicated pool for advisory locks ONLY (the session-lock service and
+ * `withXactLock`). Kept separate from `adminPool` so a held lock connection
+ * and the work it gates draw from different pools - see the deadlock note
+ * above. Sized for peak concurrent held locks INCLUDING nesting: one logical
+ * operation can hold up to two locks at once (an automation run lock wrapping
+ * an `incident.dedupe-open-for-system` lock), so this needs headroom above the
+ * raw concurrent-operation count.
+ *
+ * ## Stall backstops (server-enforced, can't be skipped by a hung process)
+ *
+ * Advisory locks lock no rows or tables - only other callers of the SAME key
+ * block. But a critical section whose `fn` hangs (e.g. an unbounded await)
+ * would hold its key + this connection open indefinitely, blocking same-key
+ * callers. Two connection-level timeouts bound that, set ONLY on this pool
+ * (where every transaction is a short lock critical section, so the limits are
+ * safe; the admin pool runs arbitrary plugin work and must NOT inherit them):
+ *
+ *   - `idle_in_transaction_session_timeout`: the lock transaction sits "idle in
+ *     transaction" for the whole time `fn` runs (it only issued BEGIN + the
+ *     lock). If `fn` hangs past this, Postgres ABORTS the session, which
+ *     auto-releases the advisory lock and frees the connection - so a stall
+ *     self-heals instead of stranding the lock forever.
+ *   - `lock_timeout`: a caller BLOCKED waiting to acquire a contended/stalled
+ *     key aborts after this (verified to apply to `pg_advisory_xact_lock`),
+ *     surfacing as a retryable error rather than an indefinite block that also
+ *     ties up a lock-pool connection.
+ *
+ * Both default high enough that a healthy short critical section never trips
+ * them; tune via env if your critical sections are unusually long.
+ */
+export const lockPool = new Pool({
+  ...COMMON_POOL_CONFIG,
+  max: intFromEnv("DATABASE_LOCK_POOL_MAX", 10),
+  idle_in_transaction_session_timeout: intFromEnv(
+    "DATABASE_LOCK_IDLE_TX_TIMEOUT_MS",
+    30_000,
+  ),
+  lock_timeout: intFromEnv("DATABASE_LOCK_TIMEOUT_MS", 30_000),
+});
+
+// A pg Pool emits 'error' on behalf of IDLE clients whose backend dies (admin
+// termination, failover, network drop). With no listener, that 'error' is
+// unhandled and CRASHES the process. Log and swallow: the pool discards the
+// dead client and hands out a fresh one on the next checkout. (Checked-out
+// clients are covered separately - the scoped-db transaction wrapper and the
+// advisory-lock service attach their own per-client handlers while held.)
+adminPool.on("error", (error) => {
+  rootLogger.warn("adminPool idle client error (recovered)", error);
+});
+lockPool.on("error", (error) => {
+  rootLogger.warn("lockPool idle client error (recovered)", error);
 });
 
 export const db = drizzle({ client: adminPool, schema });

package/src/plugin-manager/core-services.ts

@@ -14,7 +14,7 @@ import {
 import { AuthApi } from "@checkstack/auth-common";
 import type { ServiceRegistry } from "../services/service-registry";
 import { rootLogger } from "../logger";
-import { db } from "../db";
+import { db, lockPool } from "../db";
 import { jwtService } from "../services/jwt";
 import {
   CoreHealthCheckRegistry,
@@ -98,11 +98,14 @@ export function registerCoreServices({
     return createScopedDb(db, assignedSchema);
   });
 
-  // 1b. Advisory Lock Factory (server-global, backed by the shared admin
-  // pool). Session locks need connection affinity, so the service checks
-  // out a dedicated client per acquired lock and releases on the SAME
-  // client — the scoped per-query DB proxy can't provide that.
-  const advisoryLockService = createAdvisoryLockService(adminPool);
+  // 1b. Advisory Lock Factory (server-global, backed by the DEDICATED
+  // `lockPool`, NOT `adminPool`). Both session locks (`tryAcquire`) and the
+  // transaction-scoped `withXactLock` HOLD a connection for the lock's whole
+  // lifetime while the locked work runs on `adminPool`. Drawing the lock
+  // connection from a separate pool keeps the acquire graph acyclic
+  // (lockPool -> adminPool, never back), so a held lock can never starve the
+  // work pool into the `idle in transaction` deadlock. See `db.ts`.
+  const advisoryLockService = createAdvisoryLockService(lockPool);
   registry.registerFactory(
     coreServices.advisoryLock,
     () => advisoryLockService,

package/src/utils/relocate-legacy-public-objects.test.ts

@@ -0,0 +1,261 @@
+import { describe, it, expect, beforeEach, afterEach } from "bun:test";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import {
+  readPluginOwnedObjects,
+  relocateLegacyPublicObjects,
+  type RelocationClient,
+} from "./relocate-legacy-public-objects";
+
+/**
+ * A fake client that answers catalog membership queries from in-memory schema
+ * maps and records the `ALTER ... SET SCHEMA` statements it is asked to run.
+ */
+function makeFakeClient(state: {
+  // schema -> relations present, with relkind
+  relations: Record<string, Record<string, string>>;
+  // schema -> type names present
+  types: Record<string, Set<string>>;
+}) {
+  const altered: string[] = [];
+  const client: RelocationClient = {
+    async query<T = Record<string, unknown>>(
+      text: string,
+      values?: unknown[],
+    ): Promise<{ rows: T[] }> {
+      if (text.startsWith("ALTER ")) {
+        altered.push(text);
+        return { rows: [] };
+      }
+      // Catalog lookups are parameterized: relations use pg_class, types pg_type.
+      if (text.includes("pg_class")) {
+        const wantsTarget = text.includes("n.nspname = $1");
+        const schema = wantsTarget ? (values?.[0] as string) : "public";
+        const names = (wantsTarget ? values?.[1] : values?.[0]) as string[];
+        const present = state.relations[schema] ?? {};
+        const rows = names
+          .filter((n) => n in present)
+          .map((n) => ({ relname: n, relkind: present[n] }));
+        return { rows: rows as T[] };
+      }
+      if (text.includes("pg_type")) {
+        const wantsTarget = text.includes("n.nspname = $1");
+        const schema = wantsTarget ? (values?.[0] as string) : "public";
+        const names = (wantsTarget ? values?.[1] : values?.[0]) as string[];
+        const present = state.types[schema] ?? new Set<string>();
+        const rows = names
+          .filter((n) => present.has(n))
+          .map((n) => ({ typname: n }));
+        return { rows: rows as T[] };
+      }
+      return { rows: [] };
+    },
+  };
+  return { client, altered };
+}
+
+describe("relocateLegacyPublicObjects", () => {
+  it("moves tables and enums that live in public into the plugin schema", async () => {
+    const { client, altered } = makeFakeClient({
+      relations: {
+        public: {
+          health_check_configurations: "r",
+          health_check_auto_incidents: "r",
+        },
+        plugin_healthcheck: {},
+      },
+      types: {
+        public: new Set(["health_check_status", "bucket_size"]),
+        plugin_healthcheck: new Set(),
+      },
+    });
+
+    const { moved } = await relocateLegacyPublicObjects({
+      client,
+      schema: "plugin_healthcheck",
+      owned: {
+        relations: [
+          "health_check_configurations",
+          "health_check_auto_incidents",
+          "health_check_state_transitions", // not in public yet -> skipped
+        ],
+        types: ["health_check_status", "bucket_size"],
+      },
+    });
+
+    expect(altered).toEqual([
+      'ALTER TABLE public."health_check_configurations" SET SCHEMA "plugin_healthcheck"',
+      'ALTER TABLE public."health_check_auto_incidents" SET SCHEMA "plugin_healthcheck"',
+      'ALTER TYPE public."health_check_status" SET SCHEMA "plugin_healthcheck"',
+      'ALTER TYPE public."bucket_size" SET SCHEMA "plugin_healthcheck"',
+    ]);
+    expect(moved).toContain("table health_check_configurations");
+    expect(moved).toContain("type health_check_status");
+  });
+
+  it("is idempotent: skips objects already in the plugin schema or absent from public", async () => {
+    const { client, altered } = makeFakeClient({
+      relations: {
+        public: {},
+        plugin_healthcheck: { health_check_configurations: "r" },
+      },
+      types: {
+        public: new Set(),
+        plugin_healthcheck: new Set(["health_check_status"]),
+      },
+    });
+
+    const { moved } = await relocateLegacyPublicObjects({
+      client,
+      schema: "plugin_healthcheck",
+      owned: {
+        relations: ["health_check_configurations"],
+        types: ["health_check_status"],
+      },
+    });
+
+    expect(altered).toEqual([]);
+    expect(moved).toEqual([]);
+  });
+
+  it("skips objects present in BOTH public and the plugin schema (no collision ALTER)", async () => {
+    // A half-finished prior run (or a manual move) can leave an object in both
+    // schemas. Moving it again would error with "relation already exists in
+    // schema", so it must be skipped.
+    const { client, altered } = makeFakeClient({
+      relations: {
+        public: { health_check_runs: "r" },
+        plugin_healthcheck: { health_check_runs: "r" },
+      },
+      types: {
+        public: new Set(["health_check_status"]),
+        plugin_healthcheck: new Set(["health_check_status"]),
+      },
+    });
+
+    const { moved } = await relocateLegacyPublicObjects({
+      client,
+      schema: "plugin_healthcheck",
+      owned: {
+        relations: ["health_check_runs"],
+        types: ["health_check_status"],
+      },
+    });
+
+    expect(altered).toEqual([]);
+    expect(moved).toEqual([]);
+  });
+
+  it("uses the right ALTER keyword per object kind (view, matview, sequence)", async () => {
+    const { client, altered } = makeFakeClient({
+      relations: {
+        public: { my_view: "v", my_matview: "m", my_seq: "S", my_table: "r" },
+        plugin_x: {},
+      },
+      types: { public: new Set(), plugin_x: new Set() },
+    });
+
+    await relocateLegacyPublicObjects({
+      client,
+      schema: "plugin_x",
+      owned: {
+        relations: ["my_view", "my_matview", "my_seq", "my_table"],
+        types: [],
+      },
+    });
+
+    expect(altered).toEqual([
+      'ALTER VIEW public."my_view" SET SCHEMA "plugin_x"',
+      'ALTER MATERIALIZED VIEW public."my_matview" SET SCHEMA "plugin_x"',
+      'ALTER SEQUENCE public."my_seq" SET SCHEMA "plugin_x"',
+      'ALTER TABLE public."my_table" SET SCHEMA "plugin_x"',
+    ]);
+  });
+
+  it("never relocates into the public schema", async () => {
+    const { client, altered } = makeFakeClient({
+      relations: { public: { foo: "r" } },
+      types: {},
+    });
+
+    const { moved } = await relocateLegacyPublicObjects({
+      client,
+      schema: "public",
+      owned: { relations: ["foo"], types: [] },
+    });
+
+    expect(altered).toEqual([]);
+    expect(moved).toEqual([]);
+  });
+});
+
+describe("readPluginOwnedObjects", () => {
+  let dir: string;
+
+  beforeEach(() => {
+    dir = fs.mkdtempSync(path.join(os.tmpdir(), "owned-objects-"));
+    fs.mkdirSync(path.join(dir, "meta"), { recursive: true });
+  });
+
+  afterEach(() => {
+    fs.rmSync(dir, { recursive: true, force: true });
+  });
+
+  const writeSnapshot = (name: string, body: unknown) =>
+    fs.writeFileSync(
+      path.join(dir, "meta", name),
+      JSON.stringify(body),
+      "utf8",
+    );
+
+  it("unions object names across ALL snapshots (incl. created-then-dropped)", () => {
+    // Early snapshot: a table that a later migration drops.
+    writeSnapshot("0000_snapshot.json", {
+      tables: {
+        "public.kept": { name: "kept", schema: "" },
+        "public.dropped_later": { name: "dropped_later", schema: "" },
+      },
+      enums: { "public.status": { name: "status", schema: "public" } },
+    });
+    // Latest snapshot: no longer lists the dropped table, adds a new one.
+    writeSnapshot("0001_snapshot.json", {
+      tables: {
+        "public.kept": { name: "kept", schema: "" },
+        "public.added": { name: "added", schema: "" },
+      },
+      enums: { "public.status": { name: "status", schema: "public" } },
+    });
+
+    const owned = readPluginOwnedObjects(dir);
+
+    expect(new Set(owned.relations)).toEqual(
+      new Set(["kept", "dropped_later", "added"]),
+    );
+    expect(owned.types).toEqual(["status"]);
+  });
+
+  it("ignores objects declared in an explicit non-default schema", () => {
+    writeSnapshot("0000_snapshot.json", {
+      tables: {
+        "public.in_public": { name: "in_public", schema: "" },
+        "other.elsewhere": { name: "elsewhere", schema: "other" },
+      },
+    });
+
+    const owned = readPluginOwnedObjects(dir);
+    expect(owned.relations).toEqual(["in_public"]);
+  });
+
+  it("returns empty when there is no meta folder", () => {
+    const empty = fs.mkdtempSync(path.join(os.tmpdir(), "no-meta-"));
+    try {
+      expect(readPluginOwnedObjects(empty)).toEqual({
+        relations: [],
+        types: [],
+      });
+    } finally {
+      fs.rmSync(empty, { recursive: true, force: true });
+    }
+  });
+});

package/src/utils/relocate-legacy-public-objects.ts

@@ -0,0 +1,206 @@
+import fs from "node:fs";
+import path from "node:path";
+import { z } from "zod";
+
+/**
+ * Relocate a plugin's objects that were left in `public` by pre-isolation
+ * deploys into the plugin's dedicated schema.
+ *
+ * ## Why this exists
+ *
+ * Each plugin's objects are supposed to live in a dedicated schema
+ * (`plugin_<id>`). Deploys that predate schema isolation - or that ran
+ * migrations before the connection/search_path handling was correct - created
+ * the plugin's tables and enums in `public` instead, while the
+ * `__drizzle_migrations` ledger lived in the plugin schema. Runtime kept
+ * working only because the scoped-db search_path fell back to `public`.
+ *
+ * Rather than carry a permanent `public` fallback in the migration search_path
+ * (which risks new objects silently landing in `public`), the loader moves the
+ * stragglers into the plugin schema once, up front, with fully-qualified
+ * `ALTER ... SET SCHEMA` statements. After this runs, every object the plugin
+ * owns lives in `plugin_<id>`, so migrations can use a strict
+ * `search_path = "plugin_<id>"` and new objects always land in the right place.
+ *
+ * The move is by-OID, so columns, foreign keys, enum references, indexes, and
+ * owned sequences all keep working. It is idempotent: an object is moved only
+ * if it currently exists in `public` and does not already exist in the plugin
+ * schema, so fresh installs and already-migrated installs are no-ops.
+ *
+ * ## Which objects
+ *
+ * The owned-object set is the UNION of every Drizzle snapshot under the
+ * plugin's `drizzle/meta/`, not just the latest. A table that an early
+ * migration created and a later one drops (e.g. a since-removed table) still
+ * exists in `public` on an upgrading database when its `DROP` migration is
+ * pending; including it here means it gets moved into the plugin schema first,
+ * so the unqualified `DROP TABLE` resolves under the strict search_path.
+ */
+
+/** Minimal pooled-client surface this helper needs (modelled on `pg.PoolClient`). */
+export interface RelocationClient {
+  query<T = Record<string, unknown>>(
+    queryText: string,
+    values?: unknown[],
+  ): Promise<{ rows: T[] }>;
+}
+
+export interface PluginOwnedObjects {
+  /** Tables, views, and materialized views (everything in `pg_class`-land). */
+  relations: string[];
+  /** Enum / composite type names. */
+  types: string[];
+}
+
+// Snapshots carry far more than we read; accept and ignore the rest.
+const objectEntrySchema = z
+  .object({ name: z.string(), schema: z.string().optional() })
+  .passthrough();
+const snapshotSchema = z
+  .object({
+    tables: z.record(z.string(), objectEntrySchema).optional(),
+    enums: z.record(z.string(), objectEntrySchema).optional(),
+    sequences: z.record(z.string(), objectEntrySchema).optional(),
+    views: z.record(z.string(), objectEntrySchema).optional(),
+  })
+  .passthrough();
+
+/** An object belongs in the default/`public` namespace (i.e. is relocatable). */
+function isDefaultSchema(schema: string | undefined): boolean {
+  return schema === undefined || schema === "" || schema === "public";
+}
+
+/**
+ * Read the union of all object names a plugin has ever declared, across every
+ * snapshot in its `drizzle/meta/` folder.
+ */
+export function readPluginOwnedObjects(
+  migrationsFolder: string,
+): PluginOwnedObjects {
+  const metaDir = path.join(migrationsFolder, "meta");
+  if (!fs.existsSync(metaDir)) {
+    return { relations: [], types: [] };
+  }
+
+  const relations = new Set<string>();
+  const types = new Set<string>();
+
+  const snapshotFiles = fs
+    .readdirSync(metaDir)
+    .filter((f) => f.endsWith("_snapshot.json"));
+
+  for (const file of snapshotFiles) {
+    const raw = JSON.parse(fs.readFileSync(path.join(metaDir, file), "utf8"));
+    const snapshot = snapshotSchema.parse(raw);
+
+    for (const entry of Object.values(snapshot.tables ?? {})) {
+      if (isDefaultSchema(entry.schema)) relations.add(entry.name);
+    }
+    for (const entry of Object.values(snapshot.views ?? {})) {
+      if (isDefaultSchema(entry.schema)) relations.add(entry.name);
+    }
+    for (const entry of Object.values(snapshot.sequences ?? {})) {
+      if (isDefaultSchema(entry.schema)) relations.add(entry.name);
+    }
+    for (const entry of Object.values(snapshot.enums ?? {})) {
+      if (isDefaultSchema(entry.schema)) types.add(entry.name);
+    }
+  }
+
+  return { relations: [...relations], types: [...types] };
+}
+
+/** Double-quote and escape a Postgres identifier for safe interpolation. */
+function quoteIdent(name: string): string {
+  return `"${name.replaceAll('"', '""')}"`;
+}
+
+/**
+ * Map a `pg_class.relkind` to the right `ALTER ... SET SCHEMA` keyword. Kinds
+ * not in this map (indexes, toast tables, composite-type rows, etc.) are never
+ * relocated here.
+ */
+const ALTER_KEYWORD_BY_RELKIND: Record<string, string> = {
+  r: "TABLE", // ordinary table
+  p: "TABLE", // partitioned table
+  S: "SEQUENCE", // sequence
+  v: "VIEW", // view
+  m: "MATERIALIZED VIEW", // materialized view
+};
+
+/**
+ * Move the plugin's `public`-resident objects into `schema`. Returns the list
+ * of moved objects (for logging). No-op when `schema` is `public`.
+ */
+export async function relocateLegacyPublicObjects({
+  client,
+  schema,
+  owned,
+}: {
+  client: RelocationClient;
+  schema: string;
+  owned: PluginOwnedObjects;
+}): Promise<{ moved: string[] }> {
+  const moved: string[] = [];
+  if (schema === "public") return { moved };
+
+  const target = quoteIdent(schema);
+
+  // --- Relations (tables / views / matviews / standalone sequences) ---
+  if (owned.relations.length > 0) {
+    const inPublic = await client.query<{ relname: string; relkind: string }>(
+      `SELECT c.relname, c.relkind
+         FROM pg_class c
+         JOIN pg_namespace n ON n.oid = c.relnamespace
+        WHERE n.nspname = 'public' AND c.relname = ANY($1::text[])`,
+      [owned.relations],
+    );
+    const inTarget = await client.query<{ relname: string }>(
+      `SELECT c.relname
+         FROM pg_class c
+         JOIN pg_namespace n ON n.oid = c.relnamespace
+        WHERE n.nspname = $1 AND c.relname = ANY($2::text[])`,
+      [schema, owned.relations],
+    );
+    const alreadyMoved = new Set(inTarget.rows.map((r) => r.relname));
+
+    for (const { relname, relkind } of inPublic.rows) {
+      if (alreadyMoved.has(relname)) continue;
+      const keyword = ALTER_KEYWORD_BY_RELKIND[relkind];
+      if (!keyword) continue;
+      await client.query(
+        `ALTER ${keyword} public.${quoteIdent(relname)} SET SCHEMA ${target}`,
+      );
+      moved.push(`${keyword.toLowerCase()} ${relname}`);
+    }
+  }
+
+  // --- Types (enums / composite types) ---
+  if (owned.types.length > 0) {
+    const inPublic = await client.query<{ typname: string }>(
+      `SELECT t.typname
+         FROM pg_type t
+         JOIN pg_namespace n ON n.oid = t.typnamespace
+        WHERE n.nspname = 'public' AND t.typname = ANY($1::text[])`,
+      [owned.types],
+    );
+    const inTarget = await client.query<{ typname: string }>(
+      `SELECT t.typname
+         FROM pg_type t
+         JOIN pg_namespace n ON n.oid = t.typnamespace
+        WHERE n.nspname = $1 AND t.typname = ANY($2::text[])`,
+      [schema, owned.types],
+    );
+    const alreadyMoved = new Set(inTarget.rows.map((r) => r.typname));
+
+    for (const { typname } of inPublic.rows) {
+      if (alreadyMoved.has(typname)) continue;
+      await client.query(
+        `ALTER TYPE public.${quoteIdent(typname)} SET SCHEMA ${target}`,
+      );
+      moved.push(`type ${typname}`);
+    }
+  }
+
+  return { moved };
+}

package/src/utils/run-plugin-migrations.test.ts

@@ -28,8 +28,11 @@ function makeFakeClient() {
   };
 }
 
+/** A relocation step that does nothing (keeps tests off the filesystem/db). */
+const noopRelocate = async () => {};
+
 describe("runPluginMigrations", () => {
-  it("runs the migrator on a single pinned connection with search_path set first", async () => {
+  it("runs the migrator on a single pinned connection with a strict search_path set first", async () => {
     const { client, queries, isReleased } = makeFakeClient();
 
     let connectCount = 0;
@@ -49,6 +52,7 @@ describe("runPluginMigrations", () => {
       pool,
       migrationsFolder: "/plugins/healthcheck/drizzle",
       migrationsSchema: "plugin_healthcheck",
+      relocateLegacyObjects: noopRelocate,
       createMigrationDb: (c) => {
         clientPassedToFactory = c;
         return fakeDb;
@@ -70,8 +74,9 @@ describe("runPluginMigrations", () => {
     expect(clientPassedToFactory).toBe(client);
     expect(dbPassedToMigrate).toBe(fakeDb);
 
-    // search_path is pointed at the plugin schema (after creating it) BEFORE
-    // the migrator runs.
+    // The plugin schema is created BEFORE search_path points at it (so new
+    // objects can never fall through to `public`), and the search_path is
+    // STRICT - plugin schema only, no `public` fallback.
     expect(queriesBeforeMigrate).toEqual([
       'CREATE SCHEMA IF NOT EXISTS "plugin_healthcheck"',
       'SET search_path = "plugin_healthcheck"',
@@ -82,6 +87,68 @@ describe("runPluginMigrations", () => {
     expect(isReleased()).toBe(true);
   });
 
+  it("relocates legacy public objects after creating the schema and before setting search_path / migrating", async () => {
+    const { client, queries } = makeFakeClient();
+    const pool = { connect: async () => client };
+
+    const events: string[] = [];
+    let schemaPassedToRelocate: string | undefined;
+    let folderPassedToRelocate: string | undefined;
+    let clientPassedToRelocate: PoolClient | undefined;
+
+    await runPluginMigrations({
+      pool,
+      migrationsFolder: "/plugins/healthcheck/drizzle",
+      migrationsSchema: "plugin_healthcheck",
+      relocateLegacyObjects: async ({ client: c, schema, migrationsFolder }) => {
+        clientPassedToRelocate = c;
+        schemaPassedToRelocate = schema;
+        folderPassedToRelocate = migrationsFolder;
+        events.push(`relocate@${queries.length}`);
+      },
+      createMigrationDb: () => ({}) as unknown as MigrationDb,
+      migrate: async () => {
+        events.push(`migrate@${queries.length}`);
+      },
+    });
+
+    // Relocation runs on the same pinned client, with the plugin schema and
+    // the plugin's own migrations folder.
+    expect(clientPassedToRelocate).toBe(client);
+    expect(schemaPassedToRelocate).toBe("plugin_healthcheck");
+    expect(folderPassedToRelocate).toBe("/plugins/healthcheck/drizzle");
+
+    // Ordering: CREATE SCHEMA, then relocate, then SET search_path, then migrate.
+    // After CREATE SCHEMA only (1 query) relocate runs; after the SET (2
+    // queries) migrate runs.
+    expect(events).toEqual(["relocate@1", "migrate@2"]);
+    expect(queries).toEqual([
+      'CREATE SCHEMA IF NOT EXISTS "plugin_healthcheck"',
+      'SET search_path = "plugin_healthcheck"',
+      "SET search_path = public",
+    ]);
+  });
+
+  it("uses a strict plugin-only search_path with no `public` fallback", async () => {
+    const { client, queries } = makeFakeClient();
+    const pool = { connect: async () => client };
+
+    await runPluginMigrations({
+      pool,
+      migrationsFolder: "/x",
+      migrationsSchema: "plugin_healthcheck",
+      relocateLegacyObjects: noopRelocate,
+      createMigrationDb: () => ({}) as unknown as MigrationDb,
+      migrate: async () => {},
+    });
+
+    const setStatement = queries.find(
+      (q) => q.startsWith("SET search_path =") && q.includes("plugin_healthcheck"),
+    );
+    expect(setStatement).toBe('SET search_path = "plugin_healthcheck"');
+    expect(setStatement).not.toContain("public");
+  });
+
   it("resets search_path and releases the connection even when the migrator throws", async () => {
     const { client, queries, isReleased } = makeFakeClient();
     const pool = { connect: async () => client };
@@ -92,6 +159,7 @@ describe("runPluginMigrations", () => {
         pool,
         migrationsFolder: "/x",
         migrationsSchema: "plugin_x",
+        relocateLegacyObjects: noopRelocate,
         createMigrationDb: () => ({}) as unknown as MigrationDb,
         migrate: async () => {
           throw boom;
@@ -111,14 +179,15 @@ describe("runPluginMigrations", () => {
       pool,
       migrationsFolder: "/x",
       migrationsSchema: "plugin_x",
+      relocateLegacyObjects: noopRelocate,
       createMigrationDb: () => ({}) as unknown as MigrationDb,
       migrate: async () => {},
     });
 
     // The pool surface the helper depends on is exactly `connect`; everything
-    // else (CREATE SCHEMA, SET search_path, the migration itself) happens on
-    // the checked-out client. If this contract ever widens, the regression
-    // that motivated the pinned connection could creep back in.
+    // else (CREATE SCHEMA, relocation, SET search_path, the migration itself)
+    // happens on the checked-out client. If this contract ever widens, the
+    // regression that motivated the pinned connection could creep back in.
     expect(Object.keys(pool)).toEqual(["connect"]);
   });
 });

package/src/utils/run-plugin-migrations.ts

@@ -1,6 +1,10 @@
 import { drizzle, type NodePgDatabase } from "drizzle-orm/node-postgres";
 import { migrate as defaultMigrate } from "drizzle-orm/node-postgres/migrator";
 import type { Pool, PoolClient } from "pg";
+import {
+  readPluginOwnedObjects,
+  relocateLegacyPublicObjects,
+} from "./relocate-legacy-public-objects";
 
 type MigrationDb = NodePgDatabase<Record<string, unknown>>;
 
@@ -25,34 +29,68 @@ export interface RunPluginMigrationsArgs {
     db: MigrationDb,
     config: { migrationsFolder: string; migrationsSchema: string },
   ) => Promise<void>;
+  /**
+   * Moves the plugin's objects that earlier deploys left in `public` into the
+   * plugin schema, before migrations run. Injectable for tests.
+   */
+  relocateLegacyObjects?: (args: {
+    client: PoolClient;
+    schema: string;
+    migrationsFolder: string;
+  }) => Promise<void>;
 }
 
+const defaultRelocateLegacyObjects = async ({
+  client,
+  schema,
+  migrationsFolder,
+}: {
+  client: PoolClient;
+  schema: string;
+  migrationsFolder: string;
+}): Promise<void> => {
+  const owned = readPluginOwnedObjects(migrationsFolder);
+  await relocateLegacyPublicObjects({ client, schema, owned });
+};
+
 /**
  * Run a plugin's Drizzle migrations on a SINGLE pinned pool connection.
  *
- * ## Why a pinned connection is required
+ * ## Strict, isolated search_path
  *
  * Plugin migrations are schema-agnostic: they reference the plugin's tables,
- * types, and enums *unqualified* and rely on `search_path` to resolve them
- * into the plugin's schema (e.g. `plugin_healthcheck`). So `search_path` must
- * be set before the migration SQL runs.
+ * types, and enums *unqualified* and rely on `search_path` to resolve them. We
+ * run them with a STRICT `search_path = "<plugin_schema>"` (no `public`
+ * fallback) so new objects can only ever land in the plugin schema, and a
+ * migration that references something not in that schema fails loudly instead
+ * of silently reading or writing `public`.
+ *
+ * The schema is created BEFORE the `SET`. That ordering matters: if the schema
+ * did not exist, an unqualified `CREATE TABLE` would have nowhere to go and
+ * Postgres would error - which is the safe outcome we want, not a silent
+ * fallthrough.
  *
- * Setting it at the *session* level on the shared pool does NOT work, for the
- * same reason session-level advisory locks don't (see `advisory-lock.ts`):
- * Drizzle's `migrate()` wraps all pending migrations in one transaction, and
- * with a `pg.Pool` that transaction checks out a *different* physical
- * connection than the one the `SET` ran on. The migration statements then
- * execute with the default `public` search_path.
+ * ## Relocating legacy `public` objects first
  *
- * This stays invisible on a fresh database - every object (including each
- * enum) is created within that one transaction, so unqualified references
- * still resolve against whatever schema that connection happens to use. But on
- * an UPGRADE, where earlier migrations already created an enum in the plugin
- * schema and only newer migrations run, a new migration that references the
- * pre-existing enum fails with `type "..." does not exist`.
+ * Some installs created the plugin's objects in `public` (pre-isolation
+ * deploys), which is exactly what a strict search_path would choke on. Before
+ * migrating, {@link relocateLegacyPublicObjects} moves any of the plugin's
+ * objects that are still in `public` into the plugin schema using
+ * fully-qualified `ALTER ... SET SCHEMA` (so it needs no search_path of its
+ * own). After it runs, everything the plugin owns lives in the plugin schema
+ * and the strict search_path resolves cleanly. It is idempotent, so fresh and
+ * already-migrated installs are no-ops.
  *
- * Binding the migrator to ONE pinned client, on which we set `search_path`
- * first, guarantees every migration statement runs under the intended schema.
+ * ## Why a pinned connection
+ *
+ * The `search_path` must be set on the exact connection the migration
+ * statements run on. Setting it at the *session* level on the shared pool does
+ * NOT guarantee that, for the same reason session-level advisory locks don't
+ * (see `advisory-lock.ts`): Drizzle's `migrate()` wraps all pending migrations
+ * in one transaction, and with a `pg.Pool` that transaction can check out a
+ * *different* physical connection than the one the `SET` ran on. Binding the
+ * migrator (and the relocation) to ONE pinned client guarantees every statement
+ * runs on the connection we prepared.
  */
 export async function runPluginMigrations({
   pool,
@@ -60,13 +98,23 @@ export async function runPluginMigrations({
   migrationsSchema,
   createMigrationDb = (client) => drizzle(client),
   migrate = defaultMigrate,
+  relocateLegacyObjects = defaultRelocateLegacyObjects,
 }: RunPluginMigrationsArgs): Promise<void> {
   const client = await pool.connect();
   try {
-    // Ensure the schema exists before pointing search_path at it. SET to a
-    // missing schema silently falls back to `public` at resolution time, which
-    // would recreate the very bug this helper exists to prevent.
+    // Create the schema BEFORE anything points at it, so relocated and newly
+    // created objects have a home and never fall through to `public`.
     await client.query(`CREATE SCHEMA IF NOT EXISTS "${migrationsSchema}"`);
+
+    // Move any of this plugin's objects that earlier deploys left in `public`
+    // into the plugin schema, so the strict search_path below resolves them.
+    await relocateLegacyObjects({
+      client,
+      schema: migrationsSchema,
+      migrationsFolder,
+    });
+
+    // Strict search_path: plugin schema only, no `public` fallback.
     await client.query(`SET search_path = "${migrationsSchema}"`);
 
     await migrate(createMigrationDb(client), {