@effectstream/db 0.104.1 → 0.200.2

package/README.md

@@ -55,6 +55,53 @@ small executable subpaths - used directly by the orchestrator.
 > `acquireDBMutex(name)` / `releaseDBMutex(name)` so concurrent generators
 > don't trample each other.
 
+### PgLite gateway lifecycle
+
+`startPglite(0)` binds only to `127.0.0.1` and reports the actual assigned port.
+Its handle has two shutdown modes:
+
+```typescript
+const database = await startPglite(0);
+
+// Compatibility default: stop accepting, preserve accepted client sockets.
+await database.close();
+
+// Explicit owner shutdown: destroy any stragglers and wait for teardown.
+await database.close({ force: true });
+```
+
+The no-argument close is bounded even if a consumer deliberately retains a raw
+`pg.Client`: it stops new accepts, unrefs and preserves accepted sockets, and
+settles without waiting for those clients. Existing clients may finish work
+while they remain connected. PGlite database cleanup is deferred until the last
+accepted socket drains, so a client retained forever can defer that cleanup
+forever. A resolved default-close promise therefore does not mean database
+cleanup has completed. Consumers remain responsible for their client lifecycle.
+
+Use `{ force: true }` only when your framework owns every connected client.
+Forced close destroys tracked sockets and waits for listener/socket teardown,
+so raw clients can emit their normal `Connection terminated unexpectedly`
+error; owners must end clients first or handle that event.
+
+Every call returns the same stored cleanup promise. The first call selects the
+mode, including when default and force calls race; a later force call cannot
+escalate cleanup already started in default mode. Startup listen failure still
+attempts database cleanup. Forced shutdown and default shutdown with no retained
+socket attempt database cleanup even if the listener fails to close. When both
+operations fail, the promise rejects with an `AggregateError` in
+listener-then-database order. If default shutdown deferred database cleanup, a
+later cleanup failure is observed and logged, but cannot change the already
+settled close promise.
+
+Migration note: forced socket destruction became the implicit default in
+`0.104.0` and was observed downstream during an upgrade to `0.200.1`. The
+corrected no-argument close restores the non-destructive compatibility default;
+PGlite cleanup waits for the last accepted socket, and forced teardown is
+explicit. The useful `0.104.0` changes remain: IPv4 loopback binding, startup
+rejection, address validation, actual-port reporting, and listener-close error
+propagation. Cleanup idempotency is strengthened from a boolean early return to
+a shared promise.
+
 ## Inside EffectStream
 
 `@effectstream/db` is the canonical access path. The runtime gets a
@@ -91,7 +138,8 @@ Dynamic table / event helpers:
 
 Subpath entry points (executable scripts):
 
-- `@effectstream/db/start-pglite`: boot a PgLite instance.
+- `@effectstream/db/start-pglite`: boot a PgLite instance and choose bounded
+  default or explicit forced teardown as described above.
 - `@effectstream/db/apply-migrations` - apply SQL migrations against the active DB.
 - `@effectstream/db/db-wait`: block until the DB accepts a connection.
 - `@effectstream/db/pgtyped-update`: regenerate pgtyped types.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@effectstream/db",
-  "version": "0.104.1",
+  "version": "0.200.2",
   "license": "(MIT OR Apache-2.0)",
   "homepage": "https://effectstream.github.io/docs/",
   "repository": {
@@ -21,11 +21,11 @@
     "./version": "./migrations/system-version.ts"
   },
   "dependencies": {
-    "@effectstream/coroutine": "0.104.1",
-    "@effectstream/log": "0.104.1",
-    "@effectstream/runtime": "0.104.1",
-    "@effectstream/sync": "0.104.1",
-    "@effectstream/utils": "0.104.1",
+    "@effectstream/coroutine": "0.200.2",
+    "@effectstream/log": "0.200.2",
+    "@effectstream/runtime": "0.200.2",
+    "@effectstream/sync": "0.200.2",
+    "@effectstream/utils": "0.200.2",
     "@electric-sql/pglite": "^0.3.14",
     "@pgtyped/runtime": "2.4.2",
     "@pgtyped/parser": "2.4.2",

package/scripts/pgtyped-update.ts

@@ -4,7 +4,9 @@
  */
 
 import { spawn } from "node:child_process";
-import { resolve } from "node:path";
+import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join, resolve } from "node:path";
 import { existsSync } from "node:fs";
 
 import pg from "pg";
@@ -34,6 +36,47 @@ export interface PgtypedUpdateOptions {
   userMigrations?: UserMigration[];
   pgtypedConfigPath?: string;
   pgtypedBin?: string;
+  pgtypedTimeoutMs?: number;
+}
+
+async function preparePgtypedConfig(
+  configPath: string,
+  port: number,
+): Promise<{ cleanup: () => Promise<void>; path: string }> {
+  const parsed = JSON.parse(await readFile(configPath, "utf8")) as unknown;
+  if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
+    throw new Error("Pgtyped config must be a JSON object.");
+  }
+
+  const config = parsed as Record<string, unknown>;
+  if (config.db !== undefined && (
+    !config.db || typeof config.db !== "object" || Array.isArray(config.db)
+  )) {
+    throw new Error("Pgtyped config db field must be an object.");
+  }
+  const { dbUrl: _ignoredDbUrl, ...configWithoutUri } = config;
+  const directory = await mkdtemp(join(tmpdir(), "effectstream-pgtyped-"));
+  const derivedPath = join(directory, "pgtypedconfig.json");
+  try {
+    await writeFile(derivedPath, JSON.stringify({
+      ...configWithoutUri,
+      db: {
+        ...(config.db as Record<string, unknown> | undefined),
+        dbName: "postgres",
+        host: "127.0.0.1",
+        port,
+        user: "postgres",
+      },
+    }));
+  } catch (error) {
+    await rm(directory, { force: true, recursive: true });
+    throw error;
+  }
+
+  return {
+    path: derivedPath,
+    cleanup: () => rm(directory, { force: true, recursive: true }),
+  };
 }
 
 async function resolveUserMigrations(): Promise<UserMigration[]> {
@@ -49,7 +92,7 @@ async function resolveUserMigrations(): Promise<UserMigration[]> {
   return mod.migrationTable ?? [];
 }
 
-async function applyUserMigrations(migrations: UserMigration[]): Promise<void> {
+async function applyUserMigrations(migrations: UserMigration[], port: number): Promise<void> {
   const prefix = createPrefix("user-migrations");
 
   if (migrations.length === 0) {
@@ -58,14 +101,14 @@ async function applyUserMigrations(migrations: UserMigration[]): Promise<void> {
   }
 
   const client = new pg.Client({
-    host: "localhost",
-    port: 5432,
+    host: "127.0.0.1",
+    port,
     user: "postgres",
     database: "postgres",
   });
-  await client.connect();
 
   try {
+    await client.connect();
     for (const migration of migrations) {
       console.log(`${prefix}Applying ${migration.name}`);
       await client.query(migration.sql);
@@ -76,67 +119,121 @@ async function applyUserMigrations(migrations: UserMigration[]): Promise<void> {
   }
 }
 
-async function applySystemMigrations(): Promise<void> {
+async function applySystemMigrations(port: number): Promise<void> {
   const prefix = createPrefix("apply-migrations");
-  const db = getConnection();
-  const migrations = await getMigrations();
-  for (const migration of migrations) {
-    await applyMigrations(db as any, 0, migration.version, migration.sql, true);
+  const db = getConnection({
+    host: "127.0.0.1",
+    port,
+    user: "postgres",
+    database: "postgres",
+    max: 1,
+  });
+  try {
+    const migrations = await getMigrations();
+    for (const migration of migrations) {
+      await applyMigrations(db as any, 0, migration.version, migration.sql, true);
+    }
+    console.log(`${prefix}✅ System migrations applied`);
+  } finally {
+    await db.end();
   }
-  console.log(`${prefix}✅ System migrations applied`);
-  await db.end();
 }
 
-async function runPgtyped(pgtypedBin: string, pgtypedConfigPath: string): Promise<void> {
+async function runPgtyped(
+  pgtypedBin: string,
+  pgtypedConfigPath: string,
+  port: number,
+  timeoutMs: number,
+): Promise<void> {
   const prefix = createPrefix("pgtyped");
   console.log(`${prefix}Starting...`);
 
-  return new Promise((resolve, reject) => {
-    const child = spawn("node", [pgtypedBin, "-c", pgtypedConfigPath], {
-      stdio: ["ignore", "pipe", "pipe"],
-      shell: false,
-    });
+  const preparedConfig = await preparePgtypedConfig(pgtypedConfigPath, port);
+  try {
+    await new Promise<void>((resolve, reject) => {
+      const childEnv = {
+        ...process.env,
+        PGDATABASE: "postgres",
+        PGHOST: "127.0.0.1",
+        PGPORT: String(port),
+        PGUSER: "postgres",
+      };
+      delete childEnv.DATABASE_URL;
+      delete childEnv.PGURI;
+
+      const child = spawn("node", [
+        pgtypedBin,
+        "-c",
+        preparedConfig.path,
+      ], {
+        env: childEnv,
+        stdio: ["ignore", "pipe", "pipe"],
+        shell: false,
+      });
 
-    child.stdout?.on("data", (chunk: Buffer | string) => {
-      chunk.toString().split("\n").forEach((line) => {
-        if (line.trim()) console.log(`${prefix}${line}`);
+      child.stdout?.on("data", (chunk: Buffer | string) => {
+        chunk.toString().split("\n").forEach((line) => {
+          if (line.trim()) console.log(`${prefix}${line}`);
+        });
       });
-    });
 
-    child.stderr?.on("data", (chunk: Buffer | string) => {
-      chunk.toString().split("\n").forEach((line) => {
-        if (line.trim()) console.error(`${prefix}${line}`);
+      child.stderr?.on("data", (chunk: Buffer | string) => {
+        chunk.toString().split("\n").forEach((line) => {
+          if (line.trim()) console.error(`${prefix}${line}`);
+        });
       });
-    });
 
-    child.on("close", (code) => {
-      if (code === 0) {
-        console.log(`${prefix}✅ Completed successfully`);
-        resolve();
-      } else {
-        reject(new Error(`pgtyped failed with exit code ${code}`));
-      }
+      let forceKillTimer: ReturnType<typeof setTimeout> | undefined;
+      let settled = false;
+      let timedOut = false;
+      const timeout = setTimeout(() => {
+        timedOut = true;
+        child.kill("SIGTERM");
+        forceKillTimer = setTimeout(() => child.kill("SIGKILL"), 5_000);
+      }, timeoutMs);
+      const settle = (operation: () => void) => {
+        if (settled) return;
+        settled = true;
+        clearTimeout(timeout);
+        if (forceKillTimer) clearTimeout(forceKillTimer);
+        operation();
+      };
+
+      child.once("error", (error) => settle(() => reject(error)));
+      child.once("close", (code) => settle(() => {
+        if (timedOut) {
+          reject(new Error(`pgtyped timed out after ${timeoutMs}ms`));
+        } else if (code === 0) {
+          console.log(`${prefix}✅ Completed successfully`);
+          resolve();
+        } else {
+          reject(new Error(`pgtyped failed with exit code ${code}`));
+        }
+      }));
     });
-  });
+  } finally {
+    await preparedConfig.cleanup();
+  }
 }
 
 export async function main(options?: PgtypedUpdateOptions) {
   const userMigrations = options?.userMigrations ?? await resolveUserMigrations();
   const pgtypedConfigPath = options?.pgtypedConfigPath ?? "./pgtypedconfig.json";
   const pgtypedBin = options?.pgtypedBin ?? resolve(process.cwd(), "node_modules/@paima/pgtyped-cli/lib/index.js");
+  const pgtypedTimeoutMs = options?.pgtypedTimeoutMs ?? 600_000;
 
   console.log("🚀 Starting pgtyped-update...\n");
 
   let pglite: PgliteHandle | undefined;
   try {
-    pglite = await startPglite();
-    await waitForDb();
-    await applySystemMigrations();
-    await applyUserMigrations(userMigrations);
-    await runPgtyped(pgtypedBin, pgtypedConfigPath);
+    pglite = await startPglite(0);
+    await waitForDb(pglite.port, "127.0.0.1");
+    await applySystemMigrations(pglite.port);
+    await applyUserMigrations(userMigrations, pglite.port);
+    await runPgtyped(pgtypedBin, pgtypedConfigPath, pglite.port, pgtypedTimeoutMs);
     console.log("\n✅ All steps completed successfully");
   } finally {
-    await pglite?.close();
+    await pglite?.close({ force: true });
   }
 }
 

package/scripts/start-pglite.ts

@@ -11,7 +11,104 @@ export interface PgliteHandle {
   server: net.Server;
   db: PGlite;
   port: number;
-  close: () => Promise<void>;
+  close: (options?: PgliteCloseOptions) => Promise<void>;
+}
+
+export interface PgliteCloseOptions {
+  /**
+   * Destroy accepted client sockets before closing the gateway. Use this only
+   * when the caller owns those clients and has installed their error handling.
+   */
+  force?: boolean;
+}
+
+interface ListenerCloseLifecycle {
+  complete: () => boolean;
+  completion: Promise<void>;
+}
+
+function throwCleanupErrors(errors: unknown[], message: string): void {
+  if (errors.length === 1) throw errors[0];
+  if (errors.length > 1) throw new AggregateError(errors, message);
+}
+
+async function destroyTrackedSockets(sockets: Set<net.Socket>): Promise<void> {
+  while (sockets.size > 0) {
+    const tracked = [...sockets];
+    const closed = tracked.map((socket) => new Promise<void>((resolve) => {
+      socket.once("close", resolve);
+    }));
+    for (const socket of tracked) socket.destroy();
+    await Promise.all(closed);
+  }
+}
+
+async function closeListener(
+  server: net.Server,
+  sockets: Set<net.Socket>,
+  force: boolean,
+  lifecycle: ListenerCloseLifecycle,
+  waitForAcceptedSockets = true,
+): Promise<void> {
+  const socketTeardown = force ? destroyTrackedSockets(sockets) : undefined;
+
+  if (!server.listening) {
+    await socketTeardown;
+    if (force && !lifecycle.complete()) await lifecycle.completion;
+    return;
+  }
+
+  if (!waitForAcceptedSockets) {
+    // `server.close(callback)` keeps Bun's listener-close lifecycle referenced
+    // until every preserved socket drains. The compatibility path cannot await
+    // that lifecycle, so initiate close without a callback after unrefing the
+    // listener and accepted sockets. Synchronous initiation failures still
+    // reject this operation.
+    server.close();
+    return;
+  }
+
+  let listenerError: unknown;
+  try {
+    await new Promise<void>((resolve, reject) => {
+      try {
+        server.close((error) => {
+          if (error) reject(error);
+          else resolve();
+        });
+      } catch (error) {
+        reject(error);
+      }
+    });
+  } catch (error) {
+    listenerError = error;
+  }
+
+  await socketTeardown;
+  if (listenerError !== undefined) throw listenerError;
+}
+
+async function closeResources(
+  server: net.Server,
+  db: PGlite,
+  sockets: Set<net.Socket>,
+  force: boolean,
+  lifecycle: ListenerCloseLifecycle,
+): Promise<void> {
+  const errors: unknown[] = [];
+  try {
+    await closeListener(server, sockets, force, lifecycle);
+  } catch (error) {
+    errors.push(error);
+  }
+
+  try {
+    await db.close();
+  } catch (error) {
+    errors.push(error);
+  }
+
+  throwCleanupErrors(errors, "Failed to close the PGlite gateway and database.");
 }
 
 export async function startPglite(port = 5432): Promise<PgliteHandle> {
@@ -68,44 +165,145 @@ export async function startPglite(port = 5432): Promise<PgliteHandle> {
     });
   });
 
+  let listenerCloseComplete = false;
+  const listenerCloseCompletion = new Promise<void>((resolve) => {
+    server.once("close", () => {
+      listenerCloseComplete = true;
+      resolve();
+    });
+  });
+  const listenerLifecycle: ListenerCloseLifecycle = {
+    complete: () => listenerCloseComplete,
+    completion: listenerCloseCompletion,
+  };
+
   const sockets = new Set<net.Socket>();
+  let defaultCleanupDeferred = false;
+  let databaseCleanupPromise: Promise<void> | undefined;
+  let deferredCleanupObserved = false;
+
+  const closeDatabase = (): Promise<void> => {
+    databaseCleanupPromise ??= Promise.resolve().then(async () => {
+      // Once no accepted socket can deliver more ingress, finish the final live
+      // serialized operation before closing its PGlite backend.
+      await queue;
+      await db.close();
+    });
+    return databaseCleanupPromise;
+  };
+
+  const observeDeferredDatabaseCleanup = (): void => {
+    if (deferredCleanupObserved) return;
+    deferredCleanupObserved = true;
+    void closeDatabase().catch((error) => {
+      console.error("database: deferred PGlite cleanup failed", error);
+    });
+  };
+
   server.on("connection", (socket) => {
     sockets.add(socket);
-    socket.once("close", () => sockets.delete(socket));
+    if (defaultCleanupDeferred) socket.unref();
+    socket.once("close", () => {
+      sockets.delete(socket);
+      if (defaultCleanupDeferred && sockets.size === 0) {
+        observeDeferredDatabaseCleanup();
+      }
+    });
   });
 
-  await new Promise<void>((resolve, reject) => {
-    server.once("error", reject);
-    server.listen(port, "127.0.0.1", () => {
-      server.off("error", reject);
-      resolve();
+  try {
+    await new Promise<void>((resolve, reject) => {
+      server.once("error", reject);
+      server.listen(port, "127.0.0.1", () => {
+        server.off("error", reject);
+        resolve();
+      });
     });
-  });
+  } catch (listenError) {
+    const errors: unknown[] = [listenError];
+    try {
+      await db.close();
+    } catch (dbError) {
+      errors.push(dbError);
+    }
+    throwCleanupErrors(errors, "Failed to start the PGlite gateway and close its database.");
+    throw listenError;
+  }
 
   const address = server.address();
   if (!address || typeof address === "string") {
-    await db.close();
-    throw new Error("Unable to determine the PGlite gateway port.");
+    const addressError = new Error("Unable to determine the PGlite gateway port.");
+    const errors: unknown[] = [addressError];
+    try {
+      await closeResources(server, db, sockets, true, listenerLifecycle);
+    } catch (cleanupError) {
+      if (cleanupError instanceof AggregateError) errors.push(...cleanupError.errors);
+      else errors.push(cleanupError);
+    }
+    throwCleanupErrors(errors, "Unable to determine and clean up the PGlite gateway.");
+    throw addressError;
   }
   const actualPort = address.port;
   console.info(`database: server listening on port ${actualPort}`);
 
-  let closed = false;
+  let cleanupPromise: Promise<void> | undefined;
+
+  const closeDefault = async (): Promise<void> => {
+    const errors: unknown[] = [];
+    const deferDatabaseCleanup = sockets.size > 0;
+    if (deferDatabaseCleanup) {
+      defaultCleanupDeferred = true;
+      server.unref();
+      for (const socket of sockets) socket.unref();
+    }
+
+    try {
+      await closeListener(
+        server,
+        sockets,
+        false,
+        listenerLifecycle,
+        !deferDatabaseCleanup,
+      );
+    } catch (error) {
+      errors.push(error);
+    }
+
+    if (!deferDatabaseCleanup) {
+      try {
+        await closeDatabase();
+      } catch (error) {
+        errors.push(error);
+      }
+    } else {
+      if (sockets.size === 0) observeDeferredDatabaseCleanup();
+    }
+
+    throwCleanupErrors(errors, "Failed to close the PGlite gateway and database.");
+  };
+
+  const closeForced = async (): Promise<void> => {
+    const errors: unknown[] = [];
+    try {
+      await closeListener(server, sockets, true, listenerLifecycle);
+    } catch (error) {
+      errors.push(error);
+    }
+    try {
+      await closeDatabase();
+    } catch (error) {
+      errors.push(error);
+    }
+    throwCleanupErrors(errors, "Failed to close the PGlite gateway and database.");
+  };
 
   return {
     server,
     db,
     port: actualPort,
-    close: async () => {
-      if (closed) return;
-      closed = true;
-      for (const socket of sockets) socket.destroy();
-      if (server.listening) {
-        await new Promise<void>((resolve, reject) => {
-          server.close((error) => error ? reject(error) : resolve());
-        });
-      }
-      await db.close();
+    close: (options = {}) => {
+      cleanupPromise ??= options.force === true ? closeForced() : closeDefault();
+      return cleanupPromise;
     },
   };
 }

package/scripts/wait-for-db.ts

@@ -1,6 +1,6 @@
 #!/usr/bin/env bun
-import { spawn } from "node:child_process";
-import { args, exit } from "@effectstream/utils/runtime";
+import net from "node:net";
+import { args } from "@effectstream/utils/runtime";
 
 // Get port from arguments.
 const portArgName = "--port";
@@ -12,28 +12,34 @@ if (isNaN(port)) {
   throw new Error(`Port argument ${portArgName} is not a number`);
 }
 
-async function waitForDb() {
-  try {
-    console.log("waiting for db on port", port);
-    const child = spawn("npx", ["wait-on", `tcp:${port}`], {
-      stdio: "inherit",
-      shell: true,
-    });
-
-    const code = await new Promise<number | null>((resolve) => {
-      child.on("close", (code, _sig) => resolve(code));
-    });
-
-    if (code === 0) {
-      console.log("✅ Database is ready on port 5432");
-    } else {
-      console.error("❌ Failed to connect to database on port 5432");
-      exit(code ?? 1);
+async function waitForDb(
+  targetPort = port,
+  host = "127.0.0.1",
+  timeoutMs = 10_000,
+): Promise<void> {
+  console.log(`waiting for db on ${host}:${targetPort}`);
+  const deadline = Date.now() + timeoutMs;
+  let lastError: unknown;
+  while (Date.now() < deadline) {
+    try {
+      await new Promise<void>((resolve, reject) => {
+        const socket = net.connect({ host, port: targetPort });
+        socket.once("connect", () => {
+          socket.destroy();
+          resolve();
+        });
+        socket.once("error", reject);
+      });
+      console.log(`✅ Database is ready on ${host}:${targetPort}`);
+      return;
+    } catch (error) {
+      lastError = error;
+      await new Promise((resolve) => setTimeout(resolve, 100));
     }
-  } catch (error) {
-    console.error("❌ Error waiting for database:", error);
-    exit(1);
   }
+  throw new Error(`Failed to connect to database at ${host}:${targetPort}.`, {
+    cause: lastError,
+  });
 }
 
 export { waitForDb };

package/test/fixtures/pglite-close/01-sentinel.test.ts

@@ -0,0 +1,10 @@
+import { expect, test } from "bun:test";
+
+if (process.env.EFFECTSTREAM_PGLITE_MULTIFILE_FIXTURE !== "1") {
+  test.skip("multi-file sentinel fixture runs only through its parent", () => {});
+} else {
+  console.info("PGLITE_SENTINEL_REGISTERED");
+  test("the following file still registers", () => {
+    expect(true).toBe(true);
+  });
+}