@powerhousedao/switchboard 6.2.3-dev.5 → 6.2.3-dev.50

package/test/observability-signals.test.ts

@@ -0,0 +1,131 @@
+import { EventEmitter } from "node:events";
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { installFatalErrorShutdown } from "../src/fatal-shutdown.mjs";
+import { installObservabilitySignalHandlers } from "../src/observability-signals.mjs";
+
+function makeProc() {
+  const emitter = new EventEmitter();
+  const proc = Object.assign(emitter, {
+    pid: 1234,
+    exitCode: undefined as number | undefined,
+    kill: vi.fn((_pid: number, signal: string) => {
+      emitter.emit(signal);
+      return true;
+    }),
+    exit: vi.fn(),
+  });
+  return proc;
+}
+
+const logger = {
+  error: vi.fn(),
+} as unknown as Parameters<typeof installFatalErrorShutdown>[0];
+
+/** Lets the finally callback behind the flush promise run. */
+async function settleMicrotasks(): Promise<void> {
+  await Promise.resolve();
+  await Promise.resolve();
+  await Promise.resolve();
+}
+
+describe("installObservabilitySignalHandlers", () => {
+  afterEach(() => {
+    vi.useRealTimers();
+  });
+
+  it.each(["SIGINT", "SIGTERM"])(
+    "exits 0 after the flush on an ordinary %s",
+    async (signal) => {
+      const proc = makeProc();
+      const flush = vi.fn(() => Promise.resolve());
+      installObservabilitySignalHandlers(flush, proc as never);
+
+      proc.emit(signal);
+      await settleMicrotasks();
+
+      expect(flush).toHaveBeenCalledOnce();
+      expect(proc.exit).toHaveBeenCalledExactlyOnceWith(0);
+    },
+  );
+
+  it("does not exit before the flush settles", async () => {
+    const proc = makeProc();
+    let finish: () => void = () => undefined;
+    const flush = vi.fn(
+      () => new Promise<void>((resolve) => (finish = resolve)),
+    );
+    installObservabilitySignalHandlers(flush, proc as never);
+
+    proc.emit("SIGTERM");
+    await settleMicrotasks();
+    expect(proc.exit).not.toHaveBeenCalled();
+
+    finish();
+    await settleMicrotasks();
+    expect(proc.exit).toHaveBeenCalledExactlyOnceWith(0);
+  });
+
+  it("keeps exit code 1 when a fatal error raised the SIGTERM", async () => {
+    vi.useFakeTimers();
+    const proc = makeProc();
+    const flush = vi.fn(() => Promise.resolve());
+    // Production order: observability's handlers install at module load,
+    // before the fatal handlers.
+    installObservabilitySignalHandlers(flush, proc as never);
+    installFatalErrorShutdown(logger, proc as never);
+
+    proc.emit("uncaughtException", new Error("boom"));
+    expect(proc.exitCode).toBe(1);
+    await settleMicrotasks();
+
+    // The flush ran, but observability never lowered the code to 0; the
+    // process is still alive inside the fatal path's drain budget.
+    expect(flush).toHaveBeenCalledOnce();
+    expect(proc.exit).not.toHaveBeenCalled();
+
+    // Fatal-shutdown's backstop owns the exit and preserves code 1.
+    vi.advanceTimersByTime(15_000);
+    expect(proc.exit).toHaveBeenCalledExactlyOnceWith(1);
+    expect(proc.exit).not.toHaveBeenCalledWith(0);
+    expect(proc.exitCode).toBe(1);
+  });
+
+  it("leaves the fatal drain budget intact even when the flush is slow", async () => {
+    vi.useFakeTimers();
+    const proc = makeProc();
+    let finish: () => void = () => undefined;
+    const flush = vi.fn(
+      () => new Promise<void>((resolve) => (finish = resolve)),
+    );
+    installObservabilitySignalHandlers(flush, proc as never);
+    installFatalErrorShutdown(logger, proc as never);
+
+    proc.emit("uncaughtException", new Error("boom"));
+
+    // The flush gives up at its own ~5s budget (simulated by resolving it
+    // here); that settling must not force an exit ahead of the drain.
+    vi.advanceTimersByTime(5_000);
+    finish();
+    await settleMicrotasks();
+    expect(proc.exit).not.toHaveBeenCalled();
+
+    // Only the 15s fatal backstop ends the process, with code 1.
+    vi.advanceTimersByTime(9_999);
+    expect(proc.exit).not.toHaveBeenCalled();
+    vi.advanceTimersByTime(1);
+    expect(proc.exit).toHaveBeenCalledExactlyOnceWith(1);
+  });
+
+  it("respects a non-zero exit code set by any other module", async () => {
+    const proc = makeProc();
+    proc.exitCode = 1;
+    const flush = vi.fn(() => Promise.resolve());
+    installObservabilitySignalHandlers(flush, proc as never);
+
+    proc.emit("SIGTERM");
+    await settleMicrotasks();
+
+    expect(flush).toHaveBeenCalledOnce();
+    expect(proc.exit).not.toHaveBeenCalled();
+  });
+});

package/test/pglite-dialect.test.ts

@@ -1,7 +1,54 @@
 import { PGlite } from "@electric-sql/pglite";
 import { Kysely, sql } from "kysely";
-import { afterEach, describe, expect, it } from "vitest";
-import { ClosablePGliteDialect } from "../src/pglite-dialect.js";
+import type { ILogger } from "document-model";
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { createReactorKysely } from "../src/server.mjs";
+import { EventEmitter } from "node:events";
+import { installFatalErrorShutdown } from "../src/fatal-shutdown.mjs";
+import {
+  ClosablePGliteDialect,
+  reactorPgliteDialectOptions,
+} from "../src/pglite-dialect.js";
+
+type Row = { id: number };
+type Schema = { t: Row };
+
+function stubLogger(): ILogger {
+  const logger = {
+    verbose: vi.fn(),
+    debug: vi.fn(),
+    info: vi.fn(),
+    warn: vi.fn(),
+    error: vi.fn(),
+    child: vi.fn(),
+  };
+  logger.child.mockReturnValue(logger);
+  return logger as unknown as ILogger;
+}
+
+/** COMMIT on an aborted transaction silently rolls back unless the dialect guards it. */
+async function commitOfAnAbortedTransaction(
+  db: Kysely<Schema>,
+): Promise<{ outcome: unknown; rows: Row[] }> {
+  await sql`create table t (id int primary key)`.execute(db);
+  const outcome = await db
+    .transaction()
+    .execute(async (trx) => {
+      await sql`insert into t (id) values (1)`.execute(trx);
+      try {
+        await sql`select 1 / 0`.execute(trx);
+      } catch {
+        /* swallowed, exactly as an unrelated caller would */
+      }
+      return "JOB-SUCCEEDED";
+    })
+    .then(
+      (value: unknown) => value,
+      (error: Error) => ({ error: error.message }),
+    );
+  const rows = await sql<Row>`select id from t`.execute(db);
+  return { outcome, rows: rows.rows };
+}
 
 describe("ClosablePGliteDialect", () => {
   const created: PGlite[] = [];
@@ -26,6 +73,34 @@ describe("ClosablePGliteDialect", () => {
     expect(pglite.closed).toBe(true);
   });
 
+  it("tolerates a close that rejects because the runtime aborted", async () => {
+    const pglite = new PGlite();
+    created.push(pglite);
+    const db = new Kysely({ dialect: new ClosablePGliteDialect(pglite) });
+    await sql`select 1`.execute(db);
+    const realClose = pglite.close.bind(pglite);
+    pglite.close = () => {
+      pglite.close = realClose;
+      return Promise.reject(new Error("PGlite aborted: ENOSPC"));
+    };
+
+    await expect(db.destroy()).resolves.toBeUndefined();
+  });
+
+  it("rethrows a close that fails for any other reason", async () => {
+    const pglite = new PGlite();
+    created.push(pglite);
+    const db = new Kysely({ dialect: new ClosablePGliteDialect(pglite) });
+    await sql`select 1`.execute(db);
+    const realClose = pglite.close.bind(pglite);
+    pglite.close = () => {
+      pglite.close = realClose;
+      return Promise.reject(new Error("disk gone"));
+    };
+
+    await expect(db.destroy()).rejects.toThrow("disk gone");
+  });
+
   it("is idempotent if the PGlite is already closed", async () => {
     const pglite = new PGlite();
     created.push(pglite);
@@ -37,4 +112,86 @@ describe("ClosablePGliteDialect", () => {
 
     await expect(db.destroy()).resolves.toBeUndefined();
   });
+
+  it("refuses to report a commit for an aborted transaction", async () => {
+    const pglite = new PGlite();
+    created.push(pglite);
+    const db = new Kysely<Schema>({
+      dialect: new ClosablePGliteDialect(pglite, {
+        onDiagnostic: () => undefined,
+      }),
+    });
+
+    const { outcome, rows } = await commitOfAnAbortedTransaction(db);
+    expect(outcome).not.toBe("JOB-SUCCEEDED");
+    expect(rows).toEqual([]);
+
+    await db.destroy();
+  });
+});
+
+describe("switchboard's reactor storage factory", () => {
+  it("builds a PGlite kysely with the commit guard", async () => {
+    const storage = await createReactorKysely({
+      reactorDbUrl: undefined,
+      reactorPgliteDir: ".ph/unused-by-the-in-memory-branch",
+      reactorPgliteMajor: 17,
+      inMemory: true,
+      hostPoolSize: () => {
+        throw new Error("the PGlite branch must not read the host pool size");
+      },
+      logger: stubLogger(),
+    });
+
+    const { outcome, rows } = await commitOfAnAbortedTransaction(
+      storage.kysely as unknown as Kysely<Schema>,
+    );
+    expect(outcome).not.toBe("JOB-SUCCEEDED");
+    expect(rows).toEqual([]);
+    expect(storage.poolInstrumentation).toBeUndefined();
+
+    await storage.kysely.destroy();
+  });
+});
+
+describe("a poisoned reactor PGlite session", () => {
+  it("takes the fatal shutdown path", async () => {
+    const emitter = new EventEmitter();
+    const proc = Object.assign(emitter, {
+      pid: 1234,
+      exitCode: undefined as number | undefined,
+      kill: vi.fn(() => true),
+      exit: vi.fn(),
+    });
+    const logger = stubLogger();
+    installFatalErrorShutdown(logger, proc as never);
+
+    const pglite = new PGlite();
+    const dead = new Proxy(pglite, {
+      get(target, prop, receiver) {
+        if (prop === "query") {
+          return (text: string, params?: unknown[]) =>
+            /dead_call/.test(text)
+              ? new Promise(() => undefined)
+              : target.query(text, params);
+        }
+        const value = Reflect.get(target, prop, receiver) as unknown;
+        return typeof value === "function"
+          ? (value as (...args: unknown[]) => unknown).bind(target)
+          : value;
+      },
+    });
+    const db = new Kysely<Schema>({
+      dialect: new ClosablePGliteDialect(dead, {
+        ...reactorPgliteDialectOptions(logger, proc as never),
+        statementTimeoutMs: 50,
+      }),
+    });
+
+    await expect(sql`select 1 as dead_call`.execute(db)).rejects.toThrow();
+
+    expect(proc.kill).toHaveBeenCalledWith(1234, "SIGTERM");
+    expect(proc.exitCode).toBe(1);
+    await pglite.close();
+  });
 });

package/test/pglite-preflight.test.ts

@@ -0,0 +1,313 @@
+import { createDurableNodeFs } from "@powerhousedao/reactor-api/pglite-node";
+import type { ILogger } from "document-model";
+import { promises as fs } from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import {
+  afterAll,
+  afterEach,
+  beforeAll,
+  beforeEach,
+  describe,
+  expect,
+  it,
+  vi,
+} from "vitest";
+import { runPglitePreflight } from "../src/pglite-preflight.js";
+import { loadNodeFsClass, loadPGliteModule } from "../src/pglite-version.js";
+import { writeSnapshotFromDir } from "./snapshot-writer.js";
+
+// PGlite initdb and pg_dump are several times slower on the Windows runner.
+const BOOT_TIMEOUT = 120_000;
+const RM = { recursive: true, force: true, maxRetries: 10 } as const;
+const ROWS = 10;
+const CONVERTED_LINE =
+  /^Converted PGlite snapshot at (.+): snapshot (\d+) bytes -> (\d+) bytes on disk, (\d+) entries, (\d+) ms$/;
+
+type StubLogger = ILogger & {
+  debug: ReturnType<typeof vi.fn>;
+  info: ReturnType<typeof vi.fn>;
+  warn: ReturnType<typeof vi.fn>;
+};
+
+function stubLogger(): StubLogger {
+  const logger = {
+    verbose: vi.fn(),
+    debug: vi.fn(),
+    info: vi.fn(),
+    warn: vi.fn(),
+    error: vi.fn(),
+    child: vi.fn(),
+  };
+  logger.child.mockReturnValue(logger);
+  return logger as unknown as StubLogger;
+}
+
+async function exists(p: string): Promise<boolean> {
+  try {
+    await fs.lstat(p);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+async function pgVersion(dir: string): Promise<string> {
+  return (await fs.readFile(path.join(dir, "PG_VERSION"), "utf8")).trim();
+}
+
+/** Opens `dir` the way the server does and counts the seeded rows. */
+async function countRows(dir: string, major: 16 | 17): Promise<number> {
+  const [{ PGlite: Pg }, Base] = await Promise.all([
+    loadPGliteModule(major),
+    loadNodeFsClass(major),
+  ]);
+  const pg = new Pg({
+    fs: createDurableNodeFs(Base, dir, {
+      fsync: false,
+      maintenanceIntervalMs: 0,
+    }),
+  });
+  try {
+    await pg.waitReady;
+    const { rows } = await pg.query<{ n: number }>(
+      'SELECT count(*)::int AS n FROM "Rows"',
+    );
+    return rows[0].n;
+  } finally {
+    await pg.close();
+  }
+}
+
+function convertedLines(logger: StubLogger): string[] {
+  return logger.info.mock.calls
+    .map((call) => String(call[0]))
+    .filter((line) => CONVERTED_LINE.test(line));
+}
+
+describe("runPglitePreflight", () => {
+  // A pure-snapshot PG16 store: snapshot.bin and nothing else, the layout
+  // AtomicNodeFs left for a store it created itself.
+  let template: string;
+  let root: string;
+  let logger: StubLogger;
+
+  beforeAll(async () => {
+    template = await fs.mkdtemp(path.join(os.tmpdir(), "preflight-template-"));
+    const loose = path.join(template, "loose");
+    const { PGlite: LegacyPGlite } = await loadPGliteModule(16);
+    const LegacyNodeFS = await loadNodeFsClass(16);
+    const pg = new LegacyPGlite({ fs: new LegacyNodeFS(loose) });
+    await pg.waitReady;
+    await pg.exec('CREATE TABLE "Rows" (id int PRIMARY KEY, payload text)');
+    await pg.exec(
+      `INSERT INTO "Rows" SELECT g, repeat('x', 100) FROM generate_series(1, ${ROWS}) g`,
+    );
+    await pg.close();
+    await fs.rm(path.join(loose, "postmaster.pid"), { force: true });
+    expect(await pgVersion(loose)).toBe("16");
+
+    const store = path.join(template, "store");
+    await fs.mkdir(store);
+    await writeSnapshotFromDir(loose, path.join(store, "snapshot.bin"));
+    await fs.rm(loose, RM);
+  }, BOOT_TIMEOUT);
+
+  afterAll(async () => {
+    await fs.rm(template, RM);
+  }, BOOT_TIMEOUT);
+
+  beforeEach(async () => {
+    root = await fs.mkdtemp(path.join(os.tmpdir(), "preflight-"));
+    logger = stubLogger();
+  });
+
+  afterEach(async () => {
+    await fs.rm(root, RM);
+  }, BOOT_TIMEOUT);
+
+  async function snapshotDir(name: string): Promise<string> {
+    const dir = path.join(root, name);
+    await fs.cp(path.join(template, "store"), dir, { recursive: true });
+    return dir;
+  }
+
+  async function looseDir(name: string, major = 17): Promise<string> {
+    const dir = path.join(root, name);
+    await fs.mkdir(path.join(dir, "pg_wal"), { recursive: true });
+    await fs.writeFile(path.join(dir, "PG_VERSION"), `${major}\n`);
+    return dir;
+  }
+
+  it(
+    "converts before detecting the major, so a PG16 snapshot is migrated",
+    async () => {
+      const dir = await snapshotDir("reactor-storage");
+
+      const detected = await runPglitePreflight({
+        dirs: [dir],
+        migratePglite: true,
+        inMemory: false,
+        logger,
+      });
+
+      expect(detected.get(dir)).toBe(17);
+      expect(await pgVersion(dir)).toBe("17");
+      expect(await exists(path.join(dir, "snapshot.bin"))).toBe(false);
+      expect(await exists(`${dir}.converting`)).toBe(false);
+      expect(await exists(`${dir}.old`)).toBe(false);
+      expect(convertedLines(logger)).toHaveLength(1);
+      expect(logger.info).toHaveBeenCalledWith(
+        expect.stringContaining("Migrating"),
+      );
+      expect(await countRows(dir, 17)).toBe(ROWS);
+    },
+    BOOT_TIMEOUT,
+  );
+
+  // Pins the order alone: before this change a pure-snapshot dir had no loose
+  // PG_VERSION and the migration logged "No PG_VERSION; skipping". The case
+  // above also needs the legacy pg_dump to work.
+  it(
+    "attempts the migration of a PG16 snapshot instead of skipping it",
+    async () => {
+      const dir = await snapshotDir("reactor-storage");
+
+      await runPglitePreflight({
+        dirs: [dir],
+        migratePglite: true,
+        inMemory: false,
+        logger,
+      }).catch(() => undefined);
+
+      expect(convertedLines(logger)).toHaveLength(1);
+      expect(logger.info).not.toHaveBeenCalledWith(
+        expect.stringContaining("No PG_VERSION"),
+      );
+      expect(logger.info).toHaveBeenCalledWith(
+        expect.stringContaining(`Migrating ${dir} from PG16 to PG17`),
+      );
+    },
+    BOOT_TIMEOUT,
+  );
+
+  it(
+    "boots a converted dir on the second start without converting",
+    async () => {
+      const dir = await snapshotDir("reactor-storage");
+      const first = { dirs: [dir], inMemory: false, logger };
+      await runPglitePreflight(first);
+      expect(convertedLines(logger)).toHaveLength(1);
+      expect(await pgVersion(dir)).toBe("16");
+
+      const second = stubLogger();
+      const detected = await runPglitePreflight({ ...first, logger: second });
+
+      expect(detected.get(dir)).toBe(16);
+      expect(convertedLines(second)).toHaveLength(0);
+      expect(await exists(`${dir}.converting`)).toBe(false);
+      expect(await exists(`${dir}.old`)).toBe(false);
+      expect(await countRows(dir, 16)).toBe(ROWS);
+    },
+    BOOT_TIMEOUT,
+  );
+
+  it("removes pg_wal/xlogtemp.*", async () => {
+    const dir = await looseDir("read-storage");
+    const segment = path.join(dir, "pg_wal", "000000010000000000000001");
+    await fs.writeFile(segment, "wal");
+    await fs.writeFile(path.join(dir, "pg_wal", "xlogtemp.42"), "");
+    await fs.writeFile(path.join(dir, "pg_wal", "xlogtemp.7"), "");
+
+    const detected = await runPglitePreflight({
+      dirs: [dir],
+      inMemory: false,
+      logger,
+    });
+
+    expect(detected.get(dir)).toBe(17);
+    expect(await fs.readdir(path.join(dir, "pg_wal"))).toEqual([
+      "000000010000000000000001",
+    ]);
+    expect(await exists(segment)).toBe(true);
+  });
+
+  it("removes postmaster.pid without a warn call", async () => {
+    const dir = await looseDir("read-storage");
+    const lockfile = path.join(dir, "postmaster.pid");
+    await fs.writeFile(lockfile, "42\n");
+
+    await runPglitePreflight({ dirs: [dir], inMemory: false, logger });
+
+    expect(await exists(lockfile)).toBe(false);
+    expect(logger.warn).not.toHaveBeenCalled();
+    expect(logger.debug).toHaveBeenCalledWith(
+      expect.stringContaining(lockfile),
+    );
+  });
+
+  it(
+    "logs one info line per converted dir with sizes and duration",
+    async () => {
+      const reactor = await snapshotDir("reactor-storage");
+      const readModel = await snapshotDir("read-storage");
+
+      await runPglitePreflight({
+        dirs: [reactor, readModel],
+        inMemory: false,
+        logger,
+      });
+
+      const lines = convertedLines(logger);
+      expect(lines).toHaveLength(2);
+      const byDir = new Map(
+        lines.map((line) => {
+          const match = CONVERTED_LINE.exec(line)!;
+          return [match[1], match.slice(2).map(Number)] as const;
+        }),
+      );
+      for (const dir of [reactor, readModel]) {
+        const [snapshotBytes, convertedBytes, entries] = byDir.get(dir)!;
+        expect(snapshotBytes).toBeGreaterThan(0);
+        expect(convertedBytes).toBeGreaterThan(0);
+        expect(entries).toBeGreaterThan(0);
+      }
+    },
+    BOOT_TIMEOUT,
+  );
+
+  it("touches nothing in memory", async () => {
+    const dir = await snapshotDir("reactor-storage");
+
+    const detected = await runPglitePreflight({
+      dirs: [dir],
+      inMemory: true,
+      logger,
+    });
+
+    expect(detected.size).toBe(0);
+    expect(await exists(path.join(dir, "snapshot.bin"))).toBe(true);
+  });
+
+  it("wipes the dirs and their conversion siblings under forcePgVersion", async () => {
+    const dir = await snapshotDir("reactor-storage");
+    await fs.mkdir(`${dir}.old`);
+    await fs.mkdir(`${dir}.converting`);
+
+    const detected = await runPglitePreflight({
+      dirs: [dir],
+      forcePgVersion: 17,
+      migratePglite: true,
+      inMemory: false,
+      logger,
+    });
+
+    expect(detected.size).toBe(0);
+    expect(await exists(dir)).toBe(false);
+    expect(await exists(`${dir}.old`)).toBe(false);
+    expect(await exists(`${dir}.converting`)).toBe(false);
+    expect(logger.warn).toHaveBeenCalledWith(
+      expect.stringContaining("ignoring --migrate-pglite"),
+    );
+  });
+});

package/test/pglite-version.test.ts

@@ -1,5 +1,5 @@
 import { describe, expect, it } from "vitest";
-import { parseForcePgVersion } from "../src/pglite-version.js";
+import { openForVerify, parseForcePgVersion } from "../src/pglite-version.js";
 
 describe("parseForcePgVersion", () => {
   it("returns null for undefined", () => {
@@ -35,3 +35,11 @@ describe("parseForcePgVersion", () => {
     expect(() => parseForcePgVersion("16.5")).toThrow(/PH_FORCE_PG_VERSION/);
   });
 });
+
+describe("openForVerify", () => {
+  it("throws on an unsupported major before touching disk", async () => {
+    await expect(openForVerify(15, "/nonexistent/pglite-dir")).rejects.toThrow(
+      /PG_VERSION=15 is not supported/,
+    );
+  });
+});