@powerhousedao/switchboard 6.2.3-dev.5 → 6.2.3-dev.51

package/test/fatal-shutdown.test.ts

@@ -0,0 +1,165 @@
+import { EventEmitter } from "node:events";
+import { afterEach, describe, expect, it, vi } from "vitest";
+import {
+  installFatalErrorShutdown,
+  triggerFatalShutdown,
+} from "../src/fatal-shutdown.mjs";
+
+function makeProc() {
+  const emitter = new EventEmitter();
+  const proc = Object.assign(emitter, {
+    pid: 1234,
+    exitCode: undefined as number | undefined,
+    kill: vi.fn((_pid: number, signal: string) => {
+      emitter.emit(signal);
+      return true;
+    }),
+    exit: vi.fn(),
+  });
+  return proc;
+}
+
+const logger = {
+  error: vi.fn(),
+} as unknown as Parameters<typeof installFatalErrorShutdown>[0];
+
+describe("installFatalErrorShutdown", () => {
+  afterEach(() => {
+    vi.useRealTimers();
+  });
+
+  it.each(["uncaughtException", "unhandledRejection"])(
+    "routes %s through SIGTERM with exit code 1",
+    (event) => {
+      const proc = makeProc();
+      const onSigterm = vi.fn();
+      proc.on("SIGTERM", onSigterm);
+      installFatalErrorShutdown(logger, proc as never);
+
+      proc.emit(event, new Error("boom"));
+
+      expect(proc.kill).toHaveBeenCalledWith(1234, "SIGTERM");
+      expect(onSigterm).toHaveBeenCalledOnce();
+      expect(proc.exitCode).toBe(1);
+      expect(proc.exit).not.toHaveBeenCalled();
+    },
+  );
+
+  it("leaves an unhandled rejection to another listener, such as Sentry's", () => {
+    const proc = makeProc();
+    const other = vi.fn();
+    proc.on("unhandledRejection", other);
+    installFatalErrorShutdown(logger, proc as never);
+
+    proc.emit("unhandledRejection", new Error("boom"));
+
+    expect(other).toHaveBeenCalledOnce();
+    expect(proc.kill).not.toHaveBeenCalled();
+    expect(proc.exitCode).toBeUndefined();
+  });
+
+  it("still shuts down on an uncaught exception when another listener exists", () => {
+    const proc = makeProc();
+    proc.on("uncaughtException", vi.fn());
+    installFatalErrorShutdown(logger, proc as never);
+
+    proc.emit("uncaughtException", new Error("boom"));
+
+    expect(proc.kill).toHaveBeenCalledWith(1234, "SIGTERM");
+    expect(proc.exitCode).toBe(1);
+  });
+
+  it("signals once when a second fatal error arrives during shutdown", () => {
+    const proc = makeProc();
+    installFatalErrorShutdown(logger, proc as never);
+
+    proc.emit("unhandledRejection", new Error("first"));
+    proc.emit("uncaughtException", new Error("second"));
+
+    expect(proc.kill).toHaveBeenCalledOnce();
+  });
+
+  it("forces exit 1 through the original exit when shutdown hangs", () => {
+    vi.useFakeTimers();
+    const proc = makeProc();
+    const originalExit = proc.exit;
+    installFatalErrorShutdown(logger, proc as never);
+    // The builder swaps process.exit for a recording shim during shutdown.
+    proc.exit = vi.fn();
+
+    proc.emit("unhandledRejection", new Error("boom"));
+    vi.advanceTimersByTime(15_000);
+
+    expect(originalExit).toHaveBeenCalledWith(1);
+    expect(proc.exit).not.toHaveBeenCalled();
+  });
+
+  it("still reaches exit when the logger itself throws EPIPE inside onFatal", () => {
+    vi.useFakeTimers();
+    const proc = makeProc();
+    const onSigterm = vi.fn();
+    proc.on("SIGTERM", onSigterm);
+    const error = vi.fn(() => {
+      throw Object.assign(new Error("write EPIPE"), { code: "EPIPE" });
+    });
+    const throwingLogger = { error } as unknown as Parameters<
+      typeof installFatalErrorShutdown
+    >[0];
+    installFatalErrorShutdown(throwingLogger, proc as never);
+
+    // Without the guard the EPIPE propagates out of the uncaughtException
+    // listener, which in a real process re-enters fatal handling instead of
+    // running the shutdown.
+    expect(() =>
+      proc.emit("uncaughtException", new Error("boom")),
+    ).not.toThrow();
+
+    expect(proc.kill).toHaveBeenCalledOnce();
+    expect(onSigterm).toHaveBeenCalledOnce();
+    expect(proc.exitCode).toBe(1);
+
+    vi.advanceTimersByTime(15_000);
+
+    expect(proc.exit).toHaveBeenCalledTimes(1);
+    expect(proc.exit).toHaveBeenCalledWith(1);
+    // Exactly the two guarded log attempts: the fatal report and the
+    // forced-exit notice. A re-entered handler would log again.
+    expect(error).toHaveBeenCalledTimes(2);
+  });
+
+  it("routes a triggered fatal error through SIGTERM with exit code 1", () => {
+    const proc = makeProc();
+    const onSigterm = vi.fn();
+    proc.on("SIGTERM", onSigterm);
+    installFatalErrorShutdown(logger, proc as never);
+
+    expect(
+      triggerFatalShutdown("Flush failed", new Error("ENOSPC"), proc as never),
+    ).toBe(true);
+    triggerFatalShutdown("Flush failed", new Error("ENOSPC"), proc as never);
+    proc.emit("uncaughtException", new Error("later"));
+
+    expect(proc.kill).toHaveBeenCalledOnce();
+    expect(onSigterm).toHaveBeenCalledOnce();
+    expect(proc.exitCode).toBe(1);
+  });
+
+  it("ignores a trigger when shutdown on fatal errors is not installed", () => {
+    const proc = makeProc();
+
+    expect(
+      triggerFatalShutdown("Flush failed", new Error("ENOSPC"), proc as never),
+    ).toBe(false);
+    expect(proc.kill).not.toHaveBeenCalled();
+    expect(proc.exitCode).toBeUndefined();
+  });
+
+  it("installs its listeners once per process", () => {
+    const proc = makeProc();
+    installFatalErrorShutdown(logger, proc as never);
+    installFatalErrorShutdown(logger, proc as never);
+
+    expect(proc.listenerCount("uncaughtException")).toBe(1);
+    expect(proc.listenerCount("unhandledRejection")).toBe(1);
+  });
+});

package/test/mcp-flag.test.ts

@@ -0,0 +1,56 @@
+import { afterEach, beforeAll, describe, expect, it } from "vitest";
+import { initFeatureFlags } from "../src/feature-flags.js";
+import { MCP_ENABLED, resolveMcpEnabled } from "../src/mcp-flag.mjs";
+import type { BooleanFlagSource } from "../src/workflow-runtime.mjs";
+
+describe("resolveMcpEnabled", () => {
+  let featureFlags: BooleanFlagSource;
+
+  beforeAll(async () => {
+    featureFlags = await initFeatureFlags();
+  });
+
+  afterEach(() => {
+    delete process.env[MCP_ENABLED];
+  });
+
+  it("defaults to on", async () => {
+    await expect(resolveMcpEnabled({ featureFlags })).resolves.toBe(true);
+    await expect(
+      resolveMcpEnabled({ featureFlags, option: true }),
+    ).resolves.toBe(true);
+  });
+
+  it('is turned off by MCP_ENABLED="false"', async () => {
+    process.env[MCP_ENABLED] = "false";
+
+    await expect(resolveMcpEnabled({ featureFlags })).resolves.toBe(false);
+  });
+
+  // `ph vetra` passes `mcp: true` explicitly; the env var still wins.
+  it("turns off a host that asked for MCP", async () => {
+    process.env[MCP_ENABLED] = "false";
+
+    await expect(
+      resolveMcpEnabled({ featureFlags, option: true }),
+    ).resolves.toBe(false);
+  });
+
+  it("never turns on a host that switched MCP off", async () => {
+    process.env[MCP_ENABLED] = "true";
+
+    await expect(
+      resolveMcpEnabled({ featureFlags, option: false }),
+    ).resolves.toBe(false);
+  });
+
+  it("leaves MCP on for any value but the string false", async () => {
+    for (const raw of ["true", "0", "no", ""]) {
+      process.env[MCP_ENABLED] = raw;
+      await expect(
+        resolveMcpEnabled({ featureFlags }),
+        `MCP_ENABLED=${JSON.stringify(raw)}`,
+      ).resolves.toBe(true);
+    }
+  });
+});

package/test/observability-signals.test.ts

@@ -0,0 +1,131 @@
+import { EventEmitter } from "node:events";
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { installFatalErrorShutdown } from "../src/fatal-shutdown.mjs";
+import { installObservabilitySignalHandlers } from "../src/observability-signals.mjs";
+
+function makeProc() {
+  const emitter = new EventEmitter();
+  const proc = Object.assign(emitter, {
+    pid: 1234,
+    exitCode: undefined as number | undefined,
+    kill: vi.fn((_pid: number, signal: string) => {
+      emitter.emit(signal);
+      return true;
+    }),
+    exit: vi.fn(),
+  });
+  return proc;
+}
+
+const logger = {
+  error: vi.fn(),
+} as unknown as Parameters<typeof installFatalErrorShutdown>[0];
+
+/** Lets the finally callback behind the flush promise run. */
+async function settleMicrotasks(): Promise<void> {
+  await Promise.resolve();
+  await Promise.resolve();
+  await Promise.resolve();
+}
+
+describe("installObservabilitySignalHandlers", () => {
+  afterEach(() => {
+    vi.useRealTimers();
+  });
+
+  it.each(["SIGINT", "SIGTERM"])(
+    "exits 0 after the flush on an ordinary %s",
+    async (signal) => {
+      const proc = makeProc();
+      const flush = vi.fn(() => Promise.resolve());
+      installObservabilitySignalHandlers(flush, proc as never);
+
+      proc.emit(signal);
+      await settleMicrotasks();
+
+      expect(flush).toHaveBeenCalledOnce();
+      expect(proc.exit).toHaveBeenCalledExactlyOnceWith(0);
+    },
+  );
+
+  it("does not exit before the flush settles", async () => {
+    const proc = makeProc();
+    let finish: () => void = () => undefined;
+    const flush = vi.fn(
+      () => new Promise<void>((resolve) => (finish = resolve)),
+    );
+    installObservabilitySignalHandlers(flush, proc as never);
+
+    proc.emit("SIGTERM");
+    await settleMicrotasks();
+    expect(proc.exit).not.toHaveBeenCalled();
+
+    finish();
+    await settleMicrotasks();
+    expect(proc.exit).toHaveBeenCalledExactlyOnceWith(0);
+  });
+
+  it("keeps exit code 1 when a fatal error raised the SIGTERM", async () => {
+    vi.useFakeTimers();
+    const proc = makeProc();
+    const flush = vi.fn(() => Promise.resolve());
+    // Production order: observability's handlers install at module load,
+    // before the fatal handlers.
+    installObservabilitySignalHandlers(flush, proc as never);
+    installFatalErrorShutdown(logger, proc as never);
+
+    proc.emit("uncaughtException", new Error("boom"));
+    expect(proc.exitCode).toBe(1);
+    await settleMicrotasks();
+
+    // The flush ran, but observability never lowered the code to 0; the
+    // process is still alive inside the fatal path's drain budget.
+    expect(flush).toHaveBeenCalledOnce();
+    expect(proc.exit).not.toHaveBeenCalled();
+
+    // Fatal-shutdown's backstop owns the exit and preserves code 1.
+    vi.advanceTimersByTime(15_000);
+    expect(proc.exit).toHaveBeenCalledExactlyOnceWith(1);
+    expect(proc.exit).not.toHaveBeenCalledWith(0);
+    expect(proc.exitCode).toBe(1);
+  });
+
+  it("leaves the fatal drain budget intact even when the flush is slow", async () => {
+    vi.useFakeTimers();
+    const proc = makeProc();
+    let finish: () => void = () => undefined;
+    const flush = vi.fn(
+      () => new Promise<void>((resolve) => (finish = resolve)),
+    );
+    installObservabilitySignalHandlers(flush, proc as never);
+    installFatalErrorShutdown(logger, proc as never);
+
+    proc.emit("uncaughtException", new Error("boom"));
+
+    // The flush gives up at its own ~5s budget (simulated by resolving it
+    // here); that settling must not force an exit ahead of the drain.
+    vi.advanceTimersByTime(5_000);
+    finish();
+    await settleMicrotasks();
+    expect(proc.exit).not.toHaveBeenCalled();
+
+    // Only the 15s fatal backstop ends the process, with code 1.
+    vi.advanceTimersByTime(9_999);
+    expect(proc.exit).not.toHaveBeenCalled();
+    vi.advanceTimersByTime(1);
+    expect(proc.exit).toHaveBeenCalledExactlyOnceWith(1);
+  });
+
+  it("respects a non-zero exit code set by any other module", async () => {
+    const proc = makeProc();
+    proc.exitCode = 1;
+    const flush = vi.fn(() => Promise.resolve());
+    installObservabilitySignalHandlers(flush, proc as never);
+
+    proc.emit("SIGTERM");
+    await settleMicrotasks();
+
+    expect(flush).toHaveBeenCalledOnce();
+    expect(proc.exit).not.toHaveBeenCalled();
+  });
+});

package/test/pglite-dialect.test.ts

@@ -1,7 +1,62 @@
 import { PGlite } from "@electric-sql/pglite";
 import { Kysely, sql } from "kysely";
-import { afterEach, describe, expect, it } from "vitest";
-import { ClosablePGliteDialect } from "../src/pglite-dialect.js";
+import type { ILogger } from "document-model";
+import { afterEach, describe, expect, it, vi } from "vitest";
+import {
+  IN_MEMORY_PGLITE_STORAGE_FACTS,
+  PGLITE_PATH_STORAGE_FACTS,
+  POSTGRES_STORAGE_FACTS,
+} from "@powerhousedao/reactor";
+import { mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { createReactorKysely } from "../src/server.mjs";
+import { EventEmitter } from "node:events";
+import { installFatalErrorShutdown } from "../src/fatal-shutdown.mjs";
+import {
+  ClosablePGliteDialect,
+  reactorPgliteDialectOptions,
+} from "../src/pglite-dialect.js";
+
+type Row = { id: number };
+type Schema = { t: Row };
+
+function stubLogger(): ILogger {
+  const logger = {
+    verbose: vi.fn(),
+    debug: vi.fn(),
+    info: vi.fn(),
+    warn: vi.fn(),
+    error: vi.fn(),
+    child: vi.fn(),
+  };
+  logger.child.mockReturnValue(logger);
+  return logger as unknown as ILogger;
+}
+
+/** COMMIT on an aborted transaction silently rolls back unless the dialect guards it. */
+async function commitOfAnAbortedTransaction(
+  db: Kysely<Schema>,
+): Promise<{ outcome: unknown; rows: Row[] }> {
+  await sql`create table t (id int primary key)`.execute(db);
+  const outcome = await db
+    .transaction()
+    .execute(async (trx) => {
+      await sql`insert into t (id) values (1)`.execute(trx);
+      try {
+        await sql`select 1 / 0`.execute(trx);
+      } catch {
+        /* swallowed, exactly as an unrelated caller would */
+      }
+      return "JOB-SUCCEEDED";
+    })
+    .then(
+      (value: unknown) => value,
+      (error: Error) => ({ error: error.message }),
+    );
+  const rows = await sql<Row>`select id from t`.execute(db);
+  return { outcome, rows: rows.rows };
+}
 
 describe("ClosablePGliteDialect", () => {
   const created: PGlite[] = [];
@@ -26,6 +81,34 @@ describe("ClosablePGliteDialect", () => {
     expect(pglite.closed).toBe(true);
   });
 
+  it("tolerates a close that rejects because the runtime aborted", async () => {
+    const pglite = new PGlite();
+    created.push(pglite);
+    const db = new Kysely({ dialect: new ClosablePGliteDialect(pglite) });
+    await sql`select 1`.execute(db);
+    const realClose = pglite.close.bind(pglite);
+    pglite.close = () => {
+      pglite.close = realClose;
+      return Promise.reject(new Error("PGlite aborted: ENOSPC"));
+    };
+
+    await expect(db.destroy()).resolves.toBeUndefined();
+  });
+
+  it("rethrows a close that fails for any other reason", async () => {
+    const pglite = new PGlite();
+    created.push(pglite);
+    const db = new Kysely({ dialect: new ClosablePGliteDialect(pglite) });
+    await sql`select 1`.execute(db);
+    const realClose = pglite.close.bind(pglite);
+    pglite.close = () => {
+      pglite.close = realClose;
+      return Promise.reject(new Error("disk gone"));
+    };
+
+    await expect(db.destroy()).rejects.toThrow("disk gone");
+  });
+
   it("is idempotent if the PGlite is already closed", async () => {
     const pglite = new PGlite();
     created.push(pglite);
@@ -37,4 +120,136 @@ describe("ClosablePGliteDialect", () => {
 
     await expect(db.destroy()).resolves.toBeUndefined();
   });
+
+  it("refuses to report a commit for an aborted transaction", async () => {
+    const pglite = new PGlite();
+    created.push(pglite);
+    const db = new Kysely<Schema>({
+      dialect: new ClosablePGliteDialect(pglite, {
+        onDiagnostic: () => undefined,
+      }),
+    });
+
+    const { outcome, rows } = await commitOfAnAbortedTransaction(db);
+    expect(outcome).not.toBe("JOB-SUCCEEDED");
+    expect(rows).toEqual([]);
+
+    await db.destroy();
+  });
+});
+
+describe("switchboard's reactor storage factory", () => {
+  it("builds a PGlite kysely with the commit guard", async () => {
+    const storage = await createReactorKysely({
+      reactorDbUrl: undefined,
+      reactorPgliteDir: ".ph/unused-by-the-in-memory-branch",
+      reactorPgliteMajor: 17,
+      inMemory: true,
+      hostPoolSize: () => {
+        throw new Error("the PGlite branch must not read the host pool size");
+      },
+      logger: stubLogger(),
+    });
+
+    const { outcome, rows } = await commitOfAnAbortedTransaction(
+      storage.kysely as unknown as Kysely<Schema>,
+    );
+    expect(outcome).not.toBe("JOB-SUCCEEDED");
+    expect(rows).toEqual([]);
+    expect(storage.poolInstrumentation).toBeUndefined();
+
+    await storage.kysely.destroy();
+  });
+
+  describe("storage facts", () => {
+    let dataDir: string | undefined;
+
+    afterEach(async () => {
+      if (dataDir) await rm(dataDir, { recursive: true, force: true });
+      dataDir = undefined;
+    });
+
+    const open = (opts: Partial<Parameters<typeof createReactorKysely>[0]>) =>
+      createReactorKysely({
+        reactorDbUrl: undefined,
+        reactorPgliteDir: ".ph/unused-by-the-in-memory-branch",
+        reactorPgliteMajor: 17,
+        inMemory: false,
+        hostPoolSize: () => 1,
+        logger: stubLogger(),
+        ...opts,
+      });
+
+    // Kysely closes only a driver that has run a query; an unopened PGlite
+    // would still be initialising its data dir when afterEach removes it.
+    const openAndDestroy = async (
+      kysely: Awaited<ReturnType<typeof createReactorKysely>>["kysely"],
+    ) => {
+      await sql`select 1`.execute(kysely);
+      await kysely.destroy();
+    };
+
+    it("reports in-memory PGlite as not durable", async () => {
+      const storage = await open({ inMemory: true });
+      expect(storage.storageFacts).toEqual(IN_MEMORY_PGLITE_STORAGE_FACTS);
+      await openAndDestroy(storage.kysely);
+    });
+
+    it("reports a PGlite data directory as durable on a path", async () => {
+      dataDir = await mkdtemp(join(tmpdir(), "sb-facts-"));
+      const storage = await open({ reactorPgliteDir: dataDir });
+      expect(storage.storageFacts).toEqual(PGLITE_PATH_STORAGE_FACTS);
+      await openAndDestroy(storage.kysely);
+    }, 30_000);
+
+    it("reports a Postgres url as a durable server", async () => {
+      const storage = await open({
+        reactorDbUrl: "postgres://postgres:postgres@127.0.0.1:1/never-dialed",
+      });
+      expect(storage.storageFacts).toEqual(POSTGRES_STORAGE_FACTS);
+      await storage.kysely.destroy();
+    });
+  });
+});
+
+describe("a poisoned reactor PGlite session", () => {
+  it("takes the fatal shutdown path", async () => {
+    const emitter = new EventEmitter();
+    const proc = Object.assign(emitter, {
+      pid: 1234,
+      exitCode: undefined as number | undefined,
+      kill: vi.fn(() => true),
+      exit: vi.fn(),
+    });
+    const logger = stubLogger();
+    installFatalErrorShutdown(logger, proc as never);
+
+    const pglite = new PGlite();
+    const dead = new Proxy(pglite, {
+      get(target, prop, receiver) {
+        if (prop === "query") {
+          return (text: string, params?: unknown[]) =>
+            /dead_call/.test(text)
+              ? new Promise(() => undefined)
+              : target.query(text, params);
+        }
+        const value = Reflect.get(target, prop, receiver) as unknown;
+        return typeof value === "function"
+          ? (value as (...args: unknown[]) => unknown).bind(target)
+          : value;
+      },
+    });
+    const db = new Kysely<Schema>({
+      dialect: new ClosablePGliteDialect(dead, {
+        ...reactorPgliteDialectOptions(logger, proc as never),
+        statementTimeoutMs: 50,
+      }),
+    });
+
+    await expect(sql`select 1 as dead_call`.execute(db)).rejects.toThrow();
+
+    expect(proc.kill).toHaveBeenCalledWith(1234, "SIGTERM");
+    expect(proc.exitCode).toBe(1);
+    await pglite.close();
+  });
 });