@effectstream/runtime 0.200.5 → 0.200.6

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@effectstream/runtime",
-  "version": "0.200.5",
+  "version": "0.200.6",
   "license": "(MIT OR Apache-2.0)",
   "homepage": "https://effectstream.github.io/docs/",
   "repository": {
@@ -13,17 +13,17 @@
   },
   "exports": "./src/mod.ts",
   "dependencies": {
-    "@effectstream/concise": "0.200.5",
-    "@effectstream/config": "0.200.5",
-    "@effectstream/coroutine": "0.200.5",
-    "@effectstream/crypto": "0.200.5",
-    "@effectstream/db": "0.200.5",
-    "@effectstream/event-client": "0.200.5",
-    "@effectstream/event-server": "0.200.5",
-    "@effectstream/log": "0.200.5",
-    "@effectstream/sm": "0.200.5",
-    "@effectstream/sync": "0.200.5",
-    "@effectstream/utils": "0.200.5",
+    "@effectstream/concise": "0.200.6",
+    "@effectstream/config": "0.200.6",
+    "@effectstream/coroutine": "0.200.6",
+    "@effectstream/crypto": "0.200.6",
+    "@effectstream/db": "0.200.6",
+    "@effectstream/event-client": "0.200.6",
+    "@effectstream/event-server": "0.200.6",
+    "@effectstream/log": "0.200.6",
+    "@effectstream/sm": "0.200.6",
+    "@effectstream/sync": "0.200.6",
+    "@effectstream/utils": "0.200.6",
     "@fastify/cors": "^11.0.1",
     "@metamask/json-rpc-engine": "^10.0.3",
     "pg": "8.21.0",

package/src/api/http-server.ts

@@ -1,11 +1,16 @@
-import fastify, { type FastifyInstance, type FastifyReply, type FastifyRequest } from "fastify";
+import fastify, {
+  type FastifyInstance,
+  type FastifyReply,
+  type FastifyRequest,
+  type FastifyServerOptions,
+} from "fastify";
 import { evmRpcEngine } from "./rpc-evm/eip1193.ts";
 import { appliedBlockStatus } from "./apply-status.ts";
 import { finalizedStreamStatus } from "./stream-status.ts";
 import { buildHealthReport, healthHttpStatus } from "./health.ts";
 import type { Pool } from "pg";
 import cors from "@fastify/cors";
-import { ensure, run, suspend, until } from "effection";
+import { ensure, suspend, until, useScope } from "effection";
 import {
   acquireDBMutex,
   getAllAddresses,
@@ -53,6 +58,23 @@ import {
 import { PrimitiveRegistry } from "@effectstream/sm";
 import { ConfigNetworkType, getWriteNamespace, usePaimaStaticConfig } from "@effectstream/config";
 
+/**
+ * Parse `EFFECTSTREAM_TRUST_PROXY` into Fastify's `trustProxy` option.
+ *
+ * - `true` / `1` / empty → `true` (trust every hop; the default)
+ * - `false` / `0`        → `false`
+ * - anything else        → comma-separated proxy IPs / CIDRs
+ */
+export function parseTrustProxy(raw: string | undefined): boolean | string[] {
+  const value = (raw ?? "").trim();
+  if (value === "" || value.toLowerCase() === "true" || value === "1") return true;
+  if (value.toLowerCase() === "false" || value === "0") return false;
+  return value
+    .split(",")
+    .map((entry) => entry.trim())
+    .filter((entry) => entry.length > 0);
+}
+
 function tableListContains(
   list: Array<{ table_name: string | null }>,
   name: string,
@@ -165,10 +187,22 @@ export const startHttpServer = function* (
   apiRouter?: StartConfigApiRouter,
   grammar?: GrammarDefinition,
 ) {
+  // This operation's own scope. Work a request starts is run in it rather than
+  // in a detached `run()`, so an in-flight request is visible to — and
+  // cancelled by — the shutdown that closes the server below.
+  const scope = yield* useScope();
   // Use dbConn directly; queries are executed via pgtyped PreparedQuery.run
   // Allow any webpage to access the server.
   // This node is not specific for a specific website.
-  const server = fastify({ routerOptions: { maxParamLength: 300 } });
+  // trustProxy decides what `request.ip` means. Deployments front this server
+  // with nginx/Caddy, and without it Fastify reports the proxy's address for
+  // every client — so anything keyed per IP (the rate limiter above all)
+  // shares ONE bucket across every user behind the proxy.
+  const serverOptions: FastifyServerOptions = {
+    routerOptions: { maxParamLength: 300 },
+    trustProxy: parseTrustProxy(ENV.EFFECTSTREAM_TRUST_PROXY),
+  };
+  const server = fastify(serverOptions);
   // OpenAPI Docs
   yield* registerOpenApiDocumentation(server, ENV.EFFECTSTREAM_API_PORT);
 
@@ -978,7 +1012,10 @@ export const startHttpServer = function* (
         reply,
       ) => {
         const { name } = request.query;
-        await run(() => acquireDBMutex(`http-server:${name}`));
+        // Scope-bound, not detached: waiting for the mutex must end when the
+        // server's scope does, otherwise shutdown blocks on a request that
+        // only cancellation could release.
+        await scope.run(() => acquireDBMutex(`http-server:${name}`));
         return "ok";
       },
     );
@@ -1039,8 +1076,13 @@ export const startHttpServer = function* (
     );
   });
 
+  // Close unconditionally, including when `listen()` below rejects. The Fastify
+  // instance is fully built by this point — CORS, Swagger and every plugin the
+  // host registered through `apiRouter` — and `close()` is what runs their
+  // `onClose` hooks. Gating on `server.server.listening` skipped all of it on a
+  // bind conflict and leaked the instance.
   yield* ensure(function* () {
-    if (server.server.listening) yield* until(server.close());
+    yield* until(server.close());
   });
   const address = yield* until(
     server.listen({ port: ENV.EFFECTSTREAM_API_PORT, host: "0.0.0.0" }),

package/src/api/trust-proxy.test.ts

@@ -0,0 +1,22 @@
+import { describe, expect, test } from "bun:test";
+import { parseTrustProxy } from "./http-server.ts";
+
+describe("parseTrustProxy", () => {
+  test("defaults to trusting every hop", () => {
+    expect(parseTrustProxy(undefined)).toBe(true);
+    expect(parseTrustProxy("")).toBe(true);
+    expect(parseTrustProxy("true")).toBe(true);
+    expect(parseTrustProxy("TRUE")).toBe(true);
+    expect(parseTrustProxy("1")).toBe(true);
+  });
+  test("can be switched off", () => {
+    expect(parseTrustProxy("false")).toBe(false);
+    expect(parseTrustProxy("0")).toBe(false);
+  });
+  test("a bare number that is not 0/1 is treated as an address list entry", () => {
+    expect(parseTrustProxy("2")).toEqual(["2"]);
+  });
+  test("a list becomes trimmed proxy addresses", () => {
+    expect(parseTrustProxy(" 127.0.0.1, 10.0.0.0/8 ,")).toEqual(["127.0.0.1", "10.0.0.0/8"]);
+  });
+});

package/src/main.ts

@@ -58,6 +58,46 @@ export function* init() {
   yield* initTelemetry();
 }
 
+/**
+ * Report a set of failures the way the rest of the codebase already does (see
+ * `db/scripts/start-pglite.ts` and the MQTT broker): a lone failure keeps its
+ * identity, several become one ordered `AggregateError`.
+ */
+function throwCleanupErrors(errors: unknown[], message: string): void {
+  if (errors.length === 1) throw errors[0];
+  if (errors.length > 1) throw new AggregateError(errors, message);
+}
+
+/**
+ * Spawn a child of the runtime that reports its own failure instead of only
+ * racing to become the scope's error.
+ *
+ * Effection tears the scope down on the first child failure and keeps exactly
+ * one error; anything else that fails while unwinding replaces it. Recording
+ * the failure where it happens keeps it ahead of every cleanup error in the
+ * aggregate `start()` finally throws.
+ */
+function* superviseChild(
+  causes: unknown[],
+  name: string,
+  body: () => Operation<unknown>,
+): Operation<void> {
+  yield* spawn(function* () {
+    try {
+      yield* body();
+    } catch (error) {
+      if (!causes.includes(error)) causes.push(error);
+      log.local(
+        ComponentNames.EFFECTSTREAM_RUNTIME,
+        "child-task",
+        SeverityNumber.ERROR,
+        (l) => l(`child task "${name}" failed: ${String(error)}`),
+      );
+      throw error;
+    }
+  });
+}
+
 /**
  * Main entry point to start the Paima Engine Node.
  *
@@ -67,13 +107,50 @@ export function* init() {
  * @param config - Paima Engine Node configuration object.
  */
 export function* start(config: StartConfig): Operation<void> {
-  const { syncInfo } = config;
+  // Why the runtime is stopping, in the order the failures happened.
+  const causes: unknown[] = [];
+  // What went wrong while releasing resources, in teardown order.
+  const cleanups: unknown[] = [];
+
+  // Registered first, so it runs last: one place reports everything that went
+  // wrong. Without it effection keeps only the error thrown by whichever
+  // teardown ran last, so a failing `dbConn.end()` silently replaced the real
+  // cause of the shutdown.
+  yield* ensure(function* () {
+    throwCleanupErrors(
+      [...causes, ...cleanups],
+      "Effectstream runtime did not shut down cleanly",
+    );
+  });
 
   const dbConn = getConnection();
   yield* ensure(function* () {
-    yield* call(() => dbConn.end());
+    try {
+      yield* call(() => dbConn.end());
+    } catch (error) {
+      cleanups.push(error);
+    }
   });
 
+  try {
+    // The body gets its own frame, so everything it owns — broker, HTTP
+    // server, merge/heartbeat/snapshot children — is released, and every
+    // release failure recorded, before the pool above is drained.
+    yield* call(() => runRuntime(config, dbConn, causes, cleanups));
+  } catch (error) {
+    // A body or child failure. `superviseChild` may already have recorded it.
+    if (!causes.includes(error)) causes.push(error);
+  }
+}
+
+function* runRuntime(
+  config: StartConfig,
+  dbConn: ReturnType<typeof getConnection>,
+  causes: unknown[],
+  cleanups: unknown[],
+): Operation<void> {
+  const { syncInfo } = config;
+
   const syncProtocols = yield* startup(dbConn as any, // Client,
     syncInfo, config);
 
@@ -90,9 +167,33 @@ export function* start(config: StartConfig): Operation<void> {
     yield* startSync(syncProtocol);
   }
 
-  // Create MQTT Broker
+  // Create MQTT Broker. Startup must fail fast on a bind conflict (e.g. a
+  // stale engine still holding the port) instead of running without a broker.
   if (ENV.MQTT_BROKER) {
-    new EventBroker("effectstream-engine").createServer();
+    const eventBroker = new EventBroker("effectstream-engine");
+    let startFailed = false;
+    // Registered BEFORE start(): `start()` binds its TCP listener before it
+    // resolves, so a cancellation that lands while we are still awaiting it
+    // (a SIGINT during a slow bind) must still release both listeners. The
+    // broker coordinates shutdown with its own listen outcome, so calling
+    // `shutdown()` mid-start is safe and does not deadlock.
+    yield* ensure(function* () {
+      // A failed start already ran — and reported — the broker's own cleanup.
+      if (startFailed) return;
+      try {
+        yield* call(() => eventBroker.shutdown());
+      } catch (error) {
+        // Recorded rather than logged away: the broker builds a causally
+        // ordered AggregateError that the host needs to see.
+        cleanups.push(error);
+      }
+    });
+    try {
+      yield* call(() => eventBroker.start());
+    } catch (error) {
+      startFailed = true;
+      throw error;
+    }
   }
 
   // 20× main clock block time (NTP if present, else protocol 0).
@@ -107,7 +208,7 @@ export function* start(config: StartConfig): Operation<void> {
   const lagThresholdMs = ENV.EFFECTSTREAM_LAG_THRESHOLD_MS ??
     (clockBlockTimeMS != null ? clockBlockTimeMS * 20 : 60_000);
 
-  yield* spawn(function* () {
+  yield* superviseChild(causes, "http-server", function* () {
     yield* startHttpServer(
       dbConn,
       syncProtocols,
@@ -123,10 +224,14 @@ export function* start(config: StartConfig): Operation<void> {
   const { stream: finalizedStream, subscription: finalizedBlocks } =
     yield* createBoundedFinalizedStream(ENV.EFFECTSTREAM_FINALIZED_STREAM_CAP);
 
-  yield* spawn(() => startMerge(syncProtocols, finalizedStream));
+  yield* superviseChild(
+    causes,
+    "merge",
+    () => startMerge(syncProtocols, finalizedStream),
+  );
 
   const heartbeatIntervalMs = 60_000;
-  yield* spawn(function* () {
+  yield* superviseChild(causes, "heartbeat", function* () {
     while (true) {
       yield* sleep(heartbeatIntervalMs);
       const now = Date.now();
@@ -162,7 +267,11 @@ export function* start(config: StartConfig): Operation<void> {
     ? lastHashRow.effectstream_block_hash!.toString() as EffectstreamBlockHash
     : null;
   if (config.snapshotConfig) {
-    yield* spawn(() => runSnapshotLoop(config.snapshotConfig!));
+    yield* superviseChild(
+      causes,
+      "snapshot-loop",
+      () => runSnapshotLoop(config.snapshotConfig!),
+    );
   }
 
   const coalescer = createEmptyBlockCoalescer({

package/src/process-blocks.ts

@@ -410,10 +410,12 @@ export function* processFinalizedBlockWithRetry(
   let attempt = 0;
   while (true) {
     let dbClient: PoolClient | undefined;
+    let checkout: Promise<PoolClient> | undefined;
     let transientErr: unknown;
     try {
       yield* acquireDBMutex(lockName);
-      dbClient = yield* until(pool.connect());
+      checkout = pool.connect();
+      dbClient = yield* until(checkout);
       return yield* processFinalizedBlock(
         value,
         config,
@@ -434,6 +436,23 @@ export function* processFinalizedBlockWithRetry(
         } catch {
           /* already released/destroyed */
         }
+      } else if (checkout) {
+        // Cancellation won the race with the checkout. `until` abandons the
+        // promise, but the pool still hands the client over afterwards and
+        // nobody is left in this scope to give it back — a leaked client keeps
+        // the pool one connection short and blocks `pool.end()` forever.
+        void checkout.then(
+          (client) => {
+            try {
+              client.release();
+            } catch {
+              /* already released/destroyed */
+            }
+          },
+          () => {
+            /* the checkout itself failed; nothing to return */
+          },
+        );
       }
     }
 

package/src/telemetry.ts

@@ -2,7 +2,7 @@ import { NodeSDK } from "@opentelemetry/sdk-node";
 import { getNodeAutoInstrumentations } from "@opentelemetry/auto-instrumentations-node";
 import opentelemetry from "@opentelemetry/api";
 import { defaultOtelSetup, PaimaTelemetryContext } from "@effectstream/log";
-import type { Operation } from "effection";
+import { call, ensure, type Operation } from "effection";
 import fs from "node:fs";
 import path from "node:path";
 import { parse } from "jsonc-parser";
@@ -34,4 +34,30 @@ export function* initTelemetry(): Operation<void> {
   );
 
   sdk.start();
+
+  // `initTelemetry` is delegated with `yield*`, so this registers in the
+  // caller's scope: telemetry lives exactly as long as whoever called `init()`,
+  // and — being the first thing that scope acquires — is released last, after
+  // the broker, HTTP server, database and child tasks it instruments.
+  //
+  // The SDK owns a periodic metric reader plus batch span/log processors; left
+  // running they keep exporting (and holding timers) past the lifetime of the
+  // node that created them.
+  let stopped = false;
+  yield* ensure(function* () {
+    // A re-entrant unwind must not shut the same SDK down twice: OpenTelemetry
+    // rejects the second call rather than treating it as a no-op.
+    if (stopped) return;
+    stopped = true;
+    try {
+      yield* call(() => sdk.shutdown());
+    } catch (error) {
+      // Reported, never fatal. `shutdown()` also rejects when the collector is
+      // simply unreachable, and a lost telemetry flush must not turn an
+      // otherwise clean runtime shutdown into a failed one — every resource
+      // that matters (broker, HTTP, database, child tasks) has already been
+      // released by the time this runs.
+      console.error("OpenTelemetry shutdown failed:", error);
+    }
+  });
 }

package/test/lifecycle/fixtures/runtime-lifecycle.ts

@@ -0,0 +1,246 @@
+/**
+ * Subprocess fixture driving the real `start()` from `src/main.ts` (spec 00031:
+ * G2, G3, G5, G8, G10b).
+ *
+ * A scenario is one process: the runtime boots against an in-process PGLite
+ * gateway on an OS-selected port above 10000, the scenario's failure is
+ * injected, the runtime is torn down, and the observable outcome is printed as
+ * stdout markers for the parent test to assert on.
+ *
+ * Booting `start()` in the shared `bun test` process is not an option — the
+ * existing reproduction harness documents that a second in-process boot stalls
+ * the sync loops — so every scenario gets its own process, exactly like
+ * `test/reproduction/node-runner.ts`.
+ *
+ * Usage: `bun runtime-lifecycle.ts '<json Spec>'`
+ */
+import type { Server } from "node:net";
+import pg from "pg";
+import { run, type Task } from "effection";
+import { EventBroker } from "@effectstream/event-server";
+import { startPglite, type PgliteHandle } from "@effectstream/db/start-pglite";
+import {
+  toSyncProtocolWithNetwork,
+  withEffectstreamStaticConfig,
+} from "@effectstream/config";
+import { TestChainControl } from "@effectstream/sync";
+import { init, start } from "../../../src/main.ts";
+import type { StartConfigGameStateTransitions } from "../../../src/types.ts";
+import {
+  buildConfig,
+  TEST_PRIMITIVE_TYPE,
+  TestEventPrimitive,
+} from "../../reproduction/scenario.ts";
+import { listenTcp, sleep, tcpConnects, waitUntil } from "../support.ts";
+
+export type Spec = {
+  mode:
+    | "broker-cancel-during-start"
+    | "broker-shutdown-error"
+    | "child-failure-with-pool-end-error";
+  apiPort: number;
+  pglitePort: number;
+  brokerTcpPort: number;
+  brokerWsPort: number;
+};
+
+const spec: Spec = JSON.parse(process.argv[2] ?? "{}");
+const noopStf: StartConfigGameStateTransitions = function* () {};
+
+/** Flatten an error tree into a stable, greppable one-line shape. */
+function describeError(error: unknown): string {
+  if (error instanceof AggregateError) {
+    return `Aggregate[${error.errors.map(describeError).join(" | ")}]`;
+  }
+  if (error instanceof Error) {
+    const code = (error as NodeJS.ErrnoException).code;
+    return code ? `${code}:${error.message}` : error.message;
+  }
+  return String(error);
+}
+
+/**
+ * Report whether a port is released within `ms`. Polling rather than sampling
+ * once keeps a busy machine's slow-but-correct release from being read as a
+ * leak; a genuine leak stays bound for the whole window.
+ */
+async function portState(
+  port: number,
+  ms = 5_000,
+): Promise<"bound" | "free"> {
+  const end = Date.now() + ms;
+  for (;;) {
+    if (!(await tcpConnects(port))) return "free";
+    if (Date.now() >= end) return "bound";
+    await sleep(50);
+  }
+}
+
+function say(marker: string, value: string): void {
+  console.log(`${marker}:${value}`);
+}
+
+async function main(): Promise<void> {
+  // The engine reads every one of these lazily through ENV getters, so setting
+  // them here (before the runtime is constructed) is enough.
+  process.env.PGLITE = "true";
+  process.env.PGLITE_DATA_DIR = "memory://";
+  process.env.DB_HOST = "127.0.0.1";
+  process.env.DB_PORT = String(spec.pglitePort);
+  process.env.DB_NAME = "postgres";
+  process.env.DB_USER = "postgres";
+  process.env.DB_PW = "";
+  process.env.EFFECTSTREAM_API_PORT = String(spec.apiPort);
+  process.env.EFFECTSTREAM_COALESCE_EMPTY_BLOCKS = "false";
+  process.env.MQTT_ENGINE_BROKER_PORT = String(spec.brokerTcpPort);
+  process.env.MQTT_ENGINE_BROKER_WS_PORT = String(spec.brokerWsPort);
+  const usesBroker = spec.mode !== "child-failure-with-pool-end-error";
+  process.env.MQTT_BROKER = usesBroker ? "true" : "false";
+  // Telemetry is exercised by its own fixture; here it would only add an
+  // unreachable-collector flush to every teardown.
+  process.env.OTEL_SDK_DISABLED = "true";
+
+  let brokerStartGateInstalled = false;
+  if (spec.mode === "broker-cancel-during-start") {
+    // Let the broker really bind, then hang so the runtime is cancelled while
+    // `start()` is still awaiting it.
+    const never = new Promise<void>(() => {});
+    const realStart = EventBroker.prototype.start;
+    EventBroker.prototype.start = function (this: EventBroker): Promise<void> {
+      return realStart.call(this).then(() => never);
+    };
+    brokerStartGateInstalled = true;
+  }
+
+  if (spec.mode === "broker-shutdown-error") {
+    const realShutdown = (EventBroker.prototype as unknown as {
+      shutdown: () => Promise<void>;
+    }).shutdown;
+    (EventBroker.prototype as unknown as { shutdown: () => Promise<void> })
+      .shutdown = function (this: EventBroker): Promise<void> {
+        // Release the real resources first, then fail: a broker that frees its
+        // ports but reports a cleanup error must not be reported as clean.
+        return realShutdown.call(this).then(() => {
+          throw new Error("broker-shutdown-boom");
+        });
+      };
+  }
+
+  let apiBlocker: Server | undefined;
+  if (spec.mode === "child-failure-with-pool-end-error") {
+    // Wildcard bind: the runtime listens on 0.0.0.0, and on BSD a loopback-only
+    // holder would not conflict with it.
+    apiBlocker = await listenTcp(spec.apiPort, "0.0.0.0");
+    const realEnd = pg.Pool.prototype.end;
+    void realEnd;
+    pg.Pool.prototype.end = function (this: pg.Pool): Promise<void> {
+      return Promise.reject(new Error("pool-end-boom"));
+    };
+  }
+
+  const database: PgliteHandle = await startPglite(spec.pglitePort);
+  await database.db.waitReady;
+  say("PGLITE_READY", String(database.port));
+
+  const cfg = buildConfig({
+    securityNamespace: "runtime-lifecycle",
+    events: [],
+  });
+  const syncInfo = toSyncProtocolWithNetwork(cfg);
+  // Explicit, tiny tips. Without them the synthetic TEST chain falls back to a
+  // wall-clock tip and the node tries to sync decades of blocks.
+  for (const entry of syncInfo) {
+    TestChainControl.setTip(entry.syncProtocol.name, 3);
+  }
+
+  let started = false;
+  const task: Task<unknown> = run(function* () {
+    yield* init();
+    yield* withEffectstreamStaticConfig(cfg, function* () {
+      yield* start({
+        appName: "runtime-lifecycle",
+        appVersion: "1.0.0",
+        syncInfo,
+        gameStateTransitions: noopStf,
+        userDefinedPrimitives: { [TEST_PRIMITIVE_TYPE]: TestEventPrimitive },
+        events: false,
+        dev: {
+          resetPublicData: false,
+          onStarted: () => {
+            started = true;
+          },
+        },
+      });
+    });
+  });
+  const settled = Promise.resolve(task).then(
+    () => "ok",
+    (error: unknown) => `err:${describeError(error)}`,
+  );
+
+  try {
+    if (spec.mode === "child-failure-with-pool-end-error") {
+      // The HTTP child fails at listen; the runtime tears itself down.
+      say("TASK_SETTLED", await settled);
+    } else {
+      await waitUntil(() => started, 60_000, "runtime startup");
+      // Both listeners: the broker binds TCP first, so waiting only on TCP
+      // would race the WebSocket half of its startup.
+      await waitUntil(
+        () => tcpConnects(spec.brokerTcpPort),
+        60_000,
+        "broker TCP listener",
+      );
+      await waitUntil(
+        () => tcpConnects(spec.brokerWsPort),
+        60_000,
+        "broker WebSocket listener",
+      );
+      say("BROKER_TCP_BOUND", "yes");
+      if (spec.mode === "broker-shutdown-error") {
+        // This scenario is about a *failing* shutdown, not about cancelling a
+        // start. Wait until the runtime is demonstrably past the broker block
+        // — the HTTP server is spawned after it — so the broker's own cleanup
+        // registration is not itself the thing under test.
+        await waitUntil(
+          () => tcpConnects(spec.apiPort),
+          60_000,
+          "API listener (runtime past the broker block)",
+        );
+      }
+      const halted = await task.halt().then(
+        () => "ok",
+        (error: unknown) => `err:${describeError(error)}`,
+      );
+      // Give the OS a moment to release listeners before sampling them.
+      await sleep(100);
+      say("AFTER_HALT_BROKER_TCP", await portState(spec.brokerTcpPort));
+      say("AFTER_HALT_BROKER_WS", await portState(spec.brokerWsPort));
+      say("AFTER_HALT_API", await portState(spec.apiPort));
+      say("HALT_SETTLED", halted);
+    }
+  } finally {
+    if (brokerStartGateInstalled) {
+      // Nothing else runs after this, but keep the process from holding a
+      // half-started broker if the scenario bailed out early.
+    }
+    await database.close({ force: true }).catch(() => {});
+    if (apiBlocker) {
+      await new Promise<void>((resolve) => apiBlocker!.close(() => resolve()));
+    }
+  }
+
+  say("FIXTURE_DONE", spec.mode);
+}
+
+main().then(
+  () => process.exit(0),
+  (error: unknown) => {
+    console.error(
+      `FIXTURE_FAILED:${
+        error instanceof Error ? (error.stack ?? error.message) : String(error)
+      }`,
+    );
+    process.exit(1);
+  },
+);

package/test/lifecycle/fixtures/telemetry-lifecycle.ts

@@ -0,0 +1,51 @@
+/**
+ * Subprocess fixture for the telemetry lifecycle reproductions (spec 00031, G4).
+ *
+ * `init()` installs OpenTelemetry auto-instrumentation, which monkey-patches
+ * `http`, `pg` and friends process-wide. Running it inside the shared `bun test`
+ * process would leak that patching into every other suite, so each scenario gets
+ * its own process and communicates through stdout markers.
+ *
+ * Usage: `bun telemetry-lifecycle.ts <observe|failing-shutdown>`
+ */
+import { NodeSDK } from "@opentelemetry/sdk-node";
+import { ensure, run, suspend } from "effection";
+import { init } from "../../../src/main.ts";
+
+const mode = process.argv[2] ?? "observe";
+
+const realShutdown = NodeSDK.prototype.shutdown;
+NodeSDK.prototype.shutdown = function (this: NodeSDK): Promise<void> {
+  console.log("TELEMETRY_SHUTDOWN");
+  if (mode === "failing-shutdown") {
+    return Promise.reject(new Error("telemetry-shutdown-boom"));
+  }
+  return realShutdown.call(this);
+};
+
+const realConsoleError = console.error;
+console.error = (...args: unknown[]) => {
+  realConsoleError(...args);
+  const rendered = args.map((arg) =>
+    arg instanceof Error ? arg.message : String(arg)
+  ).join(" ");
+  if (rendered.includes("telemetry-shutdown-boom")) {
+    console.log("TELEMETRY_SHUTDOWN_REPORTED");
+  }
+};
+
+const task = run(function* () {
+  yield* init();
+  // Stands in for every resource the runtime acquires after init(): broker,
+  // HTTP, database, child tasks. Registered later, so it must tear down first.
+  yield* ensure(function* () {
+    console.log("SIBLING_CLEANUP");
+  });
+  yield* suspend();
+});
+
+const outcome = await task.halt().then(
+  () => "ok",
+  (error: unknown) => `err:${error instanceof Error ? error.message : String(error)}`,
+);
+console.log(`HALT_SETTLED:${outcome}`);

package/test/lifecycle/http-server-lifecycle.test.ts

@@ -0,0 +1,219 @@
+/**
+ * Reproductions for the HTTP-server half of the runtime resource lifecycle
+ * (spec 00031: G1, G9, G10).
+ *
+ * `startHttpServer` is driven directly — it is the unit that owns the API port,
+ * the Fastify instance and every plugin the host registered through
+ * `apiRouter` — so each defect is observable without booting a database.
+ */
+import { afterEach, expect, test } from "bun:test";
+import pg from "pg";
+import { run, type Task } from "effection";
+import { withEffectstreamStaticConfig } from "@effectstream/config";
+import {
+  acquireDBMutex,
+  releaseDBMutex,
+  waitUntilFree,
+} from "@effectstream/db";
+import type { FastifyInstance } from "fastify";
+import { startHttpServer } from "../../src/api/http-server.ts";
+import type { StartConfigApiRouter } from "../../src/types.ts";
+import {
+  closeTcp,
+  connectTcp,
+  freePortAbove10000,
+  listenTcp,
+  restoreEnv,
+  saveEnv,
+  settle,
+  sleep,
+  tcpRebindable,
+  waitUntil,
+  withDeadline,
+} from "./support.ts";
+
+const ENV_KEYS = [
+  "EFFECTSTREAM_API_PORT",
+  "ENABLE_DEV_AND_DEBUG_ENDPOINTS",
+  "PGLITE",
+] as const;
+
+const STATIC_CONFIG = {
+  securityNamespace: "runtime-lifecycle",
+  allNetworks: { viemNetworks: {} },
+};
+
+/**
+ * A pool that is never queried. Every route that touches the database is only
+ * *registered* by `startHttpServer`; these tests never invoke one, so no
+ * connection is ever opened.
+ */
+function idlePool(): pg.Pool {
+  return new pg.Pool({
+    host: "127.0.0.1",
+    port: 1,
+    user: "unused",
+    database: "unused",
+    max: 1,
+  });
+}
+
+type Owned = {
+  env: ReturnType<typeof saveEnv>;
+  pools: pg.Pool[];
+  tasks: Task<unknown>[];
+  servers: Awaited<ReturnType<typeof listenTcp>>[];
+  sockets: Awaited<ReturnType<typeof connectTcp>>[];
+};
+
+const owned: Owned = {
+  env: saveEnv(ENV_KEYS),
+  pools: [],
+  tasks: [],
+  servers: [],
+  sockets: [],
+};
+
+function startServer(
+  pool: pg.Pool,
+  apiRouter?: StartConfigApiRouter,
+): Task<unknown> {
+  const task = run(function* () {
+    yield* withEffectstreamStaticConfig(STATIC_CONFIG, function* () {
+      yield* startHttpServer(pool, [], 60_000, apiRouter);
+    });
+  });
+  owned.tasks.push(task);
+  // start() rejects on a bind conflict; the tests that expect it observe the
+  // task themselves, this only keeps the rejection from going unhandled.
+  Promise.resolve(task).catch(() => {});
+  return task;
+}
+
+afterEach(async () => {
+  for (const socket of owned.sockets.splice(0)) socket.destroy();
+  for (const task of owned.tasks.splice(0)) {
+    // Bounded: on the unfixed runtime a halt can block behind an in-flight
+    // request forever, and cleanup must not take the whole file down with it.
+    await Promise.race([task.halt().catch(() => {}), sleep(2_000)]);
+  }
+  for (const server of owned.servers.splice(0)) await closeTcp(server);
+  for (const pool of owned.pools.splice(0)) await pool.end().catch(() => {});
+  // The DB mutex is a module global shared by every suite in this process. A
+  // reproduced leak must not be inherited by the next test file.
+  const mutex = waitUntilFree();
+  if (mutex.db_mutex === "locked") releaseDBMutex(mutex.running.name);
+  restoreEnv(owned.env);
+});
+
+test("G1: a rejected HTTP listen still runs the apiRouter's onClose hooks", async () => {
+  const port = await freePortAbove10000();
+  owned.servers.push(await listenTcp(port));
+  process.env.EFFECTSTREAM_API_PORT = String(port);
+
+  const closeHooks: string[] = [];
+  const apiRouter: StartConfigApiRouter = (server: FastifyInstance) => {
+    server.addHook("onClose", async () => {
+      closeHooks.push("apiRouter");
+    });
+    return Promise.resolve();
+  };
+
+  const pool = idlePool();
+  owned.pools.push(pool);
+  const outcome = await withDeadline(settle(startServer(pool, apiRouter)));
+
+  expect(outcome.ok).toBe(false);
+  expect((outcome as { error: NodeJS.ErrnoException }).error.code).toBe(
+    "EADDRINUSE",
+  );
+  // The Fastify instance was fully built (plugins registered, undici/hook
+  // state live) before listen failed. Skipping close on a failed bind leaks it
+  // and silently skips every host-registered teardown hook.
+  expect(closeHooks).toEqual(["apiRouter"]);
+});
+
+// Spec 00031 Acceptance 1 for the API port. This is an acceptance guard, not a
+// reproduction: it is already green on the PR base (see plan P1.4, G10a), and
+// it exists so a shutdown change cannot silently regress rebindability.
+test("G10a (acceptance): the API port rebinds immediately after shutdown, repeatedly, with a live keep-alive client", async () => {
+  const port = await freePortAbove10000();
+  process.env.EFFECTSTREAM_API_PORT = String(port);
+
+  for (let round = 0; round < 3; round++) {
+    const pool = idlePool();
+    owned.pools.push(pool);
+    const task = startServer(pool);
+    await waitUntil(
+      async () => (await fetch(`http://127.0.0.1:${port}/health`)
+        .then((r) => r.status === 200 || r.status === 503)
+        .catch(() => false)),
+      10_000,
+      `round ${round}: server listening`,
+    );
+
+    // A real keep-alive client: an idle HTTP/1.1 socket the kernel still holds
+    // open. `server.close()` alone waits for it, so shutdown must force it.
+    const keepAlive = await connectTcp(port);
+    owned.sockets.push(keepAlive);
+    keepAlive.write(
+      `GET /health HTTP/1.1\r\nHost: 127.0.0.1\r\nConnection: keep-alive\r\n\r\n`,
+    );
+    await new Promise<void>((resolve, reject) => {
+      keepAlive.once("data", () => resolve());
+      keepAlive.once("error", reject);
+    });
+
+    await withDeadline(task.halt(), 5_000);
+    expect(await tcpRebindable(port)).toBe(true);
+  }
+}, 30_000);
+
+test("G9: a per-request DB-mutex task does not outlive the HTTP server's scope", async () => {
+  const port = await freePortAbove10000();
+  process.env.EFFECTSTREAM_API_PORT = String(port);
+  process.env.ENABLE_DEV_AND_DEBUG_ENDPOINTS = "true";
+  // The mutex is a no-op unless PGLITE is on; the route exists to serialize
+  // PGLite's single WASM backend, so that is the configuration under test.
+  process.env.PGLITE = "true";
+
+  const pool = idlePool();
+  owned.pools.push(pool);
+  const task = startServer(pool);
+  await waitUntil(
+    async () => (await fetch(`http://127.0.0.1:${port}/health`)
+      .then((r) => r.status === 200 || r.status === 503)
+      .catch(() => false)),
+    10_000,
+    "server listening",
+  );
+
+  // Hold the mutex so the request's `run(() => acquireDBMutex(...))` cannot
+  // settle: it is now a detached Effection task with a pending acquisition.
+  await run(() => acquireDBMutex("test-holder"));
+  const request = settle(
+    fetch(`http://127.0.0.1:${port}/db_acquire_lock?name=ghost`),
+  );
+  await waitUntil(
+    () => waitUntilFree().waiting.some((w) => w.name === "http-server:ghost"),
+    5_000,
+    "request task queued on the mutex",
+  );
+
+  // Shut the runtime's HTTP scope down while that request is in flight. The
+  // request is only unblockable by cancellation, so a scope that cannot reach
+  // its own in-flight work never settles.
+  const halted = await settle(withDeadline(task.halt(), 3_000));
+
+  // Release unconditionally so the request can drain on either code path.
+  releaseDBMutex("test-holder");
+  await settle(withDeadline(request, 3_000));
+  await sleep(150);
+
+  expect(halted.ok).toBe(true);
+  // Nothing the runtime owned is running any more, so the mutex the test
+  // released must still be free. On the unfixed runtime the detached task
+  // survives the halt and grabs it — a post-shutdown side effect.
+  expect(waitUntilFree().running.name).toBe("");
+  expect(waitUntilFree().db_mutex).toBe("free");
+}, 20_000);

package/test/lifecycle/process-blocks-lifecycle.test.ts

@@ -0,0 +1,68 @@
+/**
+ * Reproduction for the block-processing checkout boundary (spec 00031: G7).
+ *
+ * `processFinalizedBlockWithRetry` is driven directly against a stub pool: the
+ * defect is entirely in the window between `pool.connect()` being issued and
+ * its client being assigned, so no database is involved.
+ */
+import { afterEach, beforeEach, expect, test } from "bun:test";
+import type { Pool, PoolClient } from "pg";
+import { run } from "effection";
+import { processFinalizedBlockWithRetry } from "../../src/process-blocks.ts";
+import type { ChainBlock } from "@effectstream/sync";
+import type { StartConfig } from "../../src/types.ts";
+import { restoreEnv, saveEnv, settle, sleep, waitUntil } from "./support.ts";
+
+let env: ReturnType<typeof saveEnv>;
+
+beforeEach(() => {
+  env = saveEnv(["PGLITE"]);
+  // `PGLITE` defaults to true, which turns the per-block mutex on. This test is
+  // about the pool checkout, so keep it out of the mutex's way entirely.
+  process.env.PGLITE = "false";
+});
+
+afterEach(() => restoreEnv(env));
+
+test("G7: halting while the pool checkout is in flight still returns the client", async () => {
+  const released: unknown[] = [];
+  let resolveConnect: ((client: PoolClient) => void) | undefined;
+
+  const pool = {
+    connect: () =>
+      new Promise<PoolClient>((resolve) => {
+        resolveConnect = resolve;
+      }),
+  } as unknown as Pool;
+
+  const task = run(() =>
+    processFinalizedBlockWithRetry(
+      { blockNumber: 1 } as unknown as ChainBlock,
+      {} as StartConfig,
+      pool,
+      null,
+    )
+  );
+  // The operation is cancelled, so its own promise settles as halted.
+  void settle(Promise.resolve(task));
+
+  await waitUntil(
+    () => resolveConnect !== undefined,
+    5_000,
+    "pool.connect() issued",
+  );
+  await task.halt();
+
+  // The pool hands the client over after the halt — exactly what happens when
+  // a real checkout wins the race with cancellation. Nobody is left to return
+  // it, so the pool is permanently one client short and `pool.end()` hangs.
+  const client = {
+    release: (error?: Error) => {
+      released.push(error ?? null);
+    },
+  } as unknown as PoolClient;
+  resolveConnect!(client);
+  await sleep(200);
+
+  expect(released.length).toBe(1);
+}, 20_000);

package/test/lifecycle/runtime-lifecycle.test.ts

@@ -0,0 +1,128 @@
+/**
+ * Reproductions for the runtime-level resource boundaries in `src/main.ts`
+ * (spec 00031: G2, G3, G5, G8, G10b).
+ *
+ * Every scenario boots the real `start()` in its own subprocess — see
+ * `fixtures/runtime-lifecycle.ts` — so these tests only allocate ports, launch
+ * the scenarios once, and assert on the markers each one printed.
+ */
+import { beforeAll, expect, test } from "bun:test";
+import { freeDistinctPorts } from "./support.ts";
+
+const FIXTURE = new URL("./fixtures/runtime-lifecycle.ts", import.meta.url)
+  .pathname;
+const CWD = new URL("../../../../..", import.meta.url).pathname;
+
+type Mode =
+  | "broker-cancel-during-start"
+  | "broker-shutdown-error"
+  | "child-failure-with-pool-end-error";
+
+const MODES: Mode[] = [
+  "broker-cancel-during-start",
+  "broker-shutdown-error",
+  "child-failure-with-pool-end-error",
+];
+
+const outputs = new Map<Mode, string>();
+
+async function runScenario(mode: Mode, ports: number[]): Promise<string> {
+  const [apiPort, pglitePort, brokerTcpPort, brokerWsPort] = ports;
+  const spec = JSON.stringify({
+    mode,
+    apiPort,
+    pglitePort,
+    brokerTcpPort,
+    brokerWsPort,
+  });
+  const child = Bun.spawn([process.execPath, FIXTURE, spec], {
+    cwd: CWD,
+    stdout: "pipe",
+    stderr: "pipe",
+  });
+  const [stdout, stderr, exitCode] = await Promise.all([
+    new Response(child.stdout).text(),
+    new Response(child.stderr).text(),
+    child.exited,
+  ]);
+  const output = `${stdout}\n${stderr}`;
+  expect(exitCode, `${mode} fixture failed:\n${output}`).toBe(0);
+  expect(output, `${mode} fixture did not finish:\n${output}`).toContain(
+    `FIXTURE_DONE:${mode}`,
+  );
+  return output;
+}
+
+/** Read a `MARKER:value` line the fixture printed. */
+function marker(mode: Mode, name: string): string {
+  const output = outputs.get(mode) ?? "";
+  const match = output.match(new RegExp(`^${name}:(.*)$`, "m"));
+  if (!match) {
+    throw new Error(`marker ${name} missing from ${mode} output:\n${output}`);
+  }
+  return match[1].trim();
+}
+
+beforeAll(async () => {
+  // Allocate every port in one sequential pass BEFORE launching anything:
+  // concurrent allocation can hand the same OS-selected port to two scenarios,
+  // and a leaked listener in one would then be misread as the other's.
+  const ports = await freeDistinctPorts(4 * MODES.length);
+  // The scenarios are independent processes on disjoint ports, so run them
+  // concurrently: each one boots a database and the whole engine.
+  const results = await Promise.all(
+    MODES.map((mode, index) =>
+      runScenario(mode, ports.slice(index * 4, index * 4 + 4))
+    ),
+  );
+  MODES.forEach((mode, index) => outputs.set(mode, results[index]));
+}, 300_000);
+
+test("G2: cancelling start() while the broker is still starting still shuts the broker down", () => {
+  // The broker really bound both listeners before start() was cancelled.
+  expect(marker("broker-cancel-during-start", "BROKER_TCP_BOUND")).toBe("yes");
+  expect(marker("broker-cancel-during-start", "AFTER_HALT_BROKER_TCP")).toBe(
+    "free",
+  );
+  expect(marker("broker-cancel-during-start", "AFTER_HALT_BROKER_WS")).toBe(
+    "free",
+  );
+});
+
+test("G10b: the broker's ports are immediately rebindable after a cancelled start", () => {
+  // Spec Acceptance 1 for the broker half: a leaked listener is exactly what
+  // makes a second run of the same node fail on a fixed port.
+  const tcp = marker("broker-cancel-during-start", "AFTER_HALT_BROKER_TCP");
+  const ws = marker("broker-cancel-during-start", "AFTER_HALT_BROKER_WS");
+  expect({ tcp, ws }).toEqual({ tcp: "free", ws: "free" });
+});
+
+test("G3: a broker shutdown failure is reported, not swallowed", () => {
+  expect(marker("broker-shutdown-error", "BROKER_TCP_BOUND")).toBe("yes");
+  // The broker released its ports and then failed; the runtime must not
+  // report that teardown as clean.
+  expect(marker("broker-shutdown-error", "AFTER_HALT_BROKER_TCP")).toBe("free");
+  expect(marker("broker-shutdown-error", "HALT_SETTLED")).toContain(
+    "broker-shutdown-boom",
+  );
+});
+
+test("G8: a dbConn.end() failure does not replace the primary error", () => {
+  // Scenario: the API port is occupied, so the spawned HTTP child fails at
+  // listen (the primary, causal failure), and the pool's end() then rejects.
+  const settled = marker("child-failure-with-pool-end-error", "TASK_SETTLED");
+  expect(settled).toContain("pool-end-boom");
+  expect(settled).toContain("EADDRINUSE");
+});
+
+test("G5: a failing child task's error survives the runtime's own cleanup", () => {
+  // Same scenario, read from the child-supervision side: the runtime has no
+  // boundary that retains a child's failure once a later frame cleanup throws,
+  // so the only error the host ever sees is the cleanup's.
+  const settled = marker("child-failure-with-pool-end-error", "TASK_SETTLED");
+  expect(settled).toContain("EADDRINUSE");
+  // ...and it must come first: the child failed before anything was cleaned up.
+  expect(settled.indexOf("EADDRINUSE")).toBeLessThan(
+    settled.indexOf("pool-end-boom"),
+  );
+});

package/test/lifecycle/support.ts

@@ -0,0 +1,153 @@
+/**
+ * Shared helpers for the runtime resource-lifecycle reproductions.
+ *
+ * Every bind in this suite uses an OS-selected port above 10000 (never the
+ * 9999/8883/8884 product defaults) so the suite is safe on a shared machine and
+ * can run concurrently with anything else.
+ */
+import net, { type AddressInfo, type Server, type Socket } from "node:net";
+
+/** Grab an OS-selected free TCP port above 10000. */
+export async function freePortAbove10000(): Promise<number> {
+  for (let attempt = 0; attempt < 40; attempt++) {
+    const server = net.createServer();
+    await new Promise<void>((resolve, reject) => {
+      server.once("error", reject);
+      server.listen(0, "127.0.0.1", resolve);
+    });
+    const port = (server.address() as AddressInfo).port;
+    await closeTcp(server);
+    if (port > 10_000) return port;
+  }
+  throw new Error("Unable to acquire an OS-selected port above 10000");
+}
+
+/** N distinct free ports above 10000. */
+export async function freeDistinctPorts(count: number): Promise<number[]> {
+  const ports = new Set<number>();
+  while (ports.size < count) ports.add(await freePortAbove10000());
+  return [...ports];
+}
+
+/**
+ * Occupy `port`. Defaults to the wildcard address because the runtime's HTTP
+ * server binds `0.0.0.0`: on BSD/macOS a loopback-only holder does NOT conflict
+ * with a wildcard bind, so a `127.0.0.1` blocker would silently fail to block.
+ */
+export async function listenTcp(
+  port: number,
+  host = "0.0.0.0",
+): Promise<Server> {
+  const server = net.createServer();
+  await new Promise<void>((resolve, reject) => {
+    server.once("error", reject);
+    server.listen(port, host, resolve);
+  });
+  return server;
+}
+
+export async function closeTcp(server?: Server): Promise<void> {
+  if (!server?.listening) return;
+  await new Promise<void>((resolve) => server.close(() => resolve()));
+}
+
+/** True when `port` can be bound again right now. */
+export async function tcpRebindable(
+  port: number,
+  host = "0.0.0.0",
+): Promise<boolean> {
+  try {
+    const server = await listenTcp(port, host);
+    await closeTcp(server);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/** True once a TCP connection to `port` succeeds; never rejects. */
+export function tcpConnects(port: number): Promise<boolean> {
+  return new Promise((resolve) => {
+    const sock = net.connect(port, "127.0.0.1");
+    sock.once("connect", () => {
+      sock.destroy();
+      resolve(true);
+    });
+    sock.once("error", () => {
+      sock.destroy();
+      resolve(false);
+    });
+  });
+}
+
+export async function connectTcp(port: number): Promise<Socket> {
+  const socket = net.connect({ port, host: "127.0.0.1" });
+  await new Promise<void>((resolve, reject) => {
+    socket.once("connect", resolve);
+    socket.once("error", reject);
+  });
+  return socket;
+}
+
+export function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+/** Poll `check` until it is true, or throw after `ms`. */
+export async function waitUntil(
+  check: () => boolean | Promise<boolean>,
+  ms = 5_000,
+  label = "condition",
+): Promise<void> {
+  const end = Date.now() + ms;
+  for (;;) {
+    if (await check()) return;
+    if (Date.now() >= end) throw new Error(`${label} not met within ${ms}ms`);
+    await sleep(5);
+  }
+}
+
+/** Resolve `promise`, or throw once `ms` has elapsed. */
+export async function withDeadline<T>(
+  promise: PromiseLike<T>,
+  ms = 10_000,
+): Promise<T> {
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  try {
+    return await Promise.race([
+      Promise.resolve(promise),
+      new Promise<never>((_, reject) => {
+        timer = setTimeout(
+          () => reject(new Error(`deadline exceeded after ${ms}ms`)),
+          ms,
+        );
+      }),
+    ]);
+  } finally {
+    if (timer !== undefined) clearTimeout(timer);
+  }
+}
+
+/** Settle a promise into `{ ok }` / `{ ok: false, error }` without throwing. */
+export async function settle<T>(
+  promise: PromiseLike<T>,
+): Promise<{ ok: true; value: T } | { ok: false; error: unknown }> {
+  try {
+    return { ok: true, value: await promise };
+  } catch (error) {
+    return { ok: false, error };
+  }
+}
+
+export type EnvSnapshot = Record<string, string | undefined>;
+
+export function saveEnv(keys: readonly string[]): EnvSnapshot {
+  return Object.fromEntries(keys.map((key) => [key, process.env[key]]));
+}
+
+export function restoreEnv(snapshot: EnvSnapshot): void {
+  for (const [key, value] of Object.entries(snapshot)) {
+    if (value === undefined) delete process.env[key];
+    else process.env[key] = value;
+  }
+}

package/test/lifecycle/telemetry-lifecycle.test.ts

@@ -0,0 +1,61 @@
+/**
+ * Reproductions for the telemetry half of the runtime resource lifecycle
+ * (spec 00031: G4, plus the spec's "a telemetry failure does not skip broker,
+ * HTTP, database, or child-task cleanup" edge case).
+ *
+ * Each scenario runs in its own subprocess — see `fixtures/telemetry-lifecycle.ts`
+ * for why — and is asserted from the marker order on its stdout.
+ */
+import { expect, test } from "bun:test";
+
+const FIXTURE = new URL("./fixtures/telemetry-lifecycle.ts", import.meta.url)
+  .pathname;
+
+async function runFixture(mode: string): Promise<string> {
+  const child = Bun.spawn([process.execPath, FIXTURE, mode], {
+    cwd: new URL("../../../../..", import.meta.url).pathname,
+    stdout: "pipe",
+    stderr: "pipe",
+  });
+  const [stdout, stderr, exitCode] = await Promise.all([
+    new Response(child.stdout).text(),
+    new Response(child.stderr).text(),
+    child.exited,
+  ]);
+  const output = `${stdout}\n${stderr}`;
+  expect(exitCode, output).toBe(0);
+  return output;
+}
+
+test("G4: halting the scope that ran init() shuts the OpenTelemetry SDK down", async () => {
+  const output = await runFixture("observe");
+
+  expect(output).toContain("SIBLING_CLEANUP");
+  expect(output).toContain("HALT_SETTLED:ok");
+  // `init()` starts a NodeSDK (periodic metric reader, batch span/log
+  // processors, auto-instrumentation) and never tears it down.
+  expect(output).toContain("TELEMETRY_SHUTDOWN");
+  // Telemetry is acquired first, so it must be released last: everything it
+  // instruments has to be able to emit while it is still shutting down.
+  expect(output.indexOf("SIBLING_CLEANUP")).toBeLessThan(
+    output.indexOf("TELEMETRY_SHUTDOWN"),
+  );
+}, 60_000);
+
+test("G4: a failing telemetry shutdown neither skips other cleanup nor disappears", async () => {
+  const output = await runFixture("failing-shutdown");
+
+  // Spec edge case: the sibling resource is torn down regardless, and it is
+  // torn down first — telemetry is released last.
+  expect(output).toContain("SIBLING_CLEANUP");
+  expect(output).toContain("TELEMETRY_SHUTDOWN");
+  expect(output.indexOf("SIBLING_CLEANUP")).toBeLessThan(
+    output.indexOf("TELEMETRY_SHUTDOWN"),
+  );
+  // The failure is reported rather than swallowed silently...
+  expect(output).toContain("TELEMETRY_SHUTDOWN_REPORTED");
+  // ...but a lost telemetry flush does not turn an otherwise clean shutdown
+  // into a failed one: `shutdown()` also rejects when the collector is simply
+  // unreachable, and every resource that matters is already released.
+  expect(output).toContain("HALT_SETTLED:ok");
+}, 60_000);