@cosmicdrift/kumiko-testing 0.367.0 → 0.369.0

package/src/stack/test-stack.ts

@@ -0,0 +1,715 @@
+// @runtime dev
+import type { AuthRoutesConfig, JwtHelper } from "@cosmicdrift/kumiko-framework/api";
+import { buildServer, createSseBroker, type SseBroker } from "@cosmicdrift/kumiko-framework/api";
+import { createRegistry, validateBoot } from "@cosmicdrift/kumiko-framework/engine";
+import type {
+  AppContext,
+  FeatureDefinition,
+  JobRunIn,
+  Registry,
+  TenantId,
+} from "@cosmicdrift/kumiko-framework/engine/types";
+import type { ErrorDocsConfig } from "@cosmicdrift/kumiko-framework/errors";
+import {
+  createArchivedStreamsTable,
+  createEventsTable,
+} from "@cosmicdrift/kumiko-framework/event-store";
+import type { Lifecycle } from "@cosmicdrift/kumiko-framework/internal/stack-support";
+import {
+  createRateLimitResolver,
+  unsafePushTables,
+  validateNavBootSubset,
+  validateOwnershipBoot,
+} from "@cosmicdrift/kumiko-framework/internal/stack-support";
+import {
+  createJobRunner,
+  type JobRunner,
+  type JobRunnerOptions,
+} from "@cosmicdrift/kumiko-framework/jobs";
+import {
+  createNoopProvider,
+  type ObservabilityProvider,
+} from "@cosmicdrift/kumiko-framework/observability";
+import type {
+  Dispatcher,
+  EventDispatcher,
+  SystemHooks,
+} from "@cosmicdrift/kumiko-framework/pipeline";
+import {
+  createEntityCache,
+  createEventDedup,
+  createIdempotencyGuard,
+  dispatcherToWriteRef,
+} from "@cosmicdrift/kumiko-framework/pipeline";
+import {
+  type ClosableCacheSyncBus,
+  createLocalCacheSyncBus,
+  createRedisCacheSyncBus,
+} from "@cosmicdrift/kumiko-framework/redis";
+import type { SearchAdapter } from "@cosmicdrift/kumiko-framework/search";
+import { createInMemorySearchAdapter } from "@cosmicdrift/kumiko-framework/search";
+import { propertyOf } from "@cosmicdrift/kumiko-types/type-guards";
+import type { Hono } from "hono";
+import { createTestDb } from "./db.js";
+import { createJobFailureTracker, drainJobs } from "./drain-jobs.js";
+import { createEventCollector, type EventCollector } from "./event-collector.js";
+import { createTestRedis, queueNamePrefixForTestRedis, type TestRedis } from "./redis.js";
+import { createRequestHelper, type RequestHelper } from "./request-helper.js";
+
+export type TestStack = {
+  app: Hono;
+  jwt: JwtHelper;
+  registry: Registry;
+  db: import("@cosmicdrift/kumiko-framework/db").DbConnection;
+  redis: TestRedis;
+  // The bus this stack publishes/subscribes on; pass to TierResolverPlugin.build
+  // so a tier-engine resolver built outside buildServer joins the same sync.
+  cacheSync: ClosableCacheSyncBus;
+  search: SearchAdapter;
+  events: EventCollector;
+  http: RequestHelper;
+  observability: ObservabilityProvider;
+  // In-memory broker backing the server's SSE routes + access-invalidation
+  // channel. Tests subscribe directly (e.g. sseBroker.subscribeAccessInvalidation)
+  // to assert a consumer pushed an invalidation without opening a real SSE
+  // connection.
+  sseBroker: SseBroker;
+  // Command-dispatcher behind the HTTP routes — for direct system-writes in
+  // tests and behind entry:"signature" extraRoutes / the `wire` hook
+  // (provider-webhook wiring).
+  dispatcher: Dispatcher;
+  // The AppContext buildServer handed the request path, incl. the fields it
+  // wires itself (_fileProviderResolver). A dev-server that starts its own
+  // lane job-runners beside this stack must hand them THIS, not a
+  // `{ db, registry }` literal — that's the #1232 drift, and it makes an
+  // event-triggered job reaching for ctx.files die where the request path works.
+  context: AppContext;
+  // Present whenever a system consumer (SSE, Search) or
+  // r.multiStreamProjection is wired. Tests drain it via runOnce() (or the
+  // drainEventConsumers helper for a named-consumer backlog) for
+  // deterministic assertion — no timer-induced flakiness.
+  eventDispatcher?: EventDispatcher;
+  // Only set when the caller passed `lifecycle` via options. Tests that
+  // exercise drain() / /health/ready wire one in; ordinary suites ignore it.
+  lifecycle?: Lifecycle;
+  // Only set when `options.jobs` is truthy AND ≥1 `r.job(...)` is
+  // registered in the mounted features. Lets integration tests call
+  // `stack.jobRunner.dispatch(...)` directly, mirroring `ctx.jobRunner`.
+  jobRunner?: JobRunner;
+  // Waits for every in-flight job and its follow-up event-consumer /
+  // job-trigger cascade to settle — instead of `waitFor` polling on a
+  // specific job's side effect. Rejects with the failing job's name and
+  // error once a job exhausts its retries, or with a clear error when this
+  // stack has no jobRunner at all. Tracked failures accumulate since the
+  // PREVIOUS drainJobs() call (or setup, if none yet) — on a shared stack
+  // (e.g. beforeAll), a test that deliberately fails a job and never calls
+  // drainJobs() itself leaves that failure for the next test's drainJobs()
+  // call to reject on.
+  drainJobs: () => Promise<void>;
+  cleanup: () => Promise<void>;
+};
+
+export type TestStackOptions = {
+  features: readonly FeatureDefinition[];
+  /** System hooks to wire up. Default: all (sse, search) */
+  systemHooks?: ("sse" | "search")[];
+  /** Pipeline system hooks handed to buildServer, e.g. the cascade-delete preDelete hook. None by default. */
+  lifecycleSystemHooks?: SystemHooks;
+  /** Search config per tenant — defaults to tenant 1 with all text fields */
+  searchConfig?: {
+    tenantId: TenantId;
+    searchableFields: string[];
+    rankingFields: string[];
+  };
+  jwtSecret?: string;
+  /** Extra fields merged into the AppContext (e.g. _notifyFactory, configResolver).
+   *  Can be a function receiving (registry, db, sseBroker) for late binding. */
+  /** Boot default locale (BCP-47) merged into AppContext before extraContext. */
+  defaultLocale?: string;
+  extraContext?:
+    | Record<string, unknown>
+    | ((deps: {
+        registry: Registry;
+        db: import("@cosmicdrift/kumiko-framework/db").DbConnection;
+        sseBroker: import("@cosmicdrift/kumiko-framework/api").SseBroker;
+        redis: import("ioredis").Redis;
+      }) => Record<string, unknown>);
+  /** Wire up auth routes (login, tenant-switch). Leave undefined to skip. */
+  authConfig?: AuthRoutesConfig;
+  /** Register a file storage provider so uploads via POST /api/files work and
+   *  `ctx.files.ref(key)` is available to hooks/MSPs. Omit to skip — tests
+   *  without file handling don't need it. */
+  files?: { storageProvider: import("@cosmicdrift/kumiko-framework/files").FileStorageProvider };
+  /** Observability provider — omit for NoopProvider (no spans/metrics).
+   *  Pass a ConsoleProvider to see the span tree in stdout, or a custom
+   *  provider (e.g. a recording provider for assertions in tests). */
+  observability?: ObservabilityProvider;
+  /** Inject a process lifecycle so tests can drain() and observe
+   *  /health/ready flipping to 503. Omit if the suite doesn't care. */
+  lifecycle?: Lifecycle;
+  /** Wire L1 (global-IP) and/or L2 (auth-endpoint) rate-limit middleware.
+   *  The resolver is auto-built from the test Redis. Mirrors
+   *  buildServer's `rateLimit` option 1:1 — see there for shape. */
+  rateLimit?: import("@cosmicdrift/kumiko-framework/api").ServerOptions["rateLimit"];
+  /** Forwarded to buildServer — Integration-Tests that exercise `extraRoutes`
+   *  MUST go through here (real HTTP via `stack.http`/`stack.app.fetch`),
+   *  never `createTestDispatcher`. */
+  extraRoutes?: import("@cosmicdrift/kumiko-framework/api").ServerOptions["extraRoutes"];
+  /** Forwarded to buildServer like runProdApp. Without a PrometheusMeter-backed
+   *  `observability` the route answers 503, so spread `resolveObservabilityWiring(token)`
+   *  rather than setting `metrics` alone. */
+  metrics?: import("@cosmicdrift/kumiko-framework/api").ServerOptions["metrics"];
+  /** Inject a MasterKeyProvider for secrets-backed tests. Lands typed in
+   *  AppContext — set/delete/get + rotation job pick it up. Omit for
+   *  suites that don't touch secrets. */
+  masterKeyProvider?: import("@cosmicdrift/kumiko-framework/secrets").MasterKeyProvider;
+  /** Feature-toggle resolver. When present the dispatcher's feature-gate,
+   *  hook-filter, and MSP-filter all consult it; absent = every feature
+   *  treated as always-on. Pass the callback from
+   *  GlobalFeatureToggleRuntime.effectiveFeatures for real DB-backed
+   *  toggles, or a plain `() => new Set<string>(registry.features.keys())`
+   *  to force a specific snapshot in a unit-style setup. */
+  effectiveFeatures?: (tenantId: TenantId) => ReadonlySet<string>;
+  /** Pin the underlying Postgres DB name instead of the default
+   *  `kumiko_test_<8chars>`. Forwarded to createTestDb. Primary use
+   *  case: dev servers that want persistent storage across restarts —
+   *  combine with `persistentDb: true`. */
+  dbName?: string;
+  /** When true, cleanup() keeps the Postgres DB around — the caller
+   *  owns its lifecycle. Default false (test contract). Used by
+   *  dev-server wiring to survive hot-reloads. */
+  persistentDb?: boolean;
+  /** Cap on the main app pool (default: driver default). Lets tests saturate it deliberately. */
+  dbPoolMax?: number;
+  /** Size of the dedicated pool behind ctx.dbOutsideTransaction / ctx.outsideTransaction. */
+  outsideTransactionPoolMax?: number;
+  /** Forwarded to buildServer — when set, requests without a JWT pass
+   *  through as anonymous instead of 401. See AnonymousAccessConfig.
+   *  Accepts either a static config object OR a factory
+   *  `({registry, db, sseBroker, redis}) => Config`, the same pattern as
+   *  `extraContext`. The factory runs once at boot; the TenantResolver inside
+   *  typically closes over `db` for subdomain lookups. */
+  anonymousAccess?:
+    | import("@cosmicdrift/kumiko-framework/api").ServerOptions["anonymousAccess"]
+    | ((deps: {
+        registry: Registry;
+        db: import("@cosmicdrift/kumiko-framework/db").DbConnection;
+        sseBroker: import("@cosmicdrift/kumiko-framework/api").SseBroker;
+        redis: import("ioredis").Redis;
+      }) => import("@cosmicdrift/kumiko-framework/api").ServerOptions["anonymousAccess"]);
+  /** Optional post-factory enricher (e.g. merge auth-foundation tenant
+   *  providers). Keeps framework free of a bundled-features dependency. */
+  enrichAnonymousAccess?: (
+    base: import("@cosmicdrift/kumiko-framework/api").ServerOptions["anonymousAccess"] | undefined,
+    deps: {
+      registry: Registry;
+      db: import("@cosmicdrift/kumiko-framework/db").DbConnection;
+    },
+  ) => Promise<
+    import("@cosmicdrift/kumiko-framework/api").ServerOptions["anonymousAccess"] | undefined
+  >;
+  /** Opt-in JobRunner wired into ctx.jobRunner and merged into
+   *  dispatcherOptions so event-triggered jobs enqueue on commit — mirrors
+   *  the prod entrypoint's `buildJobRunnerWithHook`. Unlike prod (which
+   *  always builds one), this only builds a runner when `registry.getAllJobs()`
+   *  is non-empty — a deliberate test-stack-only shortcut, not parity.
+   *  Default `undefined`. Pass `consumerLane` only when this test IS the
+   *  sole consumer (otherwise enqueuer-only, avoiding double-running
+   *  `runOnBoot`/cron jobs against a caller-owned consumer). */
+  jobs?: {
+    consumerLane?: JobRunIn;
+    /** Default: unique per stack, derived from its test Redis keyPrefix. Pass
+     *  the same value to every runner that has to share this stack's queues. */
+    queueNamePrefix?: string;
+    /** Source of active tenant ids for `perTenant: true` jobs. Omit when the
+     *  tenant feature is among `options.features` — the framework then
+     *  resolves tenants on its own via that feature's active-tenant-ids
+     *  query, same as prod. Only needed for suites that fan a job over
+     *  tenants without mounting the tenant feature. */
+    getActiveTenantIds?: JobRunnerOptions["getActiveTenantIds"];
+    /** Run-log callbacks (job_runs / tenant job failures), built once registry
+     *  and db exist. The framework can't import the bundled jobs feature, so the
+     *  dev-server wrapper supplies prod's `jobRunLoggerCallbacks` here. They are
+     *  awaited before drainJobs() observes the job's outcome. */
+    runLogger?: (deps: {
+      registry: Registry;
+      db: import("@cosmicdrift/kumiko-framework/db").DbConnection;
+    }) =>
+      | Pick<
+          JobRunnerOptions,
+          "onJobStart" | "onJobComplete" | "onJobFailed" | "onJobQueued" | "onJobDropped"
+        >
+      | undefined;
+  };
+  /** Override the event dispatcher's polling-timer interval. Default 50ms.
+   *  Tests that assert LISTEN/NOTIFY wake-up latency need this pushed far
+   *  out (e.g. 60_000) so the polling timer can't land inside the
+   *  assertion window and mask a dead subscription — see E.4 (#2042). */
+  eventDispatcherPollIntervalMs?: number;
+  /** Opt out of auto-created r.entity() tables for callers that create/migrate
+   *  them themselves. Default true — false pushes only projection/MSP/
+   *  storeTable sources, same as before fw#3102. */
+  entityTables?: boolean;
+  /** Time source of the stack's rate-limit resolver, so tests can freeze the
+   *  clock instead of racing the window. Default: system time. */
+  rateLimitNowMs?: () => number;
+  /** Forwarded to buildServer's top-level `ServerOptions.trustedProxyHops`
+   *  — see there. Default 0. */
+  trustedProxyHops?: number;
+  /** Links the app's own error reasons to its own docs (`docsUrl` in error responses); framework reasons keep the framework docs link. Passed to buildServer's `ServerOptions.errorDocs`. */
+  errorDocs?: ErrorDocsConfig;
+  /** Second stack on the Redis namespace of `owner`: own connection, same
+   *  keyPrefix, so rate-limit buckets, locks, idempotency keys and the
+   *  cache-sync channel are shared across both. The derived
+   *  queueNamePrefix is identical too, so both stacks share BullMQ queues.
+   *  Cleanup only disconnects; the owner flushes the namespace and queues, so
+   *  clean up the borrowing stack first. The DB is NOT shared — pass the same
+   *  `dbName` + `persistentDb` for that. */
+  sharedRedisWith?: TestStack;
+  /** Publish/receive cache invalidations (tenant timezone, file providers,
+   *  tier sets) over Redis Pub/Sub, namespaced by this stack's keyPrefix.
+   *  Implied by `sharedRedisWith`; set it on the owner stack too, or its writes
+   *  never reach the borrower. Without it the stack uses a process-local bus. */
+  cacheSync?: boolean;
+  /** "full" runs the prod-boot validateBoot over `features` (screens and ref
+   *  entities across features included). Use it with the app's complete feature
+   *  composition; the default only checks what is safe on a feature subset. */
+  validateBoot?: "full";
+};
+
+const DEFAULT_JWT_SECRET = "test-stack-secret-minimum-32-characters!!";
+
+export async function setupTestStack(options: TestStackOptions): Promise<TestStack> {
+  const jwtSecret = options.jwtSecret ?? DEFAULT_JWT_SECRET;
+  const enabledHooks = options.systemHooks ?? ["sse", "search"];
+
+  // Install the Temporal polyfill before feature code runs. Idempotent: the
+  // production server boot does the same. A no-op on runtimes with native Temporal.
+  const { ensureTemporalPolyfill } = await import("@cosmicdrift/kumiko-framework/time");
+  await ensureTemporalPolyfill();
+
+  // Ownership boot-guards (fw#2639 unqualified where-fragments, fw#2626
+  // where-rules on the write path) otherwise run only in the prod boot via
+  // validateBoot, leaving every test stack blind to a broken access map.
+  // Runs after the polyfill (buildEntityTable needs it to derive the probe
+  // columns) and before the ephemeral DB/Redis exist, so a throw can't leak
+  // a database that nothing will clean up.
+  validateOwnershipBoot(options.features);
+  // Nav/workspace refs fail the prod boot too; the subset check tolerates refs into unmounted features.
+  validateNavBootSubset(options.features);
+  if (options.validateBoot === "full") validateBoot(options.features);
+
+  // Forward db-name/persistent-flag through to createTestDb. The
+  // defaults (undefined dbName, persistent:false) keep the legacy
+  // test contract: fresh kumiko_test_<random> DB per setup, dropped
+  // on cleanup.
+  const [testDb, testRedis] = await Promise.all([
+    createTestDb({
+      ...(options.dbName !== undefined && { dbName: options.dbName }),
+      ...(options.persistentDb !== undefined && { persistent: options.persistentDb }),
+      ...(options.dbPoolMax !== undefined && { poolMax: options.dbPoolMax }),
+      ...(options.outsideTransactionPoolMax !== undefined && {
+        outsideTransactionPoolMax: options.outsideTransactionPoolMax,
+      }),
+    }),
+    createTestRedis(
+      options.sharedRedisWith && { borrowKeyPrefix: options.sharedRedisWith.redis.keyPrefix },
+    ),
+  ]);
+
+  // Any throw in this DDL region must still drop the just-created ephemeral
+  // DB/Redis — the caller never gets the stack object back to call
+  // cleanup() itself (same "leaked ephemeral DB" failure class as the later
+  // try/catch below, but for setup that runs before that try starts).
+  try {
+    // Every ES-entity writes events via createEventStoreExecutor in the
+    // feature's write handlers. Auto-create the events table so every
+    // setupTestStack call is ready for writes without needing a manual
+    // createEventsTable().
+    await createEventsTable(testDb.db);
+    // Archive-stream metadata — needed by ctx.appendEvent's archive guard and
+    // loadAggregate's default-skip. Idempotent, so production boot running
+    // the same call is fine.
+    await createArchivedStreamsTable(testDb.db);
+
+    // Framework state for projection rebuild/status + event-consumer cursors.
+    // Idempotent — production boot flows run the same calls.
+    const { createProjectionStateTable, createEventConsumerStateTable } = await import(
+      "@cosmicdrift/kumiko-framework/pipeline"
+    );
+    await createProjectionStateTable(testDb.db);
+    await createEventConsumerStateTable(testDb.db);
+
+    // Files support: when a provider is registered, the fileRefs table must
+    // exist before the first upload. Skipped when no provider — the table
+    // stays off tenant test DBs that never touch files.
+    if (options.files) {
+      const { fileRefsTable } = await import("@cosmicdrift/kumiko-framework/files");
+      await unsafePushTables(testDb.db, { fileRefsTable });
+    }
+
+    // Same table list as `kumiko schema generate` (collectTableMetas) — divergence
+    // was the #255 prod-crash and the #3102 missing r.entity() tables.
+    const { collectTableMetas } = await import("@cosmicdrift/kumiko-framework/db");
+    const { tableExists } = await import("@cosmicdrift/kumiko-framework/db");
+    let metas = collectTableMetas(options.features);
+    if (options.entityTables === false) {
+      const { enumerateFeatureTableSources } = await import("@cosmicdrift/kumiko-framework/db");
+      const { extractTableInfo } = await import("@cosmicdrift/kumiko-framework/db");
+      const nonEntityNames = new Set(
+        options.features.flatMap((f) =>
+          enumerateFeatureTableSources(f).map((s) => extractTableInfo(s.table).name),
+        ),
+      );
+      metas = metas.filter((m) => nonEntityNames.has(m.tableName));
+    }
+    const missing: Record<string, unknown> = {};
+    for (const meta of metas) {
+      if (await tableExists(testDb.db, `public.${meta.tableName}`)) continue;
+      missing[meta.tableName] = meta;
+    }
+    if (Object.keys(missing).length > 0) {
+      await unsafePushTables(testDb.db, missing);
+    }
+  } catch (error) {
+    try {
+      await testDb.cleanup();
+    } catch {
+      // ignore — `error` is the one that matters
+    }
+    try {
+      await testRedis.cleanup();
+    } catch {
+      // ignore — `error` is the one that matters
+    }
+    throw error;
+  }
+
+  // Redis only where two stacks can actually disagree (avoids two extra
+  // Pub/Sub connections); otherwise a local bus so invalidation still works
+  // inside this stack. Passing it explicitly keeps buildServer from using its
+  // REDIS_URL default (an unnamespaced channel).
+  const cacheSync: ClosableCacheSyncBus =
+    options.cacheSync || options.sharedRedisWith
+      ? createRedisCacheSyncBus({
+          redisUrl: testRedis.redisUrl,
+          channelPrefix: testRedis.keyPrefix,
+        })
+      : createLocalCacheSyncBus();
+
+  const searchAdapter = createInMemorySearchAdapter();
+  const events = createEventCollector();
+  const registry = createRegistry([...options.features]);
+
+  // Wire SSE broker with event collector — built early (not just before
+  // buildServer) because extraContext + the job context below both need it.
+  const sseBroker = createSseBroker();
+  sseBroker.addClient(
+    "tenant:00000000-0000-4000-8000-000000000001",
+    (event) => events.sse.push(event),
+    () => {},
+  );
+
+  const entityCache = createEntityCache(testRedis.redis, { ttlSeconds: 60 });
+
+  // A static `files.storageProvider` is wired as the per-tenant resolver — the
+  // framework test seam that doesn't require mounting config + file-foundation.
+  // (Bundled GDPR tests mount the real provider features instead.)
+  let fileProviderResolver:
+    | import("@cosmicdrift/kumiko-framework/files").FileProviderResolver
+    | undefined;
+  if (options.files) {
+    const provider = options.files.storageProvider;
+    fileProviderResolver = () => Promise.resolve(provider);
+  }
+
+  const observability = options.observability ?? createNoopProvider();
+
+  // Same AppContext buildServer() gets below (db/redis/searchAdapter/
+  // entityCache/masterKeyProvider/fileProviderResolver/extraContext) —
+  // shared so the job context can't drift from the request-path context
+  // (kumiko-framework#1232: a reduced `{ db, registry }` literal let jobs
+  // pass in tests while reaching for fields only prod's context has).
+  //
+  // effectiveFeatures deliberately stays OUT of appContext — prod never puts
+  // it there either (only dispatcherOptions.effectiveFeatures, consumed by
+  // the command-dispatcher — see buildJobRunnerWithHook in entrypoint/
+  // index.ts). Putting it here would reintroduce the exact test-vs-prod
+  // drift #1232 fixed: a job/handler reading `ctx.effectiveFeatures` would
+  // pass in tests and break in prod (kumiko-framework#1255).
+  const appContext = {
+    db: testDb.db,
+    redis: testRedis.redis,
+    searchAdapter,
+    entityCache,
+    registry,
+    ...(options.masterKeyProvider ? { masterKeyProvider: options.masterKeyProvider } : {}),
+    ...(fileProviderResolver ? { _fileProviderResolver: fileProviderResolver } : {}),
+    ...(options.defaultLocale !== undefined && { defaultLocale: options.defaultLocale }),
+    ...(typeof options.extraContext === "function"
+      ? options.extraContext({ registry, db: testDb.db, sseBroker, redis: testRedis.redis })
+      : options.extraContext),
+  };
+
+  // Built before buildServer() so it can be merged into dispatcherOptions
+  // (write handlers' `ctx.jobRunner.dispatch(...)` and the afterCommit
+  // event-trigger hook both need it present at dispatcher-construction
+  // time, same as the prod entrypoint's buildJobRunnerWithHook). Uses the
+  // test-stack's own ephemeral redis — no separate `redisUrl` seam needed.
+  //
+  // context = appContext + tracer/meter, mirroring the prod entrypoint's
+  // `contextWithObservability(options.context, observability)`.
+  let jobRunner: JobRunner | undefined;
+  // Fed to createJobRunner's onJobComplete/onJobFailed below so drainJobs()
+  // can tell a job's final outcome apart from an attempt that will retry —
+  // built before createJobRunner regardless of whether a runner ends up
+  // existing, so the returned drainJobs() closure always has one to read.
+  const jobFailureTracker = createJobFailureTracker();
+  if (options.jobs && registry.getAllJobs().size > 0) {
+    const runLogger = options.jobs.runLogger?.({ registry, db: testDb.db });
+    jobRunner = createJobRunner({
+      registry,
+      context: { ...appContext, tracer: observability.tracer, meter: observability.meter },
+      // The real REDIS_URL, not one rebuilt from `.options` — that lost
+      // password/username/tls/path (pr-review kumiko-framework #1036/2).
+      redisUrl: testRedis.redisUrl,
+      ...(options.jobs.consumerLane !== undefined && { consumerLane: options.jobs.consumerLane }),
+      queueNamePrefix:
+        options.jobs.queueNamePrefix ?? queueNamePrefixForTestRedis(testRedis.keyPrefix),
+      ...(options.jobs.getActiveTenantIds !== undefined && {
+        getActiveTenantIds: options.jobs.getActiveTenantIds,
+      }),
+      ...(runLogger?.onJobStart !== undefined && { onJobStart: runLogger.onJobStart }),
+      ...(runLogger?.onJobQueued !== undefined && { onJobQueued: runLogger.onJobQueued }),
+      ...(runLogger?.onJobDropped !== undefined && { onJobDropped: runLogger.onJobDropped }),
+      // Tracker fires in finally: drainJobs wakes on it, so the log row must
+      // be written first, and a throwing logger must not swallow the signal.
+      onJobComplete: async (jobName, jobId, ...rest) => {
+        try {
+          await runLogger?.onJobComplete?.(jobName, jobId, ...rest);
+        } finally {
+          jobFailureTracker.onJobComplete(jobName, jobId);
+        }
+      },
+      onJobFailed: async (jobName, jobId, error, ...rest) => {
+        try {
+          await runLogger?.onJobFailed?.(jobName, jobId, error, ...rest);
+        } finally {
+          jobFailureTracker.onJobFailed(jobName, jobId, error);
+        }
+      },
+    });
+  }
+
+  // From here on, any throw must stop a jobRunner that was already created
+  // above (createJobRunner() itself opens live BullMQ Queue/lock Redis
+  // connections, .start() adds a live worker) — otherwise a failing
+  // buildServer()/search-config/etc. leaks that connection and the test
+  // process hangs on exit (pr-review kumiko-framework #1036/1).
+  try {
+    if (jobRunner) await jobRunner.start();
+
+    // Auto-configure search for tenant 1 based on registry
+    if (enabledHooks.includes("search")) {
+      const searchableFields: string[] = [];
+      for (const feature of options.features) {
+        for (const [, entity] of Object.entries(feature.entities ?? {})) {
+          for (const [fieldName, field] of Object.entries(entity.fields)) {
+            if (field.type === "text" && field.searchable) {
+              searchableFields.push(fieldName);
+            }
+            if (field.type === "embedded") {
+              for (const [subName, subField] of Object.entries(field.schema)) {
+                if (subField.searchable) {
+                  searchableFields.push(`${fieldName}_${subName}`);
+                }
+              }
+            }
+          }
+        }
+      }
+
+      if (options.searchConfig) {
+        await searchAdapter.configure(options.searchConfig.tenantId, {
+          searchableFields: options.searchConfig.searchableFields,
+          rankingFields: options.searchConfig.rankingFields,
+        });
+      } else if (searchableFields.length > 0) {
+        await searchAdapter.configure("00000000-0000-4000-8000-000000000001", {
+          searchableFields,
+          rankingFields: searchableFields,
+        });
+      }
+    }
+
+    const idempotency = createIdempotencyGuard(testRedis.redis, { ttlSeconds: 60 });
+    const eventDedup = createEventDedup(testRedis.redis, { ttlSeconds: 60 });
+
+    // buildServer only auto-wires an L3 RateLimitResolver from
+    // `context.redis` — a degradation test that nulls `redis` via
+    // extraContext (to exercise a handler's own ctx.redis-down guard) also
+    // silently strips rate-limiting, even though every handler with a
+    // `rateLimit` declaration still requires a resolver at dispatch time.
+    // Since e7bf543ad, an IP-keyed bucket key is built unconditionally
+    // (falls back to a shared "unknown" IP rather than skipping), so the
+    // missing resolver now throws instead of merely skipping enforcement.
+    // Mirrors idempotency/eventDedup above: built from the test-stack's own
+    // always-real `testRedis.redis`, independent of whatever extraContext
+    // did to `appContext.redis` — an explicit `context.rateLimit` from
+    // extraContext still wins (same precedence buildServer documents).
+    const explicitRateLimit = propertyOf(appContext, "rateLimit");
+    const needsOwnRateLimitResolver =
+      explicitRateLimit === undefined &&
+      (appContext.redis === undefined || options.rateLimitNowMs !== undefined);
+    const rateLimitFallback = needsOwnRateLimitResolver
+      ? createRateLimitResolver({
+          // An app-supplied redis keeps buckets and app redis on the same instance.
+          redis: appContext.redis ?? testRedis.redis,
+          ...(options.rateLimitNowMs ? { nowMs: options.rateLimitNowMs } : {}),
+        })
+      : undefined;
+
+    const server = buildServer({
+      registry,
+      context: rateLimitFallback ? { ...appContext, rateLimit: rateLimitFallback } : appContext,
+      jwtSecret,
+      ...(options.observability ? { observability: options.observability } : {}),
+      ...(options.lifecycleSystemHooks && { systemHooks: options.lifecycleSystemHooks }),
+      dispatcherOptions: {
+        idempotency,
+        cacheSync,
+        outsideTransactionDb: testDb.outsideTransactionDb,
+        ...(options.effectiveFeatures && { effectiveFeatures: options.effectiveFeatures }),
+        ...(jobRunner && { jobRunner }),
+      },
+      eventDedup,
+      sseBroker,
+      ...(options.trustedProxyHops !== undefined && { trustedProxyHops: options.trustedProxyHops }),
+      ...(options.errorDocs && { errorDocs: options.errorDocs }),
+      ...(options.extraRoutes && { extraRoutes: options.extraRoutes }),
+      ...(options.metrics && { metrics: options.metrics }),
+      // Tests drive the dispatcher via stack.eventDispatcher.runOnce() for
+      // deterministic drains — no timer-induced flakiness. pollIntervalMs
+      // stays short anyway in case a test opts into `.start()`. pgClient
+      // plumbs through the LISTEN wake-up for tests that want to measure
+      // post-commit latency (Sprint E.4).
+      eventDispatcher: {
+        pollIntervalMs: options.eventDispatcherPollIntervalMs ?? 50,
+        pgClient: testDb.client,
+        systemConsumers: {
+          sse: enabledHooks.includes("sse"),
+          search: enabledHooks.includes("search"),
+          accessInvalidation: enabledHooks.includes("sse"),
+        },
+      },
+      // Default tests to no login rate-limiter so existing suites that loop
+      // over logins don't hit a 429 after 10 attempts. Suites specifically
+      // testing the limiter can override via authConfig.loginRateLimit.
+      ...(options.authConfig
+        ? {
+            auth: {
+              ...options.authConfig,
+              ...(options.authConfig.loginRateLimit === undefined ? { loginRateLimit: null } : {}),
+            },
+          }
+        : {}),
+      ...(options.lifecycle ? { lifecycle: options.lifecycle } : {}),
+      ...(options.rateLimit ? { rateLimit: options.rateLimit } : {}),
+      ...(await (async () => {
+        const baseAnon =
+          typeof options.anonymousAccess === "function"
+            ? options.anonymousAccess({
+                registry,
+                db: testDb.db,
+                sseBroker,
+                redis: testRedis.redis,
+              })
+            : options.anonymousAccess;
+        const resolvedAnon = options.enrichAnonymousAccess
+          ? await options.enrichAnonymousAccess(baseAnon, {
+              registry,
+              db: testDb.db,
+            })
+          : baseAnon;
+        return resolvedAnon ? { anonymousAccess: resolvedAnon } : {};
+      })()),
+    });
+
+    // Mirrors the prod entrypoints — without this, JobContext.write/writeAs/
+    // queryAs/queryAsMember throw on first use inside a test-stack job.
+    if (jobRunner) jobRunner.attachDispatcher(dispatcherToWriteRef(server.dispatcher));
+
+    const eventDispatcher: EventDispatcher | undefined = server.eventDispatcher;
+
+    // Pre-register consumer state rows so tests can call runOnce() directly
+    // without a preceding explicit start(). Timer fires at pollIntervalMs=50
+    // but each consumer runs at most one turn at a time — tests that drain via
+    // runOnce() remain deterministic. Tests that specifically exercise the
+    // timer loop call start() again (idempotent) after setup.
+    if (eventDispatcher) await eventDispatcher.ensureRegistered();
+
+    const http = createRequestHelper(server.app, server.jwt, {
+      ...(options.authConfig?.sessionCreator !== undefined && {
+        sessionCreator: options.authConfig.sessionCreator,
+      }),
+    });
+
+    return {
+      app: server.app,
+      jwt: server.jwt,
+      registry,
+      db: testDb.db,
+      redis: testRedis,
+      cacheSync,
+      search: searchAdapter,
+      events,
+      http,
+      observability: server.observability,
+      sseBroker,
+      dispatcher: server.dispatcher,
+      context: server.context,
+      ...(eventDispatcher ? { eventDispatcher } : {}),
+      ...(server.lifecycle ? { lifecycle: server.lifecycle } : {}),
+      ...(jobRunner ? { jobRunner } : {}),
+      drainJobs: () => drainJobs({ db: testDb.db, eventDispatcher, jobRunner }, jobFailureTracker),
+      cleanup: async () => {
+        if (jobRunner) await jobRunner.stop();
+        if (eventDispatcher) await eventDispatcher.stop();
+        await server.observability.shutdown();
+        await cacheSync.close();
+        await Promise.all([testDb.cleanup(), testRedis.cleanup()]);
+      },
+    };
+  } catch (error) {
+    // Best-effort — a broken jobRunner.stop()/cleanup() must never mask the
+    // real setup failure below. testDb/testRedis are created before this
+    // try even starts, so a throw anywhere in setup (buildServer,
+    // search-config, ...) would otherwise leak their open pg/ioredis
+    // sockets — the caller never gets the stack object back to call
+    // cleanup() itself, which is the same "hangs on exit" failure class
+    // this function exists to prevent.
+    if (jobRunner) {
+      try {
+        await jobRunner.stop();
+      } catch {
+        // ignore — `error` is the one that matters
+      }
+    }
+    try {
+      await cacheSync.close();
+    } catch {
+      // ignore — `error` is the one that matters
+    }
+    try {
+      await testDb.cleanup();
+    } catch {
+      // ignore — `error` is the one that matters
+    }
+    try {
+      await testRedis.cleanup();
+    } catch {
+      // ignore — `error` is the one that matters
+    }
+    throw error;
+  }
+}