@cosmicdrift/kumiko-framework 0.235.4 → 0.236.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cosmicdrift/kumiko-framework",
-  "version": "0.235.4",
+  "version": "0.236.0",
   "description": "Framework core — engine, pipeline, API, DB, and every other bit that makes Kumiko go.",
   "license": "BUSL-1.1",
   "author": "Marc Frost <marc@cosmicdriftgamestudio.com>",
@@ -198,7 +198,7 @@
     "./package.json": "./package.json"
   },
   "dependencies": {
-    "@cosmicdrift/kumiko-types": "0.235.4",
+    "@cosmicdrift/kumiko-types": "0.236.0",
     "bullmq": "^5.76.7",
     "bun-types": "^1.3.13",
     "hono": "^4.13.1",
@@ -214,7 +214,7 @@
     "zod": "^4.4.3"
   },
   "devDependencies": {
-    "@cosmicdrift/kumiko-dispatcher-live": "0.235.4",
+    "@cosmicdrift/kumiko-dispatcher-live": "0.236.0",
     "bun-types": "^1.3.13",
     "pino-pretty": "^13.1.3"
   },

package/src/api/__tests__/redis-sse-broker.integration.test.ts

@@ -0,0 +1,151 @@
+import { afterAll, afterEach, beforeAll, describe, expect, test } from "bun:test";
+import { createTestRedis, type TestRedis } from "../../stack";
+import { waitFor } from "../../testing/wait-for";
+import { generateId } from "../../utils";
+import { createRedisSseBroker, type RedisSseBroker } from "../redis-sse-broker";
+import type { SseEvent } from "../sse-broker";
+
+let testRedis: TestRedis;
+// keyPrefix on testRedis.redis does not apply to pub/sub channel names
+// (stack/redis.ts), so isolation from other parallel test files comes from
+// generateId()-suffixed channel/userId values below, not from the prefix.
+const brokers: RedisSseBroker[] = [];
+
+function trackedBroker(): RedisSseBroker {
+  const broker = createRedisSseBroker({ redisUrl: testRedis.redisUrl });
+  brokers.push(broker);
+  return broker;
+}
+
+beforeAll(async () => {
+  testRedis = await createTestRedis();
+});
+
+afterAll(async () => {
+  await testRedis.cleanup();
+});
+
+afterEach(async () => {
+  await Promise.all(brokers.splice(0).map((b) => b.close()));
+});
+
+describe("createRedisSseBroker", () => {
+  test("pushToChannel on one broker reaches a client registered on another (cross-pod fanout)", async () => {
+    const podA = trackedBroker();
+    const podB = trackedBroker();
+    const channel = `test-channel-${generateId()}`;
+    const received: SseEvent[] = [];
+
+    podA.addClient(
+      channel,
+      (event) => received.push(event),
+      () => {},
+    );
+
+    // Retry the publish inside the predicate, not just the wait: Pub/Sub
+    // does not buffer for a subscriber whose psubscribe ack hasn't landed
+    // yet (a one-time race right after broker construction, not
+    // reproducible once real traffic starts seconds after boot) — a single
+    // fire-and-check would be flaky rather than proving the real property.
+    await waitFor(() => {
+      podB.pushToChannel(channel, { type: "unit.updated", data: { id: "1" } });
+      return received.length >= 1;
+    });
+    expect(received[0]).toEqual({ type: "unit.updated", data: { id: "1" } });
+  });
+
+  test("publishAccessInvalidation on one broker fires subscribeAccessInvalidation listeners on another", async () => {
+    const podA = trackedBroker();
+    const podB = trackedBroker();
+    const userId = `user-${generateId()}`;
+    let invalidated = false;
+
+    podA.subscribeAccessInvalidation(userId, () => {
+      invalidated = true;
+    });
+
+    await waitFor(() => {
+      podB.publishAccessInvalidation(userId);
+      return invalidated;
+    });
+    expect(invalidated).toBe(true);
+  });
+
+  test("a client on channel A never receives an event published to channel B (no cross-tenant leak)", async () => {
+    const podA = trackedBroker();
+    const podB = trackedBroker();
+    const channelA = `tenant-a-${generateId()}`;
+    const channelB = `tenant-b-${generateId()}`;
+    const receivedOnA: SseEvent[] = [];
+    const receivedOnControl: SseEvent[] = [];
+
+    podA.addClient(
+      channelA,
+      (event) => receivedOnA.push(event),
+      () => {},
+    );
+    // Control channel proves the psubscribe pipe itself is alive — without
+    // this, a broken subscription would make the negative assertion below
+    // pass for the wrong reason (nothing ever arrives, on any channel).
+    podA.addClient(
+      channelB,
+      (event) => receivedOnControl.push(event),
+      () => {},
+    );
+
+    await waitFor(() => {
+      podB.pushToChannel(channelB, { type: "unit.updated", data: { id: "leak-probe" } });
+      return receivedOnControl.length >= 1;
+    });
+    expect(receivedOnA).toHaveLength(0);
+  });
+
+  test("subscribeAccessInvalidation for one user is never fired by an invalidation for another user", async () => {
+    const podA = trackedBroker();
+    const podB = trackedBroker();
+    const userA = `user-a-${generateId()}`;
+    const userB = `user-b-${generateId()}`;
+    let invalidatedA = false;
+    let invalidatedBControl = false;
+
+    podA.subscribeAccessInvalidation(userA, () => {
+      invalidatedA = true;
+    });
+    podA.subscribeAccessInvalidation(userB, () => {
+      invalidatedBControl = true;
+    });
+
+    await waitFor(() => {
+      podB.publishAccessInvalidation(userB);
+      return invalidatedBControl;
+    });
+    expect(invalidatedA).toBe(false);
+  });
+
+  test("a malformed message on the channel namespace is dropped, not thrown, and does not kill delivery", async () => {
+    const podA = trackedBroker();
+    const publisherRaw = new (await import("ioredis")).default(testRedis.redisUrl);
+    const channel = `test-channel-${generateId()}`;
+    const received: SseEvent[] = [];
+
+    podA.addClient(
+      channel,
+      (event) => received.push(event),
+      () => {},
+    );
+
+    try {
+      await waitFor(async () => {
+        // Not valid JSON — simulates a foreign/misbehaving publisher on the
+        // shared kumiko:sse:* namespace.
+        await publisherRaw.publish(`kumiko:sse:ch:${channel}`, "not-json{{{");
+        // Broker must still be alive: a real event right after must arrive.
+        podA.pushToChannel(channel, { type: "unit.updated", data: { id: "after-garbage" } });
+        return received.length >= 1;
+      });
+      expect(received[0]).toEqual({ type: "unit.updated", data: { id: "after-garbage" } });
+    } finally {
+      publisherRaw.disconnect();
+    }
+  });
+});

package/src/api/index.ts

@@ -41,6 +41,8 @@ export {
 export type { JwtHelper, JwtKeyring, JwtPayload } from "./jwt";
 export { createJwtHelper, loadJwtSecretOrKeyring } from "./jwt";
 export { patAllows, qnMatches } from "./pat-scope";
+export type { RedisSseBroker, RedisSseBrokerOptions } from "./redis-sse-broker";
+export { createRedisSseBroker } from "./redis-sse-broker";
 export { type RequestContextData, requestContext } from "./request-context";
 export { requestIdMiddleware } from "./request-id-middleware";
 export { createApiRoutes } from "./routes";

package/src/api/redis-sse-broker.ts

@@ -0,0 +1,89 @@
+import { createRedisPubSubSignal } from "../redis/pubsub-signal";
+import { createSseBroker, type SseBroker, type SseEvent } from "./sse-broker";
+
+// Channel namespace for cross-replica fanout (fw#2625). Every pod publishes
+// here and every pod's psubscribe listens here, so a push on one instance
+// reaches SSE clients connected to any other instance without a shared
+// cursor — Pub/Sub for connected-client fanout, the event-store's shared
+// cursor stays the mechanism for "read every event exactly once".
+const CHANNEL_PREFIX = "kumiko:sse:ch:";
+const INVALIDATION_PREFIX = "kumiko:sse:inval:";
+const PSUBSCRIBE_PATTERN = "kumiko:sse:*";
+
+export type RedisSseBrokerOptions = {
+  readonly redisUrl: string;
+};
+
+export type RedisSseBroker = SseBroker & {
+  // Not on SseBroker — the in-memory broker has nothing to release, but
+  // this one owns two live ioredis connections (via the shared PubSubSignal).
+  // buildServer only calls this when it created the broker itself (an
+  // app-injected sseBroker owns its own lifecycle).
+  close(): Promise<void>;
+};
+
+function isSseEvent(value: unknown): value is SseEvent {
+  return (
+    typeof value === "object" &&
+    value !== null &&
+    typeof (value as { type?: unknown }).type === "string" &&
+    typeof (value as { data?: unknown }).data === "object" &&
+    (value as { data?: unknown }).data !== null
+  );
+}
+
+// Transport layer around a local `createSseBroker()` — all client/listener
+// state lives in `inner`, this only moves events across the Redis wire via
+// the shared PubSubSignal. `pushToChannel`/`publishAccessInvalidation` never
+// call `inner` directly: they publish, and the signal's message handler
+// below calls `inner` for BOTH the publishing pod and every other pod, so
+// there's exactly one delivery path regardless of which instance receives
+// the request.
+export function createRedisSseBroker(opts: RedisSseBrokerOptions): RedisSseBroker {
+  const inner = createSseBroker();
+  const signal = createRedisPubSubSignal({
+    redisUrl: opts.redisUrl,
+    channelPattern: PSUBSCRIBE_PATTERN,
+    label: "sse-broker",
+  });
+
+  signal.onMessage((channel, payload) => {
+    if (channel.startsWith(CHANNEL_PREFIX)) {
+      if (!isSseEvent(payload)) {
+        console.error(`[kumiko:sse-broker] dropping malformed event on "${channel}"`);
+        // skip: malformed payload already logged above, nothing to deliver
+        return;
+      }
+      inner.pushToChannel(channel.slice(CHANNEL_PREFIX.length), payload);
+      // skip: channel already routed to the SSE-event branch above, the
+      // invalidation branch below is mutually exclusive with this one
+      return;
+    }
+
+    if (channel.startsWith(INVALIDATION_PREFIX)) {
+      inner.publishAccessInvalidation(channel.slice(INVALIDATION_PREFIX.length));
+    }
+  });
+
+  return {
+    addClient: inner.addClient,
+    removeClient: inner.removeClient,
+    getClientCount: inner.getClientCount,
+    getTotalClientCount: inner.getTotalClientCount,
+    subscribeAccessInvalidation: inner.subscribeAccessInvalidation,
+
+    pushToChannel(channel, event) {
+      signal.publish(`${CHANNEL_PREFIX}${channel}`, event);
+    },
+
+    // fw#1601: this is the security-critical call — a revoked session's
+    // stream must close on every replica, not just the one that observed
+    // the revocation event. Publishing (rather than calling inner directly,
+    // like the in-memory broker does) is what makes that true here.
+    publishAccessInvalidation(userId) {
+      signal.publish(`${INVALIDATION_PREFIX}${userId}`, 1);
+    },
+
+    close: signal.close,
+  };
+}

package/src/api/server.ts

@@ -57,6 +57,7 @@ import { createJwtHelper, type JwtHelper, type JwtKeyring } from "./jwt";
 import { observabilityMiddleware } from "./observability-middleware";
 import { assertOriginGuardConfig, originMiddleware } from "./origin-middleware";
 import { piiCiphertextResponseGuard } from "./pii-leak-guard";
+import { createRedisSseBroker } from "./redis-sse-broker";
 import { requestContext } from "./request-context";
 import { buildRequestContextData, requestIdMiddleware } from "./request-id-middleware";
 import {
@@ -316,7 +317,18 @@ export function buildServer(options: ServerOptions): KumikoServer {
     options.jwtIssuer,
     options.jwtTtl ?? defaultJwtTtl,
   );
-  const sseBroker = options.sseBroker ?? createSseBroker();
+  // An explicit ServerOptions.sseBroker always wins (caller owns its
+  // lifecycle). Otherwise, REDIS_URL decides the default: without
+  // cross-replica fanout, replicas > 1 silently drops SSE/access-
+  // invalidation events for whichever pod didn't win the shared cursor
+  // (fw#2625) — the safe default has to be the correct one, not the
+  // single-process one.
+  const ownedRedisSseBroker = options.sseBroker
+    ? undefined
+    : process.env["REDIS_URL"]
+      ? createRedisSseBroker({ redisUrl: process.env["REDIS_URL"] })
+      : undefined;
+  const sseBroker = options.sseBroker ?? ownedRedisSseBroker ?? createSseBroker();
 
   // Resolve the per-process instance identifier. Prefer explicit
   // ServerOptions.instanceId (tests, deliberate wiring), fall back to the
@@ -587,6 +599,15 @@ export function buildServer(options: ServerOptions): KumikoServer {
     });
   }
 
+  // Same reasoning, scoped to the broker this function created itself — an
+  // app-injected sseBroker (options.sseBroker) owns its own teardown.
+  if (options.lifecycle && ownedRedisSseBroker) {
+    const broker = ownedRedisSseBroker;
+    options.lifecycle.registerShutdownHook("redisSseBroker", async () => {
+      await broker.close();
+    });
+  }
+
   const app = new Hono();
 
   const sensitiveConfig = mergeSensitiveConfig(

package/src/api/sse-broker.ts

@@ -29,8 +29,10 @@ export type SseBroker = {
 };
 
 export function createSseBroker(): SseBroker {
-  // Cross-replica fanout lives one level up: the SSE + access-invalidation
-  // consumers (system-hooks.ts) run delivery: "per-instance" (#1718).
+  // Purely local: no cross-replica fanout. buildServer wraps this in
+  // createRedisSseBroker (fw#2625) whenever REDIS_URL is set, which is what
+  // makes pushToChannel/publishAccessInvalidation reach every replica's
+  // clients — this reference implementation stays single-process only.
   const channels = new Map<string, Map<string, SseClient>>();
   // Set, not Map<listenerId, fn> — dedup key is callback reference. Every
   // subscriber must pass a distinct closure (dispatch-stream.ts does, one

package/src/db/__tests__/list-filter-field-access.integration.test.ts

@@ -0,0 +1,214 @@
+// fw#2629: `list()` guarded filtering/sorting only against unknown columns,
+// not per-field `access.read`. A caller who cannot read a field could still
+// filter or sort by it — result count and row order become an oracle for a
+// value the response itself strips out (filterReadFields in
+// engine/field-access.ts). This proves the fix over real HTTP: a filter on
+// an unreadable field must be unsatisfiable (not merely "field stripped from
+// the response"), and a sort on an unreadable field must fall back silently
+// to the default id-ASC order.
+
+import { afterAll, beforeAll, beforeEach, describe, expect, test } from "bun:test";
+import { asRawClient } from "../../db/query";
+import { defineFeature } from "../../engine/define-feature";
+import { createEntity, createTextField } from "../../engine/factories";
+import { from } from "../../engine/ownership";
+import { setupTestStack, type TestStack, TestUsers, unsafeCreateEntityTable } from "../../stack";
+
+const filterNoteEntity = createEntity({
+  table: "fa_filter_notes",
+  fields: {
+    title: createTextField({ required: true }),
+    ownerId: createTextField({ required: true }),
+    secret: createTextField({
+      filterable: true,
+      sortable: true,
+      access: { read: { Admin: "all" } },
+    }),
+    ownedNote: createTextField({
+      filterable: true,
+      access: { read: { Admin: "all", User: from("user:id", "ownerId") } },
+    }),
+  },
+});
+
+const LIST_QN = "filternotes:query:note:list";
+
+const filterNotesFeature = defineFeature("filternotes", (r) => {
+  r.crud("note", filterNoteEntity, {
+    write: { access: { roles: ["Admin"] } },
+    read: { access: { openToAll: true } },
+  });
+});
+
+type NoteRow = {
+  readonly id: string;
+  readonly title: string;
+  readonly ownerId: string;
+  readonly secret?: string;
+  readonly ownedNote?: string;
+};
+
+type ListResult = {
+  readonly rows: readonly NoteRow[];
+  readonly total?: number;
+};
+
+describe("list() field-level read access gates filter/sort (fw#2629)", () => {
+  let stack: TestStack;
+
+  beforeAll(async () => {
+    stack = await setupTestStack({ features: [filterNotesFeature] });
+    await unsafeCreateEntityTable(stack.db, filterNoteEntity);
+  });
+
+  afterAll(async () => {
+    await stack.cleanup();
+  });
+
+  beforeEach(async () => {
+    await asRawClient(stack.db).unsafe("DELETE FROM kumiko_events");
+    await asRawClient(stack.db).unsafe('DELETE FROM "fa_filter_notes"');
+  });
+
+  const CREATE = "filternotes:write:note:create";
+
+  async function seedTwoSecrets(): Promise<void> {
+    await stack.http.write(
+      CREATE,
+      { title: "Note A", ownerId: TestUsers.user.id, secret: "alpha" },
+      TestUsers.admin,
+    );
+    await stack.http.write(
+      CREATE,
+      { title: "Note B", ownerId: TestUsers.user.id, secret: "beta" },
+      TestUsers.admin,
+    );
+  }
+
+  test("baseline: rows exist for a caller without read access to `secret`, but the field is stripped", async () => {
+    await seedTwoSecrets();
+    const result = await stack.http.queryOk<ListResult>(LIST_QN, { limit: 50 }, TestUsers.user);
+
+    expect(result.rows).toHaveLength(2);
+    expect(result.rows.every((r) => r.secret === undefined)).toBe(true);
+  });
+
+  test("count-channel oracle is closed: a matching and a non-matching filter on `secret` are indistinguishable", async () => {
+    await seedTwoSecrets();
+
+    const matching = await stack.http.queryOk<ListResult>(
+      LIST_QN,
+      { limit: 50, totalCount: true, filter: { field: "secret", op: "eq", value: "alpha" } },
+      TestUsers.user,
+    );
+    const nonExistent = await stack.http.queryOk<ListResult>(
+      LIST_QN,
+      {
+        limit: 50,
+        totalCount: true,
+        filter: { field: "secret", op: "eq", value: "does-not-exist-anywhere" },
+      },
+      TestUsers.user,
+    );
+
+    expect(matching.rows).toHaveLength(0);
+    expect(matching.total).toBe(0);
+    expect(nonExistent.rows).toHaveLength(0);
+    expect(nonExistent.total).toBe(0);
+    expect(matching).toEqual(nonExistent);
+  });
+
+  test("count-channel oracle is closed for `ne` and `in` on `secret` too", async () => {
+    await seedTwoSecrets();
+
+    const ne = await stack.http.queryOk<ListResult>(
+      LIST_QN,
+      { limit: 50, totalCount: true, filter: { field: "secret", op: "ne", value: "alpha" } },
+      TestUsers.user,
+    );
+    const inOp = await stack.http.queryOk<ListResult>(
+      LIST_QN,
+      {
+        limit: 50,
+        totalCount: true,
+        filters: [{ field: "secret", op: "in", value: ["alpha", "beta"] }],
+      },
+      TestUsers.user,
+    );
+
+    expect(ne.rows).toHaveLength(0);
+    expect(ne.total).toBe(0);
+    expect(inOp.rows).toHaveLength(0);
+    expect(inOp.total).toBe(0);
+  });
+
+  test("a legitimate filter by a caller who CAN read the field still narrows results", async () => {
+    await seedTwoSecrets();
+    const result = await stack.http.queryOk<ListResult>(
+      LIST_QN,
+      { limit: 50, totalCount: true, filter: { field: "secret", op: "eq", value: "alpha" } },
+      TestUsers.admin,
+    );
+
+    expect(result.rows).toHaveLength(1);
+    expect(result.rows[0]?.title).toBe("Note A");
+    expect(result.total).toBe(1);
+  });
+
+  test("ownership-typed filter binds the correct rule without corrupting later param placeholders", async () => {
+    await stack.http.write(
+      CREATE,
+      { title: "Mine", ownerId: TestUsers.user.id, secret: "x", ownedNote: "shared-value" },
+      TestUsers.admin,
+    );
+    await stack.http.write(
+      CREATE,
+      { title: "Theirs", ownerId: TestUsers.admin.id, secret: "y", ownedNote: "shared-value" },
+      TestUsers.admin,
+    );
+
+    const result = await stack.http.queryOk<ListResult>(
+      LIST_QN,
+      {
+        limit: 50,
+        totalCount: true,
+        filter: { field: "ownedNote", op: "eq", value: "shared-value" },
+      },
+      TestUsers.user,
+    );
+
+    expect(result.rows).toHaveLength(1);
+    expect(result.rows[0]?.title).toBe("Mine");
+    expect(result.total).toBe(1);
+  });
+
+  test("sort-channel oracle: sorting by an unreadable field falls back to id ASC", async () => {
+    // id ASC (creation order) is [first, second]; secret DESC must reverse
+    // that ("zzz" > "aaa") — the two orders disagree, so this actually
+    // distinguishes "sort applied" from "sort silently ignored".
+    const first = await stack.http.writeOk<{ id: string }>(
+      CREATE,
+      { title: "First", ownerId: TestUsers.user.id, secret: "aaa" },
+      TestUsers.admin,
+    );
+    const second = await stack.http.writeOk<{ id: string }>(
+      CREATE,
+      { title: "Second", ownerId: TestUsers.user.id, secret: "zzz" },
+      TestUsers.admin,
+    );
+
+    const asUser = await stack.http.queryOk<ListResult>(
+      LIST_QN,
+      { limit: 50, sort: "secret", sortDirection: "desc" },
+      TestUsers.user,
+    );
+    expect(asUser.rows.map((r) => r.id)).toEqual([first.id, second.id]);
+
+    const asAdmin = await stack.http.queryOk<ListResult>(
+      LIST_QN,
+      { limit: 50, sort: "secret", sortDirection: "desc" },
+      TestUsers.admin,
+    );
+    expect(asAdmin.rows.map((r) => r.id)).toEqual([second.id, first.id]);
+  });
+});

package/src/db/event-store-executor-read.ts

@@ -2,7 +2,7 @@ import { KUMIKO_NAME_SYMBOL } from "@cosmicdrift/kumiko-types/schema-table-types
 import { computeBlindIndex, configuredBlindIndexKey } from "../crypto";
 import { executeRawQueryRead } from "../db/queries/raw-sql";
 import { coerceRow, extractTableInfo } from "../db/query";
-import { buildOwnershipClause, shiftParams } from "../engine/ownership";
+import { buildOwnershipClause, normalizeAccessEntry, shiftParams } from "../engine/ownership";
 import type { EntityId } from "../engine/types";
 import { SYSTEM_TENANT_ID } from "../engine/types/identifiers";
 import { UnprocessableError } from "../errors";
@@ -269,7 +269,15 @@ export function createReadVerbs(ctx: ExecutorContext): Pick<EventStoreExecutor,
       const physicalCol = (field: string): string =>
         (table[field] as { name?: string } | undefined)?.name ?? toSnakeCase(field);
       const colSql = (field: string): string => `"${physicalCol(field)}"`;
-      const sortField = payload.sort && table[payload.sort] ? payload.sort : undefined;
+      // Field-level read access gates filtering and sorting the same way it
+      // gates the projected row: a caller who cannot read a field must not be
+      // able to probe its values through result counts or ordering (fw#2629).
+      const fieldReadClause = (field: string) =>
+        buildOwnershipClause(user, normalizeAccessEntry(entity.fields[field]?.access?.read), table);
+      const sortField =
+        payload.sort && table[payload.sort] && fieldReadClause(payload.sort).kind === "pass"
+          ? payload.sort
+          : undefined;
       const sortDescending = payload.sortDirection === "desc";
 
       // Tenant-Filter (replicates TenantDb's readWhere semantics).
@@ -317,6 +325,22 @@ export function createReadVerbs(ctx: ExecutorContext): Pick<EventStoreExecutor,
           // skip: unknown field — not a real column, drop the filter (injection guard)
           return;
         }
+        const readClause = fieldReadClause(f.field);
+        if (readClause.kind === "empty") {
+          whereSql.push("FALSE");
+          // skip: no role grants read on this field, so no row may be probed
+          // through it — an unsatisfiable filter beats an error, which would
+          // itself confirm the field exists (field-access.ts strips silently)
+          return;
+        }
+        if (readClause.kind === "sql") {
+          const shifted = shiftParams(
+            { sqlText: readClause.sqlText, params: readClause.params },
+            params.length,
+          );
+          whereSql.push(shifted.sqlText);
+          for (const p of shifted.params) params.push(p);
+        }
         if (entity.fields[f.field]?.type === "multiSelect") {
           applyMultiSelectFilter(colSql, whereSql, params, f);
           // skip: multiSelect already handled — don't fall through to applyScreenFilter

package/src/db/queries/event-consumer.ts

@@ -33,6 +33,32 @@ export async function insertConsumerIfAbsent(
   }
 }
 
+// fw#2625: a consumer that moved from delivery: "per-instance" to "shared"
+// leaves its old per-instance rows behind — no dispatcher will ever advance
+// them again, and pruneEvents stays pinned to their stale cursor forever.
+// Scoped by name + "not the shared sentinel" so a still-per-instance
+// consumer's live rows (or an already-migrated __shared__ row) are never
+// touched.
+export async function deleteOrphanedPerInstanceConsumerRows(
+  db: AnyDb,
+  consumerNames: readonly string[],
+  // Passed in rather than imported from event-consumer-state.ts (which
+  // defines SHARED_INSTANCE_SENTINEL): that module already imports from
+  // this one for the DB queries it needs, so importing back would be a
+  // require cycle. The one caller lives in that same file and has the
+  // constant in scope.
+  sharedInstanceSentinel: string,
+): Promise<void> {
+  // skip: nothing to delete — an empty ANY($1) would still hit the table,
+  // pointlessly, on every boot for an app with no migrated consumers yet.
+  if (consumerNames.length === 0) return;
+  await asRawClient(db).unsafe(
+    `DELETE FROM "kumiko_event_consumers"
+     WHERE "name" = ANY($1) AND "instance_id" != $2`,
+    [consumerNames, sharedInstanceSentinel],
+  );
+}
+
 export async function selectConsumerForUpdateSkipLocked(
   db: AnyDb,
   name: string,

package/src/entrypoint/index.ts

@@ -23,12 +23,12 @@
 // registers its own shutdown hooks (eventDispatcher.stop, jobRunner.stop)
 // in the order they were built.
 //
-// Known limitation (tracked in uebersicht.md Offene Follow-Ups): the
-// built-in SSE broker is in-memory per process. In a split api/worker
-// deploy, system-consumers that push to SSE (new-row broadcasts) run on
-// the worker and therefore can't reach clients connected to the API
-// instances. Either run all-in-one, put the SSE consumer on the API
-// side explicitly, or wait for the Redis-Pub/Sub bridge.
+// SSE fanout across processes (split api/worker deploy, replicas > 1):
+// buildServer's default sseBroker is Redis-backed whenever REDIS_URL is
+// set (fw#2625) — a push on the worker's dispatcher reaches clients
+// connected to any API instance via Redis Pub/Sub, no per-instance
+// cursor needed. Without REDIS_URL, the in-memory broker still only
+// reaches clients on the same process.
 
 import type { Hono } from "hono";
 import type { AuthRoutesConfig } from "../api/auth-routes";

package/src/pipeline/__tests__/event-consumer-state.integration.test.ts

@@ -1,4 +1,6 @@
 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
+import { insertConsumerIfAbsent } from "../../db/queries/event-consumer";
+import { asRawClient } from "../../db/query";
 import { createTestDb, type TestDb } from "../../stack";
 import { createEventConsumerStateTable } from "../event-consumer-state";
 
@@ -29,3 +31,51 @@ describe("createEventConsumerStateTable — concurrent boot", () => {
     ).resolves.toBeDefined();
   });
 });
+
+// fw#2625: sse-broadcast, access-invalidation and toggle-cache-sync moved
+// from delivery: "per-instance" to "shared" — their old per-instance rows
+// are orphaned and would pin pruneEvents forever if left behind.
+describe("createEventConsumerStateTable — orphaned per-instance row cleanup (fw#2625)", () => {
+  let testDb: TestDb;
+
+  beforeAll(async () => {
+    testDb = await createTestDb();
+    await createEventConsumerStateTable(testDb.db);
+  });
+
+  afterAll(async () => {
+    await testDb.cleanup();
+  });
+
+  async function rowExists(name: string, instanceId: string): Promise<boolean> {
+    const rows = (await asRawClient(testDb.db).unsafe(
+      `SELECT 1 FROM "kumiko_event_consumers" WHERE "name" = $1 AND "instance_id" = $2`,
+      [name, instanceId],
+    )) as ReadonlyArray<unknown>;
+    return rows.length > 0;
+  }
+
+  test("deletes only the three migrated consumers' per-instance rows, leaving unrelated and shared rows intact", async () => {
+    await insertConsumerIfAbsent(testDb.db, "system:consumer:sse-broadcast", "pod-a");
+    await insertConsumerIfAbsent(testDb.db, "system:consumer:access-invalidation", "pod-b");
+    await insertConsumerIfAbsent(
+      testDb.db,
+      "feature-toggles:projection:toggle-cache-sync",
+      "pod-c",
+    );
+    // A consumer name not in the orphan list at all — must survive.
+    await insertConsumerIfAbsent(testDb.db, "feature-toggles:projection:unrelated", "pod-d");
+    // A consumer name that happens to already run shared — must survive.
+    await insertConsumerIfAbsent(testDb.db, "system:consumer:sse-broadcast", "__shared__");
+
+    // The bootstrap's "table already exists" branch is what runs the
+    // cleanup — re-invoke it the same way a redeploying process would.
+    await createEventConsumerStateTable(testDb.db);
+
+    expect(await rowExists("system:consumer:sse-broadcast", "pod-a")).toBe(false);
+    expect(await rowExists("system:consumer:access-invalidation", "pod-b")).toBe(false);
+    expect(await rowExists("feature-toggles:projection:toggle-cache-sync", "pod-c")).toBe(false);
+    expect(await rowExists("feature-toggles:projection:unrelated", "pod-d")).toBe(true);
+    expect(await rowExists("system:consumer:sse-broadcast", "__shared__")).toBe(true);
+  });
+});

package/src/pipeline/event-consumer-state.ts

@@ -11,6 +11,7 @@ import {
   text,
 } from "../db/dialect";
 import { alterTableAddColumn } from "../db/queries/ddl";
+import { deleteOrphanedPerInstanceConsumerRows } from "../db/queries/event-consumer";
 import { tableExists } from "../db/schema-inspection";
 import { unsafePushTables } from "../stack";
 
@@ -90,6 +91,21 @@ export const eventConsumerStateTable = pgTable(
   }),
 );
 
+// fw#2625: these three consumers moved from delivery: "per-instance" to
+// "shared" (see system-hooks.ts and bundled-features/feature-toggles/
+// feature.ts). Their old per-instance rows are now orphaned — no dispatcher
+// will ever advance them again — and pruneEvents stays pinned to whichever
+// one has the lowest cursor forever. Named here (not imported from
+// system-hooks.ts / feature.ts) to avoid a pipeline → system-hooks /
+// bundled-features dependency for three string literals; those modules'
+// own exported constants (or, for feature-toggles, the qualified consumer
+// name qualify() produces) are the source of truth these must match.
+const ORPHANED_PER_INSTANCE_CONSUMER_NAMES = [
+  "system:consumer:sse-broadcast",
+  "system:consumer:access-invalidation",
+  "feature-toggles:projection:toggle-cache-sync",
+] as const;
+
 // Object-const form lets call sites write `ConsumerStatuses.disabled` instead
 // of the raw string, which keeps status checks refactor-safe. The runtime
 // value on each field is the same string the DB stores — no mapping needed.
@@ -126,8 +142,23 @@ export async function createEventConsumerStateTable(db: DbConnection): Promise<v
       " NOT NULL",
       /* ifNotExists */ true,
     );
+    // Runs on every boot, including mid-rolling-deploy: a new pod can delete
+    // an old pod's still-in-use per-instance row out from under it. That old
+    // pod's next acquireConsumer then finds no row, delivers no more SSE/
+    // access-invalidation events for the rest of its lifetime — a one-time,
+    // few-second gap that resolves itself as clients reconnect onto new pods
+    // during the same rollout. Not a replay risk: preRegisterConsumers only
+    // (re-)inserts a row at dispatcher startup, so the deleted pod won't
+    // silently get a fresh cursor-0 row and replay history mid-flight.
+    await deleteOrphanedPerInstanceConsumerRows(
+      db,
+      ORPHANED_PER_INSTANCE_CONSUMER_NAMES,
+      SHARED_INSTANCE_SENTINEL,
+    );
     // skip: table (+ any missing column) is already up to date
     return;
   }
+  // A freshly created table has no rows to clean up — the DELETE only
+  // matters for a DB that already ran the old per-instance consumers.
   await unsafePushTables(db, { kumikoEventConsumers: eventConsumerStateTable });
 }

package/src/pipeline/system-hooks.ts

@@ -294,15 +294,15 @@ export const SSE_BROADCAST_CONSUMER_NAME = "system:consumer:sse-broadcast";
 export function createSseBroadcastEventConsumer(sseBroker: SseBroker): EventConsumer {
   return {
     name: SSE_BROADCAST_CONSUMER_NAME,
-    // Per-instance delivery: each API process has its own pool of SSE
-    // clients and its own cursor. Every instance reads every event
-    // independently and pushes to its local clients. Without this, in a
-    // split-deploy (API-1/API-2 + Worker, Welle 2.5), only ONE API
-    // instance would pick up each event (shared-cursor SKIP LOCKED) and
-    // the other instance's clients would never see updates. SSE clients
-    // pin to a specific API process via the HTTP long-poll; cross-process
-    // delivery isn't solvable by sharing a cursor.
-    delivery: "per-instance",
+    // Shared delivery: one cursor reads every event exactly once (fw#2625).
+    // Fanout to every process's own SSE clients happens below, through
+    // sseBroker.pushToChannel — a Redis-backed broker (the default once
+    // REDIS_URL is set) republishes across replicas, so every pod's
+    // clients still see the event without each pod needing its own cursor.
+    delivery: "shared",
+    // History is meaningless here — an SSE push for a client that didn't
+    // exist yet has no audience to replay it to.
+    startFrom: "now",
     handler: async (event) => {
       sseBroker.pushToChannel(tenantChannel(event.tenantId), {
         type: event.type,
@@ -424,11 +424,16 @@ function readUserIdFromPreviousSnapshot(payload: Record<string, unknown>): strin
 export function createAccessInvalidationEventConsumer(sseBroker: SseBroker): EventConsumer {
   return {
     name: ACCESS_INVALIDATION_CONSUMER_NAME,
-    // Per-instance, same reasoning as SSE broadcast: subscribeAccessInvalidation
-    // listeners live in this process's in-memory sseBroker only. A shared
-    // cursor would deliver the event to exactly one instance and leave
-    // every other instance's open streams for that user un-invalidated.
-    delivery: "per-instance",
+    // Shared delivery, same reasoning as SSE broadcast (fw#2625): one
+    // cursor reads every event once and calls sseBroker.publishAccessInvalidation
+    // below, which a Redis-backed broker republishes to every replica's
+    // subscribeAccessInvalidation listeners — no per-instance cursor needed
+    // to reach every process's open streams for that user.
+    delivery: "shared",
+    // A revocation for a session/membership from before this instance
+    // existed has nothing to invalidate — replaying it would be a no-op at
+    // best and a spurious stream-close at worst.
+    startFrom: "now",
     handler: async (event) => {
       if (event.type === SESSION_REVOKED_EVENT_TYPE) {
         const userId = event.payload["userId"];

package/src/redis/client.ts

@@ -0,0 +1,77 @@
+// Thin factory for ioredis clients with Prod-sensible defaults.
+// ioredis itself doesn't enforce a command-timeout — an unreachable Redis
+// will silently hang a request forever unless the caller sets
+// `commandTimeout`. Same story for connectTimeout: the default is 10s,
+// which is too long for a /health/ready probe with a 2s budget.
+//
+// Apps are expected to build their Redis clients via this helper so
+// production defaults stay consistent and env-var wiring lives in one
+// place. Tests / one-off scripts can still `new Redis()` directly.
+//
+// Split out of index.ts (fw#2625) so pubsub-signal.ts can import this
+// without a same-directory cycle back through the barrel.
+
+import IoRedis, { type Redis, type RedisOptions } from "ioredis";
+import { readPositiveIntEnv } from "../utils/env-parse";
+
+// Connection-tuning options. The fields mirror the most consequential
+// ioredis settings; anything else can be passed through via `extra` to
+// keep the helper from re-declaring the full ioredis surface.
+export type RedisClientOptions = {
+  // Milliseconds to wait when opening the TCP connection. Default 5s —
+  // long enough for a cold cache/standby to handshake, short enough that
+  // /health/ready can surface "Redis is gone" inside its 2s-ish probe
+  // window (probe runs in parallel, so 5s here + 2s timeout on the probe
+  // side gives the probe a real answer instead of a hang).
+  readonly connectTimeoutMs?: number;
+  // Milliseconds to wait for a single command to round-trip. Default 10s.
+  // Without this, ioredis waits forever on a stalled connection — a
+  // single bad network partition quietly exhausts the whole command queue.
+  readonly commandTimeoutMs?: number;
+  // Retries per request before bubbling up as an error. Default 3 — enough
+  // to ride out a transient blip but not so many that a dead Redis keeps
+  // requests stuck waiting for retries.
+  readonly maxRetriesPerRequest?: number;
+  // Escape hatch for any ioredis option the helper doesn't expose
+  // explicitly (TLS config, Sentinel, ReadyCheck toggles, …).
+  readonly extra?: Readonly<
+    Omit<RedisOptions, "connectTimeout" | "commandTimeout" | "maxRetriesPerRequest">
+  >;
+};
+
+const DEFAULT_CONNECT_TIMEOUT_MS = 5_000;
+const DEFAULT_COMMAND_TIMEOUT_MS = 10_000;
+const DEFAULT_MAX_RETRIES_PER_REQUEST = 3;
+
+export function createRedisClient(url: string, options: RedisClientOptions = {}): Redis {
+  const connectTimeout = options.connectTimeoutMs ?? DEFAULT_CONNECT_TIMEOUT_MS;
+  const commandTimeout = options.commandTimeoutMs ?? DEFAULT_COMMAND_TIMEOUT_MS;
+  const maxRetriesPerRequest = options.maxRetriesPerRequest ?? DEFAULT_MAX_RETRIES_PER_REQUEST;
+
+  return new IoRedis(url, {
+    ...options.extra,
+    connectTimeout,
+    commandTimeout,
+    maxRetriesPerRequest,
+  });
+}
+
+// Env-var reader — mirrors dbConnectionOptionsFromEnv's contract. Strict:
+// malformed values throw at boot rather than silently falling back to
+// defaults.
+export function redisClientOptionsFromEnv(
+  env: Readonly<Record<string, string | undefined>> = process.env,
+): RedisClientOptions {
+  const opts: RedisClientOptions & {
+    connectTimeoutMs?: number;
+    commandTimeoutMs?: number;
+    maxRetriesPerRequest?: number;
+  } = {};
+  const connect = readPositiveIntEnv(env, "REDIS_CONNECT_TIMEOUT_MS");
+  const command = readPositiveIntEnv(env, "REDIS_COMMAND_TIMEOUT_MS");
+  const retries = readPositiveIntEnv(env, "REDIS_MAX_RETRIES_PER_REQUEST");
+  if (connect !== undefined) opts.connectTimeoutMs = connect;
+  if (command !== undefined) opts.commandTimeoutMs = command;
+  if (retries !== undefined) opts.maxRetriesPerRequest = retries;
+  return opts;
+}

package/src/redis/index.ts

@@ -1,74 +1,4 @@
-// Thin factory for ioredis clients with Prod-sensible defaults.
-// ioredis itself doesn't enforce a command-timeout — an unreachable Redis
-// will silently hang a request forever unless the caller sets
-// `commandTimeout`. Same story for connectTimeout: the default is 10s,
-// which is too long for a /health/ready probe with a 2s budget.
-//
-// Apps are expected to build their Redis clients via this helper so
-// production defaults stay consistent and env-var wiring lives in one
-// place. Tests / one-off scripts can still `new Redis()` directly.
-
-import IoRedis, { type Redis, type RedisOptions } from "ioredis";
-import { readPositiveIntEnv } from "../utils/env-parse";
-
-// Connection-tuning options. The fields mirror the most consequential
-// ioredis settings; anything else can be passed through via `extra` to
-// keep the helper from re-declaring the full ioredis surface.
-export type RedisClientOptions = {
-  // Milliseconds to wait when opening the TCP connection. Default 5s —
-  // long enough for a cold cache/standby to handshake, short enough that
-  // /health/ready can surface "Redis is gone" inside its 2s-ish probe
-  // window (probe runs in parallel, so 5s here + 2s timeout on the probe
-  // side gives the probe a real answer instead of a hang).
-  readonly connectTimeoutMs?: number;
-  // Milliseconds to wait for a single command to round-trip. Default 10s.
-  // Without this, ioredis waits forever on a stalled connection — a
-  // single bad network partition quietly exhausts the whole command queue.
-  readonly commandTimeoutMs?: number;
-  // Retries per request before bubbling up as an error. Default 3 — enough
-  // to ride out a transient blip but not so many that a dead Redis keeps
-  // requests stuck waiting for retries.
-  readonly maxRetriesPerRequest?: number;
-  // Escape hatch for any ioredis option the helper doesn't expose
-  // explicitly (TLS config, Sentinel, ReadyCheck toggles, …).
-  readonly extra?: Readonly<
-    Omit<RedisOptions, "connectTimeout" | "commandTimeout" | "maxRetriesPerRequest">
-  >;
-};
-
-const DEFAULT_CONNECT_TIMEOUT_MS = 5_000;
-const DEFAULT_COMMAND_TIMEOUT_MS = 10_000;
-const DEFAULT_MAX_RETRIES_PER_REQUEST = 3;
-
-export function createRedisClient(url: string, options: RedisClientOptions = {}): Redis {
-  const connectTimeout = options.connectTimeoutMs ?? DEFAULT_CONNECT_TIMEOUT_MS;
-  const commandTimeout = options.commandTimeoutMs ?? DEFAULT_COMMAND_TIMEOUT_MS;
-  const maxRetriesPerRequest = options.maxRetriesPerRequest ?? DEFAULT_MAX_RETRIES_PER_REQUEST;
-
-  return new IoRedis(url, {
-    ...options.extra,
-    connectTimeout,
-    commandTimeout,
-    maxRetriesPerRequest,
-  });
-}
-
-// Env-var reader — mirrors dbConnectionOptionsFromEnv's contract. Strict:
-// malformed values throw at boot rather than silently falling back to
-// defaults.
-export function redisClientOptionsFromEnv(
-  env: Readonly<Record<string, string | undefined>> = process.env,
-): RedisClientOptions {
-  const opts: RedisClientOptions & {
-    connectTimeoutMs?: number;
-    commandTimeoutMs?: number;
-    maxRetriesPerRequest?: number;
-  } = {};
-  const connect = readPositiveIntEnv(env, "REDIS_CONNECT_TIMEOUT_MS");
-  const command = readPositiveIntEnv(env, "REDIS_COMMAND_TIMEOUT_MS");
-  const retries = readPositiveIntEnv(env, "REDIS_MAX_RETRIES_PER_REQUEST");
-  if (connect !== undefined) opts.connectTimeoutMs = connect;
-  if (command !== undefined) opts.commandTimeoutMs = command;
-  if (retries !== undefined) opts.maxRetriesPerRequest = retries;
-  return opts;
-}
+export type { RedisClientOptions } from "./client";
+export { createRedisClient, redisClientOptionsFromEnv } from "./client";
+export type { PubSubSignal, PubSubSignalOptions } from "./pubsub-signal";
+export { createRedisPubSubSignal } from "./pubsub-signal";

package/src/redis/pubsub-signal.ts

@@ -0,0 +1,107 @@
+import { createRedisClient } from "./client";
+
+// Generic Redis Pub/Sub transport for cross-replica fanout (fw#2625).
+// Shared by the SSE broker (api/redis-sse-broker.ts, several logical
+// channels multiplexed under one prefix) and feature-toggles' cache-sync
+// signal (bundled-features, a single fixed channel) — both need the same
+// mechanics: two connections (ioredis forbids normal commands on a
+// subscriber connection), one psubscribe per instance (no per-channel
+// subscribe/unsubscribe), JSON payloads, and defensive parsing so a
+// malformed or foreign message on the pattern never crashes the listener.
+export type PubSubSignal = {
+  publish(channel: string, payload: unknown): void;
+  onMessage(listener: (channel: string, payload: unknown) => void): void;
+  close(): Promise<void>;
+};
+
+export type PubSubSignalOptions = {
+  readonly redisUrl: string;
+  // psubscribe pattern: a literal channel name is a valid (exact-match)
+  // pattern for a caller with a single fixed channel; a "prefix:*" glob
+  // is for a caller that multiplexes several logical channels under one
+  // subscription (e.g. the SSE broker's per-tenant/per-user channels).
+  readonly channelPattern: string;
+  // Included in every log line so two signals running in the same process
+  // (sse-broker, feature-toggles cache-sync) can be told apart in output.
+  readonly label: string;
+};
+
+// Redis emits 'error' on connection blips (reconnects transparently); an
+// unhandled listener crashes the process (fw#1805 — job-runner hit the same
+// class with BullMQ's internal client).
+function logConnectionError(label: string, role: string): (err: Error) => void {
+  return (err) => {
+    // biome-ignore lint/suspicious/noConsole: ops-visible fallback — no logger is threaded through PubSubSignalOptions.
+    console.error(`[kumiko:${label}] ${role} connection error:`, err.message);
+  };
+}
+
+export function createRedisPubSubSignal(opts: PubSubSignalOptions): PubSubSignal {
+  const publisher = createRedisClient(opts.redisUrl);
+  const subscriber = createRedisClient(opts.redisUrl);
+  publisher.on("error", logConnectionError(opts.label, "publisher"));
+  subscriber.on("error", logConnectionError(opts.label, "subscriber"));
+
+  const listeners = new Set<(channel: string, payload: unknown) => void>();
+  // Sole purpose: stop close() from racing the initial psubscribe (see
+  // below) and silence its logging once a shutdown is already underway —
+  // not a defense against ioredis auto-reconnecting later (quit() below
+  // never reconnects once it resolves).
+  let closed = false;
+
+  subscriber.on("pmessage", (_pattern: string, channel: string, message: string) => {
+    let payload: unknown;
+    try {
+      payload = JSON.parse(message);
+    } catch (err) {
+      // biome-ignore lint/suspicious/noConsole: ops-visible fallback, see logConnectionError above.
+      console.error(`[kumiko:${opts.label}] dropping unparseable message on "${channel}":`, err);
+      // skip: unparseable payload already logged above, nothing to deliver
+      return;
+    }
+    for (const listener of listeners) listener(channel, payload);
+  });
+
+  // ponytail: psubscribe on one pattern rather than per-channel
+  // subscribe/unsubscribe — every instance receives every message that
+  // matches, filtered by the caller. That's strictly no worse than the
+  // pre-fw#2625 per-instance-cursor model (which already read every event
+  // from Postgres in every process) and sidesteps the subscribe-ack/
+  // first-publish race a per-channel scheme would have. Revisit with
+  // per-channel subscribe only if fanout volume becomes the bottleneck.
+  //
+  // Awaited by close() below — quitting while this is still in flight (a
+  // shutdown landing immediately after construction, e.g. a test closing
+  // right after building the signal) makes ioredis rejects the queued
+  // psubscribe with "Connection is closed" instead of it completing
+  // cleanly first. Not caught here or logged when a close is already
+  // underway (`closed`), so an intentional shutdown never surfaces this
+  // as an ops-visible error.
+  const initialSubscribe = subscriber.psubscribe(opts.channelPattern).catch((err: unknown) => {
+    // skip: a close() already underway intentionally quit both connections —
+    // this rejection is the expected side effect, not an ops-visible error.
+    if (closed) return;
+    // biome-ignore lint/suspicious/noConsole: ops-visible fallback, see logConnectionError above.
+    console.error(`[kumiko:${opts.label}] psubscribe failed:`, err);
+  });
+
+  return {
+    publish(channel, payload) {
+      publisher.publish(channel, JSON.stringify(payload)).catch((err: unknown) => {
+        // skip: a close() already underway intentionally quit both
+        // connections — this rejection is the expected side effect.
+        if (closed) return;
+        // biome-ignore lint/suspicious/noConsole: ops-visible fallback, see logConnectionError above.
+        console.error(`[kumiko:${opts.label}] publish failed:`, err);
+      });
+    },
+    onMessage(listener) {
+      listeners.add(listener);
+    },
+    async close(): Promise<void> {
+      closed = true;
+      await initialSubscribe;
+      await Promise.all([publisher.quit(), subscriber.quit()]);
+    },
+  };
+}