@cosmicdrift/kumiko-framework 0.306.0 → 0.307.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (33) hide show
  1. package/package.json +4 -4
  2. package/src/api/__tests__/redis-sse-broker.integration.test.ts +66 -0
  3. package/src/api/__tests__/sse-broker.test.ts +49 -0
  4. package/src/api/redis-sse-broker.ts +17 -3
  5. package/src/api/request-context.ts +24 -0
  6. package/src/api/sse-broker.ts +29 -11
  7. package/src/changes.json +26 -0
  8. package/src/db/queries/event-consumer.ts +57 -3
  9. package/src/db/queries/event-store.ts +69 -0
  10. package/src/db/tenant-db.ts +43 -5
  11. package/src/event-store/__tests__/event-attribution.integration.test.ts +53 -3
  12. package/src/event-store/admin-api.ts +5 -0
  13. package/src/event-store/event-store.ts +16 -7
  14. package/src/jobs/__tests__/job-public-intake-origin.integration.test.ts +536 -0
  15. package/src/jobs/job-runner.ts +61 -6
  16. package/src/pipeline/__tests__/dispatcher-utils.test.ts +8 -0
  17. package/src/pipeline/__tests__/event-dispatcher-commit-order.integration.test.ts +278 -0
  18. package/src/pipeline/__tests__/event-dispatcher-delivery-max-attempts.test.ts +1 -0
  19. package/src/pipeline/__tests__/event-dispatcher-lifecycle.integration.test.ts +6 -6
  20. package/src/pipeline/__tests__/event-dispatcher-per-consumer-turns.integration.test.ts +126 -0
  21. package/src/pipeline/__tests__/public-intake-runtime-gate.integration.test.ts +261 -2
  22. package/src/pipeline/dispatch-batch.ts +49 -19
  23. package/src/pipeline/dispatch-stream.ts +7 -3
  24. package/src/pipeline/dispatcher-utils.ts +21 -2
  25. package/src/pipeline/dispatcher.ts +71 -6
  26. package/src/pipeline/event-consumer-state.ts +26 -0
  27. package/src/pipeline/event-dispatcher-admin.ts +32 -5
  28. package/src/pipeline/event-dispatcher-delivery.ts +109 -57
  29. package/src/pipeline/event-dispatcher.ts +167 -50
  30. package/src/pipeline/pending-gap-ranges.ts +72 -0
  31. package/src/pipeline/system-hooks.ts +8 -1
  32. package/src/pipeline/write-origin.ts +31 -10
  33. package/src/stack/test-stack.ts +1 -1
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@cosmicdrift/kumiko-framework",
3
- "version": "0.306.0",
3
+ "version": "0.307.0",
4
4
  "description": "Framework core — engine, pipeline, API, DB, and every other bit that makes Kumiko go.",
5
5
  "license": "BUSL-1.1",
6
6
  "author": "Marc Frost <marc@cosmicdriftgamestudio.com>",
@@ -198,8 +198,8 @@
198
198
  "./package.json": "./package.json"
199
199
  },
200
200
  "dependencies": {
201
- "@cosmicdrift/kumiko-http": "0.306.0",
202
- "@cosmicdrift/kumiko-types": "0.306.0",
201
+ "@cosmicdrift/kumiko-http": "0.307.0",
202
+ "@cosmicdrift/kumiko-types": "0.307.0",
203
203
  "bullmq": "^5.76.7",
204
204
  "bun-types": "^1.3.13",
205
205
  "hono": "^4.13.1",
@@ -215,7 +215,7 @@
215
215
  "zod": "^4.4.3"
216
216
  },
217
217
  "devDependencies": {
218
- "@cosmicdrift/kumiko-dispatcher-live": "0.306.0",
218
+ "@cosmicdrift/kumiko-dispatcher-live": "0.307.0",
219
219
  "bun-types": "^1.3.13",
220
220
  "pino-pretty": "^13.1.3"
221
221
  },
@@ -127,6 +127,72 @@ describe("createRedisSseBroker", () => {
127
127
  expect(invalidatedA).toBe(false);
128
128
  });
129
129
 
130
+ test("publishAccessInvalidation with a keptSessionId spares only the listener whose own sid matches exactly", async () => {
131
+ const podA = trackedBroker();
132
+ const podB = trackedBroker();
133
+ const userId = `user-${generateId()}`;
134
+ const sidKept = `sid-kept-${generateId()}`;
135
+ const sidAlreadyRevoked = `sid-already-revoked-${generateId()}`;
136
+ let invalidatedKept = false;
137
+ let invalidatedSidless = false;
138
+ let invalidatedAlreadyRevoked = false;
139
+
140
+ podA.subscribeAccessInvalidation(
141
+ userId,
142
+ () => {
143
+ invalidatedKept = true;
144
+ },
145
+ sidKept,
146
+ );
147
+ // No ownSid — mirrors a PAT/bearer stream, always invalidated (fail-closed).
148
+ podA.subscribeAccessInvalidation(userId, () => {
149
+ invalidatedSidless = true;
150
+ });
151
+ // Not the kept sid — mirrors a session already revoked through an
152
+ // eventless path (plain logout): the keep-list must still close it.
153
+ podA.subscribeAccessInvalidation(
154
+ userId,
155
+ () => {
156
+ invalidatedAlreadyRevoked = true;
157
+ },
158
+ sidAlreadyRevoked,
159
+ );
160
+
161
+ await waitFor(() => {
162
+ podB.publishAccessInvalidation(userId, sidKept);
163
+ return invalidatedSidless && invalidatedAlreadyRevoked;
164
+ });
165
+ // Asserted only after the control listeners above already fired for the
166
+ // same publish — proves the pipe delivered the message at all, so a
167
+ // false here means the sid really was spared, not that delivery is slow.
168
+ expect(invalidatedKept).toBe(false);
169
+ });
170
+
171
+ test("a legacy unscoped invalidation message (bare `1`, pre-scoping pod) still invalidates every listener, sid or not", async () => {
172
+ const podA = trackedBroker();
173
+ const publisherRaw = new (await import("ioredis")).default(testRedis.redisUrl);
174
+ const userId = `user-${generateId()}`;
175
+ let invalidatedWithSid = false;
176
+
177
+ podA.subscribeAccessInvalidation(
178
+ userId,
179
+ () => {
180
+ invalidatedWithSid = true;
181
+ },
182
+ `sid-${generateId()}`,
183
+ );
184
+
185
+ try {
186
+ await waitFor(async () => {
187
+ await publisherRaw.publish(`kumiko:sse:inval:${userId}`, JSON.stringify(1));
188
+ return invalidatedWithSid;
189
+ });
190
+ expect(invalidatedWithSid).toBe(true);
191
+ } finally {
192
+ publisherRaw.disconnect();
193
+ }
194
+ });
195
+
130
196
  test("a malformed message on the channel namespace is dropped, not thrown, and does not kill delivery", async () => {
131
197
  const podA = trackedBroker();
132
198
  const publisherRaw = new (await import("ioredis")).default(testRedis.redisUrl);
@@ -121,4 +121,53 @@ describe("SSE broker", () => {
121
121
  const { publish } = requireAccessInvalidation(createSseBroker());
122
122
  expect(() => publish("nobody-listening")).not.toThrow();
123
123
  });
124
+
125
+ test("publishAccessInvalidation with a keptSessionId spares only the listener whose own sid matches exactly", () => {
126
+ const { subscribe, publish } = requireAccessInvalidation(createSseBroker());
127
+ const spared = mock();
128
+ const unrelated = mock();
129
+
130
+ subscribe("user-a", spared, "sid-kept");
131
+ subscribe("user-a", unrelated, "sid-other");
132
+ publish("user-a", "sid-kept");
133
+
134
+ expect(spared).not.toHaveBeenCalled();
135
+ expect(unrelated).toHaveBeenCalledTimes(1);
136
+ });
137
+
138
+ test("publishAccessInvalidation with a keptSessionId still closes a listener whose sid was already revoked through an eventless path (keep-list, not a kill-list)", () => {
139
+ const { subscribe, publish } = requireAccessInvalidation(createSseBroker());
140
+ const alreadyRevoked = mock();
141
+
142
+ // Simulates a stream from a session logged out earlier via a path that
143
+ // never appended session-revoked — the keep-list must not accidentally
144
+ // exempt it just because its sid isn't the freshly-kept one.
145
+ subscribe("user-a", alreadyRevoked, "sid-logged-out-earlier");
146
+ publish("user-a", "sid-kept");
147
+
148
+ expect(alreadyRevoked).toHaveBeenCalledTimes(1);
149
+ });
150
+
151
+ test("publishAccessInvalidation with a keptSessionId still fires a listener with no sid of its own (fail-closed, e.g. a PAT/bearer stream)", () => {
152
+ const { subscribe, publish } = requireAccessInvalidation(createSseBroker());
153
+ const sidless = mock();
154
+
155
+ subscribe("user-a", sidless);
156
+ publish("user-a", "sid-kept");
157
+
158
+ expect(sidless).toHaveBeenCalledTimes(1);
159
+ });
160
+
161
+ test("publishAccessInvalidation with no keptSessionId (unscoped) still invalidates every listener, matching pre-scoping behavior", () => {
162
+ const { subscribe, publish } = requireAccessInvalidation(createSseBroker());
163
+ const first = mock();
164
+ const second = mock();
165
+
166
+ subscribe("user-a", first, "sid-1");
167
+ subscribe("user-a", second, "sid-2");
168
+ publish("user-a");
169
+
170
+ expect(first).toHaveBeenCalledTimes(1);
171
+ expect(second).toHaveBeenCalledTimes(1);
172
+ });
124
173
  });
@@ -60,6 +60,17 @@ function isSseEvent(value: unknown): value is SseEvent {
60
60
  );
61
61
  }
62
62
 
63
+ // Invalidation payload: `1` means userwide, `{ keptSessionId }` spares that one
64
+ // session. Anything else falls back to userwide. Older pods never read the
65
+ // payload and invalidate userwide, which keeps a mixed rolling deploy safe.
66
+ function extractInvalidationKeptSessionId(payload: unknown): string | undefined {
67
+ if (typeof payload !== "object" || payload === null || !("keptSessionId" in payload)) {
68
+ return undefined;
69
+ }
70
+ const { keptSessionId } = payload;
71
+ return typeof keptSessionId === "string" && keptSessionId.length > 0 ? keptSessionId : undefined;
72
+ }
73
+
63
74
  // Transport layer around a local `createSseBroker()` — all client/listener
64
75
  // state lives in `inner`, this only moves events across the Redis wire via
65
76
  // the shared PubSubSignal. `pushToChannel`/`publishAccessInvalidation` never
@@ -89,7 +100,10 @@ export function createRedisSseBroker(opts: RedisSseBrokerOptions): RedisSseBroke
89
100
  }
90
101
 
91
102
  if (channel.startsWith(INVALIDATION_PREFIX)) {
92
- inner.publishAccessInvalidation(channel.slice(INVALIDATION_PREFIX.length));
103
+ inner.publishAccessInvalidation(
104
+ channel.slice(INVALIDATION_PREFIX.length),
105
+ extractInvalidationKeptSessionId(payload),
106
+ );
93
107
  }
94
108
  });
95
109
 
@@ -108,8 +122,8 @@ export function createRedisSseBroker(opts: RedisSseBrokerOptions): RedisSseBroke
108
122
  // stream must close on every replica, not just the one that observed
109
123
  // the revocation event. Publishing (rather than calling inner directly,
110
124
  // like the in-memory broker does) is what makes that true here.
111
- publishAccessInvalidation(userId) {
112
- signal.publish(`${INVALIDATION_PREFIX}${userId}`, 1);
125
+ publishAccessInvalidation(userId, keptSessionId) {
126
+ signal.publish(`${INVALIDATION_PREFIX}${userId}`, keptSessionId ? { keptSessionId } : 1);
113
127
  },
114
128
 
115
129
  close: signal.close,
@@ -1,4 +1,5 @@
1
1
  import { AsyncLocalStorage } from "node:async_hooks";
2
+ import { isPersonalDataGated, type WriteOrigin } from "@cosmicdrift/kumiko-types/event-store-types";
2
3
  import { generateId } from "../utils";
3
4
 
4
5
  // Request-scoped propagation. Populated by the HTTP middleware and by the
@@ -48,6 +49,9 @@ export type RequestContextData = {
48
49
  // performance.now() at request entry, so a failing request can report how
49
50
  // long it ran. Monotonic — a wall-clock step cannot make it negative.
50
51
  readonly startedAt?: number;
52
+ // Only ever a gated origin. Read by job enqueue and event-store.append();
53
+ // dispatch roots never narrow from it, only jobs inherit explicitly.
54
+ readonly writeOrigin?: WriteOrigin;
51
55
  };
52
56
 
53
57
  const storage = new AsyncLocalStorage<RequestContextData>();
@@ -87,3 +91,23 @@ export function runWithOrigin<T>(
87
91
  fn,
88
92
  );
89
93
  }
94
+
95
+ // An ungated origin never mints a scope: seeds and boot writes keep their missing context.
96
+ export function runWithWriteOrigin<T>(origin: WriteOrigin, fn: () => T): T {
97
+ const current = requestContext.get();
98
+ if (!isPersonalDataGated(origin)) {
99
+ if (!current?.writeOrigin) return fn();
100
+ const { writeOrigin: _writeOrigin, ...rest } = current;
101
+ return requestContext.run(rest, fn);
102
+ }
103
+ const requestId = current?.requestId ?? requestContext.generateId();
104
+ return requestContext.run(
105
+ {
106
+ ...current,
107
+ requestId,
108
+ correlationId: current?.correlationId ?? requestId,
109
+ writeOrigin: origin,
110
+ },
111
+ fn,
112
+ );
113
+ }
@@ -24,21 +24,38 @@ export type SseBroker = {
24
24
  // access-teardown security control (#1561) into a no-op — a revoked
25
25
  // session keeps receiving live SSE data with no error or log. A no-op
26
26
  // stub is one line for a broker that genuinely doesn't need it.
27
- subscribeAccessInvalidation(userId: string, onInvalidate: () => void): () => void;
28
- publishAccessInvalidation(userId: string): void;
27
+ subscribeAccessInvalidation(
28
+ userId: string,
29
+ onInvalidate: () => void,
30
+ ownSid?: string,
31
+ ): () => void;
32
+ // `keptSessionId` spares exactly one stream: the caller's own session on a
33
+ // "revoke all others" write. It is a keep-list, not a list of revoked
34
+ // sessions, so a stream of a session already revoked without an event
35
+ // (plain logout) still closes. Every other reason stays userwide.
36
+ publishAccessInvalidation(userId: string, keptSessionId?: string): void;
29
37
  };
30
38
 
39
+ // Fail-closed: without a kept sid, or for a listener without its own sid
40
+ // (PAT/bearer), nothing is spared.
41
+ function isSparedByKeptSessionId(
42
+ ownSid: string | undefined,
43
+ keptSessionId: string | undefined,
44
+ ): boolean {
45
+ return keptSessionId !== undefined && ownSid === keptSessionId;
46
+ }
47
+
31
48
  export function createSseBroker(): SseBroker {
32
49
  // Purely local: no cross-replica fanout. buildServer wraps this in
33
50
  // createRedisSseBroker (fw#2625) whenever REDIS_URL is set, which is what
34
51
  // makes pushToChannel/publishAccessInvalidation reach every replica's
35
52
  // clients — this reference implementation stays single-process only.
36
53
  const channels = new Map<string, Map<string, SseClient>>();
37
- // Set, not Map<listenerId, fn> — dedup key is callback reference. Every
38
- // subscriber must pass a distinct closure (dispatch-stream.ts does, one
39
- // per stream). Two subscribes with the SAME reference for the same user
54
+ // Keyed by callback reference, value is the subscriber's own sid. Every
55
+ // subscriber must pass a distinct closure (dispatch-stream.ts does, one per
56
+ // stream). Two subscribes with the SAME reference for the same user
40
57
  // collapse into one listener, and the first unsubscribe kills both.
41
- const accessInvalidationListeners = new Map<string, Set<() => void>>();
58
+ const accessInvalidationListeners = new Map<string, Map<() => void, string | undefined>>();
42
59
 
43
60
  function getOrCreateChannel(channel: string): Map<string, SseClient> {
44
61
  let clients = channels.get(channel);
@@ -86,14 +103,14 @@ export function createSseBroker(): SseBroker {
86
103
  return total;
87
104
  },
88
105
 
89
- subscribeAccessInvalidation(userId, onInvalidate) {
106
+ subscribeAccessInvalidation(userId, onInvalidate, ownSid) {
90
107
  const channel = userAccessChannel(userId);
91
108
  let listeners = accessInvalidationListeners.get(channel);
92
109
  if (!listeners) {
93
- listeners = new Set();
110
+ listeners = new Map();
94
111
  accessInvalidationListeners.set(channel, listeners);
95
112
  }
96
- listeners.add(onInvalidate);
113
+ listeners.set(onInvalidate, ownSid);
97
114
  return () => {
98
115
  const current = accessInvalidationListeners.get(channel);
99
116
  // skip: already unsubscribed (e.g. stream ended after a publish already fired)
@@ -103,14 +120,15 @@ export function createSseBroker(): SseBroker {
103
120
  };
104
121
  },
105
122
 
106
- publishAccessInvalidation(userId) {
123
+ publishAccessInvalidation(userId, keptSessionId) {
107
124
  const channel = userAccessChannel(userId);
108
125
  const listeners = accessInvalidationListeners.get(channel);
109
126
  // skip: no live stream is watching this user right now
110
127
  if (!listeners) return;
111
128
  // Snapshot before iterating — a fired listener unsubscribes itself,
112
129
  // which would mutate `listeners` mid-iteration otherwise.
113
- for (const onInvalidate of [...listeners]) {
130
+ for (const [onInvalidate, ownSid] of [...listeners]) {
131
+ if (isSparedByKeptSessionId(ownSid, keptSessionId)) continue;
114
132
  onInvalidate();
115
133
  }
116
134
  },
package/src/changes.json CHANGED
@@ -1,4 +1,30 @@
1
1
  [
2
+ {
3
+ "version": "0.307.0",
4
+ "type": "fix",
5
+ "title": "Event dispatcher no longer skips events that commit out of id order"
6
+ },
7
+ {
8
+ "version": "0.307.0",
9
+ "type": "fix",
10
+ "title": "A slow event consumer no longer delays every other consumer"
11
+ },
12
+ {
13
+ "version": "0.307.0",
14
+ "type": "breaking",
15
+ "title": "The public-intake gate now also covers jobs, event-triggered jobs, anonymous query roots and TenantDbs built from ctx.db.unsafeRaw (fw#3185)",
16
+ "migration": "An anonymous handler (roles include \"anonymous\") without personalData: \"public-intake\" now fails when personal data (pii / userOwned / recordOwned) is written through any of these paths: a job it enqueues (write or query root), a job triggered by its handler event or by an r.defineEvent it appends, a job chained from such a job, a job ctx.write/writeAs into another handler, or a TenantDb built with createTenantDb on a ctx.db.unsafeRaw / ctx.systemDb.unsafeRaw runner (including savepoints on it). The error is AccessDeniedError with details.reason \"public_intake_required\"; for jobs details.job names the job and the job run fails. Declare access: { roles: [..., \"anonymous\"], personalData: \"public-intake\" } on the root write handler if the anonymous intake is intended, otherwise stop writing the field from that path. A query root cannot declare public-intake: move the enqueue from an anonymous query handler into a write handler that declares it. Raw SQL through unsafeRaw stays escapeHatch plus audit. Jobs without a stamped origin (cron, boot, jobs dispatched outside a request, jobs queued before this release) run as before; a job whose _writeOrigin is present but invalid fails before its handler runs."
17
+ },
18
+ {
19
+ "version": "0.307.0",
20
+ "type": "fix",
21
+ "title": "Revoking all other sessions no longer closes the caller's own live stream"
22
+ },
23
+ {
24
+ "version": "0.307.0",
25
+ "type": "fix",
26
+ "title": "Version conflicts from custom writes return 409 instead of 500"
27
+ },
2
28
  {
3
29
  "version": "0.306.0",
4
30
  "type": "breaking",
@@ -1,6 +1,30 @@
1
+ import type { PendingGapEntry } from "../../pipeline/event-consumer-state";
1
2
  import type { AnyDb } from "../query";
2
3
  import { asRawClient } from "../query";
3
4
 
5
+ // Per-turn snapshot bounds for pending-gap finality (event-dispatcher.ts's
6
+ // processConsumer). pg_current_snapshot() is this transaction's MVCC view;
7
+ // xmin is the oldest still-in-progress xact id in it, xmax the next
8
+ // unassigned one. Both travel as strings — bigint/xid8 doesn't round-trip
9
+ // through the driver as a JS number.
10
+ export async function selectSnapshotXmin(db: AnyDb): Promise<string> {
11
+ const rows = (await asRawClient(db).unsafe(
12
+ `SELECT pg_snapshot_xmin(pg_current_snapshot())::text AS xmin`,
13
+ )) as ReadonlyArray<{ xmin: string }>;
14
+ const xmin = rows[0]?.xmin;
15
+ if (xmin === undefined) throw new Error("selectSnapshotXmin: no row returned");
16
+ return xmin;
17
+ }
18
+
19
+ export async function selectSnapshotXmax(db: AnyDb): Promise<string> {
20
+ const rows = (await asRawClient(db).unsafe(
21
+ `SELECT pg_snapshot_xmax(pg_current_snapshot())::text AS xmax`,
22
+ )) as ReadonlyArray<{ xmax: string }>;
23
+ const xmax = rows[0]?.xmax;
24
+ if (xmax === undefined) throw new Error("selectSnapshotXmax: no row returned");
25
+ return xmax;
26
+ }
27
+
4
28
  /** Serialise against consumer-bootstrap INSERTs during event retention prune. */
5
29
  export async function lockEventConsumersShareMode(db: AnyDb): Promise<void> {
6
30
  await asRawClient(db).unsafe(`LOCK TABLE "kumiko_event_consumers" IN SHARE MODE`);
@@ -116,6 +140,7 @@ export type ConsumerDeliveryOutcome = {
116
140
  readonly lastError: string | null;
117
141
  readonly deadLettered: boolean;
118
142
  readonly processed: number;
143
+ readonly pendingGaps: readonly PendingGapEntry[];
119
144
  };
120
145
 
121
146
  export async function updateConsumerDeliveryOutcome(
@@ -136,14 +161,17 @@ export async function updateConsumerDeliveryOutcome(
136
161
  "status" = $3,
137
162
  "last_error" = $4,
138
163
  "rearm_count" = CASE WHEN $5 THEN 0 ELSE "rearm_count" END,
164
+ -- text param + cast: a JS string bound straight to ::jsonb double-encodes under Bun.SQL
165
+ "pending_gaps" = $6::text::jsonb,
139
166
  "updated_at" = now()
140
- WHERE "name" = $6 AND "instance_id" = $7`,
167
+ WHERE "name" = $7 AND "instance_id" = $8`,
141
168
  [
142
169
  outcome.cursor,
143
170
  outcome.attempts,
144
171
  outcome.deadLettered ? "dead" : "idle",
145
172
  outcome.lastError,
146
173
  resetRearmCount,
174
+ JSON.stringify(outcome.pendingGaps),
147
175
  name,
148
176
  instanceId,
149
177
  ],
@@ -192,13 +220,14 @@ export async function resetConsumerForMspRebuild(
192
220
  instanceId: string,
193
221
  ): Promise<void> {
194
222
  await asRawClient(db).unsafe(
195
- `INSERT INTO "kumiko_event_consumers" ("name", "instance_id", "last_processed_event_id", "status")
196
- VALUES ($1, $2, 0, 'idle')
223
+ `INSERT INTO "kumiko_event_consumers" ("name", "instance_id", "last_processed_event_id", "status", "pending_gaps")
224
+ VALUES ($1, $2, 0, 'idle', '[]'::jsonb)
197
225
  ON CONFLICT ("name", "instance_id") DO UPDATE SET
198
226
  "last_processed_event_id" = 0,
199
227
  "status" = 'idle',
200
228
  "attempts" = 0,
201
229
  "last_error" = NULL,
230
+ "pending_gaps" = '[]'::jsonb,
202
231
  "updated_at" = now()`,
203
232
  [name, instanceId],
204
233
  );
@@ -269,3 +298,28 @@ export async function rearmDeadConsumer(
269
298
  )) as ReadonlyArray<Record<string, unknown>>;
270
299
  return rows[0];
271
300
  }
301
+
302
+ // skipPoisonEvent's pending-gap branch: the poison is a pending id below the
303
+ // cursor, so it's removed from pending_gaps directly instead of advancing
304
+ // last_processed_event_id (advancing it here would be a regression — the
305
+ // cursor already sits above this id).
306
+ export async function removePendingGapReturning(
307
+ db: AnyDb,
308
+ name: string,
309
+ instanceId: string,
310
+ newPendingGaps: readonly PendingGapEntry[],
311
+ ): Promise<Record<string, unknown> | undefined> {
312
+ const rows = (await asRawClient(db).unsafe(
313
+ `UPDATE "kumiko_event_consumers" SET
314
+ "pending_gaps" = $1::text::jsonb,
315
+ "status" = 'idle',
316
+ "attempts" = 0,
317
+ "last_error" = NULL,
318
+ "rearm_count" = 0,
319
+ "updated_at" = now()
320
+ WHERE "name" = $2 AND "instance_id" = $3
321
+ RETURNING *`,
322
+ [JSON.stringify(newPendingGaps), name, instanceId],
323
+ )) as ReadonlyArray<Record<string, unknown>>;
324
+ return rows[0];
325
+ }
@@ -7,6 +7,15 @@ import {
7
7
  import type { AnyDb } from "../query";
8
8
  import { asRawClient, unsafeReadRetrying } from "../query";
9
9
 
10
+ // Gap-finality (event-dispatcher pending_gaps) needs every holder of an
11
+ // event id to already have a *real* xact id by the time it inserts — Postgres
12
+ // assigns those lazily on first write, so a bare INSERT alone doesn't
13
+ // guarantee one exists yet for comparison against a later snapshot's
14
+ // xmin/xmax. pg_current_xact_id() forces the allocation.
15
+ export async function claimXactId(db: AnyDb): Promise<void> {
16
+ await asRawClient(db).unsafe(`SELECT pg_current_xact_id()`);
17
+ }
18
+
10
19
  /** NOTIFY on commit — wakes LISTEN subscribers (event-dispatcher). */
11
20
  export async function notifyPgChannel(db: AnyDb, channel: string): Promise<void> {
12
21
  await asRawClient(db).unsafe(`SELECT pg_notify($1, '')`, [channel]);
@@ -283,3 +292,63 @@ export async function upsertArchivedStream(db: AnyDb, params: ArchiveStreamParam
283
292
  [params.tenantId, params.aggregateId, params.aggregateType, params.archivedBy, params.reason],
284
293
  );
285
294
  }
295
+
296
+ export type PendingIdRange = { readonly from: bigint; readonly to: bigint };
297
+
298
+ // Per-consumer turn fetch: the plain `id > cursor` window plus any ranges the
299
+ // consumer is still watching as pending gaps (ids invisible on an earlier
300
+ // turn that may have become visible since). Raw SQL — the typed builder's
301
+ // WhereObject is an AND of fields, it can't express this OR.
302
+ export async function selectPendingAndNewEventRows(
303
+ db: AnyDb,
304
+ cursor: bigint,
305
+ pendingRanges: readonly PendingIdRange[],
306
+ batchSize: number,
307
+ ): Promise<ReadonlyArray<Record<string, unknown>>> {
308
+ if (pendingRanges.length === 0) {
309
+ return unsafeReadRetrying(
310
+ db,
311
+ `SELECT * FROM "kumiko_events" WHERE "id" > $1 ORDER BY "id" ASC LIMIT $2`,
312
+ [cursor, batchSize],
313
+ );
314
+ }
315
+ return unsafeReadRetrying(
316
+ db,
317
+ // UNION ALL of two primary-key range scans: an `id > $1 OR EXISTS (…)`
318
+ // predicate can't use the index and would seq-scan kumiko_events per turn.
319
+ // Ranges sit below $1, so the branches never overlap.
320
+ `(SELECT e.* FROM unnest($2::bigint[], $3::bigint[]) AS g(f, t)
321
+ CROSS JOIN LATERAL (
322
+ SELECT * FROM "kumiko_events" WHERE "id" BETWEEN g.f AND g.t ORDER BY "id" ASC LIMIT $4
323
+ ) e)
324
+ UNION ALL
325
+ (SELECT * FROM "kumiko_events" WHERE "id" > $1 ORDER BY "id" ASC LIMIT $4)
326
+ ORDER BY "id" ASC LIMIT $4`,
327
+ [
328
+ cursor,
329
+ pendingRanges.map((r) => r.from.toString()),
330
+ pendingRanges.map((r) => r.to.toString()),
331
+ batchSize,
332
+ ],
333
+ );
334
+ }
335
+
336
+ // Smallest committed-and-visible id inside any pending range — lets
337
+ // skipPoisonEvent (event-dispatcher-admin.ts) tell a live poison apart from
338
+ // a still-invisible gap without pulling full rows.
339
+ export async function selectSmallestVisibleIdInRanges(
340
+ db: AnyDb,
341
+ pendingRanges: readonly PendingIdRange[],
342
+ ): Promise<bigint | null> {
343
+ if (pendingRanges.length === 0) return null;
344
+ const rows = (await unsafeReadRetrying(
345
+ db,
346
+ `SELECT MIN("id")::text AS id FROM "kumiko_events"
347
+ WHERE EXISTS (
348
+ SELECT 1 FROM unnest($1::bigint[], $2::bigint[]) AS g(f, t) WHERE "id" BETWEEN g.f AND g.t
349
+ )`,
350
+ [pendingRanges.map((r) => r.from.toString()), pendingRanges.map((r) => r.to.toString())],
351
+ )) as ReadonlyArray<{ id: string | null }>;
352
+ const id = rows[0]?.id;
353
+ return id === null || id === undefined ? null : BigInt(id);
354
+ }
@@ -51,6 +51,41 @@ const declaredUnsafeRawRunners = new WeakMap<
51
51
  // (withUnsafeRawGrant, acknowledgeConventionCrossTenant) carry it too.
52
52
  const personalDataGates = new WeakMap<TenantDb, PersonalDataGate>();
53
53
 
54
+ // Lets createTenantDb(ctx.db.unsafeRaw(reason), ...) inherit the gate. Keyed by a
55
+ // per-grant proxy, never the shared pool/tx: tagging that would gate every sibling TenantDb.
56
+ const runnerPersonalDataGates = new WeakMap<DbRunner, PersonalDataGate>();
57
+
58
+ function gatedRunner(runner: DbRunner, gate: PersonalDataGate): DbRunner {
59
+ const proxy = new Proxy(runner as object, {
60
+ // Tagged-template calls need the real driver object as `this`.
61
+ apply(target, _thisArg, args) {
62
+ return Reflect.apply(target as (...callArgs: unknown[]) => unknown, target, args);
63
+ },
64
+ get(target, prop, _receiver) {
65
+ const value = Reflect.get(target, prop, target);
66
+ if (typeof value !== "function") return value;
67
+ if (prop === "begin" || prop === "savepoint") {
68
+ // createTenantDb(tx, ...) inside the callback must inherit the gate too.
69
+ return (...args: unknown[]) => {
70
+ const callback = args[args.length - 1];
71
+ if (typeof callback !== "function") {
72
+ return Reflect.apply(value, target, args);
73
+ }
74
+ const gatedArgs = [
75
+ ...args.slice(0, -1),
76
+ (tx: unknown) => callback(gatedRunner(tx as DbRunner, gate)),
77
+ ];
78
+ return Reflect.apply(value, target, gatedArgs);
79
+ };
80
+ }
81
+ return value.bind(target);
82
+ },
83
+ // @cast-boundary proxy-erasure — Proxy<object> re-tags as the wrapped DbRunner shape.
84
+ }) as DbRunner;
85
+ runnerPersonalDataGates.set(proxy, gate);
86
+ return proxy;
87
+ }
88
+
54
89
  // The executor passes its entity so the check does not depend on the table-name lookup.
55
90
  export function assertPersonalDataWrite(
56
91
  db: TenantDb,
@@ -151,7 +186,9 @@ function buildUncheckedSystemDb(
151
186
  });
152
187
  }
153
188
  report("unsafe-raw", reason);
154
- return tenantDbRunner(db);
189
+ const runner = tenantDbRunner(db);
190
+ const personalDataGate = personalDataGates.get(db);
191
+ return personalDataGate ? gatedRunner(runner, personalDataGate) : runner;
155
192
  }
156
193
 
157
194
  const uncheckedSystemDb: UncheckedSystemDb = {
@@ -352,6 +389,7 @@ export function createTenantDb(
352
389
  ): TenantDb {
353
390
  if (meter) registerStandardMetrics(meter);
354
391
  const report = grants?.report ?? fallbackEscapeHatchReporter(tenantId);
392
+ const personalDataGate = grants?.personalDataGate ?? runnerPersonalDataGates.get(db);
355
393
 
356
394
  function withDbSpan<T>(
357
395
  operation: "select" | "insert" | "update" | "delete",
@@ -452,9 +490,9 @@ export function createTenantDb(
452
490
  table: Table | EntityTableMeta,
453
491
  keys: readonly string[],
454
492
  ): AccessDeniedError | undefined {
455
- if (!grants?.personalDataGate) return undefined;
493
+ if (!personalDataGate) return undefined;
456
494
  try {
457
- grants.personalDataGate(tableNameOf(table), keys);
495
+ personalDataGate(tableNameOf(table), keys);
458
496
  return undefined;
459
497
  } catch (e) {
460
498
  if (e instanceof AccessDeniedError) return e;
@@ -553,7 +591,7 @@ export function createTenantDb(
553
591
  });
554
592
  }
555
593
  report("unsafe-raw", reason);
556
- return db;
594
+ return personalDataGate ? gatedRunner(db, personalDataGate) : db;
557
595
  }
558
596
 
559
597
  const tenantDb: TenantDb = {
@@ -648,7 +686,7 @@ export function createTenantDb(
648
686
  return createTenantDb(db, tenantId, "system", tracer, meter, signal, grants);
649
687
  });
650
688
  bindTenantDbRunner(tenantDb, db);
651
- if (grants?.personalDataGate) personalDataGates.set(tenantDb, grants.personalDataGate);
689
+ if (personalDataGate) personalDataGates.set(tenantDb, personalDataGate);
652
690
  return tenantDb;
653
691
  }
654
692