@cosmicdrift/kumiko-framework 0.307.0 → 0.309.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (174) hide show
  1. package/package.json +5 -5
  2. package/src/__tests__/anonymous-access.integration.test.ts +1 -1
  3. package/src/__tests__/error-contract.integration.test.ts +1 -1
  4. package/src/__tests__/field-access.integration.test.ts +1 -1
  5. package/src/__tests__/fixtures/migrate-open-to-all/manual-only.expected.ts +1 -1
  6. package/src/__tests__/fixtures/migrate-open-to-all/manual-only.input.ts +1 -1
  7. package/src/__tests__/fixtures/migrate-open-to-all/test-simple.expected.ts +1 -1
  8. package/src/__tests__/fixtures/migrate-open-to-all/test-simple.input.ts +1 -1
  9. package/src/__tests__/full-stack.integration.test.ts +1 -1
  10. package/src/__tests__/transition-guard.integration.test.ts +1 -1
  11. package/src/api/__tests__/api.test.ts +1 -1
  12. package/src/api/__tests__/batch.integration.test.ts +1 -1
  13. package/src/api/__tests__/body-limit.test.ts +1 -1
  14. package/src/api/__tests__/dispatcher-live.integration.test.ts +1 -1
  15. package/src/api/__tests__/extra-routes.integration.test.ts +1 -1
  16. package/src/api/__tests__/http-route-rate-limit.integration.test.ts +1 -1
  17. package/src/api/__tests__/nested-write-ownership.integration.test.ts +1 -1
  18. package/src/api/__tests__/nested-write.integration.test.ts +1 -1
  19. package/src/api/__tests__/pii-leak-guard.integration.test.ts +1 -1
  20. package/src/api/__tests__/redis-sse-broker.integration.test.ts +73 -6
  21. package/src/api/__tests__/request-locale.integration.test.ts +1 -1
  22. package/src/api/__tests__/server-boot-guards.test.ts +1 -1
  23. package/src/api/__tests__/server-error-logging.test.ts +1 -1
  24. package/src/api/__tests__/sse-broker.test.ts +107 -13
  25. package/src/api/__tests__/stream-escape-hatch.integration.test.ts +1 -1
  26. package/src/api/auth-routes.ts +1 -1
  27. package/src/api/redis-sse-broker.ts +51 -11
  28. package/src/api/server.ts +11 -0
  29. package/src/api/sse-broker.ts +61 -22
  30. package/src/bun-db/__tests__/postgres-closed-socket-write.integration.test.ts +87 -0
  31. package/src/bun-db/connection.ts +2 -0
  32. package/src/changes.json +61 -0
  33. package/src/compliance/override-schema.ts +1 -1
  34. package/src/crypto/__tests__/event-pii.test.ts +1 -1
  35. package/src/crypto/kms-adapter.ts +1 -1
  36. package/src/db/__tests__/feature-table-sources.test.ts +2 -2
  37. package/src/db/__tests__/tenant-db-where-merge.integration.test.ts +1 -1
  38. package/src/db/collect-table-metas.ts +3 -6
  39. package/src/db/feature-table-sources.ts +3 -4
  40. package/src/db/queries/event-consumer.ts +98 -6
  41. package/src/derivatives/__tests__/derivatives-context.integration.test.ts +1 -1
  42. package/src/derivatives/delete-stored-file.ts +86 -0
  43. package/src/derivatives/index.ts +2 -0
  44. package/src/engine/__tests__/boot-validator-action-wiring.test.ts +1 -1
  45. package/src/engine/__tests__/boot-validator-dashboard.test.ts +1 -1
  46. package/src/engine/__tests__/boot-validator-nav-area.test.ts +1 -1
  47. package/src/engine/__tests__/boot-validator-pii-retention.test.ts +1 -1
  48. package/src/engine/__tests__/boot-validator-projection-list.test.ts +1 -1
  49. package/src/engine/__tests__/boot-validator-query-output-schema.test.ts +1 -1
  50. package/src/engine/__tests__/boot-validator-query-refs.test.ts +1 -1
  51. package/src/engine/__tests__/boot-validator-ref-entity.test.ts +1 -1
  52. package/src/engine/__tests__/boot-validator-s0-integration.test.ts +1 -1
  53. package/src/engine/__tests__/boot-validator.test.ts +35 -1
  54. package/src/engine/__tests__/build-app-schema.test.ts +1 -1
  55. package/src/engine/__tests__/build-config-feature-schema-secrets.test.ts +1 -1
  56. package/src/engine/__tests__/define-feature-entity-mapping.test.ts +1 -1
  57. package/src/engine/__tests__/engine.test.ts +1 -1
  58. package/src/engine/__tests__/event-helpers.test.ts +1 -1
  59. package/src/engine/__tests__/event-migration-declarative.test.ts +1 -1
  60. package/src/engine/__tests__/extend-entity-projection.test.ts +1 -1
  61. package/src/engine/__tests__/extends-registrar.test.ts +1 -1
  62. package/src/engine/__tests__/hook-phases.test.ts +1 -1
  63. package/src/engine/__tests__/lifecycle-hooks.test.ts +1 -1
  64. package/src/engine/__tests__/pipeline-engine.test.ts +1 -1
  65. package/src/engine/__tests__/pipeline-handler.integration.test.ts +1 -1
  66. package/src/engine/__tests__/pipeline-observability.integration.test.ts +1 -1
  67. package/src/engine/__tests__/pipeline-performance.integration.test.ts +1 -1
  68. package/src/engine/__tests__/pipeline-sub-pipelines.test.ts +1 -1
  69. package/src/engine/__tests__/post-query-hook.test.ts +1 -1
  70. package/src/engine/__tests__/projection-detail-actions.test.ts +1 -1
  71. package/src/engine/__tests__/projection-detail-singleton.test.ts +1 -1
  72. package/src/engine/__tests__/projection-detail-tabs.test.ts +1 -1
  73. package/src/engine/__tests__/read-steps-tenant-filter.integration.test.ts +1 -1
  74. package/src/engine/__tests__/registry.test.ts +1 -1
  75. package/src/engine/__tests__/screen.test.ts +1 -1
  76. package/src/engine/__tests__/validate-projection-allowlist.test.ts +1 -1
  77. package/src/engine/boot-validator/__tests__/access-declarations.test.ts +1 -1
  78. package/src/engine/boot-validator/__tests__/action-icon.test.ts +81 -0
  79. package/src/engine/boot-validator/__tests__/anonymous-rate-limit-required.test.ts +1 -1
  80. package/src/engine/boot-validator/__tests__/no-all-role-in-handler-access.test.ts +1 -1
  81. package/src/engine/boot-validator/__tests__/parent-ref.test.ts +1 -1
  82. package/src/engine/boot-validator/__tests__/rate-limit-disabled-reason.test.ts +1 -1
  83. package/src/engine/boot-validator/screens.ts +83 -0
  84. package/src/engine/define-handler.ts +2 -1
  85. package/src/engine/entity-handlers.ts +2 -1
  86. package/src/engine/feature-ast/__tests__/pattern-change-schema.test.ts +1 -1
  87. package/src/engine/feature-ast/pattern-change-schema.ts +1 -1
  88. package/src/engine/feature-builder-state.ts +2 -1
  89. package/src/engine/feature-config-events-jobs.ts +2 -1
  90. package/src/engine/feature-entity-handlers.ts +2 -1
  91. package/src/engine/schema-builder.ts +1 -1
  92. package/src/entrypoint/__tests__/entrypoint-attach-dispatcher.integration.test.ts +1 -1
  93. package/src/entrypoint/__tests__/entrypoint-job-wiring.integration.test.ts +1 -1
  94. package/src/entrypoint/__tests__/split-deploy.integration.test.ts +1 -1
  95. package/src/env/__tests__/compose-env-schema.test.ts +1 -1
  96. package/src/env/__tests__/dry-run.test.ts +1 -1
  97. package/src/env/__tests__/parse-env-dry-run.test.ts +1 -1
  98. package/src/env/_zod-introspect.ts +1 -1
  99. package/src/env/dry-run.ts +1 -1
  100. package/src/env/index.ts +1 -1
  101. package/src/errors/__tests__/classes.test.ts +1 -1
  102. package/src/es-ops/__tests__/runner.integration.test.ts +1 -1
  103. package/src/event-store/__tests__/backfill-pii.integration.test.ts +1 -1
  104. package/src/event-store/__tests__/event-attribution.integration.test.ts +1 -1
  105. package/src/event-store/__tests__/upcaster.integration.test.ts +1 -1
  106. package/src/jobs/__tests__/job-backoff.integration.test.ts +1 -1
  107. package/src/jobs/__tests__/job-event-trigger.integration.test.ts +1 -1
  108. package/src/jobs/__tests__/job-multi-trigger.integration.test.ts +1 -1
  109. package/src/jobs/__tests__/job-public-intake-origin.integration.test.ts +1 -1
  110. package/src/jobs/__tests__/job-retention.integration.test.ts +1 -1
  111. package/src/jobs/__tests__/job-write-as.integration.test.ts +1 -1
  112. package/src/jobs/__tests__/jobs.integration.test.ts +1 -1
  113. package/src/observability/__tests__/metrics-wiring.test.ts +1 -1
  114. package/src/observability/__tests__/observability.integration.test.ts +1 -1
  115. package/src/observability/metrics-wiring.ts +1 -1
  116. package/src/pipeline/__tests__/archive-stream.integration.test.ts +1 -1
  117. package/src/pipeline/__tests__/causation-chain.integration.test.ts +1 -1
  118. package/src/pipeline/__tests__/ctx-bridge.integration.test.ts +1 -1
  119. package/src/pipeline/__tests__/ctx-systemdb.integration.test.ts +1 -1
  120. package/src/pipeline/__tests__/dispatcher.test.ts +1 -1
  121. package/src/pipeline/__tests__/domain-events-projections.integration.test.ts +1 -1
  122. package/src/pipeline/__tests__/escape-hatch-audit.integration.test.ts +1 -1
  123. package/src/pipeline/__tests__/escape-hatch.integration.test.ts +1 -1
  124. package/src/pipeline/__tests__/event-dedup.integration.test.ts +1 -1
  125. package/src/pipeline/__tests__/event-define-event-strict.integration.test.ts +1 -1
  126. package/src/pipeline/__tests__/event-dispatcher-batch-delivery.test.ts +178 -0
  127. package/src/pipeline/__tests__/event-dispatcher-batch-handler.integration.test.ts +143 -0
  128. package/src/pipeline/__tests__/event-dispatcher-db-outage.integration.test.ts +192 -0
  129. package/src/pipeline/__tests__/event-dispatcher-idle.integration.test.ts +94 -0
  130. package/src/pipeline/__tests__/event-dispatcher-start-from.integration.test.ts +103 -3
  131. package/src/pipeline/__tests__/fetch-for-writing.integration.test.ts +1 -1
  132. package/src/pipeline/__tests__/idempotency-transient-failure.integration.test.ts +1 -1
  133. package/src/pipeline/__tests__/job-trigger-consumer.integration.test.ts +1 -1
  134. package/src/pipeline/__tests__/lifecycle-pipeline.test.ts +1 -1
  135. package/src/pipeline/__tests__/load-aggregate-query.integration.test.ts +1 -1
  136. package/src/pipeline/__tests__/msp-multi-hop.integration.test.ts +1 -1
  137. package/src/pipeline/__tests__/msp-rebuild.integration.test.ts +116 -1
  138. package/src/pipeline/__tests__/multi-stream-projection.integration.test.ts +1 -1
  139. package/src/pipeline/__tests__/pending-gap-ranges.test.ts +94 -0
  140. package/src/pipeline/__tests__/post-query-hook.integration.test.ts +1 -1
  141. package/src/pipeline/__tests__/public-intake-runtime-gate.integration.test.ts +1 -1
  142. package/src/pipeline/__tests__/query-projection.integration.test.ts +1 -1
  143. package/src/pipeline/__tests__/rate-limit-default.integration.test.ts +1 -1
  144. package/src/pipeline/__tests__/rebuild-poison-quarantine.integration.test.ts +1 -1
  145. package/src/pipeline/__tests__/search-consumer-named-events.integration.test.ts +1 -1
  146. package/src/pipeline/__tests__/system-identity-switch.integration.test.ts +1 -1
  147. package/src/pipeline/__tests__/try-append-event.integration.test.ts +1 -1
  148. package/src/pipeline/__tests__/unsafe-raw-escape-hatch.integration.test.ts +1 -1
  149. package/src/pipeline/dispatch-stream.ts +6 -1
  150. package/src/pipeline/event-dispatcher-delivery.ts +132 -37
  151. package/src/pipeline/event-dispatcher.ts +121 -17
  152. package/src/pipeline/index.ts +1 -0
  153. package/src/pipeline/msp-rebuild.ts +39 -1
  154. package/src/pipeline/pending-gap-ranges.ts +48 -0
  155. package/src/pipeline/system-hooks.ts +223 -68
  156. package/src/pipeline/write-origin.ts +1 -1
  157. package/src/rate-limit/__tests__/dispatcher-l3.integration.test.ts +1 -1
  158. package/src/search/__tests__/derive-search-adapter-config.test.ts +65 -0
  159. package/src/search/__tests__/meilisearch-adapter.integration.test.ts +91 -0
  160. package/src/search/__tests__/search-consumer-batch.integration.test.ts +306 -0
  161. package/src/search/derive-search-adapter-config.ts +27 -0
  162. package/src/search/index.ts +1 -0
  163. package/src/search/meilisearch-adapter.ts +96 -14
  164. package/src/secrets/__tests__/contains-secret.test.ts +1 -1
  165. package/src/stack/__tests__/request-helper.integration.test.ts +1 -1
  166. package/src/stack/__tests__/setup-test-stack-entity-tables.integration.test.ts +65 -0
  167. package/src/stack/__tests__/setup-test-stack-jobs.integration.test.ts +1 -1
  168. package/src/stack/__tests__/setup-test-stack-metrics.integration.test.ts +1 -1
  169. package/src/stack/test-stack.ts +61 -54
  170. package/src/testing/__tests__/e2e-generator.test.ts +1 -1
  171. package/src/testing/__tests__/real-providers.test.ts +9 -1
  172. package/src/testing/e2e-generator.ts +1 -1
  173. package/src/testing/index.ts +1 -1
  174. package/src/testing/real-providers.ts +12 -4
@@ -17,7 +17,8 @@
17
17
 
18
18
  import { afterAll, afterEach, beforeAll, describe, expect, test } from "bun:test";
19
19
  import { sql } from "@cosmicdrift/kumiko-framework/db";
20
- import { z } from "zod";
20
+ import * as z from "zod";
21
+ import type { DbConnection, DbTx } from "../../db/connection";
21
22
  import { integer as pgInteger, table as pgTable, uuid as pgUuid } from "../../db/dialect";
22
23
  import { createEventStoreExecutor } from "../../db/event-store-executor";
23
24
  import { asRawClient, selectMany, updateMany } from "../../db/query";
@@ -35,6 +36,8 @@ import {
35
36
  TestUsers,
36
37
  unsafeCreateEntityTable,
37
38
  } from "../../stack";
39
+ import { waitFor } from "../../testing";
40
+ import { SHARED_INSTANCE_SENTINEL } from "../event-consumer-state";
38
41
 
39
42
  // --- Fixtures: two aggregates feeding one MSP + two cornered MSPs ---
40
43
 
@@ -69,6 +72,10 @@ const sagaStateTable = pgTable("read_mspreb_saga_state", {
69
72
  id: pgUuid("id").primaryKey(),
70
73
  });
71
74
 
75
+ // Captured for the raw-SQL out-of-order-commit test below, which appends
76
+ // events directly (bypassing the write handler) to control commit timing.
77
+ let invoiceBilledEventName = "";
78
+
72
79
  const feature = defineFeature("mspreb", (r) => {
73
80
  r.entity("msp-reb-invoice", invoiceEntity);
74
81
  r.entity("msp-reb-payment", paymentEntity);
@@ -78,6 +85,7 @@ const feature = defineFeature("mspreb", (r) => {
78
85
  z.object({ customer: z.uuid(), cents: z.number().int() }),
79
86
  { piiFields: "none" },
80
87
  );
88
+ invoiceBilledEventName = invoiceBilled.name;
81
89
  const paymentReceived = r.defineEvent(
82
90
  "payment-received",
83
91
  z.object({ customer: z.uuid(), cents: z.number().int() }),
@@ -231,6 +239,47 @@ async function runFullDispatcher(): Promise<void> {
231
239
  await stack.eventDispatcher?.runOnce();
232
240
  }
233
241
 
242
+ // Appends an invoice-billed event directly (not via the write handler) so
243
+ // the caller controls exactly when its id is committed and visible. `version`
244
+ // must be distinct per call for the SAME aggregateId — a repeated version on
245
+ // an uncommitted row makes Postgres block the second INSERT until the first
246
+ // transaction resolves (unique-constraint ambiguity), which would deadlock
247
+ // against a caller still holding that first transaction open.
248
+ async function insertInvoiceBilledEvent(
249
+ db: DbConnection | DbTx,
250
+ customer: string,
251
+ cents: number,
252
+ version: number,
253
+ ): Promise<void> {
254
+ await asRawClient(db).unsafe(
255
+ `INSERT INTO "kumiko_events"
256
+ (aggregate_id, aggregate_type, tenant_id, version, type, payload, metadata, created_by)
257
+ VALUES ($1::uuid, 'msp-reb-invoice', $2::uuid, $3, $4, $5::jsonb, '{}'::jsonb, 'test')`,
258
+ [
259
+ customer,
260
+ admin.tenantId,
261
+ version,
262
+ invoiceBilledEventName,
263
+ JSON.stringify({ customer, cents }),
264
+ ],
265
+ );
266
+ }
267
+
268
+ async function readPendingGaps(
269
+ consumerName: string,
270
+ ): Promise<ReadonlyArray<{ from: string; to: string; xmax: string }>> {
271
+ const rows = (await asRawClient(stack.db).unsafe(
272
+ `SELECT "pending_gaps", jsonb_typeof("pending_gaps") AS kind FROM "kumiko_event_consumers" WHERE "name" = $1 AND "instance_id" = $2`,
273
+ [consumerName, SHARED_INSTANCE_SENTINEL],
274
+ )) as ReadonlyArray<{
275
+ pending_gaps: ReadonlyArray<{ from: string; to: string; xmax: string }>;
276
+ kind: string;
277
+ }>;
278
+ // A double-encoded write lands as a jsonb string scalar, not an array.
279
+ if (rows[0] && rows[0].kind !== "array") throw new Error(`pending_gaps is jsonb ${rows[0].kind}`);
280
+ return rows[0]?.pending_gaps ?? [];
281
+ }
282
+
234
283
  // --- Tests ---
235
284
 
236
285
  describe("rebuildMultiStreamProjection — rebuildable read-model", () => {
@@ -310,6 +359,72 @@ describe("rebuildMultiStreamProjection — rebuildable read-model", () => {
310
359
  const [row] = await selectMany(stack.db, balanceTable, { customer: carol });
311
360
  expect(row).toMatchObject({ invoicesCents: 42_00, paymentsCents: 0 });
312
361
  });
362
+
363
+ test("rebuild handoff seeds a pending gap for a lower-id write still open during replay, and the live dispatcher delivers it exactly once (#3231)", async () => {
364
+ const frank = "00000000-0000-4000-8000-00000000f106";
365
+ await updateMany(
366
+ stack.db,
367
+ eventConsumerStateTable,
368
+ { status: "disabled", updatedAt: sql`now()` },
369
+ { name: SAGA_MSP },
370
+ );
371
+
372
+ // Tx A grabs the LOW id first but holds its transaction open —
373
+ // uncommitted, so it's invisible to the rebuild's horizon read below.
374
+ let releaseA!: () => void;
375
+ const aGate = new Promise<void>((resolve) => {
376
+ releaseA = resolve;
377
+ });
378
+ let markAInserted!: () => void;
379
+ const aInserted = new Promise<void>((resolve) => {
380
+ markAInserted = resolve;
381
+ });
382
+ const aDone = stack.db
383
+ .begin(async (tx: DbTx) => {
384
+ await insertInvoiceBilledEvent(tx, frank, 1_00, 1);
385
+ markAInserted();
386
+ await aGate;
387
+ })
388
+ .catch(() => {});
389
+ await aInserted;
390
+
391
+ // Commits AFTER A grabbed its id, so it's the higher, only-visible id
392
+ // when the rebuild reads its horizon.
393
+ await insertInvoiceBilledEvent(stack.db, frank, 100_00, 2);
394
+
395
+ try {
396
+ const result = await rebuildMultiStreamProjection(BALANCE_MSP, {
397
+ db: stack.db,
398
+ registry: stack.registry,
399
+ });
400
+ // Only the visible (higher-id) event was replayed — A's row is still
401
+ // invisible, so it must not be double-counted once it does arrive.
402
+ const [afterRebuild] = await selectMany(stack.db, balanceTable, { customer: frank });
403
+ expect(afterRebuild).toMatchObject({ invoicesCents: 100_00 });
404
+ expect(result.lastProcessedEventId).toBeGreaterThan(0n);
405
+
406
+ const gapsAfterRebuild = await readPendingGaps(BALANCE_MSP);
407
+ expect(gapsAfterRebuild).not.toEqual([]);
408
+ } finally {
409
+ releaseA();
410
+ await aDone;
411
+ }
412
+
413
+ // A commits now — the live dispatcher must deliver it exactly once via
414
+ // the seeded pending gap, landing at 1_00 + 100_00, not double-applied.
415
+ await waitFor(
416
+ async () => {
417
+ await runFullDispatcher();
418
+ const [row] = await selectMany(stack.db, balanceTable, { customer: frank });
419
+ expect(row).toMatchObject({ invoicesCents: 101_00 });
420
+ expect(await readPendingGaps(BALANCE_MSP)).toEqual([]);
421
+ },
422
+ { delays: [20, 100, 500, 1000, 3000] },
423
+ );
424
+
425
+ const [finalRow] = await selectMany(stack.db, balanceTable, { customer: frank });
426
+ expect(finalRow).toMatchObject({ invoicesCents: 101_00 });
427
+ });
313
428
  });
314
429
 
315
430
  describe("rebuildMultiStreamProjection — guard rails", () => {
@@ -7,7 +7,7 @@
7
7
  // per MSP consumer, dead-letters on repeated handler failures.
8
8
 
9
9
  import { afterAll, afterEach, beforeAll, describe, expect, test } from "bun:test";
10
- import { z } from "zod";
10
+ import * as z from "zod";
11
11
  import { integer as pgInteger, table as pgTable, uuid as pgUuid } from "../../db/dialect";
12
12
  import { createEventStoreExecutor } from "../../db/event-store-executor";
13
13
  import { asRawClient, selectMany } from "../../db/query";
@@ -0,0 +1,94 @@
1
+ import { describe, expect, test } from "bun:test";
2
+ import type { PendingGapEntry } from "../event-consumer-state";
3
+ import { capPendingGapsBelowCursor, subtractSortedIdsFromRanges } from "../pending-gap-ranges";
4
+
5
+ function gap(from: string, to: string, xmax = "100"): PendingGapEntry {
6
+ return { from, to, xmax };
7
+ }
8
+
9
+ describe("subtractSortedIdsFromRanges", () => {
10
+ test("no ids leaves ranges unchanged", () => {
11
+ const ranges = [gap("10", "20")];
12
+ expect(subtractSortedIdsFromRanges(ranges, [])).toEqual([gap("10", "20")]);
13
+ });
14
+
15
+ test("id in the middle of a range splits it in two", () => {
16
+ const ranges = [gap("10", "20")];
17
+ expect(subtractSortedIdsFromRanges(ranges, [15n])).toEqual([gap("10", "14"), gap("16", "20")]);
18
+ });
19
+
20
+ test("ids at from- and to-boundary shrink the range from inside", () => {
21
+ const ranges = [gap("10", "20")];
22
+ expect(subtractSortedIdsFromRanges(ranges, [10n, 20n])).toEqual([gap("11", "19")]);
23
+ });
24
+
25
+ test("all ids of a range make it disappear", () => {
26
+ const ranges = [gap("10", "12")];
27
+ expect(subtractSortedIdsFromRanges(ranges, [10n, 11n, 12n])).toEqual([]);
28
+ });
29
+
30
+ test("ids between, before and after ranges affect nothing", () => {
31
+ const ranges = [gap("10", "20"), gap("30", "40")];
32
+ expect(subtractSortedIdsFromRanges(ranges, [5n, 25n, 45n])).toEqual([
33
+ gap("10", "20"),
34
+ gap("30", "40"),
35
+ ]);
36
+ });
37
+
38
+ test("multiple ranges with ids spread across them use a single pointer pass", () => {
39
+ const ranges = [gap("10", "20"), gap("30", "40"), gap("50", "60")];
40
+ const ids = [15n, 30n, 40n, 55n];
41
+ expect(subtractSortedIdsFromRanges(ranges, ids)).toEqual([
42
+ gap("10", "14"),
43
+ gap("16", "20"),
44
+ gap("31", "39"),
45
+ gap("50", "54"),
46
+ gap("56", "60"),
47
+ ]);
48
+ });
49
+
50
+ test("xmax is preserved on every surviving part", () => {
51
+ const ranges = [gap("10", "20", "999")];
52
+ expect(subtractSortedIdsFromRanges(ranges, [15n])).toEqual([
53
+ gap("10", "14", "999"),
54
+ gap("16", "20", "999"),
55
+ ]);
56
+ });
57
+ });
58
+
59
+ describe("capPendingGapsBelowCursor", () => {
60
+ test("cursor 0n caps everything away", () => {
61
+ const ranges = [gap("10", "20")];
62
+ expect(capPendingGapsBelowCursor(ranges, 0n)).toEqual([]);
63
+ });
64
+
65
+ test("range fully below cursor is unchanged", () => {
66
+ const ranges = [gap("10", "20")];
67
+ expect(capPendingGapsBelowCursor(ranges, 100n)).toEqual([gap("10", "20")]);
68
+ });
69
+
70
+ test("range straddling the cursor is truncated to cursor - 1", () => {
71
+ const ranges = [gap("10", "20")];
72
+ expect(capPendingGapsBelowCursor(ranges, 15n)).toEqual([gap("10", "14")]);
73
+ });
74
+
75
+ test("range with to == cursor is truncated to cursor - 1", () => {
76
+ const ranges = [gap("10", "20")];
77
+ expect(capPendingGapsBelowCursor(ranges, 20n)).toEqual([gap("10", "19")]);
78
+ });
79
+
80
+ test("range with from == cursor is dropped entirely", () => {
81
+ const ranges = [gap("20", "30")];
82
+ expect(capPendingGapsBelowCursor(ranges, 20n)).toEqual([]);
83
+ });
84
+
85
+ test("range with from > cursor is dropped entirely", () => {
86
+ const ranges = [gap("25", "30")];
87
+ expect(capPendingGapsBelowCursor(ranges, 20n)).toEqual([]);
88
+ });
89
+
90
+ test("xmax is preserved on surviving ranges", () => {
91
+ const ranges = [gap("10", "20", "777")];
92
+ expect(capPendingGapsBelowCursor(ranges, 15n)).toEqual([gap("10", "14", "777")]);
93
+ });
94
+ });
@@ -10,7 +10,7 @@
10
10
  // nicht createTestDispatcher.
11
11
 
12
12
  import { afterAll, beforeAll, describe, expect, test } from "bun:test";
13
- import { z } from "zod";
13
+ import * as z from "zod";
14
14
  import { createEntity, createTextField, defineFeature } from "../../engine";
15
15
  import type { PostQueryHookFn } from "../../engine/types";
16
16
  import { setupTestStack, type TestStack, TestUsers } from "../../stack";
@@ -2,7 +2,7 @@
2
2
  // so the static boot check passes and only the runtime gate can stop the write.
3
3
 
4
4
  import { afterAll, beforeAll, beforeEach, describe, expect, test } from "bun:test";
5
- import { z } from "zod";
5
+ import * as z from "zod";
6
6
  import type { SchemaTable } from "../../db";
7
7
  import type { DbRunner } from "../../db/connection";
8
8
  import { createEventStoreExecutor } from "../../db/event-store-executor";
@@ -5,7 +5,7 @@
5
5
  // grant — r.systemScope() or a declared escapeHatch on the handler.
6
6
 
7
7
  import { afterAll, afterEach, beforeAll, describe, expect, test } from "bun:test";
8
- import { z } from "zod";
8
+ import * as z from "zod";
9
9
  import {
10
10
  integer as pgInteger,
11
11
  table as pgTable,
@@ -4,7 +4,7 @@
4
4
  // rate-limit/__tests__/dispatcher-l3.integration.test.ts.
5
5
 
6
6
  import { afterAll, beforeAll, describe, expect, test } from "bun:test";
7
- import { z } from "zod";
7
+ import * as z from "zod";
8
8
  import { createSystemUser, defineFeature } from "../../engine";
9
9
  import type { RateLimitConfig, RateLimitDecision, RateLimitResolver } from "../../rate-limit";
10
10
  import { createTestUser, setupTestStack, type TestStack, testTenantId } from "../../stack";
@@ -15,7 +15,7 @@
15
15
  // - SQL error (aborts the tx → proves the savepoint is load-bearing)
16
16
 
17
17
  import { afterAll, beforeAll, beforeEach, describe, expect, test } from "bun:test";
18
- import { z } from "zod";
18
+ import * as z from "zod";
19
19
  import { integer as pgInteger, table as pgTable, uuid as pgUuid } from "../../db/dialect";
20
20
  import { createEventStoreExecutor } from "../../db/event-store-executor";
21
21
  import { asRawClient } from "../../db/query";
@@ -3,7 +3,7 @@
3
3
  // the search index. The event payload only carries the named event's own
4
4
  // field slice, so the consumer reads the live projection row instead.
5
5
  import { afterAll, afterEach, beforeAll, describe, expect, test } from "bun:test";
6
- import { z } from "zod";
6
+ import * as z from "zod";
7
7
  import { asRawClient, buildEntityTable, createEventStoreExecutor, createTenantDb } from "../../db";
8
8
  import { createEntity, createTextField, defineFeature } from "../../engine";
9
9
  import {
@@ -2,7 +2,7 @@
2
2
  // r.systemScope() or { escapeHatch }. Real HTTP calls + setupTestStack — never createTestDispatcher.
3
3
 
4
4
  import { afterAll, beforeAll, describe, expect, test } from "bun:test";
5
- import { z } from "zod";
5
+ import * as z from "zod";
6
6
  import { createEventStoreExecutor } from "../../db/event-store-executor";
7
7
  import { buildEntityTable } from "../../db/table-builder";
8
8
  import {
@@ -21,7 +21,7 @@
21
21
  // may pick either writer as the winner.
22
22
 
23
23
  import { afterAll, beforeAll, beforeEach, describe, expect, test } from "bun:test";
24
- import { z } from "zod";
24
+ import * as z from "zod";
25
25
  import { createEventStoreExecutor } from "../../db/event-store-executor";
26
26
  import { asRawClient, selectMany } from "../../db/query";
27
27
  import { buildEntityTable } from "../../db/table-builder";
@@ -4,7 +4,7 @@
4
4
  // escape-hatch.integration.test.ts (fw#2855).
5
5
 
6
6
  import { afterAll, beforeAll, describe, expect, test } from "bun:test";
7
- import { z } from "zod";
7
+ import * as z from "zod";
8
8
  import { executeRawQuery } from "../../db/queries/raw-sql";
9
9
  import { defineFeature } from "../../engine";
10
10
  import { createTestUser, setupTestStack, type TestStack } from "../../stack";
@@ -1,3 +1,4 @@
1
+ import type { AccessInvalidationCredential } from "../api/sse-broker";
1
2
  import { hasAccess } from "../engine/access";
2
3
  import type { SessionUser } from "../engine/types";
3
4
  import {
@@ -74,12 +75,16 @@ async function* executeStreamInner(
74
75
  const invalidated = new Promise<void>((resolve) => {
75
76
  resolveInvalidated = resolve;
76
77
  });
78
+ const streamCredential: AccessInvalidationCredential = {
79
+ ...(user.sid !== undefined && { sid: user.sid }),
80
+ ...(user.pat?.tokenId !== undefined && { patTokenId: user.pat.tokenId }),
81
+ };
77
82
  const unsubscribeAccessInvalidation = ctx.sseBroker?.subscribeAccessInvalidation(
78
83
  user.id,
79
84
  () => {
80
85
  resolveInvalidated?.();
81
86
  },
82
- user.sid,
87
+ streamCredential,
83
88
  );
84
89
 
85
90
  let iterator: AsyncIterator<unknown> | undefined;
@@ -2,7 +2,7 @@
2
2
  // `Temporal` TYPE that ConsumerStateRow.updatedAt/StoredEventRow.createdAt
3
3
  // resolve against (same #1438 dual-package-hazard pattern as event-store.ts).
4
4
  import { Temporal as TemporalPolyfill } from "temporal-polyfill";
5
- import { requestContext } from "../api/request-context";
5
+ import { type RequestContextData, requestContext } from "../api/request-context";
6
6
  import type { DbConnection, DbTx } from "../db/connection";
7
7
  import {
8
8
  insertConsumerIfAbsent,
@@ -10,6 +10,7 @@ import {
10
10
  rearmDeadConsumer,
11
11
  recordConsumerPassFailure,
12
12
  selectConsumerForUpdateSkipLocked,
13
+ selectProvablyIdleConsumerPairs,
13
14
  updateConsumerDeliveryOutcome,
14
15
  } from "../db/queries/event-consumer";
15
16
  import {
@@ -20,7 +21,7 @@ import {
20
21
  import { coerceRow, extractTableInfo } from "../db/query";
21
22
  import { qnScope } from "../engine/qualified-name";
22
23
  import type { AppContext } from "../engine/types";
23
- import { eventsTable, toStoredEvent as rowToStoredEvent } from "../event-store";
24
+ import { eventsTable, toStoredEvent as rowToStoredEvent, type StoredEvent } from "../event-store";
24
25
  import {
25
26
  emitDispatcherError,
26
27
  emitEventConsumerLag,
@@ -159,6 +160,39 @@ export async function acquireConsumerState(
159
160
  return { state, skip: null };
160
161
  }
161
162
 
163
+ // Read-only pre-check run once per doPass, before any consumer's turn opens
164
+ // a transaction (event-dispatcher.ts's doPass/runConsumerTurn). Keyed like
165
+ // consumerBackoff/inFlightTurns: `${name}:${instanceId}`. Consumers not
166
+ // provably idle (missing row, dead, pending gaps, or an event past cursor)
167
+ // simply don't appear in the result — the caller falls through to the
168
+ // existing acquireConsumerState/FOR UPDATE SKIP LOCKED path unchanged.
169
+ //
170
+ // Race: an event that commits between this query and the next tick CAN
171
+ // make this snapshot stale — that's expected, not a bug. What it can't do
172
+ // is get silently lost: the append fires NOTIFY (or the next poll tick
173
+ // fires regardless), triggering a fresh doPass. That pass's own pre-check
174
+ // sees the new head id and excludes the consumer from "idle"; a new
175
+ // pending_gap only ever gets created when a later turn's fetch (id > the
176
+ // old cursor) walks past the row that committed late, and this same
177
+ // pre-check reads pending_gaps live on every pass, so it observes that gap
178
+ // too. In short: this can under-report idleness for one tick, never
179
+ // over-report it.
180
+ export async function selectIdleConsumerKeys(
181
+ db: DbConnection,
182
+ pairs: ReadonlyArray<{ readonly name: string; readonly instanceId: string }>,
183
+ ): Promise<ReadonlySet<string>> {
184
+ if (pairs.length === 0) return new Set();
185
+ const names = pairs.map((p) => p.name);
186
+ const instanceIds = pairs.map((p) => p.instanceId);
187
+ const idlePairs = await selectProvablyIdleConsumerPairs(
188
+ db,
189
+ names,
190
+ instanceIds,
191
+ ConsumerStatuses.dead,
192
+ );
193
+ return new Set(idlePairs.map((p) => `${p.name}:${p.instanceId}`));
194
+ }
195
+
162
196
  // Shared pre-registration: one row per (consumer, shard), cursor = 0,
163
197
  // status = idle. Shared-delivery consumers use SHARED_INSTANCE_SENTINEL;
164
198
  // per-instance consumers use the dispatcher's instanceId. Idempotent
@@ -230,12 +264,15 @@ export type DeliveryOutcome = {
230
264
  readonly resolvedPendingIds: readonly bigint[];
231
265
  };
232
266
 
233
- // Deliver events to the consumer's handler in events.id order. Halt-on-
234
- // poison: a throw breaks the loop, the cursor stays at the last successful
235
- // event, and attempts climb. At the consumer's effectiveMaxAttempts
236
- // (errorPolicy.maxAttempts ?? maxAttempts) the caller persists status=
237
- // "dead" and the consumer is parked until ops intervenes (see
238
- // restartConsumer / skipPoisonEvent).
267
+ // Deliver events to the consumer's handler in events.id order, or — when
268
+ // the consumer wires a batchHandler — through it in one call first. Halt-
269
+ // on-poison: a throw breaks the per-event loop, the cursor stays at the
270
+ // last successful event, and attempts climb. At the consumer's
271
+ // effectiveMaxAttempts (errorPolicy.maxAttempts ?? maxAttempts) the caller
272
+ // persists status="dead" and the consumer is parked until ops intervenes
273
+ // (see restartConsumer / skipPoisonEvent). A batch that throws falls back
274
+ // to the same per-event loop for the same events — the batch failure
275
+ // itself does not bump attempts, only a per-event failure does.
239
276
  export async function deliverEvents(
240
277
  consumer: EventConsumer,
241
278
  events: ReadonlyArray<StoredEventRow>,
@@ -262,6 +299,17 @@ export async function deliverEvents(
262
299
  lastError = null;
263
300
  };
264
301
 
302
+ if (consumer.batchHandler && events.length > 0) {
303
+ const batchSucceeded = await tryApplyBatch(consumer, events, context);
304
+ if (batchSucceeded) {
305
+ for (const row of events) resolve(row.id);
306
+ processed = events.length;
307
+ return { cursor, attempts, lastError, deadLettered, processed, failed, resolvedPendingIds };
308
+ }
309
+ // fall through to the per-event loop below; the batch failure itself
310
+ // does NOT bump attempts — only a per-event failure does.
311
+ }
312
+
265
313
  for (const row of events) {
266
314
  try {
267
315
  await applyEvent(consumer, row, context);
@@ -285,42 +333,89 @@ export async function deliverEvents(
285
333
  return { cursor, attempts, lastError, deadLettered, processed, failed, resolvedPendingIds };
286
334
  }
287
335
 
288
- async function applyEvent(
336
+ // Shared requestContext scope for one apply — event-derived fields
337
+ // (correlationId, causationId, writeOrigin) come from `causationSource`;
338
+ // applyEvent uses the event itself, applyBatch uses the LAST event of the
339
+ // turn (see applyBatch for why). requestId falls back to a fresh id
340
+ // because the dispatcher runs outside any HTTP request (background poll),
341
+ // and a stable log-correlation handle is still useful for debugging.
342
+ // #3043 — an event this apply writes is attributed to the consumer, not
343
+ // to whatever wrote the triggering event; causationId already links back.
344
+ function buildConsumerRequestScope(
289
345
  consumer: EventConsumer,
290
- row: StoredEventRow,
291
- context: AppContext,
292
- ): Promise<void> {
293
- // Propagate causation: if the handler calls ctx.appendEvent, the new
294
- // event should record THIS event as its cause. correlationId is
295
- // inherited unchanged — it survives the hop across streams by design.
296
- // requestId falls back to a fresh id because the dispatcher runs
297
- // outside any HTTP request (background poll), and a stable log-
298
- // correlation handle is still useful for debugging.
299
- const stored = rowToStoredEvent(row);
300
- const correlationId = stored.metadata.correlationId ?? requestContext.generateId();
301
- const causationId = String(stored.id);
346
+ causationSource: StoredEvent,
347
+ ): RequestContextData {
348
+ const correlationId = causationSource.metadata.correlationId ?? requestContext.generateId();
349
+ const causationId = String(causationSource.id);
302
350
  const requestId = requestContext.generateId();
303
351
  // The job-trigger consumer's handleEvent stamps event-triggered jobs from this.
304
- const rawStoredWriteOrigin = stored.metadata.writeOrigin;
352
+ const rawStoredWriteOrigin = causationSource.metadata.writeOrigin;
305
353
  const writeOrigin =
306
354
  rawStoredWriteOrigin === undefined
307
355
  ? undefined
308
356
  : (parseWriteOrigin(rawStoredWriteOrigin) ?? UNPARSEABLE_STORED_WRITE_ORIGIN);
309
- // #3043 — an event this apply writes is attributed to the consumer, not
310
- // to whatever wrote the triggering event; causationId already links back.
311
- await requestContext.run(
312
- {
313
- requestId,
314
- correlationId,
315
- causationId,
316
- handler: consumer.name,
317
- feature: consumer.featureName ?? qnScope(consumer.name),
318
- writeOrigin,
319
- },
320
- async () => {
321
- await consumer.handler(stored, context);
322
- },
323
- );
357
+ return {
358
+ requestId,
359
+ correlationId,
360
+ causationId,
361
+ handler: consumer.name,
362
+ feature: consumer.featureName ?? qnScope(consumer.name),
363
+ writeOrigin,
364
+ };
365
+ }
366
+
367
+ async function applyEvent(
368
+ consumer: EventConsumer,
369
+ row: StoredEventRow,
370
+ context: AppContext,
371
+ ): Promise<void> {
372
+ const stored = rowToStoredEvent(row);
373
+ await requestContext.run(buildConsumerRequestScope(consumer, stored), async () => {
374
+ await consumer.handler(stored, context);
375
+ });
376
+ }
377
+
378
+ // Runs the consumer's batchHandler for the whole turn under one
379
+ // requestContext scope, attempted before the per-event loop. Returns
380
+ // false on failure instead of rethrowing — the caller falls back to
381
+ // per-event delivery for the exact same events, so a batch failure must
382
+ // not itself count as an applyEvent failure or bump attempts.
383
+ async function tryApplyBatch(
384
+ consumer: EventConsumer,
385
+ events: ReadonlyArray<StoredEventRow>,
386
+ context: AppContext,
387
+ ): Promise<boolean> {
388
+ try {
389
+ await applyBatch(consumer, events, context);
390
+ return true;
391
+ } catch (e) {
392
+ const errMessage = e instanceof Error ? e.message : String(e);
393
+ context.log?.warn(
394
+ `event-dispatcher: ${consumer.name} batch of ${events.length} failed, falling back to per-event delivery: ${errMessage}`,
395
+ );
396
+ return false;
397
+ }
398
+ }
399
+
400
+ async function applyBatch(
401
+ consumer: EventConsumer,
402
+ events: ReadonlyArray<StoredEventRow>,
403
+ context: AppContext,
404
+ ): Promise<void> {
405
+ const batchHandler = consumer.batchHandler;
406
+ // skip: deliverEvents only calls this for consumers that wire a batchHandler.
407
+ if (!batchHandler) return;
408
+ const stored = events.map((row) => rowToStoredEvent(row));
409
+ // correlationId/causationId/writeOrigin come from the LAST row — a
410
+ // batch-appended event's causation is attributed to the turn's most
411
+ // recent input, same as chaining N applyEvent calls would leave the
412
+ // requestContext at after the final one.
413
+ const lastStored = stored.at(-1);
414
+ // skip: deliverEvents never calls this with an empty turn.
415
+ if (!lastStored) return;
416
+ await requestContext.run(buildConsumerRequestScope(consumer, lastStored), async () => {
417
+ await batchHandler(stored, context);
418
+ });
324
419
  }
325
420
 
326
421
  // Best-effort mode: record the error on the skip counter so ops can alert on