@cosmicdrift/kumiko-framework 0.307.0 → 0.309.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (174) hide show
  1. package/package.json +5 -5
  2. package/src/__tests__/anonymous-access.integration.test.ts +1 -1
  3. package/src/__tests__/error-contract.integration.test.ts +1 -1
  4. package/src/__tests__/field-access.integration.test.ts +1 -1
  5. package/src/__tests__/fixtures/migrate-open-to-all/manual-only.expected.ts +1 -1
  6. package/src/__tests__/fixtures/migrate-open-to-all/manual-only.input.ts +1 -1
  7. package/src/__tests__/fixtures/migrate-open-to-all/test-simple.expected.ts +1 -1
  8. package/src/__tests__/fixtures/migrate-open-to-all/test-simple.input.ts +1 -1
  9. package/src/__tests__/full-stack.integration.test.ts +1 -1
  10. package/src/__tests__/transition-guard.integration.test.ts +1 -1
  11. package/src/api/__tests__/api.test.ts +1 -1
  12. package/src/api/__tests__/batch.integration.test.ts +1 -1
  13. package/src/api/__tests__/body-limit.test.ts +1 -1
  14. package/src/api/__tests__/dispatcher-live.integration.test.ts +1 -1
  15. package/src/api/__tests__/extra-routes.integration.test.ts +1 -1
  16. package/src/api/__tests__/http-route-rate-limit.integration.test.ts +1 -1
  17. package/src/api/__tests__/nested-write-ownership.integration.test.ts +1 -1
  18. package/src/api/__tests__/nested-write.integration.test.ts +1 -1
  19. package/src/api/__tests__/pii-leak-guard.integration.test.ts +1 -1
  20. package/src/api/__tests__/redis-sse-broker.integration.test.ts +73 -6
  21. package/src/api/__tests__/request-locale.integration.test.ts +1 -1
  22. package/src/api/__tests__/server-boot-guards.test.ts +1 -1
  23. package/src/api/__tests__/server-error-logging.test.ts +1 -1
  24. package/src/api/__tests__/sse-broker.test.ts +107 -13
  25. package/src/api/__tests__/stream-escape-hatch.integration.test.ts +1 -1
  26. package/src/api/auth-routes.ts +1 -1
  27. package/src/api/redis-sse-broker.ts +51 -11
  28. package/src/api/server.ts +11 -0
  29. package/src/api/sse-broker.ts +61 -22
  30. package/src/bun-db/__tests__/postgres-closed-socket-write.integration.test.ts +87 -0
  31. package/src/bun-db/connection.ts +2 -0
  32. package/src/changes.json +61 -0
  33. package/src/compliance/override-schema.ts +1 -1
  34. package/src/crypto/__tests__/event-pii.test.ts +1 -1
  35. package/src/crypto/kms-adapter.ts +1 -1
  36. package/src/db/__tests__/feature-table-sources.test.ts +2 -2
  37. package/src/db/__tests__/tenant-db-where-merge.integration.test.ts +1 -1
  38. package/src/db/collect-table-metas.ts +3 -6
  39. package/src/db/feature-table-sources.ts +3 -4
  40. package/src/db/queries/event-consumer.ts +98 -6
  41. package/src/derivatives/__tests__/derivatives-context.integration.test.ts +1 -1
  42. package/src/derivatives/delete-stored-file.ts +86 -0
  43. package/src/derivatives/index.ts +2 -0
  44. package/src/engine/__tests__/boot-validator-action-wiring.test.ts +1 -1
  45. package/src/engine/__tests__/boot-validator-dashboard.test.ts +1 -1
  46. package/src/engine/__tests__/boot-validator-nav-area.test.ts +1 -1
  47. package/src/engine/__tests__/boot-validator-pii-retention.test.ts +1 -1
  48. package/src/engine/__tests__/boot-validator-projection-list.test.ts +1 -1
  49. package/src/engine/__tests__/boot-validator-query-output-schema.test.ts +1 -1
  50. package/src/engine/__tests__/boot-validator-query-refs.test.ts +1 -1
  51. package/src/engine/__tests__/boot-validator-ref-entity.test.ts +1 -1
  52. package/src/engine/__tests__/boot-validator-s0-integration.test.ts +1 -1
  53. package/src/engine/__tests__/boot-validator.test.ts +35 -1
  54. package/src/engine/__tests__/build-app-schema.test.ts +1 -1
  55. package/src/engine/__tests__/build-config-feature-schema-secrets.test.ts +1 -1
  56. package/src/engine/__tests__/define-feature-entity-mapping.test.ts +1 -1
  57. package/src/engine/__tests__/engine.test.ts +1 -1
  58. package/src/engine/__tests__/event-helpers.test.ts +1 -1
  59. package/src/engine/__tests__/event-migration-declarative.test.ts +1 -1
  60. package/src/engine/__tests__/extend-entity-projection.test.ts +1 -1
  61. package/src/engine/__tests__/extends-registrar.test.ts +1 -1
  62. package/src/engine/__tests__/hook-phases.test.ts +1 -1
  63. package/src/engine/__tests__/lifecycle-hooks.test.ts +1 -1
  64. package/src/engine/__tests__/pipeline-engine.test.ts +1 -1
  65. package/src/engine/__tests__/pipeline-handler.integration.test.ts +1 -1
  66. package/src/engine/__tests__/pipeline-observability.integration.test.ts +1 -1
  67. package/src/engine/__tests__/pipeline-performance.integration.test.ts +1 -1
  68. package/src/engine/__tests__/pipeline-sub-pipelines.test.ts +1 -1
  69. package/src/engine/__tests__/post-query-hook.test.ts +1 -1
  70. package/src/engine/__tests__/projection-detail-actions.test.ts +1 -1
  71. package/src/engine/__tests__/projection-detail-singleton.test.ts +1 -1
  72. package/src/engine/__tests__/projection-detail-tabs.test.ts +1 -1
  73. package/src/engine/__tests__/read-steps-tenant-filter.integration.test.ts +1 -1
  74. package/src/engine/__tests__/registry.test.ts +1 -1
  75. package/src/engine/__tests__/screen.test.ts +1 -1
  76. package/src/engine/__tests__/validate-projection-allowlist.test.ts +1 -1
  77. package/src/engine/boot-validator/__tests__/access-declarations.test.ts +1 -1
  78. package/src/engine/boot-validator/__tests__/action-icon.test.ts +81 -0
  79. package/src/engine/boot-validator/__tests__/anonymous-rate-limit-required.test.ts +1 -1
  80. package/src/engine/boot-validator/__tests__/no-all-role-in-handler-access.test.ts +1 -1
  81. package/src/engine/boot-validator/__tests__/parent-ref.test.ts +1 -1
  82. package/src/engine/boot-validator/__tests__/rate-limit-disabled-reason.test.ts +1 -1
  83. package/src/engine/boot-validator/screens.ts +83 -0
  84. package/src/engine/define-handler.ts +2 -1
  85. package/src/engine/entity-handlers.ts +2 -1
  86. package/src/engine/feature-ast/__tests__/pattern-change-schema.test.ts +1 -1
  87. package/src/engine/feature-ast/pattern-change-schema.ts +1 -1
  88. package/src/engine/feature-builder-state.ts +2 -1
  89. package/src/engine/feature-config-events-jobs.ts +2 -1
  90. package/src/engine/feature-entity-handlers.ts +2 -1
  91. package/src/engine/schema-builder.ts +1 -1
  92. package/src/entrypoint/__tests__/entrypoint-attach-dispatcher.integration.test.ts +1 -1
  93. package/src/entrypoint/__tests__/entrypoint-job-wiring.integration.test.ts +1 -1
  94. package/src/entrypoint/__tests__/split-deploy.integration.test.ts +1 -1
  95. package/src/env/__tests__/compose-env-schema.test.ts +1 -1
  96. package/src/env/__tests__/dry-run.test.ts +1 -1
  97. package/src/env/__tests__/parse-env-dry-run.test.ts +1 -1
  98. package/src/env/_zod-introspect.ts +1 -1
  99. package/src/env/dry-run.ts +1 -1
  100. package/src/env/index.ts +1 -1
  101. package/src/errors/__tests__/classes.test.ts +1 -1
  102. package/src/es-ops/__tests__/runner.integration.test.ts +1 -1
  103. package/src/event-store/__tests__/backfill-pii.integration.test.ts +1 -1
  104. package/src/event-store/__tests__/event-attribution.integration.test.ts +1 -1
  105. package/src/event-store/__tests__/upcaster.integration.test.ts +1 -1
  106. package/src/jobs/__tests__/job-backoff.integration.test.ts +1 -1
  107. package/src/jobs/__tests__/job-event-trigger.integration.test.ts +1 -1
  108. package/src/jobs/__tests__/job-multi-trigger.integration.test.ts +1 -1
  109. package/src/jobs/__tests__/job-public-intake-origin.integration.test.ts +1 -1
  110. package/src/jobs/__tests__/job-retention.integration.test.ts +1 -1
  111. package/src/jobs/__tests__/job-write-as.integration.test.ts +1 -1
  112. package/src/jobs/__tests__/jobs.integration.test.ts +1 -1
  113. package/src/observability/__tests__/metrics-wiring.test.ts +1 -1
  114. package/src/observability/__tests__/observability.integration.test.ts +1 -1
  115. package/src/observability/metrics-wiring.ts +1 -1
  116. package/src/pipeline/__tests__/archive-stream.integration.test.ts +1 -1
  117. package/src/pipeline/__tests__/causation-chain.integration.test.ts +1 -1
  118. package/src/pipeline/__tests__/ctx-bridge.integration.test.ts +1 -1
  119. package/src/pipeline/__tests__/ctx-systemdb.integration.test.ts +1 -1
  120. package/src/pipeline/__tests__/dispatcher.test.ts +1 -1
  121. package/src/pipeline/__tests__/domain-events-projections.integration.test.ts +1 -1
  122. package/src/pipeline/__tests__/escape-hatch-audit.integration.test.ts +1 -1
  123. package/src/pipeline/__tests__/escape-hatch.integration.test.ts +1 -1
  124. package/src/pipeline/__tests__/event-dedup.integration.test.ts +1 -1
  125. package/src/pipeline/__tests__/event-define-event-strict.integration.test.ts +1 -1
  126. package/src/pipeline/__tests__/event-dispatcher-batch-delivery.test.ts +178 -0
  127. package/src/pipeline/__tests__/event-dispatcher-batch-handler.integration.test.ts +143 -0
  128. package/src/pipeline/__tests__/event-dispatcher-db-outage.integration.test.ts +192 -0
  129. package/src/pipeline/__tests__/event-dispatcher-idle.integration.test.ts +94 -0
  130. package/src/pipeline/__tests__/event-dispatcher-start-from.integration.test.ts +103 -3
  131. package/src/pipeline/__tests__/fetch-for-writing.integration.test.ts +1 -1
  132. package/src/pipeline/__tests__/idempotency-transient-failure.integration.test.ts +1 -1
  133. package/src/pipeline/__tests__/job-trigger-consumer.integration.test.ts +1 -1
  134. package/src/pipeline/__tests__/lifecycle-pipeline.test.ts +1 -1
  135. package/src/pipeline/__tests__/load-aggregate-query.integration.test.ts +1 -1
  136. package/src/pipeline/__tests__/msp-multi-hop.integration.test.ts +1 -1
  137. package/src/pipeline/__tests__/msp-rebuild.integration.test.ts +116 -1
  138. package/src/pipeline/__tests__/multi-stream-projection.integration.test.ts +1 -1
  139. package/src/pipeline/__tests__/pending-gap-ranges.test.ts +94 -0
  140. package/src/pipeline/__tests__/post-query-hook.integration.test.ts +1 -1
  141. package/src/pipeline/__tests__/public-intake-runtime-gate.integration.test.ts +1 -1
  142. package/src/pipeline/__tests__/query-projection.integration.test.ts +1 -1
  143. package/src/pipeline/__tests__/rate-limit-default.integration.test.ts +1 -1
  144. package/src/pipeline/__tests__/rebuild-poison-quarantine.integration.test.ts +1 -1
  145. package/src/pipeline/__tests__/search-consumer-named-events.integration.test.ts +1 -1
  146. package/src/pipeline/__tests__/system-identity-switch.integration.test.ts +1 -1
  147. package/src/pipeline/__tests__/try-append-event.integration.test.ts +1 -1
  148. package/src/pipeline/__tests__/unsafe-raw-escape-hatch.integration.test.ts +1 -1
  149. package/src/pipeline/dispatch-stream.ts +6 -1
  150. package/src/pipeline/event-dispatcher-delivery.ts +132 -37
  151. package/src/pipeline/event-dispatcher.ts +121 -17
  152. package/src/pipeline/index.ts +1 -0
  153. package/src/pipeline/msp-rebuild.ts +39 -1
  154. package/src/pipeline/pending-gap-ranges.ts +48 -0
  155. package/src/pipeline/system-hooks.ts +223 -68
  156. package/src/pipeline/write-origin.ts +1 -1
  157. package/src/rate-limit/__tests__/dispatcher-l3.integration.test.ts +1 -1
  158. package/src/search/__tests__/derive-search-adapter-config.test.ts +65 -0
  159. package/src/search/__tests__/meilisearch-adapter.integration.test.ts +91 -0
  160. package/src/search/__tests__/search-consumer-batch.integration.test.ts +306 -0
  161. package/src/search/derive-search-adapter-config.ts +27 -0
  162. package/src/search/index.ts +1 -0
  163. package/src/search/meilisearch-adapter.ts +96 -14
  164. package/src/secrets/__tests__/contains-secret.test.ts +1 -1
  165. package/src/stack/__tests__/request-helper.integration.test.ts +1 -1
  166. package/src/stack/__tests__/setup-test-stack-entity-tables.integration.test.ts +65 -0
  167. package/src/stack/__tests__/setup-test-stack-jobs.integration.test.ts +1 -1
  168. package/src/stack/__tests__/setup-test-stack-metrics.integration.test.ts +1 -1
  169. package/src/stack/test-stack.ts +61 -54
  170. package/src/testing/__tests__/e2e-generator.test.ts +1 -1
  171. package/src/testing/__tests__/real-providers.test.ts +9 -1
  172. package/src/testing/e2e-generator.ts +1 -1
  173. package/src/testing/index.ts +1 -1
  174. package/src/testing/real-providers.ts +12 -4
@@ -0,0 +1,143 @@
1
+ // #3227 — batchHandler wired into a real dispatcher pass: a successful
2
+ // batch is delivered in one call (handler never runs, cursor lands at the
3
+ // last event) and a batch+handler failure still dead-letters through the
4
+ // normal errorPolicy machinery, delivering the poisoned event via handler
5
+ // exactly once before it does.
6
+
7
+ import { afterEach, beforeAll, describe, expect, test } from "bun:test";
8
+ import { createEventStoreExecutor } from "../../db/event-store-executor";
9
+ import { createTenantDb, type TenantDb } from "../../db/tenant-db";
10
+ import { defineFeature } from "../../engine";
11
+ import type { StoredEvent } from "../../event-store";
12
+ import {
13
+ createEventDispatcher,
14
+ type EventConsumer,
15
+ type EventDispatcher,
16
+ getConsumerState,
17
+ } from "../../pipeline";
18
+ import {
19
+ resetEventStore,
20
+ setupTestStack,
21
+ type TestStack,
22
+ TestUsers,
23
+ unsafeCreateEntityTable,
24
+ } from "../../stack";
25
+ import { sharedWidgetEntity, sharedWidgetTable } from "../../testing";
26
+
27
+ const executor = createEventStoreExecutor(sharedWidgetTable, sharedWidgetEntity, {
28
+ entityName: "widget",
29
+ });
30
+
31
+ const feature = defineFeature("batchhandlerintegration", (r) => {
32
+ r.entity("widget", sharedWidgetEntity);
33
+ });
34
+
35
+ const admin = TestUsers.admin;
36
+ let stack: TestStack;
37
+ let tdb: TenantDb;
38
+
39
+ beforeAll(async () => {
40
+ stack = await setupTestStack({ features: [feature], systemHooks: [] });
41
+ await unsafeCreateEntityTable(stack.db, sharedWidgetEntity, "widget");
42
+ tdb = createTenantDb(stack.db, admin.tenantId);
43
+ });
44
+
45
+ afterEach(async () => {
46
+ await resetEventStore(stack, ["read_widgets"]);
47
+ });
48
+
49
+ async function appendWidget(name: string): Promise<string> {
50
+ const created = await executor.create({ name }, admin, tdb);
51
+ if (!created.isSuccess) throw new Error("create failed");
52
+ return String(created.data.id);
53
+ }
54
+
55
+ function buildDispatcher(consumers: readonly EventConsumer[]): EventDispatcher {
56
+ return createEventDispatcher({
57
+ db: stack.db,
58
+ consumers,
59
+ context: { db: stack.db, redis: stack.redis.redis, registry: stack.registry },
60
+ batchSize: 200,
61
+ // No timer/LISTEN involved — runOnce() below drives every pass
62
+ // explicitly, so pollIntervalMs never fires a competing pass.
63
+ pollIntervalMs: 60_000,
64
+ });
65
+ }
66
+
67
+ describe("event-dispatcher — batchHandler", () => {
68
+ test("a successful batch delivers all events in one call, handler stays unused, cursor lands on the last event", async () => {
69
+ const consumerName = "batchhandlerintegration:success";
70
+ const batchSizes: number[] = [];
71
+ const batchIds: bigint[] = [];
72
+ const handlerIds: string[] = [];
73
+
74
+ const consumer: EventConsumer = {
75
+ name: consumerName,
76
+ batchHandler: async (events) => {
77
+ batchSizes.push(events.length);
78
+ for (const event of events) batchIds.push(BigInt(event.id));
79
+ },
80
+ handler: async (event) => {
81
+ handlerIds.push(event.aggregateId);
82
+ },
83
+ };
84
+
85
+ const dispatcher = buildDispatcher([consumer]);
86
+ await dispatcher.ensureRegistered();
87
+
88
+ await appendWidget("one");
89
+ await appendWidget("two");
90
+ await appendWidget("three");
91
+
92
+ const result = await dispatcher.runOnce();
93
+
94
+ expect(batchSizes).toEqual([3]);
95
+ expect(handlerIds).toEqual([]);
96
+ expect(result.byConsumer[consumerName]).toEqual({ processed: 3, failed: 0 });
97
+
98
+ const state = await getConsumerState(stack.db, consumerName);
99
+ expect(state?.status).toBe("idle");
100
+ expect(state?.lastProcessedEventId).toBe(batchIds.at(-1));
101
+ });
102
+
103
+ test("batch throws, handler poisons the 2nd event: dead-letters at maxAttempts, first event delivered exactly once via handler", async () => {
104
+ const consumerName = "batchhandlerintegration:poison";
105
+ const handlerCalls: StoredEvent[] = [];
106
+ let poisonAggregateId: string | undefined;
107
+
108
+ const consumer: EventConsumer = {
109
+ name: consumerName,
110
+ errorPolicy: { maxAttempts: 2 },
111
+ batchHandler: async () => {
112
+ throw new Error("batch always fails in this test");
113
+ },
114
+ handler: async (event) => {
115
+ handlerCalls.push(event);
116
+ if (event.aggregateId === poisonAggregateId) {
117
+ throw new Error("poisoned event");
118
+ }
119
+ },
120
+ };
121
+
122
+ const dispatcher = buildDispatcher([consumer]);
123
+ await dispatcher.ensureRegistered();
124
+
125
+ await appendWidget("first");
126
+ poisonAggregateId = await appendWidget("second");
127
+
128
+ const firstPass = await dispatcher.runOnce();
129
+ expect(firstPass.byConsumer[consumerName]?.failed).toBeGreaterThan(0);
130
+
131
+ const secondPass = await dispatcher.runOnce();
132
+ expect(secondPass.byConsumer[consumerName]).toBeDefined();
133
+
134
+ const state = await getConsumerState(stack.db, consumerName);
135
+ expect(state?.status).toBe("dead");
136
+
137
+ const firstEventDeliveries = handlerCalls.filter(
138
+ (event) => event.aggregateId === handlerCalls[0]?.aggregateId,
139
+ );
140
+ expect(firstEventDeliveries).toHaveLength(1);
141
+ expect(state?.lastProcessedEventId).toBe(BigInt(handlerCalls[0]?.id ?? "0"));
142
+ });
143
+ });
@@ -0,0 +1,192 @@
1
+ // kumiko-framework#3243: a DB outage must not crash the dispatcher and must
2
+ // not stay silent forever. The idle pre-check's error log (doPass) fires once
3
+ // per outage, and a matching "recovered" line must fire once the DB answers
4
+ // again, so ops can see the return without restarting the process. Routes the
5
+ // dispatcher's own db connection through a TCP proxy (same pattern as
6
+ // bun-db/__tests__/postgres-closed-socket-write.integration.test.ts) so the
7
+ // outage can be simulated without touching the shared test DB other suites
8
+ // depend on.
9
+
10
+ import { afterEach, beforeAll, describe, expect, test } from "bun:test";
11
+ import net from "node:net";
12
+ import postgres from "postgres";
13
+ import { createDbConnection } from "../../db/connection";
14
+ import { createEventStoreExecutor } from "../../db/event-store-executor";
15
+ import { createTenantDb, type TenantDb } from "../../db/tenant-db";
16
+ import { defineFeature } from "../../engine";
17
+ import type { StoredEvent } from "../../event-store";
18
+ import type { Logger } from "../../logging/types";
19
+ import { type MetricEvent, RecordingMeter, registerStandardMetrics } from "../../observability";
20
+ import { createEventDispatcher, type EventConsumer, type EventDispatcher } from "../../pipeline";
21
+ import {
22
+ resetEventStore,
23
+ setupTestStack,
24
+ type TestStack,
25
+ TestUsers,
26
+ unsafeCreateEntityTable,
27
+ } from "../../stack";
28
+ import { sharedWidgetEntity, sharedWidgetTable, waitFor } from "../../testing";
29
+ import { testDatabaseUrl } from "../../testing/closed-connection-error";
30
+
31
+ const executor = createEventStoreExecutor(sharedWidgetTable, sharedWidgetEntity, {
32
+ entityName: "widget",
33
+ });
34
+
35
+ const feature = defineFeature("dboutage", (r) => {
36
+ r.entity("widget", sharedWidgetEntity);
37
+ });
38
+
39
+ const admin = TestUsers.admin;
40
+ let stack: TestStack;
41
+ let tdb: TenantDb;
42
+ // setupTestStack creates its own ephemeral kumiko_test_<random> DB and never
43
+ // exposes its name/URL. Passing an explicit name here so the proxy below
44
+ // can build the exact connection string setupTestStack itself would have.
45
+ const testDbName = `kumiko_test_dboutage_${crypto.randomUUID().slice(0, 8)}`;
46
+
47
+ beforeAll(async () => {
48
+ stack = await setupTestStack({ features: [feature], systemHooks: [], dbName: testDbName });
49
+ await unsafeCreateEntityTable(stack.db, sharedWidgetEntity, "widget");
50
+ tdb = createTenantDb(stack.db, admin.tenantId);
51
+ });
52
+
53
+ afterEach(async () => {
54
+ await resetEventStore(stack, ["read_widgets"]);
55
+ });
56
+
57
+ async function appendWidget(name: string): Promise<void> {
58
+ await executor.create({ name }, admin, tdb);
59
+ }
60
+
61
+ function recordingLogger(): Logger & { readonly lines: string[] } {
62
+ const lines: string[] = [];
63
+ const logger: Logger & { lines: string[] } = {
64
+ lines,
65
+ info: (msg) => lines.push(`info:${msg}`),
66
+ warn: (msg) => lines.push(`warn:${msg}`),
67
+ error: (msg) => lines.push(`error:${msg}`),
68
+ debug: () => {},
69
+ child: () => logger,
70
+ };
71
+ return logger;
72
+ }
73
+
74
+ // A TCP passthrough to the real test DB. Stopping it (server.close() +
75
+ // destroying every live socket) simulates an outage without touching the
76
+ // shared test DB; restarting on the SAME port simulates the DB coming back
77
+ // without the dispatcher's client needing a new connection string.
78
+ function startDbProxy(dbUrl: URL): { server: net.Server; stop: () => Promise<void> } {
79
+ const sockets = new Set<net.Socket>();
80
+ const server = net.createServer((downstream) => {
81
+ sockets.add(downstream);
82
+ const upstream = net.connect(Number(dbUrl.port), dbUrl.hostname);
83
+ sockets.add(upstream);
84
+ downstream.pipe(upstream).pipe(downstream);
85
+ downstream.on("error", () => upstream.destroy());
86
+ upstream.on("error", () => downstream.destroy());
87
+ downstream.on("close", () => sockets.delete(downstream));
88
+ upstream.on("close", () => sockets.delete(upstream));
89
+ });
90
+ return {
91
+ server,
92
+ stop: async () => {
93
+ for (const socket of sockets) socket.destroy();
94
+ await new Promise<void>((resolve) => server.close(() => resolve()));
95
+ },
96
+ };
97
+ }
98
+
99
+ async function listenOn(server: net.Server, port: number): Promise<void> {
100
+ await new Promise<void>((resolve, reject) => {
101
+ server.once("error", reject);
102
+ server.listen(port, resolve);
103
+ });
104
+ }
105
+
106
+ describe("E: dispatcher survives a DB outage and logs the recovery", () => {
107
+ test("idle pre-check logs the outage once and the recovery once, then resumes delivery", async () => {
108
+ const dbUrl = new URL(testDatabaseUrl().replace(/\/[^/]+$/, `/${testDbName}`));
109
+ const proxy1 = startDbProxy(dbUrl);
110
+ await listenOn(proxy1.server, 0);
111
+ const address = proxy1.server.address();
112
+ if (address === null || typeof address === "string") {
113
+ throw new Error("proxy did not bind to a TCP port");
114
+ }
115
+ const proxyPort = address.port;
116
+
117
+ const proxiedUrl = new URL(dbUrl.toString());
118
+ proxiedUrl.hostname = "127.0.0.1";
119
+ proxiedUrl.port = String(proxyPort);
120
+ const proxiedDb = createDbConnection(proxiedUrl.toString(), { maxConnections: 1 });
121
+
122
+ const observed: string[] = [];
123
+ const consumer: EventConsumer = {
124
+ name: "dboutage:consumer",
125
+ handler: async (event: StoredEvent) => {
126
+ observed.push(String(event.payload["name"]));
127
+ },
128
+ };
129
+
130
+ const logger = recordingLogger();
131
+ const metricEvents: MetricEvent[] = [];
132
+ const lastListenGauge = (): number | undefined =>
133
+ metricEvents
134
+ .filter(
135
+ (e) => e.type === "gauge.set" && e.name === "kumiko_event_dispatcher_listen_connected",
136
+ )
137
+ .pop()?.value;
138
+ const meter = new RecordingMeter((e) => metricEvents.push(e));
139
+ registerStandardMetrics(meter);
140
+ const listenClient = postgres(proxiedUrl.toString(), { max: 1 });
141
+ const dispatcher: EventDispatcher = createEventDispatcher({
142
+ db: proxiedDb.db,
143
+ consumers: [consumer],
144
+ context: { db: proxiedDb.db, log: logger },
145
+ pollIntervalMs: 50,
146
+ pgClient: listenClient,
147
+ meter,
148
+ });
149
+
150
+ try {
151
+ await dispatcher.start();
152
+ expect(lastListenGauge()).toBe(1);
153
+
154
+ await appendWidget("before-outage");
155
+ await waitFor(() => observed.includes("before-outage"));
156
+ // Delivery is at-least-once, so killing the proxy mid-commit would
157
+ // redeliver "before-outage" after recovery. Drain first so the turn's
158
+ // TX has already committed.
159
+ await dispatcher.drain();
160
+
161
+ await proxy1.stop();
162
+
163
+ await waitFor(() => logger.lines.some((line) => line.includes("idle pre-check failed")));
164
+ expect(lastListenGauge()).toBe(0);
165
+
166
+ const proxy2 = startDbProxy(dbUrl);
167
+ await listenOn(proxy2.server, proxyPort);
168
+ try {
169
+ await waitFor(() => logger.lines.some((line) => line.includes("idle pre-check recovered")));
170
+
171
+ await appendWidget("after-recovery");
172
+ await waitFor(() => observed.includes("after-recovery"));
173
+ } finally {
174
+ await proxy2.stop();
175
+ }
176
+
177
+ const failedCount = logger.lines.filter((line) =>
178
+ line.includes("idle pre-check failed"),
179
+ ).length;
180
+ const recoveredCount = logger.lines.filter((line) =>
181
+ line.includes("idle pre-check recovered"),
182
+ ).length;
183
+ expect(failedCount).toBe(1);
184
+ expect(recoveredCount).toBe(1);
185
+ expect(observed).toEqual(["before-outage", "after-recovery"]);
186
+ } finally {
187
+ await dispatcher.stop();
188
+ await listenClient.end({ timeout: 1 });
189
+ await proxiedDb.close();
190
+ }
191
+ });
192
+ });
@@ -0,0 +1,94 @@
1
+ // A provably-idle consumer's doPass turn must not take the state row's
2
+ // FOR UPDATE SKIP LOCKED lock: that lock sets xmax on the row, requires an
3
+ // xid, writes WAL, and forces a commit fsync — on every poll tick, forever,
4
+ // even with zero events in the system. The pre-check in
5
+ // selectIdleConsumerKeys (event-dispatcher-delivery.ts) must keep an idle
6
+ // consumer's row untouched entirely, verified here via the row's own
7
+ // system column `xmax`: unchanged across an idle runOnce() means no lock
8
+ // was ever taken on it.
9
+
10
+ import { afterEach, beforeAll, describe, expect, test } from "bun:test";
11
+ import { createEventStoreExecutor } from "../../db/event-store-executor";
12
+ import { asRawClient } from "../../db/query";
13
+ import { createTenantDb, type TenantDb } from "../../db/tenant-db";
14
+ import { defineFeature } from "../../engine";
15
+ import {
16
+ resetEventStore,
17
+ setupTestStack,
18
+ type TestStack,
19
+ TestUsers,
20
+ unsafeCreateEntityTable,
21
+ } from "../../stack";
22
+ import { sharedWidgetEntity, sharedWidgetTable } from "../../testing";
23
+ import { SHARED_INSTANCE_SENTINEL } from "../event-consumer-state";
24
+
25
+ const executor = createEventStoreExecutor(sharedWidgetTable, sharedWidgetEntity, {
26
+ entityName: "widget",
27
+ });
28
+
29
+ let observed: Array<{ name: string }> = [];
30
+
31
+ const idleFeature = defineFeature("idletest", (r) => {
32
+ r.entity("widget", sharedWidgetEntity);
33
+
34
+ r.multiStreamProjection({
35
+ name: "observer",
36
+ apply: {
37
+ "widget.created": async (event) => {
38
+ observed.push({ name: event.payload["name"] as string });
39
+ },
40
+ },
41
+ });
42
+ });
43
+
44
+ const admin = TestUsers.admin;
45
+ const qn = "idletest:projection:observer";
46
+ let stack: TestStack;
47
+ let tdb: TenantDb;
48
+
49
+ beforeAll(async () => {
50
+ stack = await setupTestStack({
51
+ features: [idleFeature],
52
+ systemHooks: [],
53
+ });
54
+ await unsafeCreateEntityTable(stack.db, sharedWidgetEntity, "widget");
55
+ tdb = createTenantDb(stack.db, admin.tenantId);
56
+ });
57
+
58
+ afterEach(async () => {
59
+ observed = [];
60
+ await resetEventStore(stack, ["read_widgets"]);
61
+ });
62
+
63
+ async function appendWidget(name: string): Promise<void> {
64
+ await executor.create({ name }, admin, tdb);
65
+ }
66
+
67
+ async function readConsumerRowXmax(): Promise<string> {
68
+ const rows = (await asRawClient(stack.db).unsafe(
69
+ `SELECT xmax::text FROM "kumiko_event_consumers" WHERE "name" = $1 AND "instance_id" = $2`,
70
+ [qn, SHARED_INSTANCE_SENTINEL],
71
+ )) as ReadonlyArray<{ xmax: string }>;
72
+ const xmax = rows[0]?.xmax;
73
+ if (xmax === undefined) throw new Error("consumer state row missing");
74
+ return xmax;
75
+ }
76
+
77
+ describe("idle event-dispatcher passes take no row lock", () => {
78
+ test("xmax is unchanged across an idle runOnce() after draining, then a new event is still delivered", async () => {
79
+ await appendWidget("drain-me");
80
+ await stack.eventDispatcher?.runOnce();
81
+ expect(observed).toEqual([{ name: "drain-me" }]);
82
+
83
+ const xmaxBeforeIdlePass = await readConsumerRowXmax();
84
+
85
+ await stack.eventDispatcher?.runOnce();
86
+ const xmaxAfterIdlePass = await readConsumerRowXmax();
87
+ expect(xmaxAfterIdlePass).toBe(xmaxBeforeIdlePass);
88
+
89
+ await appendWidget("wake-me");
90
+ const result = await stack.eventDispatcher?.runOnce();
91
+ expect(result?.processed).toBeGreaterThan(0);
92
+ expect(observed).toEqual([{ name: "drain-me" }, { name: "wake-me" }]);
93
+ });
94
+ });
@@ -13,12 +13,12 @@
13
13
  // event-triggered workflow hits in production.
14
14
 
15
15
  import { afterAll, beforeAll, describe, expect, test } from "bun:test";
16
- import type { DbTx } from "../../db/connection";
17
- import { insertOne } from "../../db/query";
16
+ import type { DbConnection, DbTx } from "../../db/connection";
17
+ import { asRawClient, insertOne } from "../../db/query";
18
18
  import type { AppContext } from "../../engine/types";
19
19
  import { eventsTable } from "../../event-store";
20
20
  import { createTestDb, type TestDb, TestUsers } from "../../stack";
21
- import { createEventConsumerStateTable } from "../event-consumer-state";
21
+ import { createEventConsumerStateTable, SHARED_INSTANCE_SENTINEL } from "../event-consumer-state";
22
22
  import { createEventDispatcher, type EventConsumer, getConsumerState } from "../event-dispatcher";
23
23
  import { fetchPendingEvents } from "../event-dispatcher-delivery";
24
24
 
@@ -49,6 +49,40 @@ async function appendHistoricalEvent(type: string): Promise<void> {
49
49
  historicalEventCount += 1;
50
50
  }
51
51
 
52
+ // Grabs an id (like appendHistoricalEvent) but never commits — the row
53
+ // stays invisible until the caller releases and awaits the enclosing tx.
54
+ async function appendEventRawHoldingTx(tx: DbTx, type: string): Promise<void> {
55
+ await asRawClient(tx).unsafe(
56
+ `INSERT INTO "kumiko_events"
57
+ (aggregate_id, aggregate_type, tenant_id, version, type, payload, metadata, created_by)
58
+ VALUES ($1::uuid, 'start-from-test-source', $2::uuid, 1, $3, '{}'::jsonb, '{}'::jsonb, 'test')`,
59
+ [crypto.randomUUID(), admin.tenantId, type],
60
+ );
61
+ }
62
+
63
+ async function selectVisibleMaxEventId(): Promise<bigint> {
64
+ const rows = (await asRawClient(testDb.db).unsafe(
65
+ `SELECT COALESCE(MAX("id"), 0)::text AS max FROM "kumiko_events"`,
66
+ )) as ReadonlyArray<{ max: string }>;
67
+ return BigInt(rows[0]?.max ?? "0");
68
+ }
69
+
70
+ async function readPendingGaps(
71
+ db: DbConnection,
72
+ consumerName: string,
73
+ ): Promise<ReadonlyArray<{ from: string; to: string; xmax: string }>> {
74
+ const rows = (await asRawClient(db).unsafe(
75
+ `SELECT "pending_gaps", jsonb_typeof("pending_gaps") AS kind FROM "kumiko_event_consumers" WHERE "name" = $1 AND "instance_id" = $2`,
76
+ [consumerName, SHARED_INSTANCE_SENTINEL],
77
+ )) as ReadonlyArray<{
78
+ pending_gaps: ReadonlyArray<{ from: string; to: string; xmax: string }>;
79
+ kind: string;
80
+ }>;
81
+ // A double-encoded write lands as a jsonb string scalar, not an array.
82
+ if (rows[0] && rows[0].kind !== "array") throw new Error(`pending_gaps is jsonb ${rows[0].kind}`);
83
+ return rows[0]?.pending_gaps ?? [];
84
+ }
85
+
52
86
  beforeAll(async () => {
53
87
  // createTestDb() already materializes kumiko_events (see stack/db.ts).
54
88
  testDb = await createTestDb();
@@ -173,4 +207,70 @@ describe("EventConsumer.startFrom", () => {
173
207
  const afterRedeploy = await getConsumerState(testDb.db, consumer.name);
174
208
  expect(afterRedeploy?.lastProcessedEventId).toBe(advanced?.lastProcessedEventId);
175
209
  });
210
+
211
+ test("'now' registered while a lower-id write is still uncommitted seeds a pending gap and later delivers it", async () => {
212
+ const captured: string[] = [];
213
+ const consumer: EventConsumer = {
214
+ name: "test:consumer:now-out-of-order",
215
+ startFrom: "now",
216
+ handler: async (event) => {
217
+ captured.push(event.type);
218
+ },
219
+ };
220
+
221
+ // Tx A grabs the LOW id first but holds its transaction open — invisible
222
+ // to insertConsumerIfAbsent's horizon read below, same mechanic as the
223
+ // live-dispatcher out-of-order case in event-dispatcher-commit-order.
224
+ let releaseA!: () => void;
225
+ const aGate = new Promise<void>((resolve) => {
226
+ releaseA = resolve;
227
+ });
228
+ let markAInserted!: () => void;
229
+ const aInserted = new Promise<void>((resolve) => {
230
+ markAInserted = resolve;
231
+ });
232
+ const aDone = testDb.db
233
+ .begin(async (tx: DbTx) => {
234
+ await appendEventRawHoldingTx(tx, "start-from-test.low-id");
235
+ markAInserted();
236
+ await aGate;
237
+ })
238
+ .catch(() => {});
239
+ await aInserted;
240
+
241
+ // Tx B commits AFTER A grabbed its id, so B's id is the higher, visible
242
+ // MAX(id) when the consumer is registered below. A's still-open tx
243
+ // already consumed a sequence value, so B's actual id is one past
244
+ // historicalEventCount, not equal to it — read the real MAX(id).
245
+ await appendHistoricalEvent("start-from-test.high-id");
246
+ const headAtMount = await selectVisibleMaxEventId();
247
+
248
+ const dispatcher = createEventDispatcher({
249
+ db: testDb.db,
250
+ consumers: [consumer],
251
+ context: stubContext(),
252
+ });
253
+
254
+ try {
255
+ await dispatcher.ensureRegistered();
256
+
257
+ const seeded = await getConsumerState(testDb.db, consumer.name);
258
+ expect(seeded?.lastProcessedEventId).toBe(headAtMount);
259
+ expect(await readPendingGaps(testDb.db, consumer.name)).not.toEqual([]);
260
+
261
+ // Nothing to deliver yet — A's transaction hasn't committed.
262
+ const passWhileOpen = await dispatcher.runOnce();
263
+ expect(passWhileOpen.byConsumer[consumer.name]).toEqual({ processed: 0, failed: 0 });
264
+ } finally {
265
+ releaseA();
266
+ await aDone;
267
+ }
268
+
269
+ // A commits now — its lower id is delivered as a resolved pending gap,
270
+ // even though the cursor already sits above it from registration.
271
+ const pass = await dispatcher.runOnce();
272
+ expect(pass.byConsumer[consumer.name]).toEqual({ processed: 1, failed: 0 });
273
+ expect(captured).toEqual(["start-from-test.low-id"]);
274
+ expect(await readPendingGaps(testDb.db, consumer.name)).toEqual([]);
275
+ });
176
276
  });
@@ -8,7 +8,7 @@
8
8
  // consecutive versions without re-reading the DB.
9
9
 
10
10
  import { afterAll, afterEach, beforeAll, describe, expect, test } from "bun:test";
11
- import { z } from "zod";
11
+ import * as z from "zod";
12
12
  import { createEventStoreExecutor } from "../../db/event-store-executor";
13
13
  import { asRawClient } from "../../db/query-api";
14
14
  import { buildEntityTable } from "../../db/table-builder";
@@ -1,5 +1,5 @@
1
1
  import { afterAll, beforeAll, beforeEach, describe, expect, test } from "bun:test";
2
- import { z } from "zod";
2
+ import * as z from "zod";
3
3
  import { createEventStoreExecutor } from "../../db/event-store-executor";
4
4
  import { asRawClient, selectMany } from "../../db/query";
5
5
  import { buildEntityTable } from "../../db/table-builder";
@@ -17,7 +17,7 @@
17
17
  // produce, not something exercised end-to-end by any test today.
18
18
 
19
19
  import { afterAll, beforeAll, beforeEach, describe, expect, test } from "bun:test";
20
- import { z } from "zod";
20
+ import * as z from "zod";
21
21
  import { entityEventName } from "../../db";
22
22
  import { defineFeature } from "../../engine";
23
23
  import { createInMemoryFileProvider, type InMemoryFileProvider } from "../../files";
@@ -1,5 +1,5 @@
1
1
  import { describe, expect, spyOn, test } from "bun:test";
2
- import { z } from "zod";
2
+ import * as z from "zod";
3
3
  import {
4
4
  createEntity,
5
5
  createRegistry,
@@ -10,7 +10,7 @@
10
10
 
11
11
  import { afterAll, afterEach, beforeAll, describe, expect, test } from "bun:test";
12
12
  import { seedRow } from "@cosmicdrift/kumiko-framework/testing";
13
- import { z } from "zod";
13
+ import * as z from "zod";
14
14
  import type { DbConnection, DbTx } from "../../db/connection";
15
15
  import { createEventStoreExecutor } from "../../db/event-store-executor";
16
16
  import { buildEntityTable } from "../../db/table-builder";
@@ -11,7 +11,7 @@
11
11
  // 5. ctx.loadAggregate reads the triggering stream's full history.
12
12
 
13
13
  import { afterAll, afterEach, beforeAll, describe, expect, test } from "bun:test";
14
- import { z } from "zod";
14
+ import * as z from "zod";
15
15
  import { createEventStoreExecutor } from "../../db/event-store-executor";
16
16
  import { selectMany } from "../../db/query";
17
17
  import { buildEntityTable } from "../../db/table-builder";