@cosmicdrift/kumiko-framework 0.164.0 → 0.165.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (160) hide show
  1. package/package.json +5 -3
  2. package/src/__tests__/consumer-cli.integration.test.ts +62 -0
  3. package/src/__tests__/schema-cli.integration.test.ts +1 -1
  4. package/src/api/__tests__/api.test.ts +326 -18
  5. package/src/api/__tests__/auth-middleware-anonymous-access-boot.test.ts +40 -0
  6. package/src/api/__tests__/auth-routes-invalid-body-invite.test.ts +16 -0
  7. package/src/api/__tests__/auth-routes-mfa-preauth-confirm.test.ts +18 -1
  8. package/src/api/__tests__/auth-routes-mfa-preauth-enable-start.test.ts +64 -1
  9. package/src/api/__tests__/auth-routes-trusted-proxy.test.ts +146 -0
  10. package/src/api/__tests__/batch.integration.test.ts +21 -2
  11. package/src/api/__tests__/jwt.test.ts +52 -2
  12. package/src/api/__tests__/login-rate-limiter-sweep.test.ts +27 -18
  13. package/src/api/__tests__/redis-login-rate-limiter.integration.test.ts +72 -0
  14. package/src/api/__tests__/sse-broker.test.ts +57 -0
  15. package/src/api/__tests__/sse-route.test.ts +4 -0
  16. package/src/api/auth-routes.ts +178 -33
  17. package/src/api/index.ts +1 -0
  18. package/src/api/jwt.ts +22 -1
  19. package/src/api/routes.ts +117 -30
  20. package/src/api/server.ts +39 -7
  21. package/src/api/sse-broker.ts +39 -0
  22. package/src/bun-db/connection.ts +3 -3
  23. package/src/bun-db/index.ts +1 -0
  24. package/src/bun-db/query.ts +12 -3
  25. package/src/consumer-cli.ts +60 -13
  26. package/src/db/__tests__/event-store-executor-write-verbs.integration.test.ts +19 -13
  27. package/src/db/__tests__/located-timestamp.test.ts +19 -0
  28. package/src/db/__tests__/migrate-runner.test.ts +61 -0
  29. package/src/db/__tests__/replay-migration-sql.test.ts +131 -2
  30. package/src/db/__tests__/tenant-db-where-merge.test.ts +6 -2
  31. package/src/db/api.ts +2 -2
  32. package/src/db/bun-provider.ts +2 -2
  33. package/src/db/connection.ts +6 -3
  34. package/src/db/dialect.ts +1 -6
  35. package/src/db/entity-table-meta-types.ts +1 -1
  36. package/src/db/event-store-executor-context.ts +2 -3
  37. package/src/db/event-store-executor-read.ts +2 -3
  38. package/src/db/event-store-executor-write.ts +8 -0
  39. package/src/db/index.ts +8 -3
  40. package/src/db/located-timestamp.ts +4 -0
  41. package/src/db/migrate-runner.ts +107 -11
  42. package/src/db/pg-error.ts +8 -0
  43. package/src/db/postgres-provider.ts +2 -2
  44. package/src/db/queries/__tests__/event-store-idempotency-index.integration.test.ts +80 -0
  45. package/src/db/queries/ddl.ts +45 -0
  46. package/src/db/queries/event-store.ts +97 -21
  47. package/src/db/queries/test-stack.ts +4 -30
  48. package/src/db/reference-data.ts +2 -3
  49. package/src/db/replay-migration-sql.ts +114 -12
  50. package/src/db/tenant-db.ts +2 -4
  51. package/src/engine/__tests__/boot-validator.test.ts +14 -0
  52. package/src/engine/__tests__/engine.test.ts +30 -0
  53. package/src/engine/__tests__/registry.test.ts +36 -0
  54. package/src/engine/__tests__/schema-builder.test.ts +18 -0
  55. package/src/engine/__tests__/store-table.test.ts +2 -2
  56. package/src/engine/boot-validator/entity-handler.ts +14 -25
  57. package/src/engine/boot-validator/nav.ts +5 -0
  58. package/src/engine/constants.ts +32 -6
  59. package/src/engine/create-app.ts +11 -0
  60. package/src/engine/effective-features.ts +12 -2
  61. package/src/engine/extensions/user-data.ts +12 -4
  62. package/src/engine/feature-ast/__tests__/render-roundtrip.test.ts +4 -0
  63. package/src/engine/feature-ast/extractors/handlers.ts +13 -18
  64. package/src/engine/feature-ui-extensions.ts +2 -2
  65. package/src/engine/hook-helpers.ts +3 -1
  66. package/src/engine/index.ts +1 -1
  67. package/src/engine/ownership.ts +4 -3
  68. package/src/engine/registry-ingest.ts +14 -14
  69. package/src/engine/registry-state.ts +4 -1
  70. package/src/engine/registry-validate.ts +7 -2
  71. package/src/engine/schema-builder.ts +1 -0
  72. package/src/engine/steps/__tests__/duration-utils.test.ts +20 -0
  73. package/src/engine/steps/_duration-utils.ts +2 -0
  74. package/src/engine/steps/unsafe-projection-upsert.ts +1 -4
  75. package/src/engine/types/config.ts +1 -1
  76. package/src/engine/types/define-handler.ts +1 -1
  77. package/src/engine/types/entity-handlers.ts +1 -1
  78. package/src/engine/types/event-type-map.ts +1 -1
  79. package/src/engine/types/feature.ts +1 -1
  80. package/src/engine/types/fields.ts +1 -1
  81. package/src/engine/types/handlers.ts +1 -1
  82. package/src/engine/types/hooks.ts +1 -1
  83. package/src/engine/types/http-route.ts +1 -1
  84. package/src/engine/types/index.ts +0 -2
  85. package/src/engine/types/nav.ts +1 -1
  86. package/src/engine/types/ownership.ts +1 -1
  87. package/src/engine/types/projection.ts +1 -1
  88. package/src/engine/types/relations.ts +1 -1
  89. package/src/engine/types/screen.ts +1 -1
  90. package/src/engine/types/step.ts +1 -1
  91. package/src/engine/types/target-ref.ts +1 -1
  92. package/src/engine/types/tree-node.ts +1 -1
  93. package/src/engine/types/workspace.ts +1 -1
  94. package/src/engine/validate-projection-allowlist.ts +5 -5
  95. package/src/errors/classes.ts +21 -0
  96. package/src/errors/index.ts +1 -0
  97. package/src/errors/write-error-info.ts +6 -2
  98. package/src/event-store/__tests__/admin-api.integration.test.ts +27 -1
  99. package/src/event-store/__tests__/event-store.integration.test.ts +60 -14
  100. package/src/event-store/__tests__/unscoped-stream-primitives.guard.test.ts +27 -12
  101. package/src/event-store/admin-api.ts +11 -4
  102. package/src/event-store/event-store.ts +20 -21
  103. package/src/event-store/index.ts +0 -1
  104. package/src/event-store/types.ts +1 -1
  105. package/src/files/__tests__/build-storage-key.test.ts +28 -0
  106. package/src/files/__tests__/local-provider.test.ts +31 -0
  107. package/src/files/__tests__/write-stream.test.ts +3 -3
  108. package/src/files/index.ts +1 -1
  109. package/src/files/local-provider.ts +6 -1
  110. package/src/files/types.ts +8 -1
  111. package/src/jobs/__tests__/jobs.integration.test.ts +167 -7
  112. package/src/jobs/job-runner.ts +41 -11
  113. package/src/logging/types.ts +1 -1
  114. package/src/observability/index.ts +1 -0
  115. package/src/observability/standard-metrics.ts +35 -2
  116. package/src/observability/types/index.ts +1 -1
  117. package/src/observability/types/metric.ts +1 -1
  118. package/src/observability/types/provider.ts +1 -1
  119. package/src/observability/types/span.ts +1 -1
  120. package/src/pipeline/__tests__/dispatcher.test.ts +203 -0
  121. package/src/pipeline/__tests__/event-consumer-state.integration.test.ts +31 -0
  122. package/src/pipeline/__tests__/event-dispatcher-rearm.integration.test.ts +83 -0
  123. package/src/pipeline/__tests__/job-trigger-consumer.integration.test.ts +106 -0
  124. package/src/pipeline/__tests__/lifecycle-pipeline.test.ts +111 -88
  125. package/src/pipeline/dispatch-shared.ts +59 -6
  126. package/src/pipeline/dispatch-stream.ts +44 -17
  127. package/src/pipeline/dispatcher.ts +7 -1
  128. package/src/pipeline/event-consumer-state.ts +16 -13
  129. package/src/pipeline/event-dispatcher-delivery.ts +22 -7
  130. package/src/pipeline/event-dispatcher.ts +22 -0
  131. package/src/pipeline/index.ts +2 -0
  132. package/src/pipeline/system-hooks.ts +137 -1
  133. package/src/rate-limit/__tests__/resolver.integration.test.ts +18 -0
  134. package/src/rate-limit/resolver.ts +6 -2
  135. package/src/schema-cli.ts +24 -12
  136. package/src/search/__tests__/reindex-entity.integration.test.ts +24 -1
  137. package/src/search/reindex-entity.ts +31 -2
  138. package/src/search/types.ts +1 -1
  139. package/src/stack/__tests__/setup-test-stack-jobs.integration.test.ts +6 -2
  140. package/src/stack/db.ts +2 -1
  141. package/src/stack/push-entity-projection-tables.ts +2 -1
  142. package/src/stack/request-helper.ts +20 -1
  143. package/src/stack/table-helpers.ts +6 -4
  144. package/src/stack/test-stack.ts +18 -15
  145. package/src/testing/__tests__/late-bound.test.ts +7 -0
  146. package/src/testing/__tests__/wait-for.test.ts +6 -0
  147. package/src/testing/file-provider-contract.ts +26 -6
  148. package/src/testing/index.ts +1 -0
  149. package/src/testing/late-bound.ts +5 -3
  150. package/src/testing/wait-for.ts +3 -0
  151. package/src/testing/without-ambient-temporal.ts +14 -0
  152. package/src/time/__tests__/polyfill-reinstall.test.ts +17 -0
  153. package/src/time/geo-tz.ts +1 -1
  154. package/src/time/polyfill.ts +28 -39
  155. package/src/time/tz-context.ts +30 -24
  156. package/src/utils/__tests__/safe-json-temporal.test.ts +14 -0
  157. package/src/utils/safe-json.ts +3 -2
  158. package/src/db/__tests__/encryption.test.ts +0 -39
  159. package/src/db/encryption.ts +0 -45
  160. package/src/engine/__tests__/registry-facade-sweep.test.ts +0 -80
package/src/db/dialect.ts CHANGED
@@ -19,6 +19,7 @@
19
19
  import {
20
20
  type ColumnHandle,
21
21
  KUMIKO_COLUMNS_SYMBOL,
22
+ KUMIKO_META_SYMBOL,
22
23
  KUMIKO_NAME_SYMBOL,
23
24
  type SchemaTable,
24
25
  } from "@cosmicdrift/kumiko-types/schema-table-types";
@@ -41,12 +42,6 @@ export type TableColumns<_T = any> = SchemaTable;
41
42
  // biome-ignore lint/suspicious/noExplicitAny: legacy type — chain API is gone
42
43
  export type SelectQuery = any;
43
44
 
44
- // Shadow-proof handle on the EntityTableMeta. The column handles below are
45
- // spread as enumerable props, so an entity field named `source`/`columns`/
46
- // `tableName`/… would overwrite the matching meta key. extractTableInfo reads
47
- // the canonical meta from this symbol instead of the (shadowable) props.
48
- const KUMIKO_META_SYMBOL = Symbol.for("kumiko:schema:Meta");
49
-
50
45
  function isNumericPgType(t: PgType): t is `numeric(${number},${number})` {
51
46
  return t.startsWith("numeric(");
52
47
  }
@@ -1,2 +1,2 @@
1
1
  // Legacy path — re-exported for callers still importing this module directly.
2
- export * from "@cosmicdrift/kumiko-types/entity-table-meta-types";
2
+ export type * from "@cosmicdrift/kumiko-types/entity-table-meta-types";
@@ -1,3 +1,4 @@
1
+ import { KUMIKO_NAME_SYMBOL } from "@cosmicdrift/kumiko-types/schema-table-types";
1
2
  import { requestContext } from "../api/request-context";
2
3
  import {
3
4
  collectPiiSubjectFields,
@@ -359,9 +360,7 @@ export function buildExecutorContext(
359
360
  }
360
361
  // ownership has raw SQL — splice it into a raw query alongside the
361
362
  // idFilter + tenant-filter that TenantDb would have added.
362
- const tableName = String(
363
- (table as unknown as Record<symbol, unknown>)[Symbol.for("kumiko:schema:Name")],
364
- );
363
+ const tableName = String((table as unknown as Record<symbol, unknown>)[KUMIKO_NAME_SYMBOL]);
365
364
  const colSql = (field: string): string =>
366
365
  `"${(table[field] as { name?: string } | undefined)?.name ?? toSnakeCase(field)}"`;
367
366
  const whereParts: string[] = [];
@@ -1,3 +1,4 @@
1
+ import { KUMIKO_NAME_SYMBOL } from "@cosmicdrift/kumiko-types/schema-table-types";
1
2
  import { computeBlindIndex, configuredBlindIndexKey } from "../crypto";
2
3
  import { executeRawQuery } from "../db/queries/raw-sql";
3
4
  import { coerceRow, extractTableInfo } from "../db/query";
@@ -66,9 +67,7 @@ export function createReadVerbs(ctx: ExecutorContext): Pick<EventStoreExecutor,
66
67
  // Build the WHERE clause as raw SQL — ownership produces a
67
68
  // parameterised fragment that we splice in alongside simple WhereObject
68
69
  // conditions (cursor, search-filter-IDs, screen-filter, tenant-scope).
69
- const tableName = String(
70
- (table as unknown as Record<symbol, unknown>)[Symbol.for("kumiko:schema:Name")],
71
- );
70
+ const tableName = String((table as unknown as Record<symbol, unknown>)[KUMIKO_NAME_SYMBOL]);
72
71
  const whereSql: string[] = [];
73
72
  const params: unknown[] = [];
74
73
  const colSql = (field: string): string =>
@@ -3,6 +3,7 @@ import { userCanCreateFieldRow, userCanWriteFieldRow } from "../engine/ownership
3
3
  import type { EntityId } from "../engine/types";
4
4
  import {
5
5
  VersionConflictError as FrameworkVersionConflict,
6
+ IdempotentReplayError,
6
7
  InternalError,
7
8
  NotFoundError,
8
9
  UnprocessableError,
@@ -10,6 +11,7 @@ import {
10
11
  } from "../errors";
11
12
  import {
12
13
  append,
14
+ IdempotentAppendConflictError as EventStoreIdempotentAppendConflict,
13
15
  VersionConflictError as EventStoreVersionConflict,
14
16
  getStreamVersion,
15
17
  } from "../event-store";
@@ -160,6 +162,9 @@ export function createWriteVerbs(
160
162
  }),
161
163
  );
162
164
  }
165
+ if (e instanceof EventStoreIdempotentAppendConflict) {
166
+ return writeFailure(new IdempotentReplayError({ idempotencyKey: e.idempotencyKey }));
167
+ }
163
168
  throw e;
164
169
  }
165
170
 
@@ -390,6 +395,9 @@ export function createWriteVerbs(
390
395
  }),
391
396
  );
392
397
  }
398
+ if (e instanceof EventStoreIdempotentAppendConflict) {
399
+ return writeFailure(new IdempotentReplayError({ idempotencyKey: e.idempotencyKey }));
400
+ }
393
401
  throw e;
394
402
  }
395
403
  },
package/src/db/index.ts CHANGED
@@ -3,7 +3,14 @@ export { nullBlindIndexesForSubject } from "./blind-index-cleanup";
3
3
  export { collectTableMetas } from "./collect-table-metas";
4
4
  export { flattenCompoundTypes, rehydrateCompoundTypes } from "./compound-types";
5
5
  export { seedConfigValues } from "./config-seed";
6
- export type { DbConnection, DbConnectionOptions, DbRow, DbRunner, DbTx } from "./connection";
6
+ export type {
7
+ DbConnection,
8
+ DbConnectionOptions,
9
+ DbPoolHandle,
10
+ DbRow,
11
+ DbRunner,
12
+ DbTx,
13
+ } from "./connection";
7
14
  export { createDbConnection, dbConnectionOptionsFromEnv } from "./connection";
8
15
  export type { CursorQueryOptions, CursorResult } from "./cursor";
9
16
  export { decodeCursor, encodeCursor } from "./cursor";
@@ -35,8 +42,6 @@ export {
35
42
  enrichRowWithReferences,
36
43
  enrichWithReferences,
37
44
  } from "./eagerload";
38
- export type { EncryptionProvider } from "./encryption";
39
- export { createEncryptionProvider } from "./encryption";
40
45
  export {
41
46
  collectEncryptedFieldNames,
42
47
  configuredEntityFieldEncryption,
@@ -9,6 +9,10 @@
9
9
  // gespeicherter tz). Server kennt User-TZ nicht — User-spezifische
10
10
  // Anzeige passiert client-seitig aus utc.
11
11
 
12
+ // Static import, not the ambient global: Bun doesn't expose Temporal on
13
+ // globalThis, so this crashed with "Temporal is not defined" outside boot
14
+ // paths that install it (#1480).
15
+ import { Temporal } from "temporal-polyfill";
12
16
  import type { EntityDefinition } from "../engine/types";
13
17
 
14
18
  // Sprint F: <name>Utc-Spalte ist jetzt instant() (siehe dialect.ts) —
@@ -74,14 +74,103 @@ CREATE TABLE IF NOT EXISTS "_kumiko_migrations" (
74
74
  )
75
75
  `.trim();
76
76
 
77
- // Splittet SQL-File-Text in einzelne Statements. Pragma: simpler `;`-Split,
78
- // reicht weil unsere generierten SQL-Files keine eingebetteten `;` in String-
79
- // Literalen haben. App-Author der hand-editiert + tricky SQL einfügt sollte
80
- // das wissen sonst pg-Parser einziehen.
77
+ // Splits SQL-file text into individual statements on top-level `;`. A plain
78
+ // `text.split(";")` breaks the moment a `--` line comment or `/* */` block
79
+ // comment contains a semicolon (#1542) it splits mid-comment before the
80
+ // comment is ever stripped. This scans char-by-char tracking whether we're
81
+ // inside a line comment, block comment, single-quoted string, or
82
+ // double-quoted identifier, so `;` only ends a statement in plain SQL text;
83
+ // comments are dropped, quoted/identifier content (incl. `''`/`""` escapes)
84
+ // is kept verbatim. Does not handle dollar-quoted (`$$...$$`) bodies — none
85
+ // of this repo's checked-in migrations use them; add that state if one ever
86
+ // does.
87
+ type SqlScanState = "normal" | "lineComment" | "blockComment" | "singleQuote" | "doubleQuote";
88
+
81
89
  export function splitSqlStatements(sqlText: string): readonly string[] {
82
- return sqlText
83
- .split(";")
84
- .map((s) => s.replace(/--[^\n]*/g, "").trim())
90
+ const statements: string[] = [];
91
+ let current = "";
92
+ let state: SqlScanState = "normal";
93
+
94
+ for (let i = 0; i < sqlText.length; i++) {
95
+ const ch = sqlText.charAt(i);
96
+ const next = sqlText.charAt(i + 1);
97
+
98
+ if (state === "lineComment") {
99
+ if (ch === "\n") {
100
+ state = "normal";
101
+ current += ch;
102
+ }
103
+ continue;
104
+ }
105
+ if (state === "blockComment") {
106
+ if (ch === "*" && next === "/") {
107
+ state = "normal";
108
+ i++;
109
+ }
110
+ continue;
111
+ }
112
+ if (state === "singleQuote") {
113
+ current += ch;
114
+ if (ch === "'") {
115
+ if (next === "'") {
116
+ current += next;
117
+ i++;
118
+ } else {
119
+ state = "normal";
120
+ }
121
+ }
122
+ continue;
123
+ }
124
+ if (state === "doubleQuote") {
125
+ current += ch;
126
+ if (ch === '"') {
127
+ if (next === '"') {
128
+ current += next;
129
+ i++;
130
+ } else {
131
+ state = "normal";
132
+ }
133
+ }
134
+ continue;
135
+ }
136
+
137
+ // state === "normal"
138
+ if (ch === "-" && next === "-") {
139
+ state = "lineComment";
140
+ i++;
141
+ continue;
142
+ }
143
+ if (ch === "/" && next === "*") {
144
+ state = "blockComment";
145
+ i++;
146
+ continue;
147
+ }
148
+ if (ch === "'") {
149
+ state = "singleQuote";
150
+ current += ch;
151
+ continue;
152
+ }
153
+ if (ch === '"') {
154
+ state = "doubleQuote";
155
+ current += ch;
156
+ continue;
157
+ }
158
+ if (ch === ";") {
159
+ statements.push(current);
160
+ current = "";
161
+ continue;
162
+ }
163
+ current += ch;
164
+ }
165
+ if (state === "blockComment" || state === "singleQuote" || state === "doubleQuote") {
166
+ throw new Error(
167
+ `splitSqlStatements: unterminated ${state} — migration SQL is malformed, refusing to split`,
168
+ );
169
+ }
170
+ statements.push(current);
171
+
172
+ return statements
173
+ .map((s) => s.trim())
85
174
  .filter((s) => s.length > 0)
86
175
  .map((s) => `${s};`);
87
176
  }
@@ -90,14 +179,21 @@ function sha256Hex(content: string): string {
90
179
  return createHash("sha256").update(content).digest("hex");
91
180
  }
92
181
 
93
- // Liest <dir>/*.sql, sortiert lex (z.B. 0001_init.sql, 0002_add_locale.sql),
94
- // returnt Migration[] mit id + checksum + statements.
95
- export function loadMigrationsFromDir(dir: string): readonly Migration[] {
182
+ // Reads <dir>/*.sql, sorted lexically (e.g. 0001_init.sql, 0002_add_locale.sql),
183
+ // returns Migration[] with id + checksum + statements. `preprocess` is an
184
+ // optional hook applied to each file's raw content before splitting/hashing
185
+ // — used by replayMigrationsDir to expand its DESTRUCTIVE-marker comments
186
+ // without a second, drifting copy of this file-discovery logic (#1522/9).
187
+ export function loadMigrationsFromDir(
188
+ dir: string,
189
+ preprocess?: (sql: string) => string,
190
+ ): readonly Migration[] {
96
191
  const files = readdirSync(dir)
97
192
  .filter((f) => f.endsWith(".sql"))
98
193
  .sort();
99
194
  return files.map((file) => {
100
- const content = readFileSync(join(dir, file), "utf8");
195
+ const raw = readFileSync(join(dir, file), "utf8");
196
+ const content = preprocess ? preprocess(raw) : raw;
101
197
  return {
102
198
  id: file.replace(/\.sql$/, ""),
103
199
  checksum: sha256Hex(content),
@@ -41,6 +41,14 @@ export function isTableAlreadyExists(e: unknown): boolean {
41
41
  return extractPgError(e)?.code === "42P07";
42
42
  }
43
43
 
44
+ // PG SQLSTATE 55P03 — "lock not available". Raised by a NOWAIT-style lock
45
+ // request that couldn't be granted; a CREATE/DROP INDEX CONCURRENTLY racing
46
+ // another session's DDL on the same relation can hit this instead of a
47
+ // duplicate-relation error, depending on exact timing.
48
+ export function isLockNotAvailable(e: unknown): boolean {
49
+ return extractPgError(e)?.code === "55P03";
50
+ }
51
+
44
52
  export function constraintOf(e: unknown): string | undefined {
45
53
  return extractPgError(e)?.constraint_name;
46
54
  }
@@ -3,9 +3,9 @@
3
3
  // `asRawClient(db)` wrappt .unsafe und .begin transparent.
4
4
 
5
5
  import postgres from "postgres";
6
- import type { DbConnection, DbConnectionOptions } from "./api";
6
+ import type { DbConnectionOptions, DbPoolHandle } from "./api";
7
7
 
8
- export function createPgConnection(url: string, options: DbConnectionOptions = {}): DbConnection {
8
+ export function createPgConnection(url: string, options: DbConnectionOptions = {}): DbPoolHandle {
9
9
  const pgOptions: Parameters<typeof postgres>[1] = {};
10
10
  if (options.maxConnections !== undefined) pgOptions.max = options.maxConnections;
11
11
  if (options.idleTimeoutSeconds !== undefined) pgOptions.idle_timeout = options.idleTimeoutSeconds;
@@ -0,0 +1,80 @@
1
+ import { afterAll, beforeAll, describe, expect, test } from "bun:test";
2
+ import { type BunTestDb, createTestDb } from "../../../bun-db/__tests__/bun-test-db";
3
+ import { createEventsTable } from "../../../event-store/events-schema";
4
+ import { asRawClient } from "../../query";
5
+ import { ensureIdempotencyKeyIndex } from "../event-store";
6
+
7
+ let testDb: BunTestDb;
8
+
9
+ beforeAll(async () => {
10
+ testDb = await createTestDb();
11
+ await createEventsTable(testDb.db);
12
+ });
13
+
14
+ afterAll(async () => {
15
+ await testDb.cleanup();
16
+ });
17
+
18
+ async function indexValidity(): Promise<boolean | undefined> {
19
+ const rows = (await asRawClient(testDb.db).unsafe(
20
+ `SELECT i.indisvalid FROM pg_class c JOIN pg_index i ON i.indexrelid = c.oid ` +
21
+ `WHERE c.relname = 'events_idempotency_uq'`,
22
+ )) as ReadonlyArray<{ indisvalid: boolean }>;
23
+ return rows[0]?.indisvalid;
24
+ }
25
+
26
+ describe("ensureIdempotencyKeyIndex (#1499)", () => {
27
+ test("repeated calls stay idempotent — index ends up valid, no throw", async () => {
28
+ await ensureIdempotencyKeyIndex(testDb.db);
29
+ await ensureIdempotencyKeyIndex(testDb.db);
30
+ expect(await indexValidity()).toBe(true);
31
+ });
32
+
33
+ // Regression for the rolling-deploy race: two pods booting concurrently
34
+ // against the same DB both try to build the same CONCURRENTLY index.
35
+ // A plain CREATE INDEX IF NOT EXISTS no-ops the loser; a CONCURRENTLY
36
+ // build can instead raise a duplicate-relation error mid-build — that
37
+ // must be swallowed the same way createEventsTable already tolerates a
38
+ // racing CREATE TABLE, not crash-loop the booting pod.
39
+ test("concurrent calls from two racing boots don't throw", async () => {
40
+ // The prior test already built the index — without dropping it here,
41
+ // both calls below would just hit the IF NOT EXISTS no-op path (indeed
42
+ // what the original single-connection version of this test silently
43
+ // did) and never actually contend on a real CONCURRENTLY build.
44
+ await asRawClient(testDb.db).unsafe(
45
+ `DROP INDEX CONCURRENTLY IF EXISTS "events_idempotency_uq"`,
46
+ );
47
+
48
+ // Two independent connections against the SAME database, mirroring two
49
+ // pods — a single shared `testDb.db` pool would serialize both calls
50
+ // and never pin the race (kumiko-framework#1522).
51
+ const secondPodDb = await createTestDb({ dbName: testDb.dbName, persistent: true });
52
+ try {
53
+ await Promise.all([
54
+ ensureIdempotencyKeyIndex(testDb.db),
55
+ ensureIdempotencyKeyIndex(secondPodDb.db),
56
+ ]);
57
+ expect(await indexValidity()).toBe(true);
58
+ } finally {
59
+ // persistent: true → cleanup() only closes this pool, doesn't drop the
60
+ // shared DB (testDb's afterAll still owns that).
61
+ await secondPodDb.cleanup();
62
+ }
63
+ });
64
+
65
+ test("an INVALID leftover from a killed CONCURRENTLY build gets dropped and rebuilt", async () => {
66
+ // Simulates a build interrupted mid-flight (crash, deploy restart): the
67
+ // catalog entry exists but never finished — pg_index.indisvalid=false.
68
+ // Postgres has no direct DDL to force this state, so flip it straight
69
+ // in the catalog, matching what a real killed CONCURRENTLY build leaves
70
+ // behind.
71
+ await asRawClient(testDb.db).unsafe(
72
+ `UPDATE pg_index SET indisvalid = false WHERE indexrelid = 'events_idempotency_uq'::regclass`,
73
+ );
74
+ expect(await indexValidity()).toBe(false);
75
+
76
+ await ensureIdempotencyKeyIndex(testDb.db);
77
+
78
+ expect(await indexValidity()).toBe(true);
79
+ });
80
+ });
@@ -0,0 +1,45 @@
1
+ import type { AnyDb } from "../query";
2
+ import { asRawClient } from "../query";
3
+ import { quoteTableIdent } from "./table-ops";
4
+
5
+ // Generic DDL helpers used on the prod-boot path (stack/table-helpers.ts's
6
+ // unsafePushTables, event-consumer-state.ts's multi-instance backfill) —
7
+ // split out of queries/test-stack.ts so a prod-boot import doesn't point at
8
+ // a module named for test-only concerns (truncate/create/drop-database).
9
+
10
+ export async function executeDdlStatement(db: AnyDb, sqlText: string): Promise<void> {
11
+ await asRawClient(db).unsafe(sqlText);
12
+ }
13
+
14
+ export async function alterTableAddColumn(
15
+ db: AnyDb,
16
+ tableName: string,
17
+ columnName: string,
18
+ columnType: string,
19
+ defaultClause: string,
20
+ notNull: string,
21
+ // table-helpers.ts's unmanaged-table sync relies on the plain form
22
+ // throwing when a column already exists with a different shape than
23
+ // EntityTableMeta expects — that's how it surfaces drift. Callers that
24
+ // just need an idempotent, race-safe backfill (e.g. event-consumer-state's
25
+ // multi-instance boot path, #1362) opt in explicitly.
26
+ ifNotExists = false,
27
+ ): Promise<void> {
28
+ await asRawClient(db).unsafe(
29
+ `ALTER TABLE ${quoteTableIdent(tableName)} ADD COLUMN ${ifNotExists ? "IF NOT EXISTS " : ""}${quoteTableIdent(columnName)} ${columnType}${defaultClause}${notNull}`,
30
+ );
31
+ }
32
+
33
+ export async function createIndexIfNotExists(
34
+ db: AnyDb,
35
+ indexKind: "UNIQUE INDEX" | "INDEX",
36
+ indexName: string,
37
+ tableName: string,
38
+ columnList: string,
39
+ whereSql?: string,
40
+ ): Promise<void> {
41
+ const where = whereSql !== undefined ? ` WHERE ${whereSql}` : "";
42
+ await asRawClient(db).unsafe(
43
+ `CREATE ${indexKind} IF NOT EXISTS ${quoteTableIdent(indexName)} ON ${quoteTableIdent(tableName)} (${columnList})${where}`,
44
+ );
45
+ }
@@ -1,3 +1,9 @@
1
+ import {
2
+ constraintOf,
3
+ isLockNotAvailable,
4
+ isTableAlreadyExists,
5
+ isUniqueViolation,
6
+ } from "../pg-error";
1
7
  import type { AnyDb } from "../query";
2
8
  import { asRawClient } from "../query";
3
9
 
@@ -12,12 +18,98 @@ export async function notifyPgChannel(db: AnyDb, channel: string): Promise<void>
12
18
  // (same metadata jsonb). CREATE ... IF NOT EXISTS makes this safe to call
13
19
  // on every boot, same "ensure" pattern as ensureSnapshotVersionColumn: heals
14
20
  // installs that predate the index without a table rebuild.
15
- export async function ensureIdempotencyKeyIndex(db: AnyDb): Promise<void> {
16
- await asRawClient(db).unsafe(
17
- `CREATE UNIQUE INDEX IF NOT EXISTS "events_idempotency_uq" ON "kumiko_events" ` +
18
- `("tenant_id", (("metadata"->>'idempotencyKey'))) ` +
19
- `WHERE "metadata"->>'idempotencyKey' IS NOT NULL`,
21
+ //
22
+ // CONCURRENTLY (not a plain CREATE): a non-concurrent build takes a SHARE
23
+ // lock for the full table scan on kumiko_events the hottest table in the
24
+ // framework — blocking every append() for however long that scan takes on
25
+ // an existing installation's event history. CONCURRENTLY avoids that at the
26
+ // cost of needing to tolerate two failure modes a plain build doesn't have.
27
+ // Neither CREATE nor DROP ... CONCURRENTLY may run inside a transaction —
28
+ // no caller of this function (dev-server, schema-cli.ts, stack/db.ts) may
29
+ // wrap it in one, or Postgres raises 25001.
30
+ // undefined = index doesn't exist at all (nothing to drop, CREATE below
31
+ // handles it); false = exists but INVALID (crashed mid-build, needs DROP +
32
+ // rebuild); true = exists and valid.
33
+ async function indexValidity(client: ReturnType<typeof asRawClient>): Promise<boolean | undefined> {
34
+ const rows = await client.unsafe(
35
+ `SELECT i.indisvalid FROM pg_class c JOIN pg_index i ON i.indexrelid = c.oid ` +
36
+ `WHERE c.relname = 'events_idempotency_uq' AND i.indrelid = '"kumiko_events"'::regclass`,
20
37
  );
38
+ return (rows[0] as { indisvalid?: boolean } | undefined)?.indisvalid;
39
+ }
40
+
41
+ export async function ensureIdempotencyKeyIndex(db: AnyDb): Promise<void> {
42
+ const client = asRawClient(db);
43
+
44
+ try {
45
+ // 1) A prior CONCURRENTLY build that got killed mid-flight (crash, deploy
46
+ // restart) leaves an INVALID index: the catalog entry exists, so
47
+ // IF NOT EXISTS below would silently skip forever, but the index is
48
+ // incomplete and not plannable for queries — it does NOT mean the
49
+ // constraint enforces nothing; Postgres keeps maintaining an INVALID
50
+ // index on every insert, it just refuses to use it for planning.
51
+ // Detect + rebuild it. Scoped to kumiko_events specifically (indrelid),
52
+ // not just the relname, so a same-named index in another schema can't
53
+ // false-positive this DROP.
54
+ if ((await indexValidity(client)) === false) {
55
+ await client.unsafe(`DROP INDEX CONCURRENTLY IF EXISTS "events_idempotency_uq"`);
56
+ }
57
+
58
+ await client.unsafe(
59
+ `CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS "events_idempotency_uq" ON "kumiko_events" ` +
60
+ `("tenant_id", (("metadata"->>'idempotencyKey'))) ` +
61
+ `WHERE "metadata"->>'idempotencyKey' IS NOT NULL`,
62
+ );
63
+ } catch (e) {
64
+ if (isBenignConcurrentIndexBuildRace(e)) {
65
+ // "Benign" only means the losing side of a race, not that the index
66
+ // actually landed valid — a lock_timeout on the CREATE (55P03) backs
67
+ // off the same way a genuine duplicate-build race does, but leaves no
68
+ // valid index at all. Re-check before declaring victory instead of
69
+ // trusting the error class alone.
70
+ // skip: sibling pod already built a valid index — this race loser is done.
71
+ if ((await indexValidity(client)) === true) return;
72
+ console.warn(
73
+ `ensureIdempotencyKeyIndex: backed off on a "benign" race (${String(e)}) but ` +
74
+ `"events_idempotency_uq" is still missing/invalid afterward — likely a ` +
75
+ `lock_timeout during CREATE INDEX CONCURRENTLY, not an actual winner/loser race.`,
76
+ );
77
+ throw e;
78
+ }
79
+ throw duplicateIdempotencyKeyErrorOr(e);
80
+ }
81
+ }
82
+
83
+ // Two pods booting concurrently against the same DB (rolling deploy): both
84
+ // see the index missing/invalid and both start a DROP/CREATE CONCURRENTLY
85
+ // build. The loser typically does NOT get the plain duplicate-relation
86
+ // no-op IF NOT EXISTS normally gives — it can instead see a unique-violation
87
+ // on Postgres' own pg_class catalog insert (23505, constraint
88
+ // pg_class_relname_nsp_index) or a lock-not-available (55P03) from the
89
+ // racing DDL. Both are benign: the other pod's build wins and this one just
90
+ // backs off. A 23505 against "events_idempotency_uq" itself is NOT this
91
+ // race — see duplicateIdempotencyKeyErrorOr.
92
+ function isBenignConcurrentIndexBuildRace(e: unknown): boolean {
93
+ if (isTableAlreadyExists(e) || isLockNotAvailable(e)) return true;
94
+ return isUniqueViolation(e) && constraintOf(e) === "pg_class_relname_nsp_index";
95
+ }
96
+
97
+ // A 23505 against "events_idempotency_uq" means real duplicate
98
+ // metadata->>'idempotencyKey' values for the same tenant already exist —
99
+ // CONCURRENTLY still enforces uniqueness on live inserts against the
100
+ // not-yet-valid index. That needs an operator to find + resolve the
101
+ // duplicates, not a crash-loop on every subsequent boot, so re-throw a
102
+ // distinguishable error instead of the raw driver error.
103
+ function duplicateIdempotencyKeyErrorOr(e: unknown): unknown {
104
+ if (isUniqueViolation(e) && constraintOf(e) === "events_idempotency_uq") {
105
+ return new Error(
106
+ "ensureIdempotencyKeyIndex: duplicate metadata->>'idempotencyKey' values exist for at least " +
107
+ "one tenant in kumiko_events — CREATE UNIQUE INDEX CONCURRENTLY cannot complete. Find and " +
108
+ "resolve the duplicate idempotencyKey rows, then restart to retry.",
109
+ { cause: e },
110
+ );
111
+ }
112
+ return e;
21
113
  }
22
114
 
23
115
  export type SubsequentEventInsertParams = {
@@ -99,22 +191,6 @@ export async function selectAggregateMaxVersion(db: AnyDb, aggregateId: string):
99
191
  return rows[0]?.v ?? 0;
100
192
  }
101
193
 
102
- /** tenant_id the aggregate's events were written under — no membership/tenant
103
- * filter. A r.systemScope() aggregate (e.g. user) lives in whichever tenant
104
- * its creating executor used, which need not be a tenant the subject holds a
105
- * membership in. Returns null for unknown streams. */
106
- export async function selectAggregateStreamTenant(
107
- db: AnyDb,
108
- aggregateId: string,
109
- aggregateType: string,
110
- ): Promise<string | null> {
111
- const rows = (await asRawClient(db).unsafe(
112
- `SELECT "tenant_id" AS t FROM "kumiko_events" WHERE "aggregate_id" = $1 AND "aggregate_type" = $2 ORDER BY "version" LIMIT 1`,
113
- [aggregateId, aggregateType],
114
- )) as ReadonlyArray<{ t: string | null }>;
115
- return rows[0]?.t ?? null;
116
- }
117
-
118
194
  export async function selectEventsHighWaterMark(db: AnyDb): Promise<bigint> {
119
195
  const rows = (await asRawClient(db).unsafe(
120
196
  `SELECT COALESCE(MAX("id"), 0)::bigint AS max FROM "kumiko_events"`,
@@ -2,36 +2,10 @@ import type { AnyDb } from "../query";
2
2
  import { asRawClient } from "../query";
3
3
  import { quoteTableIdent } from "./table-ops";
4
4
 
5
- export async function executeDdlStatement(db: AnyDb, sqlText: string): Promise<void> {
6
- await asRawClient(db).unsafe(sqlText);
7
- }
8
-
9
- export async function alterTableAddColumn(
10
- db: AnyDb,
11
- tableName: string,
12
- columnName: string,
13
- columnType: string,
14
- defaultClause: string,
15
- notNull: string,
16
- ): Promise<void> {
17
- await asRawClient(db).unsafe(
18
- `ALTER TABLE ${quoteTableIdent(tableName)} ADD COLUMN ${quoteTableIdent(columnName)} ${columnType}${defaultClause}${notNull}`,
19
- );
20
- }
21
-
22
- export async function createIndexIfNotExists(
23
- db: AnyDb,
24
- indexKind: "UNIQUE INDEX" | "INDEX",
25
- indexName: string,
26
- tableName: string,
27
- columnList: string,
28
- whereSql?: string,
29
- ): Promise<void> {
30
- const where = whereSql !== undefined ? ` WHERE ${whereSql}` : "";
31
- await asRawClient(db).unsafe(
32
- `CREATE ${indexKind} IF NOT EXISTS ${quoteTableIdent(indexName)} ON ${quoteTableIdent(tableName)} (${columnList})${where}`,
33
- );
34
- }
5
+ // Re-exported for back-compat the generic DDL helpers moved to ./ddl so
6
+ // the prod-boot path (stack/table-helpers.ts, pipeline/event-consumer-state.ts)
7
+ // doesn't import from a module named for test-only concerns.
8
+ export { alterTableAddColumn, createIndexIfNotExists, executeDdlStatement } from "./ddl";
35
9
 
36
10
  export async function truncateTablesRestartIdentity(
37
11
  db: AnyDb,
@@ -1,3 +1,4 @@
1
+ import { KUMIKO_COLUMNS_SYMBOL } from "@cosmicdrift/kumiko-types/schema-table-types";
1
2
  import { fetchOne, insertOne, updateMany } from "../db/query";
2
3
  import type { ReferenceDataDef } from "../engine/types";
3
4
  import { SYSTEM_TENANT_ID } from "../engine/types";
@@ -7,10 +8,8 @@ import type { TableColumns } from "./dialect";
7
8
  // biome-ignore lint/suspicious/noExplicitAny: Drizzle dynamic tables
8
9
  type Table = TableColumns<any>;
9
10
 
10
- const KUMIKO_COLUMNS_SYMBOL = Symbol.for("kumiko:schema:Columns");
11
-
12
11
  function hasColumn(table: Table, field: string): boolean {
13
- const cols = (table as Record<symbol, unknown>)[KUMIKO_COLUMNS_SYMBOL];
12
+ const cols = table[KUMIKO_COLUMNS_SYMBOL];
14
13
  if (typeof cols !== "object" || cols === null) return false;
15
14
  return field in (cols as Record<string, unknown>);
16
15
  }