@cosmicdrift/kumiko-framework 0.164.0 → 0.165.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +5 -3
- package/src/__tests__/consumer-cli.integration.test.ts +62 -0
- package/src/__tests__/schema-cli.integration.test.ts +1 -1
- package/src/api/__tests__/api.test.ts +326 -18
- package/src/api/__tests__/auth-middleware-anonymous-access-boot.test.ts +40 -0
- package/src/api/__tests__/auth-routes-invalid-body-invite.test.ts +16 -0
- package/src/api/__tests__/auth-routes-mfa-preauth-confirm.test.ts +18 -1
- package/src/api/__tests__/auth-routes-mfa-preauth-enable-start.test.ts +64 -1
- package/src/api/__tests__/auth-routes-trusted-proxy.test.ts +146 -0
- package/src/api/__tests__/batch.integration.test.ts +21 -2
- package/src/api/__tests__/jwt.test.ts +52 -2
- package/src/api/__tests__/login-rate-limiter-sweep.test.ts +27 -18
- package/src/api/__tests__/redis-login-rate-limiter.integration.test.ts +72 -0
- package/src/api/__tests__/sse-broker.test.ts +57 -0
- package/src/api/__tests__/sse-route.test.ts +4 -0
- package/src/api/auth-routes.ts +178 -33
- package/src/api/index.ts +1 -0
- package/src/api/jwt.ts +22 -1
- package/src/api/routes.ts +117 -30
- package/src/api/server.ts +39 -7
- package/src/api/sse-broker.ts +39 -0
- package/src/bun-db/connection.ts +3 -3
- package/src/bun-db/index.ts +1 -0
- package/src/bun-db/query.ts +12 -3
- package/src/consumer-cli.ts +60 -13
- package/src/db/__tests__/event-store-executor-write-verbs.integration.test.ts +19 -13
- package/src/db/__tests__/located-timestamp.test.ts +19 -0
- package/src/db/__tests__/migrate-runner.test.ts +61 -0
- package/src/db/__tests__/replay-migration-sql.test.ts +131 -2
- package/src/db/__tests__/tenant-db-where-merge.test.ts +6 -2
- package/src/db/api.ts +2 -2
- package/src/db/bun-provider.ts +2 -2
- package/src/db/connection.ts +6 -3
- package/src/db/dialect.ts +1 -6
- package/src/db/entity-table-meta-types.ts +1 -1
- package/src/db/event-store-executor-context.ts +2 -3
- package/src/db/event-store-executor-read.ts +2 -3
- package/src/db/event-store-executor-write.ts +8 -0
- package/src/db/index.ts +8 -3
- package/src/db/located-timestamp.ts +4 -0
- package/src/db/migrate-runner.ts +107 -11
- package/src/db/pg-error.ts +8 -0
- package/src/db/postgres-provider.ts +2 -2
- package/src/db/queries/__tests__/event-store-idempotency-index.integration.test.ts +80 -0
- package/src/db/queries/ddl.ts +45 -0
- package/src/db/queries/event-store.ts +97 -21
- package/src/db/queries/test-stack.ts +4 -30
- package/src/db/reference-data.ts +2 -3
- package/src/db/replay-migration-sql.ts +114 -12
- package/src/db/tenant-db.ts +2 -4
- package/src/engine/__tests__/boot-validator.test.ts +14 -0
- package/src/engine/__tests__/engine.test.ts +30 -0
- package/src/engine/__tests__/registry.test.ts +36 -0
- package/src/engine/__tests__/schema-builder.test.ts +18 -0
- package/src/engine/__tests__/store-table.test.ts +2 -2
- package/src/engine/boot-validator/entity-handler.ts +14 -25
- package/src/engine/boot-validator/nav.ts +5 -0
- package/src/engine/constants.ts +32 -6
- package/src/engine/create-app.ts +11 -0
- package/src/engine/effective-features.ts +12 -2
- package/src/engine/extensions/user-data.ts +12 -4
- package/src/engine/feature-ast/__tests__/render-roundtrip.test.ts +4 -0
- package/src/engine/feature-ast/extractors/handlers.ts +13 -18
- package/src/engine/feature-ui-extensions.ts +2 -2
- package/src/engine/hook-helpers.ts +3 -1
- package/src/engine/index.ts +1 -1
- package/src/engine/ownership.ts +4 -3
- package/src/engine/registry-ingest.ts +14 -14
- package/src/engine/registry-state.ts +4 -1
- package/src/engine/registry-validate.ts +7 -2
- package/src/engine/schema-builder.ts +1 -0
- package/src/engine/steps/__tests__/duration-utils.test.ts +20 -0
- package/src/engine/steps/_duration-utils.ts +2 -0
- package/src/engine/steps/unsafe-projection-upsert.ts +1 -4
- package/src/engine/types/config.ts +1 -1
- package/src/engine/types/define-handler.ts +1 -1
- package/src/engine/types/entity-handlers.ts +1 -1
- package/src/engine/types/event-type-map.ts +1 -1
- package/src/engine/types/feature.ts +1 -1
- package/src/engine/types/fields.ts +1 -1
- package/src/engine/types/handlers.ts +1 -1
- package/src/engine/types/hooks.ts +1 -1
- package/src/engine/types/http-route.ts +1 -1
- package/src/engine/types/index.ts +0 -2
- package/src/engine/types/nav.ts +1 -1
- package/src/engine/types/ownership.ts +1 -1
- package/src/engine/types/projection.ts +1 -1
- package/src/engine/types/relations.ts +1 -1
- package/src/engine/types/screen.ts +1 -1
- package/src/engine/types/step.ts +1 -1
- package/src/engine/types/target-ref.ts +1 -1
- package/src/engine/types/tree-node.ts +1 -1
- package/src/engine/types/workspace.ts +1 -1
- package/src/engine/validate-projection-allowlist.ts +5 -5
- package/src/errors/classes.ts +21 -0
- package/src/errors/index.ts +1 -0
- package/src/errors/write-error-info.ts +6 -2
- package/src/event-store/__tests__/admin-api.integration.test.ts +27 -1
- package/src/event-store/__tests__/event-store.integration.test.ts +60 -14
- package/src/event-store/__tests__/unscoped-stream-primitives.guard.test.ts +27 -12
- package/src/event-store/admin-api.ts +11 -4
- package/src/event-store/event-store.ts +20 -21
- package/src/event-store/index.ts +0 -1
- package/src/event-store/types.ts +1 -1
- package/src/files/__tests__/build-storage-key.test.ts +28 -0
- package/src/files/__tests__/local-provider.test.ts +31 -0
- package/src/files/__tests__/write-stream.test.ts +3 -3
- package/src/files/index.ts +1 -1
- package/src/files/local-provider.ts +6 -1
- package/src/files/types.ts +8 -1
- package/src/jobs/__tests__/jobs.integration.test.ts +167 -7
- package/src/jobs/job-runner.ts +41 -11
- package/src/logging/types.ts +1 -1
- package/src/observability/index.ts +1 -0
- package/src/observability/standard-metrics.ts +35 -2
- package/src/observability/types/index.ts +1 -1
- package/src/observability/types/metric.ts +1 -1
- package/src/observability/types/provider.ts +1 -1
- package/src/observability/types/span.ts +1 -1
- package/src/pipeline/__tests__/dispatcher.test.ts +203 -0
- package/src/pipeline/__tests__/event-consumer-state.integration.test.ts +31 -0
- package/src/pipeline/__tests__/event-dispatcher-rearm.integration.test.ts +83 -0
- package/src/pipeline/__tests__/job-trigger-consumer.integration.test.ts +106 -0
- package/src/pipeline/__tests__/lifecycle-pipeline.test.ts +111 -88
- package/src/pipeline/dispatch-shared.ts +59 -6
- package/src/pipeline/dispatch-stream.ts +44 -17
- package/src/pipeline/dispatcher.ts +7 -1
- package/src/pipeline/event-consumer-state.ts +16 -13
- package/src/pipeline/event-dispatcher-delivery.ts +22 -7
- package/src/pipeline/event-dispatcher.ts +22 -0
- package/src/pipeline/index.ts +2 -0
- package/src/pipeline/system-hooks.ts +137 -1
- package/src/rate-limit/__tests__/resolver.integration.test.ts +18 -0
- package/src/rate-limit/resolver.ts +6 -2
- package/src/schema-cli.ts +24 -12
- package/src/search/__tests__/reindex-entity.integration.test.ts +24 -1
- package/src/search/reindex-entity.ts +31 -2
- package/src/search/types.ts +1 -1
- package/src/stack/__tests__/setup-test-stack-jobs.integration.test.ts +6 -2
- package/src/stack/db.ts +2 -1
- package/src/stack/push-entity-projection-tables.ts +2 -1
- package/src/stack/request-helper.ts +20 -1
- package/src/stack/table-helpers.ts +6 -4
- package/src/stack/test-stack.ts +18 -15
- package/src/testing/__tests__/late-bound.test.ts +7 -0
- package/src/testing/__tests__/wait-for.test.ts +6 -0
- package/src/testing/file-provider-contract.ts +26 -6
- package/src/testing/index.ts +1 -0
- package/src/testing/late-bound.ts +5 -3
- package/src/testing/wait-for.ts +3 -0
- package/src/testing/without-ambient-temporal.ts +14 -0
- package/src/time/__tests__/polyfill-reinstall.test.ts +17 -0
- package/src/time/geo-tz.ts +1 -1
- package/src/time/polyfill.ts +28 -39
- package/src/time/tz-context.ts +30 -24
- package/src/utils/__tests__/safe-json-temporal.test.ts +14 -0
- package/src/utils/safe-json.ts +3 -2
- package/src/db/__tests__/encryption.test.ts +0 -39
- package/src/db/encryption.ts +0 -45
- package/src/engine/__tests__/registry-facade-sweep.test.ts +0 -80
package/src/db/dialect.ts
CHANGED
|
@@ -19,6 +19,7 @@
|
|
|
19
19
|
import {
|
|
20
20
|
type ColumnHandle,
|
|
21
21
|
KUMIKO_COLUMNS_SYMBOL,
|
|
22
|
+
KUMIKO_META_SYMBOL,
|
|
22
23
|
KUMIKO_NAME_SYMBOL,
|
|
23
24
|
type SchemaTable,
|
|
24
25
|
} from "@cosmicdrift/kumiko-types/schema-table-types";
|
|
@@ -41,12 +42,6 @@ export type TableColumns<_T = any> = SchemaTable;
|
|
|
41
42
|
// biome-ignore lint/suspicious/noExplicitAny: legacy type — chain API is gone
|
|
42
43
|
export type SelectQuery = any;
|
|
43
44
|
|
|
44
|
-
// Shadow-proof handle on the EntityTableMeta. The column handles below are
|
|
45
|
-
// spread as enumerable props, so an entity field named `source`/`columns`/
|
|
46
|
-
// `tableName`/… would overwrite the matching meta key. extractTableInfo reads
|
|
47
|
-
// the canonical meta from this symbol instead of the (shadowable) props.
|
|
48
|
-
const KUMIKO_META_SYMBOL = Symbol.for("kumiko:schema:Meta");
|
|
49
|
-
|
|
50
45
|
function isNumericPgType(t: PgType): t is `numeric(${number},${number})` {
|
|
51
46
|
return t.startsWith("numeric(");
|
|
52
47
|
}
|
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
// Legacy path — re-exported for callers still importing this module directly.
|
|
2
|
-
export * from "@cosmicdrift/kumiko-types/entity-table-meta-types";
|
|
2
|
+
export type * from "@cosmicdrift/kumiko-types/entity-table-meta-types";
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { KUMIKO_NAME_SYMBOL } from "@cosmicdrift/kumiko-types/schema-table-types";
|
|
1
2
|
import { requestContext } from "../api/request-context";
|
|
2
3
|
import {
|
|
3
4
|
collectPiiSubjectFields,
|
|
@@ -359,9 +360,7 @@ export function buildExecutorContext(
|
|
|
359
360
|
}
|
|
360
361
|
// ownership has raw SQL — splice it into a raw query alongside the
|
|
361
362
|
// idFilter + tenant-filter that TenantDb would have added.
|
|
362
|
-
const tableName = String(
|
|
363
|
-
(table as unknown as Record<symbol, unknown>)[Symbol.for("kumiko:schema:Name")],
|
|
364
|
-
);
|
|
363
|
+
const tableName = String((table as unknown as Record<symbol, unknown>)[KUMIKO_NAME_SYMBOL]);
|
|
365
364
|
const colSql = (field: string): string =>
|
|
366
365
|
`"${(table[field] as { name?: string } | undefined)?.name ?? toSnakeCase(field)}"`;
|
|
367
366
|
const whereParts: string[] = [];
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { KUMIKO_NAME_SYMBOL } from "@cosmicdrift/kumiko-types/schema-table-types";
|
|
1
2
|
import { computeBlindIndex, configuredBlindIndexKey } from "../crypto";
|
|
2
3
|
import { executeRawQuery } from "../db/queries/raw-sql";
|
|
3
4
|
import { coerceRow, extractTableInfo } from "../db/query";
|
|
@@ -66,9 +67,7 @@ export function createReadVerbs(ctx: ExecutorContext): Pick<EventStoreExecutor,
|
|
|
66
67
|
// Build the WHERE clause as raw SQL — ownership produces a
|
|
67
68
|
// parameterised fragment that we splice in alongside simple WhereObject
|
|
68
69
|
// conditions (cursor, search-filter-IDs, screen-filter, tenant-scope).
|
|
69
|
-
const tableName = String(
|
|
70
|
-
(table as unknown as Record<symbol, unknown>)[Symbol.for("kumiko:schema:Name")],
|
|
71
|
-
);
|
|
70
|
+
const tableName = String((table as unknown as Record<symbol, unknown>)[KUMIKO_NAME_SYMBOL]);
|
|
72
71
|
const whereSql: string[] = [];
|
|
73
72
|
const params: unknown[] = [];
|
|
74
73
|
const colSql = (field: string): string =>
|
|
@@ -3,6 +3,7 @@ import { userCanCreateFieldRow, userCanWriteFieldRow } from "../engine/ownership
|
|
|
3
3
|
import type { EntityId } from "../engine/types";
|
|
4
4
|
import {
|
|
5
5
|
VersionConflictError as FrameworkVersionConflict,
|
|
6
|
+
IdempotentReplayError,
|
|
6
7
|
InternalError,
|
|
7
8
|
NotFoundError,
|
|
8
9
|
UnprocessableError,
|
|
@@ -10,6 +11,7 @@ import {
|
|
|
10
11
|
} from "../errors";
|
|
11
12
|
import {
|
|
12
13
|
append,
|
|
14
|
+
IdempotentAppendConflictError as EventStoreIdempotentAppendConflict,
|
|
13
15
|
VersionConflictError as EventStoreVersionConflict,
|
|
14
16
|
getStreamVersion,
|
|
15
17
|
} from "../event-store";
|
|
@@ -160,6 +162,9 @@ export function createWriteVerbs(
|
|
|
160
162
|
}),
|
|
161
163
|
);
|
|
162
164
|
}
|
|
165
|
+
if (e instanceof EventStoreIdempotentAppendConflict) {
|
|
166
|
+
return writeFailure(new IdempotentReplayError({ idempotencyKey: e.idempotencyKey }));
|
|
167
|
+
}
|
|
163
168
|
throw e;
|
|
164
169
|
}
|
|
165
170
|
|
|
@@ -390,6 +395,9 @@ export function createWriteVerbs(
|
|
|
390
395
|
}),
|
|
391
396
|
);
|
|
392
397
|
}
|
|
398
|
+
if (e instanceof EventStoreIdempotentAppendConflict) {
|
|
399
|
+
return writeFailure(new IdempotentReplayError({ idempotencyKey: e.idempotencyKey }));
|
|
400
|
+
}
|
|
393
401
|
throw e;
|
|
394
402
|
}
|
|
395
403
|
},
|
package/src/db/index.ts
CHANGED
|
@@ -3,7 +3,14 @@ export { nullBlindIndexesForSubject } from "./blind-index-cleanup";
|
|
|
3
3
|
export { collectTableMetas } from "./collect-table-metas";
|
|
4
4
|
export { flattenCompoundTypes, rehydrateCompoundTypes } from "./compound-types";
|
|
5
5
|
export { seedConfigValues } from "./config-seed";
|
|
6
|
-
export type {
|
|
6
|
+
export type {
|
|
7
|
+
DbConnection,
|
|
8
|
+
DbConnectionOptions,
|
|
9
|
+
DbPoolHandle,
|
|
10
|
+
DbRow,
|
|
11
|
+
DbRunner,
|
|
12
|
+
DbTx,
|
|
13
|
+
} from "./connection";
|
|
7
14
|
export { createDbConnection, dbConnectionOptionsFromEnv } from "./connection";
|
|
8
15
|
export type { CursorQueryOptions, CursorResult } from "./cursor";
|
|
9
16
|
export { decodeCursor, encodeCursor } from "./cursor";
|
|
@@ -35,8 +42,6 @@ export {
|
|
|
35
42
|
enrichRowWithReferences,
|
|
36
43
|
enrichWithReferences,
|
|
37
44
|
} from "./eagerload";
|
|
38
|
-
export type { EncryptionProvider } from "./encryption";
|
|
39
|
-
export { createEncryptionProvider } from "./encryption";
|
|
40
45
|
export {
|
|
41
46
|
collectEncryptedFieldNames,
|
|
42
47
|
configuredEntityFieldEncryption,
|
|
@@ -9,6 +9,10 @@
|
|
|
9
9
|
// gespeicherter tz). Server kennt User-TZ nicht — User-spezifische
|
|
10
10
|
// Anzeige passiert client-seitig aus utc.
|
|
11
11
|
|
|
12
|
+
// Static import, not the ambient global: Bun doesn't expose Temporal on
|
|
13
|
+
// globalThis, so this crashed with "Temporal is not defined" outside boot
|
|
14
|
+
// paths that install it (#1480).
|
|
15
|
+
import { Temporal } from "temporal-polyfill";
|
|
12
16
|
import type { EntityDefinition } from "../engine/types";
|
|
13
17
|
|
|
14
18
|
// Sprint F: <name>Utc-Spalte ist jetzt instant() (siehe dialect.ts) —
|
package/src/db/migrate-runner.ts
CHANGED
|
@@ -74,14 +74,103 @@ CREATE TABLE IF NOT EXISTS "_kumiko_migrations" (
|
|
|
74
74
|
)
|
|
75
75
|
`.trim();
|
|
76
76
|
|
|
77
|
-
//
|
|
78
|
-
//
|
|
79
|
-
//
|
|
80
|
-
//
|
|
77
|
+
// Splits SQL-file text into individual statements on top-level `;`. A plain
|
|
78
|
+
// `text.split(";")` breaks the moment a `--` line comment or `/* */` block
|
|
79
|
+
// comment contains a semicolon (#1542) — it splits mid-comment before the
|
|
80
|
+
// comment is ever stripped. This scans char-by-char tracking whether we're
|
|
81
|
+
// inside a line comment, block comment, single-quoted string, or
|
|
82
|
+
// double-quoted identifier, so `;` only ends a statement in plain SQL text;
|
|
83
|
+
// comments are dropped, quoted/identifier content (incl. `''`/`""` escapes)
|
|
84
|
+
// is kept verbatim. Does not handle dollar-quoted (`$$...$$`) bodies — none
|
|
85
|
+
// of this repo's checked-in migrations use them; add that state if one ever
|
|
86
|
+
// does.
|
|
87
|
+
type SqlScanState = "normal" | "lineComment" | "blockComment" | "singleQuote" | "doubleQuote";
|
|
88
|
+
|
|
81
89
|
export function splitSqlStatements(sqlText: string): readonly string[] {
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
90
|
+
const statements: string[] = [];
|
|
91
|
+
let current = "";
|
|
92
|
+
let state: SqlScanState = "normal";
|
|
93
|
+
|
|
94
|
+
for (let i = 0; i < sqlText.length; i++) {
|
|
95
|
+
const ch = sqlText.charAt(i);
|
|
96
|
+
const next = sqlText.charAt(i + 1);
|
|
97
|
+
|
|
98
|
+
if (state === "lineComment") {
|
|
99
|
+
if (ch === "\n") {
|
|
100
|
+
state = "normal";
|
|
101
|
+
current += ch;
|
|
102
|
+
}
|
|
103
|
+
continue;
|
|
104
|
+
}
|
|
105
|
+
if (state === "blockComment") {
|
|
106
|
+
if (ch === "*" && next === "/") {
|
|
107
|
+
state = "normal";
|
|
108
|
+
i++;
|
|
109
|
+
}
|
|
110
|
+
continue;
|
|
111
|
+
}
|
|
112
|
+
if (state === "singleQuote") {
|
|
113
|
+
current += ch;
|
|
114
|
+
if (ch === "'") {
|
|
115
|
+
if (next === "'") {
|
|
116
|
+
current += next;
|
|
117
|
+
i++;
|
|
118
|
+
} else {
|
|
119
|
+
state = "normal";
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
continue;
|
|
123
|
+
}
|
|
124
|
+
if (state === "doubleQuote") {
|
|
125
|
+
current += ch;
|
|
126
|
+
if (ch === '"') {
|
|
127
|
+
if (next === '"') {
|
|
128
|
+
current += next;
|
|
129
|
+
i++;
|
|
130
|
+
} else {
|
|
131
|
+
state = "normal";
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
continue;
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
// state === "normal"
|
|
138
|
+
if (ch === "-" && next === "-") {
|
|
139
|
+
state = "lineComment";
|
|
140
|
+
i++;
|
|
141
|
+
continue;
|
|
142
|
+
}
|
|
143
|
+
if (ch === "/" && next === "*") {
|
|
144
|
+
state = "blockComment";
|
|
145
|
+
i++;
|
|
146
|
+
continue;
|
|
147
|
+
}
|
|
148
|
+
if (ch === "'") {
|
|
149
|
+
state = "singleQuote";
|
|
150
|
+
current += ch;
|
|
151
|
+
continue;
|
|
152
|
+
}
|
|
153
|
+
if (ch === '"') {
|
|
154
|
+
state = "doubleQuote";
|
|
155
|
+
current += ch;
|
|
156
|
+
continue;
|
|
157
|
+
}
|
|
158
|
+
if (ch === ";") {
|
|
159
|
+
statements.push(current);
|
|
160
|
+
current = "";
|
|
161
|
+
continue;
|
|
162
|
+
}
|
|
163
|
+
current += ch;
|
|
164
|
+
}
|
|
165
|
+
if (state === "blockComment" || state === "singleQuote" || state === "doubleQuote") {
|
|
166
|
+
throw new Error(
|
|
167
|
+
`splitSqlStatements: unterminated ${state} — migration SQL is malformed, refusing to split`,
|
|
168
|
+
);
|
|
169
|
+
}
|
|
170
|
+
statements.push(current);
|
|
171
|
+
|
|
172
|
+
return statements
|
|
173
|
+
.map((s) => s.trim())
|
|
85
174
|
.filter((s) => s.length > 0)
|
|
86
175
|
.map((s) => `${s};`);
|
|
87
176
|
}
|
|
@@ -90,14 +179,21 @@ function sha256Hex(content: string): string {
|
|
|
90
179
|
return createHash("sha256").update(content).digest("hex");
|
|
91
180
|
}
|
|
92
181
|
|
|
93
|
-
//
|
|
94
|
-
//
|
|
95
|
-
|
|
182
|
+
// Reads <dir>/*.sql, sorted lexically (e.g. 0001_init.sql, 0002_add_locale.sql),
|
|
183
|
+
// returns Migration[] with id + checksum + statements. `preprocess` is an
|
|
184
|
+
// optional hook applied to each file's raw content before splitting/hashing
|
|
185
|
+
// — used by replayMigrationsDir to expand its DESTRUCTIVE-marker comments
|
|
186
|
+
// without a second, drifting copy of this file-discovery logic (#1522/9).
|
|
187
|
+
export function loadMigrationsFromDir(
|
|
188
|
+
dir: string,
|
|
189
|
+
preprocess?: (sql: string) => string,
|
|
190
|
+
): readonly Migration[] {
|
|
96
191
|
const files = readdirSync(dir)
|
|
97
192
|
.filter((f) => f.endsWith(".sql"))
|
|
98
193
|
.sort();
|
|
99
194
|
return files.map((file) => {
|
|
100
|
-
const
|
|
195
|
+
const raw = readFileSync(join(dir, file), "utf8");
|
|
196
|
+
const content = preprocess ? preprocess(raw) : raw;
|
|
101
197
|
return {
|
|
102
198
|
id: file.replace(/\.sql$/, ""),
|
|
103
199
|
checksum: sha256Hex(content),
|
package/src/db/pg-error.ts
CHANGED
|
@@ -41,6 +41,14 @@ export function isTableAlreadyExists(e: unknown): boolean {
|
|
|
41
41
|
return extractPgError(e)?.code === "42P07";
|
|
42
42
|
}
|
|
43
43
|
|
|
44
|
+
// PG SQLSTATE 55P03 — "lock not available". Raised by a NOWAIT-style lock
|
|
45
|
+
// request that couldn't be granted; a CREATE/DROP INDEX CONCURRENTLY racing
|
|
46
|
+
// another session's DDL on the same relation can hit this instead of a
|
|
47
|
+
// duplicate-relation error, depending on exact timing.
|
|
48
|
+
export function isLockNotAvailable(e: unknown): boolean {
|
|
49
|
+
return extractPgError(e)?.code === "55P03";
|
|
50
|
+
}
|
|
51
|
+
|
|
44
52
|
export function constraintOf(e: unknown): string | undefined {
|
|
45
53
|
return extractPgError(e)?.constraint_name;
|
|
46
54
|
}
|
|
@@ -3,9 +3,9 @@
|
|
|
3
3
|
// `asRawClient(db)` wrappt .unsafe und .begin transparent.
|
|
4
4
|
|
|
5
5
|
import postgres from "postgres";
|
|
6
|
-
import type {
|
|
6
|
+
import type { DbConnectionOptions, DbPoolHandle } from "./api";
|
|
7
7
|
|
|
8
|
-
export function createPgConnection(url: string, options: DbConnectionOptions = {}):
|
|
8
|
+
export function createPgConnection(url: string, options: DbConnectionOptions = {}): DbPoolHandle {
|
|
9
9
|
const pgOptions: Parameters<typeof postgres>[1] = {};
|
|
10
10
|
if (options.maxConnections !== undefined) pgOptions.max = options.maxConnections;
|
|
11
11
|
if (options.idleTimeoutSeconds !== undefined) pgOptions.idle_timeout = options.idleTimeoutSeconds;
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
import { afterAll, beforeAll, describe, expect, test } from "bun:test";
|
|
2
|
+
import { type BunTestDb, createTestDb } from "../../../bun-db/__tests__/bun-test-db";
|
|
3
|
+
import { createEventsTable } from "../../../event-store/events-schema";
|
|
4
|
+
import { asRawClient } from "../../query";
|
|
5
|
+
import { ensureIdempotencyKeyIndex } from "../event-store";
|
|
6
|
+
|
|
7
|
+
let testDb: BunTestDb;
|
|
8
|
+
|
|
9
|
+
beforeAll(async () => {
|
|
10
|
+
testDb = await createTestDb();
|
|
11
|
+
await createEventsTable(testDb.db);
|
|
12
|
+
});
|
|
13
|
+
|
|
14
|
+
afterAll(async () => {
|
|
15
|
+
await testDb.cleanup();
|
|
16
|
+
});
|
|
17
|
+
|
|
18
|
+
async function indexValidity(): Promise<boolean | undefined> {
|
|
19
|
+
const rows = (await asRawClient(testDb.db).unsafe(
|
|
20
|
+
`SELECT i.indisvalid FROM pg_class c JOIN pg_index i ON i.indexrelid = c.oid ` +
|
|
21
|
+
`WHERE c.relname = 'events_idempotency_uq'`,
|
|
22
|
+
)) as ReadonlyArray<{ indisvalid: boolean }>;
|
|
23
|
+
return rows[0]?.indisvalid;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
describe("ensureIdempotencyKeyIndex (#1499)", () => {
|
|
27
|
+
test("repeated calls stay idempotent — index ends up valid, no throw", async () => {
|
|
28
|
+
await ensureIdempotencyKeyIndex(testDb.db);
|
|
29
|
+
await ensureIdempotencyKeyIndex(testDb.db);
|
|
30
|
+
expect(await indexValidity()).toBe(true);
|
|
31
|
+
});
|
|
32
|
+
|
|
33
|
+
// Regression for the rolling-deploy race: two pods booting concurrently
|
|
34
|
+
// against the same DB both try to build the same CONCURRENTLY index.
|
|
35
|
+
// A plain CREATE INDEX IF NOT EXISTS no-ops the loser; a CONCURRENTLY
|
|
36
|
+
// build can instead raise a duplicate-relation error mid-build — that
|
|
37
|
+
// must be swallowed the same way createEventsTable already tolerates a
|
|
38
|
+
// racing CREATE TABLE, not crash-loop the booting pod.
|
|
39
|
+
test("concurrent calls from two racing boots don't throw", async () => {
|
|
40
|
+
// The prior test already built the index — without dropping it here,
|
|
41
|
+
// both calls below would just hit the IF NOT EXISTS no-op path (indeed
|
|
42
|
+
// what the original single-connection version of this test silently
|
|
43
|
+
// did) and never actually contend on a real CONCURRENTLY build.
|
|
44
|
+
await asRawClient(testDb.db).unsafe(
|
|
45
|
+
`DROP INDEX CONCURRENTLY IF EXISTS "events_idempotency_uq"`,
|
|
46
|
+
);
|
|
47
|
+
|
|
48
|
+
// Two independent connections against the SAME database, mirroring two
|
|
49
|
+
// pods — a single shared `testDb.db` pool would serialize both calls
|
|
50
|
+
// and never pin the race (kumiko-framework#1522).
|
|
51
|
+
const secondPodDb = await createTestDb({ dbName: testDb.dbName, persistent: true });
|
|
52
|
+
try {
|
|
53
|
+
await Promise.all([
|
|
54
|
+
ensureIdempotencyKeyIndex(testDb.db),
|
|
55
|
+
ensureIdempotencyKeyIndex(secondPodDb.db),
|
|
56
|
+
]);
|
|
57
|
+
expect(await indexValidity()).toBe(true);
|
|
58
|
+
} finally {
|
|
59
|
+
// persistent: true → cleanup() only closes this pool, doesn't drop the
|
|
60
|
+
// shared DB (testDb's afterAll still owns that).
|
|
61
|
+
await secondPodDb.cleanup();
|
|
62
|
+
}
|
|
63
|
+
});
|
|
64
|
+
|
|
65
|
+
test("an INVALID leftover from a killed CONCURRENTLY build gets dropped and rebuilt", async () => {
|
|
66
|
+
// Simulates a build interrupted mid-flight (crash, deploy restart): the
|
|
67
|
+
// catalog entry exists but never finished — pg_index.indisvalid=false.
|
|
68
|
+
// Postgres has no direct DDL to force this state, so flip it straight
|
|
69
|
+
// in the catalog, matching what a real killed CONCURRENTLY build leaves
|
|
70
|
+
// behind.
|
|
71
|
+
await asRawClient(testDb.db).unsafe(
|
|
72
|
+
`UPDATE pg_index SET indisvalid = false WHERE indexrelid = 'events_idempotency_uq'::regclass`,
|
|
73
|
+
);
|
|
74
|
+
expect(await indexValidity()).toBe(false);
|
|
75
|
+
|
|
76
|
+
await ensureIdempotencyKeyIndex(testDb.db);
|
|
77
|
+
|
|
78
|
+
expect(await indexValidity()).toBe(true);
|
|
79
|
+
});
|
|
80
|
+
});
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import type { AnyDb } from "../query";
|
|
2
|
+
import { asRawClient } from "../query";
|
|
3
|
+
import { quoteTableIdent } from "./table-ops";
|
|
4
|
+
|
|
5
|
+
// Generic DDL helpers used on the prod-boot path (stack/table-helpers.ts's
|
|
6
|
+
// unsafePushTables, event-consumer-state.ts's multi-instance backfill) —
|
|
7
|
+
// split out of queries/test-stack.ts so a prod-boot import doesn't point at
|
|
8
|
+
// a module named for test-only concerns (truncate/create/drop-database).
|
|
9
|
+
|
|
10
|
+
export async function executeDdlStatement(db: AnyDb, sqlText: string): Promise<void> {
|
|
11
|
+
await asRawClient(db).unsafe(sqlText);
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
export async function alterTableAddColumn(
|
|
15
|
+
db: AnyDb,
|
|
16
|
+
tableName: string,
|
|
17
|
+
columnName: string,
|
|
18
|
+
columnType: string,
|
|
19
|
+
defaultClause: string,
|
|
20
|
+
notNull: string,
|
|
21
|
+
// table-helpers.ts's unmanaged-table sync relies on the plain form
|
|
22
|
+
// throwing when a column already exists with a different shape than
|
|
23
|
+
// EntityTableMeta expects — that's how it surfaces drift. Callers that
|
|
24
|
+
// just need an idempotent, race-safe backfill (e.g. event-consumer-state's
|
|
25
|
+
// multi-instance boot path, #1362) opt in explicitly.
|
|
26
|
+
ifNotExists = false,
|
|
27
|
+
): Promise<void> {
|
|
28
|
+
await asRawClient(db).unsafe(
|
|
29
|
+
`ALTER TABLE ${quoteTableIdent(tableName)} ADD COLUMN ${ifNotExists ? "IF NOT EXISTS " : ""}${quoteTableIdent(columnName)} ${columnType}${defaultClause}${notNull}`,
|
|
30
|
+
);
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export async function createIndexIfNotExists(
|
|
34
|
+
db: AnyDb,
|
|
35
|
+
indexKind: "UNIQUE INDEX" | "INDEX",
|
|
36
|
+
indexName: string,
|
|
37
|
+
tableName: string,
|
|
38
|
+
columnList: string,
|
|
39
|
+
whereSql?: string,
|
|
40
|
+
): Promise<void> {
|
|
41
|
+
const where = whereSql !== undefined ? ` WHERE ${whereSql}` : "";
|
|
42
|
+
await asRawClient(db).unsafe(
|
|
43
|
+
`CREATE ${indexKind} IF NOT EXISTS ${quoteTableIdent(indexName)} ON ${quoteTableIdent(tableName)} (${columnList})${where}`,
|
|
44
|
+
);
|
|
45
|
+
}
|
|
@@ -1,3 +1,9 @@
|
|
|
1
|
+
import {
|
|
2
|
+
constraintOf,
|
|
3
|
+
isLockNotAvailable,
|
|
4
|
+
isTableAlreadyExists,
|
|
5
|
+
isUniqueViolation,
|
|
6
|
+
} from "../pg-error";
|
|
1
7
|
import type { AnyDb } from "../query";
|
|
2
8
|
import { asRawClient } from "../query";
|
|
3
9
|
|
|
@@ -12,12 +18,98 @@ export async function notifyPgChannel(db: AnyDb, channel: string): Promise<void>
|
|
|
12
18
|
// (same metadata jsonb). CREATE ... IF NOT EXISTS makes this safe to call
|
|
13
19
|
// on every boot, same "ensure" pattern as ensureSnapshotVersionColumn: heals
|
|
14
20
|
// installs that predate the index without a table rebuild.
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
21
|
+
//
|
|
22
|
+
// CONCURRENTLY (not a plain CREATE): a non-concurrent build takes a SHARE
|
|
23
|
+
// lock for the full table scan on kumiko_events — the hottest table in the
|
|
24
|
+
// framework — blocking every append() for however long that scan takes on
|
|
25
|
+
// an existing installation's event history. CONCURRENTLY avoids that at the
|
|
26
|
+
// cost of needing to tolerate two failure modes a plain build doesn't have.
|
|
27
|
+
// Neither CREATE nor DROP ... CONCURRENTLY may run inside a transaction —
|
|
28
|
+
// no caller of this function (dev-server, schema-cli.ts, stack/db.ts) may
|
|
29
|
+
// wrap it in one, or Postgres raises 25001.
|
|
30
|
+
// undefined = index doesn't exist at all (nothing to drop, CREATE below
|
|
31
|
+
// handles it); false = exists but INVALID (crashed mid-build, needs DROP +
|
|
32
|
+
// rebuild); true = exists and valid.
|
|
33
|
+
async function indexValidity(client: ReturnType<typeof asRawClient>): Promise<boolean | undefined> {
|
|
34
|
+
const rows = await client.unsafe(
|
|
35
|
+
`SELECT i.indisvalid FROM pg_class c JOIN pg_index i ON i.indexrelid = c.oid ` +
|
|
36
|
+
`WHERE c.relname = 'events_idempotency_uq' AND i.indrelid = '"kumiko_events"'::regclass`,
|
|
20
37
|
);
|
|
38
|
+
return (rows[0] as { indisvalid?: boolean } | undefined)?.indisvalid;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export async function ensureIdempotencyKeyIndex(db: AnyDb): Promise<void> {
|
|
42
|
+
const client = asRawClient(db);
|
|
43
|
+
|
|
44
|
+
try {
|
|
45
|
+
// 1) A prior CONCURRENTLY build that got killed mid-flight (crash, deploy
|
|
46
|
+
// restart) leaves an INVALID index: the catalog entry exists, so
|
|
47
|
+
// IF NOT EXISTS below would silently skip forever, but the index is
|
|
48
|
+
// incomplete and not plannable for queries — it does NOT mean the
|
|
49
|
+
// constraint enforces nothing; Postgres keeps maintaining an INVALID
|
|
50
|
+
// index on every insert, it just refuses to use it for planning.
|
|
51
|
+
// Detect + rebuild it. Scoped to kumiko_events specifically (indrelid),
|
|
52
|
+
// not just the relname, so a same-named index in another schema can't
|
|
53
|
+
// false-positive this DROP.
|
|
54
|
+
if ((await indexValidity(client)) === false) {
|
|
55
|
+
await client.unsafe(`DROP INDEX CONCURRENTLY IF EXISTS "events_idempotency_uq"`);
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
await client.unsafe(
|
|
59
|
+
`CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS "events_idempotency_uq" ON "kumiko_events" ` +
|
|
60
|
+
`("tenant_id", (("metadata"->>'idempotencyKey'))) ` +
|
|
61
|
+
`WHERE "metadata"->>'idempotencyKey' IS NOT NULL`,
|
|
62
|
+
);
|
|
63
|
+
} catch (e) {
|
|
64
|
+
if (isBenignConcurrentIndexBuildRace(e)) {
|
|
65
|
+
// "Benign" only means the losing side of a race, not that the index
|
|
66
|
+
// actually landed valid — a lock_timeout on the CREATE (55P03) backs
|
|
67
|
+
// off the same way a genuine duplicate-build race does, but leaves no
|
|
68
|
+
// valid index at all. Re-check before declaring victory instead of
|
|
69
|
+
// trusting the error class alone.
|
|
70
|
+
// skip: sibling pod already built a valid index — this race loser is done.
|
|
71
|
+
if ((await indexValidity(client)) === true) return;
|
|
72
|
+
console.warn(
|
|
73
|
+
`ensureIdempotencyKeyIndex: backed off on a "benign" race (${String(e)}) but ` +
|
|
74
|
+
`"events_idempotency_uq" is still missing/invalid afterward — likely a ` +
|
|
75
|
+
`lock_timeout during CREATE INDEX CONCURRENTLY, not an actual winner/loser race.`,
|
|
76
|
+
);
|
|
77
|
+
throw e;
|
|
78
|
+
}
|
|
79
|
+
throw duplicateIdempotencyKeyErrorOr(e);
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
// Two pods booting concurrently against the same DB (rolling deploy): both
|
|
84
|
+
// see the index missing/invalid and both start a DROP/CREATE CONCURRENTLY
|
|
85
|
+
// build. The loser typically does NOT get the plain duplicate-relation
|
|
86
|
+
// no-op IF NOT EXISTS normally gives — it can instead see a unique-violation
|
|
87
|
+
// on Postgres' own pg_class catalog insert (23505, constraint
|
|
88
|
+
// pg_class_relname_nsp_index) or a lock-not-available (55P03) from the
|
|
89
|
+
// racing DDL. Both are benign: the other pod's build wins and this one just
|
|
90
|
+
// backs off. A 23505 against "events_idempotency_uq" itself is NOT this
|
|
91
|
+
// race — see duplicateIdempotencyKeyErrorOr.
|
|
92
|
+
function isBenignConcurrentIndexBuildRace(e: unknown): boolean {
|
|
93
|
+
if (isTableAlreadyExists(e) || isLockNotAvailable(e)) return true;
|
|
94
|
+
return isUniqueViolation(e) && constraintOf(e) === "pg_class_relname_nsp_index";
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
// A 23505 against "events_idempotency_uq" means real duplicate
|
|
98
|
+
// metadata->>'idempotencyKey' values for the same tenant already exist —
|
|
99
|
+
// CONCURRENTLY still enforces uniqueness on live inserts against the
|
|
100
|
+
// not-yet-valid index. That needs an operator to find + resolve the
|
|
101
|
+
// duplicates, not a crash-loop on every subsequent boot, so re-throw a
|
|
102
|
+
// distinguishable error instead of the raw driver error.
|
|
103
|
+
function duplicateIdempotencyKeyErrorOr(e: unknown): unknown {
|
|
104
|
+
if (isUniqueViolation(e) && constraintOf(e) === "events_idempotency_uq") {
|
|
105
|
+
return new Error(
|
|
106
|
+
"ensureIdempotencyKeyIndex: duplicate metadata->>'idempotencyKey' values exist for at least " +
|
|
107
|
+
"one tenant in kumiko_events — CREATE UNIQUE INDEX CONCURRENTLY cannot complete. Find and " +
|
|
108
|
+
"resolve the duplicate idempotencyKey rows, then restart to retry.",
|
|
109
|
+
{ cause: e },
|
|
110
|
+
);
|
|
111
|
+
}
|
|
112
|
+
return e;
|
|
21
113
|
}
|
|
22
114
|
|
|
23
115
|
export type SubsequentEventInsertParams = {
|
|
@@ -99,22 +191,6 @@ export async function selectAggregateMaxVersion(db: AnyDb, aggregateId: string):
|
|
|
99
191
|
return rows[0]?.v ?? 0;
|
|
100
192
|
}
|
|
101
193
|
|
|
102
|
-
/** tenant_id the aggregate's events were written under — no membership/tenant
|
|
103
|
-
* filter. A r.systemScope() aggregate (e.g. user) lives in whichever tenant
|
|
104
|
-
* its creating executor used, which need not be a tenant the subject holds a
|
|
105
|
-
* membership in. Returns null for unknown streams. */
|
|
106
|
-
export async function selectAggregateStreamTenant(
|
|
107
|
-
db: AnyDb,
|
|
108
|
-
aggregateId: string,
|
|
109
|
-
aggregateType: string,
|
|
110
|
-
): Promise<string | null> {
|
|
111
|
-
const rows = (await asRawClient(db).unsafe(
|
|
112
|
-
`SELECT "tenant_id" AS t FROM "kumiko_events" WHERE "aggregate_id" = $1 AND "aggregate_type" = $2 ORDER BY "version" LIMIT 1`,
|
|
113
|
-
[aggregateId, aggregateType],
|
|
114
|
-
)) as ReadonlyArray<{ t: string | null }>;
|
|
115
|
-
return rows[0]?.t ?? null;
|
|
116
|
-
}
|
|
117
|
-
|
|
118
194
|
export async function selectEventsHighWaterMark(db: AnyDb): Promise<bigint> {
|
|
119
195
|
const rows = (await asRawClient(db).unsafe(
|
|
120
196
|
`SELECT COALESCE(MAX("id"), 0)::bigint AS max FROM "kumiko_events"`,
|
|
@@ -2,36 +2,10 @@ import type { AnyDb } from "../query";
|
|
|
2
2
|
import { asRawClient } from "../query";
|
|
3
3
|
import { quoteTableIdent } from "./table-ops";
|
|
4
4
|
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
export async function alterTableAddColumn(
|
|
10
|
-
db: AnyDb,
|
|
11
|
-
tableName: string,
|
|
12
|
-
columnName: string,
|
|
13
|
-
columnType: string,
|
|
14
|
-
defaultClause: string,
|
|
15
|
-
notNull: string,
|
|
16
|
-
): Promise<void> {
|
|
17
|
-
await asRawClient(db).unsafe(
|
|
18
|
-
`ALTER TABLE ${quoteTableIdent(tableName)} ADD COLUMN ${quoteTableIdent(columnName)} ${columnType}${defaultClause}${notNull}`,
|
|
19
|
-
);
|
|
20
|
-
}
|
|
21
|
-
|
|
22
|
-
export async function createIndexIfNotExists(
|
|
23
|
-
db: AnyDb,
|
|
24
|
-
indexKind: "UNIQUE INDEX" | "INDEX",
|
|
25
|
-
indexName: string,
|
|
26
|
-
tableName: string,
|
|
27
|
-
columnList: string,
|
|
28
|
-
whereSql?: string,
|
|
29
|
-
): Promise<void> {
|
|
30
|
-
const where = whereSql !== undefined ? ` WHERE ${whereSql}` : "";
|
|
31
|
-
await asRawClient(db).unsafe(
|
|
32
|
-
`CREATE ${indexKind} IF NOT EXISTS ${quoteTableIdent(indexName)} ON ${quoteTableIdent(tableName)} (${columnList})${where}`,
|
|
33
|
-
);
|
|
34
|
-
}
|
|
5
|
+
// Re-exported for back-compat — the generic DDL helpers moved to ./ddl so
|
|
6
|
+
// the prod-boot path (stack/table-helpers.ts, pipeline/event-consumer-state.ts)
|
|
7
|
+
// doesn't import from a module named for test-only concerns.
|
|
8
|
+
export { alterTableAddColumn, createIndexIfNotExists, executeDdlStatement } from "./ddl";
|
|
35
9
|
|
|
36
10
|
export async function truncateTablesRestartIdentity(
|
|
37
11
|
db: AnyDb,
|
package/src/db/reference-data.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { KUMIKO_COLUMNS_SYMBOL } from "@cosmicdrift/kumiko-types/schema-table-types";
|
|
1
2
|
import { fetchOne, insertOne, updateMany } from "../db/query";
|
|
2
3
|
import type { ReferenceDataDef } from "../engine/types";
|
|
3
4
|
import { SYSTEM_TENANT_ID } from "../engine/types";
|
|
@@ -7,10 +8,8 @@ import type { TableColumns } from "./dialect";
|
|
|
7
8
|
// biome-ignore lint/suspicious/noExplicitAny: Drizzle dynamic tables
|
|
8
9
|
type Table = TableColumns<any>;
|
|
9
10
|
|
|
10
|
-
const KUMIKO_COLUMNS_SYMBOL = Symbol.for("kumiko:schema:Columns");
|
|
11
|
-
|
|
12
11
|
function hasColumn(table: Table, field: string): boolean {
|
|
13
|
-
const cols =
|
|
12
|
+
const cols = table[KUMIKO_COLUMNS_SYMBOL];
|
|
14
13
|
if (typeof cols !== "object" || cols === null) return false;
|
|
15
14
|
return field in (cols as Record<string, unknown>);
|
|
16
15
|
}
|