@cosmicdrift/kumiko-framework 0.113.1 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -2
- package/src/__tests__/schema-cli.integration.test.ts +0 -29
- package/src/api/__tests__/batch.integration.test.ts +1 -4
- package/src/api/__tests__/request-id-middleware.test.ts +0 -51
- package/src/api/auth-middleware.ts +1 -65
- package/src/api/auth-routes.ts +0 -11
- package/src/api/index.ts +1 -3
- package/src/api/jwt.ts +4 -1
- package/src/api/request-context.ts +0 -3
- package/src/api/request-id-middleware.ts +0 -2
- package/src/api/routes.ts +1 -26
- package/src/api/server.ts +1 -29
- package/src/bun-db/__tests__/write-brand.test.ts +0 -21
- package/src/bun-db/query.ts +0 -17
- package/src/db/__tests__/config-seed.integration.test.ts +5 -13
- package/src/db/__tests__/entity-field-encryption.test.ts +7 -7
- package/src/db/__tests__/event-store-executor.integration.test.ts +4 -3
- package/src/db/__tests__/tenant-db-where-merge.test.ts +1 -4
- package/src/db/__tests__/tenant-db.integration.test.ts +2 -6
- package/src/db/config-seed.ts +9 -9
- package/src/db/encryption.ts +0 -7
- package/src/db/entity-field-encryption.ts +24 -81
- package/src/db/event-store-executor.ts +24 -37
- package/src/db/index.ts +0 -3
- package/src/db/query.ts +0 -1
- package/src/db/tenant-db.ts +9 -14
- package/src/engine/__tests__/boot-validator-gdpr-storage.test.ts +5 -6
- package/src/engine/__tests__/boot-validator.test.ts +9 -33
- package/src/engine/__tests__/build-config-feature-schema.test.ts +0 -21
- package/src/engine/__tests__/define-feature-entity-mapping.test.ts +0 -6
- package/src/engine/__tests__/registry.test.ts +0 -28
- package/src/engine/__tests__/screen.test.ts +0 -40
- package/src/engine/__tests__/soft-delete-cleanup.test.ts +0 -3
- package/src/engine/boot-validator/gdpr-storage.ts +1 -2
- package/src/engine/boot-validator/index.ts +2 -7
- package/src/engine/boot-validator/screens-nav.ts +0 -21
- package/src/engine/define-feature.ts +0 -17
- package/src/engine/define-handler.ts +2 -16
- package/src/engine/extensions/user-data.ts +1 -12
- package/src/engine/index.ts +2 -5
- package/src/engine/registry.ts +21 -44
- package/src/engine/soft-delete-cleanup.ts +2 -6
- package/src/engine/types/feature.ts +21 -29
- package/src/engine/types/fields.ts +1 -3
- package/src/engine/types/handlers.ts +2 -16
- package/src/engine/types/index.ts +0 -2
- package/src/engine/types/projection.ts +0 -22
- package/src/engine/types/screen.ts +0 -29
- package/src/event-store/__tests__/row-to-stored-event.test.ts +2 -2
- package/src/event-store/index.ts +0 -8
- package/src/files/provider-resolver.ts +15 -28
- package/src/i18n/required-surface-keys.ts +0 -12
- package/src/observability/__tests__/observability.integration.test.ts +1 -4
- package/src/pipeline/dispatcher.ts +6 -2
- package/src/pipeline/msp-rebuild.ts +3 -36
- package/src/pipeline/projection-rebuild.ts +3 -61
- package/src/schema-cli.ts +15 -24
- package/src/secrets/__tests__/contains-secret.test.ts +0 -34
- package/src/secrets/dek-cache.ts +5 -26
- package/src/secrets/envelope.ts +3 -5
- package/src/secrets/index.ts +1 -13
- package/src/secrets/types.ts +3 -18
- package/src/testing/e2e-generator.ts +0 -3
- package/src/testing/index.ts +0 -2
- package/src/testing/mutable-master-key-provider.ts +3 -29
- package/src/time/__tests__/tz-dateline.test.ts +10 -8
- package/src/ui-types/index.ts +0 -1
- package/src/api/__tests__/pat-scope.test.ts +0 -36
- package/src/api/pat-scope.ts +0 -14
- package/src/engine/__tests__/extend-entity-projection.test.ts +0 -123
- package/src/event-store/rebuild-dead-letter.ts +0 -111
- package/src/files/__tests__/provider-resolver.test.ts +0 -70
- package/src/pipeline/__tests__/rebuild-poison-quarantine.integration.test.ts +0 -274
- package/src/secrets/__tests__/envelope-cipher.test.ts +0 -87
- package/src/secrets/envelope-cipher.ts +0 -81
- package/src/secrets/stored-envelope.ts +0 -46
|
@@ -296,34 +296,6 @@ export type EntityListScreenDefinition = {
|
|
|
296
296
|
readonly access?: AccessRule;
|
|
297
297
|
};
|
|
298
298
|
|
|
299
|
-
// --- projectionList ---
|
|
300
|
-
|
|
301
|
-
// Like entityList, but bound to an EXPLICIT query instead of an entity. The
|
|
302
|
-
// list-query is taken verbatim from `query` (a fully qualified QN like
|
|
303
|
-
// "ledger:query:schedule:list") — NOT derived from the screen's own feature —
|
|
304
|
-
// so a screen can render any read-projection, including one owned by a
|
|
305
|
-
// different feature (the entityList feature-local resolution can't). Columns
|
|
306
|
-
// carry their own labels (no entity to derive field-labels from); there is no
|
|
307
|
-
// auto create-navigation (a projection isn't an editable entity list). Row
|
|
308
|
-
// interaction is explicit via `rowActions`. The query must return the same
|
|
309
|
-
// paged envelope as an entity list-query: `{ rows, nextCursor, total? }`.
|
|
310
|
-
export type ProjectionListScreenDefinition = {
|
|
311
|
-
readonly id: string;
|
|
312
|
-
readonly type: "projectionList";
|
|
313
|
-
readonly query: string;
|
|
314
|
-
readonly columns: readonly ListColumnSpec[];
|
|
315
|
-
readonly rowRenderer?: PlatformComponent;
|
|
316
|
-
readonly cardRenderer?: PlatformComponent;
|
|
317
|
-
readonly rowActions?: readonly RowAction[];
|
|
318
|
-
readonly toolbarActions?: readonly ToolbarAction[];
|
|
319
|
-
readonly pagination?: ListPaginationMode;
|
|
320
|
-
readonly pageSize?: number;
|
|
321
|
-
readonly defaultSort?: ListSortSpec;
|
|
322
|
-
readonly searchable?: boolean;
|
|
323
|
-
readonly slots?: ScreenSlots;
|
|
324
|
-
readonly access?: AccessRule;
|
|
325
|
-
};
|
|
326
|
-
|
|
327
299
|
// --- entityEdit ---
|
|
328
300
|
|
|
329
301
|
// camelCase `readOnly` instead of the spec's lowercase `readonly`: TS's
|
|
@@ -554,7 +526,6 @@ export type ScreenSlots = {
|
|
|
554
526
|
|
|
555
527
|
export type ScreenDefinition =
|
|
556
528
|
| EntityListScreenDefinition
|
|
557
|
-
| ProjectionListScreenDefinition
|
|
558
529
|
| EntityEditScreenDefinition
|
|
559
530
|
| ActionFormScreenDefinition
|
|
560
531
|
| ConfigEditScreenDefinition
|
|
@@ -48,8 +48,8 @@ describe("toStoredEvent", () => {
|
|
|
48
48
|
});
|
|
49
49
|
|
|
50
50
|
test("Feld-Vollständigkeit: das Mapping deckt genau die StoredEvent-Keys ab", () => {
|
|
51
|
-
//
|
|
52
|
-
//
|
|
51
|
+
// Pin gegen versehentliches Weglassen eines Feldes im Mapping. Muss mit
|
|
52
|
+
// der StoredEvent-Definition (event-store.ts) synchron bleiben.
|
|
53
53
|
const expectedKeys: ReadonlyArray<keyof StoredEvent> = [
|
|
54
54
|
"id",
|
|
55
55
|
"aggregateId",
|
package/src/event-store/index.ts
CHANGED
|
@@ -24,14 +24,6 @@ export {
|
|
|
24
24
|
streamAllEventsByType,
|
|
25
25
|
} from "./event-store";
|
|
26
26
|
export { createEventsTable, eventsTable } from "./events-schema";
|
|
27
|
-
export {
|
|
28
|
-
createRebuildDeadLetterTable,
|
|
29
|
-
listRebuildDeadLetters,
|
|
30
|
-
type RebuildDeadLetterRow,
|
|
31
|
-
rebuildDeadLetterTable,
|
|
32
|
-
recordRebuildDeadLetters,
|
|
33
|
-
type SkippedApply,
|
|
34
|
-
} from "./rebuild-dead-letter";
|
|
35
27
|
export { toStoredEvent } from "./row-to-stored-event";
|
|
36
28
|
export {
|
|
37
29
|
createSnapshotsTable,
|
|
@@ -138,34 +138,21 @@ export type FileProviderResolverDeps = {
|
|
|
138
138
|
// SYSTEM identity: provider construction is an infra read, distinct from the
|
|
139
139
|
// request user's authorization which stays at the route accessGuard.
|
|
140
140
|
export function makeFileProviderResolver(deps: FileProviderResolverDeps): FileProviderResolver {
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
const cache = new Map<string, Promise<FileStorageProvider>>();
|
|
147
|
-
return (tenantId) => {
|
|
148
|
-
const cached = cache.get(tenantId);
|
|
149
|
-
if (cached) return cached;
|
|
150
|
-
const built = (async () => {
|
|
151
|
-
if (!deps.registry || !deps._configAccessorFactory || !deps.db) {
|
|
152
|
-
throw new Error(
|
|
153
|
-
"makeFileProviderResolver: registry/_configAccessorFactory/db missing — " +
|
|
154
|
-
"mount the config + file-foundation features and wire context.db",
|
|
155
|
-
);
|
|
156
|
-
}
|
|
157
|
-
const config = deps._configAccessorFactory({
|
|
158
|
-
user: { id: SYSTEM_USER_ID, tenantId },
|
|
159
|
-
db: deps.db,
|
|
160
|
-
secrets: deps.secrets,
|
|
161
|
-
});
|
|
162
|
-
return createFileProviderForTenant(
|
|
163
|
-
{ config, registry: deps.registry, secrets: deps.secrets, _userId: SYSTEM_USER_ID },
|
|
164
|
-
tenantId,
|
|
141
|
+
return async (tenantId) => {
|
|
142
|
+
if (!deps.registry || !deps._configAccessorFactory || !deps.db) {
|
|
143
|
+
throw new Error(
|
|
144
|
+
"makeFileProviderResolver: registry/_configAccessorFactory/db missing — " +
|
|
145
|
+
"mount the config + file-foundation features and wire context.db",
|
|
165
146
|
);
|
|
166
|
-
}
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
147
|
+
}
|
|
148
|
+
const config = deps._configAccessorFactory({
|
|
149
|
+
user: { id: SYSTEM_USER_ID, tenantId },
|
|
150
|
+
db: deps.db,
|
|
151
|
+
secrets: deps.secrets,
|
|
152
|
+
});
|
|
153
|
+
return createFileProviderForTenant(
|
|
154
|
+
{ config, registry: deps.registry, secrets: deps.secrets, _userId: SYSTEM_USER_ID },
|
|
155
|
+
tenantId,
|
|
156
|
+
);
|
|
170
157
|
};
|
|
171
158
|
}
|
|
@@ -5,7 +5,6 @@ import type {
|
|
|
5
5
|
EntityListScreenDefinition,
|
|
6
6
|
FeatureDefinition,
|
|
7
7
|
NavDefinition,
|
|
8
|
-
ProjectionListScreenDefinition,
|
|
9
8
|
RowAction,
|
|
10
9
|
ScreenDefinition,
|
|
11
10
|
ToolbarAction,
|
|
@@ -77,17 +76,6 @@ export function requiredKeysFromScreen(
|
|
|
77
76
|
for (const action of list.toolbarActions ?? []) pushToolbarActionKeys(out, action);
|
|
78
77
|
break;
|
|
79
78
|
}
|
|
80
|
-
case "projectionList": {
|
|
81
|
-
const list = screen as ProjectionListScreenDefinition;
|
|
82
|
-
// Keine Entity → keine field-label-Fallbacks; nur explizite Column-Labels.
|
|
83
|
-
for (const col of list.columns) {
|
|
84
|
-
const normalized = normalizeListColumn(col);
|
|
85
|
-
if (normalized.label !== undefined) pushKey(out, normalized.label);
|
|
86
|
-
}
|
|
87
|
-
for (const action of list.rowActions ?? []) pushRowActionKeys(out, action);
|
|
88
|
-
for (const action of list.toolbarActions ?? []) pushToolbarActionKeys(out, action);
|
|
89
|
-
break;
|
|
90
|
-
}
|
|
91
79
|
case "entityEdit": {
|
|
92
80
|
const edit = screen as EntityEditScreenDefinition;
|
|
93
81
|
pushKey(out, edit.submitLabel);
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from "bun:test";
|
|
2
2
|
import { z } from "zod";
|
|
3
|
-
import type { TableColumns } from "../../db/dialect";
|
|
4
3
|
import { buildEntityTable } from "../../db/table-builder";
|
|
5
4
|
import { createRegistry, defineFeature } from "../../engine";
|
|
6
5
|
import type { AppContext, SaveContext } from "../../engine/types";
|
|
@@ -66,9 +65,7 @@ const todoEntity = {
|
|
|
66
65
|
|
|
67
66
|
let postSaveInvocations = 0;
|
|
68
67
|
const todoFeature = defineFeature("todo", (r) => {
|
|
69
|
-
|
|
70
|
-
// method-form insert (and its observability span) working — runtime shape is identical.
|
|
71
|
-
const todoTable: TableColumns = buildEntityTable("todo", todoEntity);
|
|
68
|
+
const todoTable = buildEntityTable("todo", todoEntity);
|
|
72
69
|
r.entity("todo", todoEntity);
|
|
73
70
|
|
|
74
71
|
r.writeHandler(
|
|
@@ -717,8 +717,12 @@ export function createDispatcher(
|
|
|
717
717
|
user,
|
|
718
718
|
ip: reqCtx?.ip,
|
|
719
719
|
});
|
|
720
|
-
// skip: ip-
|
|
721
|
-
//
|
|
720
|
+
// skip: ip-bucketed handler called from a non-HTTP entry point (job, seed,
|
|
721
|
+
// MSP-apply) — no client IP to bucket on, nothing to enforce. Pass
|
|
722
|
+
// through BEFORE requiring a resolver, so system/seed writes through
|
|
723
|
+
// such a handler don't need a RateLimitResolver wired (the es-ops
|
|
724
|
+
// seed dispatcher has none). L1/L2 middleware handle the HTTP-side
|
|
725
|
+
// ip caps.
|
|
722
726
|
if (bucket.kind === "skip") return;
|
|
723
727
|
if (!context.rateLimit) {
|
|
724
728
|
throw new InternalError({
|
|
@@ -11,16 +11,11 @@ import {
|
|
|
11
11
|
rebuildMetaOrThrow,
|
|
12
12
|
swapShadowIntoLive,
|
|
13
13
|
} from "../db/queries/shadow-swap";
|
|
14
|
-
import {
|
|
14
|
+
import { selectMany } from "../db/query";
|
|
15
15
|
import type { Registry, TenantId } from "../engine/types";
|
|
16
16
|
import { InternalError } from "../errors";
|
|
17
17
|
import { eventsTable, type StoredEvent, upcastStoredEvent } from "../event-store";
|
|
18
18
|
import { loadAggregate, loadAggregateAsOf } from "../event-store/event-store";
|
|
19
|
-
import {
|
|
20
|
-
createRebuildDeadLetterTable,
|
|
21
|
-
recordRebuildDeadLetters,
|
|
22
|
-
type SkippedApply,
|
|
23
|
-
} from "../event-store/rebuild-dead-letter";
|
|
24
19
|
import { upcastStoredEvents } from "../event-store/upcaster";
|
|
25
20
|
import { emitProjectionRebuild } from "../observability/standard-metrics";
|
|
26
21
|
import type { Meter } from "../observability/types/metric";
|
|
@@ -128,21 +123,12 @@ export async function rebuildMultiStreamProjection(
|
|
|
128
123
|
|
|
129
124
|
const meta = rebuildMetaOrThrow(msp.table, mspName);
|
|
130
125
|
|
|
131
|
-
// Declared per-projection (MspErrorMode): the rebuild policy falls back to
|
|
132
|
-
// continuous when omitted — previously declared but never honored here.
|
|
133
|
-
const skipApplyErrors =
|
|
134
|
-
(msp.errorMode?.rebuild ?? msp.errorMode?.continuous)?.skipApplyErrors === true;
|
|
135
|
-
|
|
136
126
|
const startedAt = Date.now();
|
|
137
127
|
let eventsProcessed = 0;
|
|
138
128
|
let lastProcessedEventId = 0n;
|
|
139
|
-
const skipped: SkippedApply[] = [];
|
|
140
129
|
|
|
141
130
|
try {
|
|
142
131
|
await ensureRebuildSchema(db);
|
|
143
|
-
// Outside the rebuild tx, like the schema: idempotent DDL colliding
|
|
144
|
-
// inside the tx would roll the whole replay back.
|
|
145
|
-
if (skipApplyErrors) await createRebuildDeadLetterTable(db);
|
|
146
132
|
await db.begin(async (tx: DbTx) => {
|
|
147
133
|
await resetConsumerForMspRebuild(tx, mspName, SHARED_INSTANCE_SENTINEL);
|
|
148
134
|
await selectConsumerForUpdate(tx, mspName, SHARED_INSTANCE_SENTINEL);
|
|
@@ -192,31 +178,13 @@ export async function rebuildMultiStreamProjection(
|
|
|
192
178
|
});
|
|
193
179
|
const applyFn = msp.apply[row.type];
|
|
194
180
|
if (!applyFn) continue;
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
// otherwise poison the rebuild tx (25P02) AND make the driver
|
|
198
|
-
// reject the whole begin() even after a caught error. Apply +
|
|
199
|
-
// ctx run on the savepoint-scoped handle.
|
|
200
|
-
try {
|
|
201
|
-
await runInSavepoint(tx, async (sp) => {
|
|
202
|
-
const rebuildCtx = createRebuildCtx(registry, sp as DbRunner, row.tenantId);
|
|
203
|
-
await applyFn(storedEvent, sp as DbRunner, rebuildCtx);
|
|
204
|
-
});
|
|
205
|
-
} catch (e) {
|
|
206
|
-
skipped.push({ event: storedEvent, error: e });
|
|
207
|
-
}
|
|
208
|
-
} else {
|
|
209
|
-
const rebuildCtx = createRebuildCtx(registry, tx, row.tenantId);
|
|
210
|
-
await applyFn(storedEvent, tx, rebuildCtx);
|
|
211
|
-
}
|
|
181
|
+
const rebuildCtx = createRebuildCtx(registry, tx, row.tenantId);
|
|
182
|
+
await applyFn(storedEvent, tx, rebuildCtx);
|
|
212
183
|
eventsProcessed++;
|
|
213
184
|
lastProcessedEventId = row.id;
|
|
214
185
|
}
|
|
215
186
|
}
|
|
216
187
|
|
|
217
|
-
if (skipped.length > 0) {
|
|
218
|
-
await recordRebuildDeadLetters(tx, mspName, skipped);
|
|
219
|
-
}
|
|
220
188
|
await updateConsumerRebuildCursor(
|
|
221
189
|
tx,
|
|
222
190
|
mspName,
|
|
@@ -242,7 +210,6 @@ export async function rebuildMultiStreamProjection(
|
|
|
242
210
|
const result: RebuildResult = {
|
|
243
211
|
projection: mspName,
|
|
244
212
|
eventsProcessed,
|
|
245
|
-
eventsSkipped: skipped.length,
|
|
246
213
|
lastProcessedEventId,
|
|
247
214
|
durationMs: Date.now() - startedAt,
|
|
248
215
|
};
|
|
@@ -13,7 +13,7 @@ import {
|
|
|
13
13
|
rebuildMetaOrThrow,
|
|
14
14
|
swapShadowIntoLive,
|
|
15
15
|
} from "../db/queries/shadow-swap";
|
|
16
|
-
import { coerceRow, extractTableInfo,
|
|
16
|
+
import { coerceRow, extractTableInfo, selectMany } from "../db/query";
|
|
17
17
|
import type { Registry, TenantId } from "../engine/types";
|
|
18
18
|
import {
|
|
19
19
|
eventsTable,
|
|
@@ -22,11 +22,6 @@ import {
|
|
|
22
22
|
upcastStoredEvent,
|
|
23
23
|
} from "../event-store";
|
|
24
24
|
import type { EventMetadata } from "../event-store/event-store";
|
|
25
|
-
import {
|
|
26
|
-
createRebuildDeadLetterTable,
|
|
27
|
-
recordRebuildDeadLetters,
|
|
28
|
-
type SkippedApply,
|
|
29
|
-
} from "../event-store/rebuild-dead-letter";
|
|
30
25
|
import { emitProjectionRebuild } from "../observability/standard-metrics";
|
|
31
26
|
import type { Meter } from "../observability/types/metric";
|
|
32
27
|
import { projectionStateTable } from "./projection-state";
|
|
@@ -124,12 +119,7 @@ function rowToStoredEvent(row: StoredEventRow): StoredEvent {
|
|
|
124
119
|
|
|
125
120
|
export type RebuildResult = {
|
|
126
121
|
readonly projection: string;
|
|
127
|
-
// Events consumed from the log — includes quarantined ones (the cursor
|
|
128
|
-
// advanced past them).
|
|
129
122
|
readonly eventsProcessed: number;
|
|
130
|
-
// Poison events quarantined into kumiko_rebuild_dead_letters this run.
|
|
131
|
-
// Always 0 unless quarantine mode was active (#760).
|
|
132
|
-
readonly eventsSkipped: number;
|
|
133
123
|
readonly lastProcessedEventId: bigint;
|
|
134
124
|
readonly durationMs: number;
|
|
135
125
|
};
|
|
@@ -155,15 +145,6 @@ type RebuildDeps = {
|
|
|
155
145
|
// on the live table; if a long-running writer holds it past this, the rebuild
|
|
156
146
|
// fails loud instead of hanging. Defaults to DEFAULT_FENCE_LOCK_TIMEOUT_MS.
|
|
157
147
|
readonly fenceLockTimeoutMs?: number;
|
|
158
|
-
// Apply-error policy for THIS run. Default strict: the first throwing
|
|
159
|
-
// apply aborts the rebuild (tx rollback, status "failed") — a poison
|
|
160
|
-
// event makes the projection permanently un-rebuildable (#760).
|
|
161
|
-
// skipApplyErrors: true confines every apply to a savepoint; a throwing
|
|
162
|
-
// apply is rolled back, recorded into kumiko_rebuild_dead_letters and
|
|
163
|
-
// skipped, and the rebuild completes without its effect. Opt-in for
|
|
164
|
-
// operators unblocking a pinned rebuild — the projection then knowingly
|
|
165
|
-
// misses the quarantined events until they are repaired and replayed.
|
|
166
|
-
readonly errorPolicy?: { readonly skipApplyErrors?: boolean };
|
|
167
148
|
// Test-only seam: fires once after the unlocked bulk drain and before the
|
|
168
149
|
// cutover fence. Lets a concurrency test inject a committed write into the
|
|
169
150
|
// replay window deterministically. The `__test_` prefix marks it test-only:
|
|
@@ -193,7 +174,7 @@ export async function rebuildProjection(
|
|
|
193
174
|
const meta = rebuildMetaOrThrow(projection.table, projectionName);
|
|
194
175
|
|
|
195
176
|
const sources = Array.isArray(projection.source) ? projection.source : [projection.source];
|
|
196
|
-
const sourcesList = [...sources
|
|
177
|
+
const sourcesList = [...sources];
|
|
197
178
|
const subscribedList = Object.keys(projection.apply);
|
|
198
179
|
// Upcasters run at read time: older stored payloads get walked through the
|
|
199
180
|
// registered r.eventMigration chain until their shape matches the current
|
|
@@ -204,11 +185,6 @@ export async function rebuildProjection(
|
|
|
204
185
|
const startedAt = Date.now();
|
|
205
186
|
let eventsProcessed = 0;
|
|
206
187
|
let lastProcessedEventId = 0n;
|
|
207
|
-
const skipApplyErrors = deps.errorPolicy?.skipApplyErrors === true;
|
|
208
|
-
// Quarantined events, collected in memory and written once after the
|
|
209
|
-
// replay settles — the #443 full-re-replay path would otherwise duplicate
|
|
210
|
-
// the dead-letter rows.
|
|
211
|
-
let skipped: SkippedApply[] = [];
|
|
212
188
|
|
|
213
189
|
// One chronological batch of events after lastProcessedEventId, applied into
|
|
214
190
|
// the shadow. Returns the batch size so the caller can detect the tail
|
|
@@ -232,21 +208,7 @@ export async function rebuildProjection(
|
|
|
232
208
|
// skip: apply-key validation ensures every subscribed type has a handler;
|
|
233
209
|
// defensive check against runtime-mutated registry
|
|
234
210
|
if (!applyFn) continue;
|
|
235
|
-
|
|
236
|
-
// Driver-native savepoint: a throwing SQL statement would otherwise
|
|
237
|
-
// poison the rebuild tx (25P02) AND make the driver reject the whole
|
|
238
|
-
// begin() even after a caught error. The apply runs on the
|
|
239
|
-
// savepoint-scoped handle so its statements are confined.
|
|
240
|
-
try {
|
|
241
|
-
await runInSavepoint(tx, async (sp) => {
|
|
242
|
-
await applyFn(storedEvent, sp as DbTx, projection.table);
|
|
243
|
-
});
|
|
244
|
-
} catch (e) {
|
|
245
|
-
skipped.push({ event: storedEvent, error: e });
|
|
246
|
-
}
|
|
247
|
-
} else {
|
|
248
|
-
await applyFn(storedEvent, tx, projection.table);
|
|
249
|
-
}
|
|
211
|
+
await applyFn(storedEvent, tx, projection.table);
|
|
250
212
|
eventsProcessed++;
|
|
251
213
|
lastProcessedEventId = row.id;
|
|
252
214
|
}
|
|
@@ -255,9 +217,6 @@ export async function rebuildProjection(
|
|
|
255
217
|
|
|
256
218
|
try {
|
|
257
219
|
await ensureRebuildSchema(db);
|
|
258
|
-
// Outside the rebuild tx, like the schema: idempotent DDL colliding
|
|
259
|
-
// inside the tx would roll the whole replay back.
|
|
260
|
-
if (skipApplyErrors) await createRebuildDeadLetterTable(db);
|
|
261
220
|
await db.begin(async (tx: DbTx) => {
|
|
262
221
|
await markProjectionRebuilding(tx, projectionName);
|
|
263
222
|
await buildShadowTable(tx, meta);
|
|
@@ -302,28 +261,12 @@ export async function rebuildProjection(
|
|
|
302
261
|
deps.__test_onBuildShadowTable?.();
|
|
303
262
|
lastProcessedEventId = 0n;
|
|
304
263
|
eventsProcessed = 0;
|
|
305
|
-
skipped = [];
|
|
306
264
|
while ((await drainBatch(tx)) === REBUILD_BATCH_SIZE) {
|
|
307
265
|
// re-replay the full log into the fresh shadow
|
|
308
266
|
}
|
|
309
|
-
// 656/2: drainBatch increments eventsProcessed for every drained row
|
|
310
|
-
// regardless of quarantine (skipApplyErrors), so this must hold
|
|
311
|
-
// exactly under the fence — a mismatch means drainBatch silently
|
|
312
|
-
// dropped a row instead of counting it, which the rest of this
|
|
313
|
-
// function has no other way to detect.
|
|
314
|
-
if (eventsProcessed < Number(expectedEvents)) {
|
|
315
|
-
throw new Error(
|
|
316
|
-
`projection-rebuild "${projectionName}": fenced re-replay processed ` +
|
|
317
|
-
`${eventsProcessed} events but ${expectedEvents} were expected — drainBatch ` +
|
|
318
|
-
"silently dropped rows instead of counting them.",
|
|
319
|
-
);
|
|
320
|
-
}
|
|
321
267
|
}
|
|
322
268
|
}
|
|
323
269
|
|
|
324
|
-
if (skipped.length > 0) {
|
|
325
|
-
await recordRebuildDeadLetters(tx, projectionName, skipped);
|
|
326
|
-
}
|
|
327
270
|
await finalizeProjectionRebuild(tx, projectionName, lastProcessedEventId);
|
|
328
271
|
await swapShadowIntoLive(tx, meta.tableName);
|
|
329
272
|
});
|
|
@@ -349,7 +292,6 @@ export async function rebuildProjection(
|
|
|
349
292
|
const result: RebuildResult = {
|
|
350
293
|
projection: projectionName,
|
|
351
294
|
eventsProcessed,
|
|
352
|
-
eventsSkipped: skipped.length,
|
|
353
295
|
lastProcessedEventId,
|
|
354
296
|
durationMs: Date.now() - startedAt,
|
|
355
297
|
};
|
package/src/schema-cli.ts
CHANGED
|
@@ -44,28 +44,17 @@ export type SchemaCliOut = {
|
|
|
44
44
|
|
|
45
45
|
const SNAPSHOT_FILENAME = ".snapshot.json";
|
|
46
46
|
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
};
|
|
51
|
-
|
|
52
|
-
// Single load-path for kumiko/schema.ts (512/3) — `generate` and `validate`
|
|
53
|
-
// both need ENTITY_METAS (and validate additionally wants FEATURES); this is
|
|
54
|
-
// the one place that enforces "ENTITY_METAS must be an array" so the two
|
|
55
|
-
// commands can't drift on the check or its wording. Throws — callers that
|
|
56
|
-
// want a graceful exit code (validate) catch it themselves.
|
|
57
|
-
async function loadSchemaModFromApp(schemaFile: string): Promise<SchemaMod> {
|
|
47
|
+
async function loadEntityMetasFromApp(
|
|
48
|
+
schemaFile: string,
|
|
49
|
+
): Promise<Parameters<typeof renderTablesDdl>[0]> {
|
|
58
50
|
// bun imports TS directly — no spawn needed.
|
|
59
|
-
const mod = (await import(schemaFile)) as { ENTITY_METAS?: unknown
|
|
51
|
+
const mod = (await import(schemaFile)) as { ENTITY_METAS?: unknown };
|
|
60
52
|
if (!Array.isArray(mod.ENTITY_METAS)) {
|
|
61
53
|
throw new Error(
|
|
62
54
|
`Schema file ${schemaFile} muss \`export const ENTITY_METAS: EntityTableMeta[]\` haben.`,
|
|
63
55
|
);
|
|
64
56
|
}
|
|
65
|
-
return
|
|
66
|
-
entityMetas: mod.ENTITY_METAS as Parameters<typeof renderTablesDdl>[0],
|
|
67
|
-
features: mod.FEATURES,
|
|
68
|
-
};
|
|
57
|
+
return mod.ENTITY_METAS as Parameters<typeof renderTablesDdl>[0];
|
|
69
58
|
}
|
|
70
59
|
|
|
71
60
|
function nextSequenceNumber(migrationsDir: string): number {
|
|
@@ -143,7 +132,7 @@ export async function runSchemaCli(
|
|
|
143
132
|
out.err(" export const ENTITY_METAS: EntityTableMeta[] = [...]");
|
|
144
133
|
return 1;
|
|
145
134
|
}
|
|
146
|
-
const
|
|
135
|
+
const metas = await loadEntityMetasFromApp(schemaFile);
|
|
147
136
|
const snapshotPath = join(migrationsDir, SNAPSHOT_FILENAME);
|
|
148
137
|
const prevSnapshot = existsSync(snapshotPath) ? loadSnapshotJson(snapshotPath) : null;
|
|
149
138
|
const result = generateMigration({
|
|
@@ -202,18 +191,20 @@ export async function runSchemaCli(
|
|
|
202
191
|
out.err(" export const ENTITY_METAS: EntityTableMeta[] = [...]");
|
|
203
192
|
return 1;
|
|
204
193
|
}
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
}
|
|
209
|
-
|
|
194
|
+
const mod = (await import(schemaFile)) as {
|
|
195
|
+
ENTITY_METAS?: unknown;
|
|
196
|
+
FEATURES?: unknown;
|
|
197
|
+
};
|
|
198
|
+
if (!Array.isArray(mod.ENTITY_METAS)) {
|
|
199
|
+
out.err(
|
|
200
|
+
` Schema file ${schemaFile} muss \`export const ENTITY_METAS: EntityTableMeta[]\` haben.`,
|
|
201
|
+
);
|
|
210
202
|
return 1;
|
|
211
203
|
}
|
|
212
|
-
const mod = { FEATURES: schemaMod.features };
|
|
213
204
|
let ok = true;
|
|
214
205
|
|
|
215
206
|
// 1. Schema drift — compute the diff, never write.
|
|
216
|
-
const metas =
|
|
207
|
+
const metas = mod.ENTITY_METAS as Parameters<typeof renderTablesDdl>[0];
|
|
217
208
|
const snapshotPath = join(migrationsDir, SNAPSHOT_FILENAME);
|
|
218
209
|
const prevSnapshot = existsSync(snapshotPath) ? loadSnapshotJson(snapshotPath) : null;
|
|
219
210
|
const drift = generateMigration({
|
|
@@ -32,18 +32,6 @@ export type _R6TypeAssertions = [
|
|
|
32
32
|
// the backstop. This is what keeps generic-over-response handlers compiling.
|
|
33
33
|
Expect<Equal<ContainsSecret<unknown>, false>>,
|
|
34
34
|
Expect<Equal<ContainsSecret<never>, false>>,
|
|
35
|
-
// 556/3: ContainsSecret<A | B> distributes over the union (naked type
|
|
36
|
-
// param) to `ContainsSecret<A> | ContainsSecret<B>` = `boolean`, not the
|
|
37
|
-
// literal `true` — pinned here so a future change to this predicate can't
|
|
38
|
-
// silently regress it. The actual leak-guard in define-handler.ts compares
|
|
39
|
-
// via `true extends ContainsSecret<TData>` (556/1) instead of `ContainsSecret<
|
|
40
|
-
// TData> extends true`, which is fail-closed for `boolean` either way.
|
|
41
|
-
Expect<Equal<ContainsSecret<{ ok: true } | { s: Secret<string> }>, boolean>>,
|
|
42
|
-
// 556/2: Map/Set are explicit SafeLeaf members now — pins that the runtime
|
|
43
|
-
// guard's "walk Map/Set entries separately" intent is mirrored, not an
|
|
44
|
-
// accidental compile-time blind spot.
|
|
45
|
-
Expect<Equal<ContainsSecret<Map<string, Secret<string>>>, false>>,
|
|
46
|
-
Expect<Equal<ContainsSecret<Set<Secret<string>>>, false>>,
|
|
47
35
|
];
|
|
48
36
|
|
|
49
37
|
const schema = z.object({ q: z.string() });
|
|
@@ -69,28 +57,6 @@ function genericResponseHandler<K extends string>(kind: K) {
|
|
|
69
57
|
}
|
|
70
58
|
void genericResponseHandler;
|
|
71
59
|
|
|
72
|
-
// 556/1: ContainsSecret<A | B> distributes over the union (naked type param
|
|
73
|
-
// in the condition) to `ContainsSecret<A> | ContainsSecret<B>` = `boolean`,
|
|
74
|
-
// not the literal `true` — a check written as `ContainsSecret<TData> extends
|
|
75
|
-
// true` silently passes a union with a leak hidden in ONE branch. Both
|
|
76
|
-
// assertions below must hold: the guard rejects the leaking union AND still
|
|
77
|
-
// accepts the clean one, proving the fix (membership form on the phantom
|
|
78
|
-
// param) is fail-closed for unions, not just for a bare Secret<>.
|
|
79
|
-
declare const leakingUnion: { ok: true } | { apiKey: Secret<string> };
|
|
80
|
-
// @ts-expect-error — R6: a Secret<> hidden in one union branch must still leak-guard
|
|
81
|
-
defineQueryHandler({
|
|
82
|
-
name: "t:query:union-leak",
|
|
83
|
-
schema,
|
|
84
|
-
handler: async () => leakingUnion,
|
|
85
|
-
});
|
|
86
|
-
|
|
87
|
-
declare const cleanUnion: { ok: true } | { id: string };
|
|
88
|
-
defineQueryHandler({
|
|
89
|
-
name: "t:query:union-clean",
|
|
90
|
-
schema,
|
|
91
|
-
handler: async () => cleanUnion,
|
|
92
|
-
});
|
|
93
|
-
|
|
94
60
|
describe("R6 ContainsSecret", () => {
|
|
95
61
|
test("a clean query handler still builds — the guard param is invisible", () => {
|
|
96
62
|
const def = defineQueryHandler({
|
package/src/secrets/dek-cache.ts
CHANGED
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
// - maxEntries (default 1000): LRU eviction kicks in on insert when full.
|
|
10
10
|
|
|
11
11
|
import { createHash } from "node:crypto";
|
|
12
|
-
import type {
|
|
12
|
+
import type { MasterKeyProvider } from "./types";
|
|
13
13
|
|
|
14
14
|
const DEFAULT_TTL_MS = 5 * 60 * 1000; // 5 minutes
|
|
15
15
|
const DEFAULT_MAX_ENTRIES = 1000;
|
|
@@ -25,16 +25,8 @@ export type DekCacheOptions = {
|
|
|
25
25
|
|
|
26
26
|
export type DekCache = {
|
|
27
27
|
// Unwrap via the provider, caching the result. Second call within TTL
|
|
28
|
-
// returns the cached DEK without hitting the provider.
|
|
29
|
-
|
|
30
|
-
// encryptedDek bytes are unique per value, so scoped providers can't
|
|
31
|
-
// collide on the key either.
|
|
32
|
-
unwrapDek(
|
|
33
|
-
encryptedDek: Buffer,
|
|
34
|
-
kekVersion: number,
|
|
35
|
-
provider: MasterKeyProvider,
|
|
36
|
-
scope?: KeyScope,
|
|
37
|
-
): Promise<Buffer>;
|
|
28
|
+
// returns the cached DEK without hitting the provider.
|
|
29
|
+
unwrapDek(encryptedDek: Buffer, kekVersion: number, provider: MasterKeyProvider): Promise<Buffer>;
|
|
38
30
|
|
|
39
31
|
// Drop every entry. Call after KEK rotation so old cached DEKs (still
|
|
40
32
|
// valid, but referencing the old kekVersion) don't serve reads that
|
|
@@ -45,19 +37,6 @@ export type DekCache = {
|
|
|
45
37
|
size(): number;
|
|
46
38
|
};
|
|
47
39
|
|
|
48
|
-
// Wrap a provider so its unwrapDek goes through the cache. Callers keep the
|
|
49
|
-
// full MasterKeyProvider contract without knowing about caching —
|
|
50
|
-
// decryptValue handles crypto, the cache handles cost.
|
|
51
|
-
export function withDekCache(provider: MasterKeyProvider, cache: DekCache): MasterKeyProvider {
|
|
52
|
-
return {
|
|
53
|
-
wrapDek: (dek, scope) => provider.wrapDek(dek, scope),
|
|
54
|
-
unwrapDek: (encryptedDek, version, scope) =>
|
|
55
|
-
cache.unwrapDek(encryptedDek, version, provider, scope),
|
|
56
|
-
currentVersion: () => provider.currentVersion(),
|
|
57
|
-
isAvailable: () => provider.isAvailable(),
|
|
58
|
-
};
|
|
59
|
-
}
|
|
60
|
-
|
|
61
40
|
export function createDekCache(opts: DekCacheOptions = {}): DekCache {
|
|
62
41
|
const ttlMs = opts.ttlMs ?? DEFAULT_TTL_MS;
|
|
63
42
|
const maxEntries = opts.maxEntries ?? DEFAULT_MAX_ENTRIES;
|
|
@@ -95,7 +74,7 @@ export function createDekCache(opts: DekCacheOptions = {}): DekCache {
|
|
|
95
74
|
}
|
|
96
75
|
|
|
97
76
|
return {
|
|
98
|
-
async unwrapDek(encryptedDek, kekVersion, provider
|
|
77
|
+
async unwrapDek(encryptedDek, kekVersion, provider) {
|
|
99
78
|
const key = cacheKey(encryptedDek, kekVersion);
|
|
100
79
|
const hit = entries.get(key);
|
|
101
80
|
if (hit && hit.expiresAt > now()) {
|
|
@@ -113,7 +92,7 @@ export function createDekCache(opts: DekCacheOptions = {}): DekCache {
|
|
|
113
92
|
entries.delete(key);
|
|
114
93
|
}
|
|
115
94
|
|
|
116
|
-
const dek = await provider.unwrapDek(encryptedDek, kekVersion
|
|
95
|
+
const dek = await provider.unwrapDek(encryptedDek, kekVersion);
|
|
117
96
|
evictOldestIfFull();
|
|
118
97
|
// Store a copy — caller can zero its own buffer after use.
|
|
119
98
|
entries.set(key, { dek: Buffer.from(dek), expiresAt: now() + ttlMs });
|
package/src/secrets/envelope.ts
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
// carries everything needed to decrypt later.
|
|
4
4
|
|
|
5
5
|
import { createCipheriv, createDecipheriv, randomBytes } from "node:crypto";
|
|
6
|
-
import type { Envelope,
|
|
6
|
+
import type { Envelope, MasterKeyProvider } from "./types";
|
|
7
7
|
|
|
8
8
|
const ALGORITHM = "aes-256-gcm";
|
|
9
9
|
const DEK_LENGTH = 32; // AES-256
|
|
@@ -14,7 +14,6 @@ const IV_LENGTH = 12; // GCM standard nonce length
|
|
|
14
14
|
export async function encryptValue(
|
|
15
15
|
plaintext: string,
|
|
16
16
|
provider: MasterKeyProvider,
|
|
17
|
-
scope?: KeyScope,
|
|
18
17
|
): Promise<Envelope> {
|
|
19
18
|
// Fresh DEK per value. Reusing DEKs across rows would break forward
|
|
20
19
|
// secrecy (one compromised ciphertext-IV pair leaks information about
|
|
@@ -27,7 +26,7 @@ export async function encryptValue(
|
|
|
27
26
|
const ciphertext = Buffer.concat([cipher.update(plaintext, "utf8"), cipher.final()]);
|
|
28
27
|
const authTag = cipher.getAuthTag();
|
|
29
28
|
|
|
30
|
-
const { encryptedDek, kekVersion } = await provider.wrapDek(dek
|
|
29
|
+
const { encryptedDek, kekVersion } = await provider.wrapDek(dek);
|
|
31
30
|
return { ciphertext, iv, authTag, encryptedDek, kekVersion };
|
|
32
31
|
} finally {
|
|
33
32
|
// Zero the DEK on best effort regardless of success — if provider.wrapDek
|
|
@@ -43,9 +42,8 @@ export async function encryptValue(
|
|
|
43
42
|
export async function decryptValue(
|
|
44
43
|
envelope: Envelope,
|
|
45
44
|
provider: MasterKeyProvider,
|
|
46
|
-
scope?: KeyScope,
|
|
47
45
|
): Promise<string> {
|
|
48
|
-
const dek = await provider.unwrapDek(envelope.encryptedDek, envelope.kekVersion
|
|
46
|
+
const dek = await provider.unwrapDek(envelope.encryptedDek, envelope.kekVersion);
|
|
49
47
|
try {
|
|
50
48
|
const decipher = createDecipheriv(ALGORITHM, dek, envelope.iv);
|
|
51
49
|
decipher.setAuthTag(envelope.authTag);
|
package/src/secrets/index.ts
CHANGED
|
@@ -1,29 +1,17 @@
|
|
|
1
|
-
export { createDekCache, type DekCache, type DekCacheOptions
|
|
1
|
+
export { createDekCache, type DekCache, type DekCacheOptions } from "./dek-cache";
|
|
2
2
|
export {
|
|
3
3
|
createEnvMasterKeyProvider,
|
|
4
4
|
type EnvMasterKeyProviderOptions,
|
|
5
5
|
type Keyring,
|
|
6
6
|
} from "./env-master-key-provider";
|
|
7
7
|
export { decryptValue, encryptValue } from "./envelope";
|
|
8
|
-
export {
|
|
9
|
-
createEnvelopeCipher,
|
|
10
|
-
type EnvelopeCipher,
|
|
11
|
-
type EnvelopeCipherOptions,
|
|
12
|
-
} from "./envelope-cipher";
|
|
13
8
|
export { assertNoSecretLeak } from "./leak-guard";
|
|
14
9
|
export { rewrapDek } from "./rotation";
|
|
15
|
-
export {
|
|
16
|
-
decodeStoredEnvelope,
|
|
17
|
-
encodeStoredEnvelope,
|
|
18
|
-
isStoredEnvelope,
|
|
19
|
-
type StoredEnvelope,
|
|
20
|
-
} from "./stored-envelope";
|
|
21
10
|
export {
|
|
22
11
|
type ContainsSecret,
|
|
23
12
|
createSecret,
|
|
24
13
|
type Envelope,
|
|
25
14
|
isSecret,
|
|
26
|
-
type KeyScope,
|
|
27
15
|
type MasterKeyProvider,
|
|
28
16
|
type Secret,
|
|
29
17
|
type SecretAuditContext,
|