@cosmicdrift/kumiko-framework 0.327.0 → 0.328.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +4 -4
- package/src/bun-db/__tests__/closed-connection-retry.integration.test.ts +94 -1
- package/src/bun-db/query.ts +153 -18
- package/src/changes.json +36 -0
- package/src/db/queries/backfill-pii.ts +184 -85
- package/src/db/queries/pii-backfill-state.ts +33 -0
- package/src/engine/__tests__/build-app-schema.test.ts +26 -0
- package/src/engine/build-app-schema.ts +2 -0
- package/src/engine/types/index.ts +1 -0
- package/src/event-store/index.ts +4 -0
- package/src/i18n/required-surface-keys.ts +8 -0
- package/src/migrations/__tests__/pending-rebuilds.integration.test.ts +45 -3
- package/src/migrations/__tests__/pii-event-backfill.integration.test.ts +457 -0
- package/src/migrations/index.ts +11 -0
- package/src/migrations/pending-rebuilds.ts +29 -3
- package/src/migrations/pii-event-backfill.ts +422 -0
- package/src/pipeline/msp-rebuild.ts +7 -1
- package/src/search/__tests__/meilisearch-adapter.integration.test.ts +65 -0
- package/src/search/__tests__/search-adapter.test.ts +15 -0
- package/src/search/in-memory-adapter.ts +6 -0
- package/src/search/meilisearch-adapter.ts +24 -0
- package/src/stack/test-stack.ts +26 -2
- package/src/ui-types/index.ts +1 -0
|
@@ -37,8 +37,8 @@
|
|
|
37
37
|
// Both stages are opt-in and off by default; without them an unresolvable
|
|
38
38
|
// subject still fails loud into `failures`, as before.
|
|
39
39
|
//
|
|
40
|
-
// Snapshots of touched aggregates are dropped (they may cache
|
|
41
|
-
// the next snapshotting load recreates them. AFTER a run, rebuild the
|
|
40
|
+
// Snapshots of touched aggregates are dropped per batch (they may cache
|
|
41
|
+
// plaintext); the next snapshotting load recreates them. AFTER a run, rebuild the
|
|
42
42
|
// affected projections — applyEntityEvent materializes ciphertext AND the
|
|
43
43
|
// blind-index columns, which keeps equality lookups (login by email) alive.
|
|
44
44
|
|
|
@@ -50,6 +50,7 @@ import {
|
|
|
50
50
|
KeyErasedError,
|
|
51
51
|
KeyNotFoundError,
|
|
52
52
|
type KmsContext,
|
|
53
|
+
type LocalKeyKmsAdapter,
|
|
53
54
|
type SubjectId,
|
|
54
55
|
} from "../../crypto/kms-adapter";
|
|
55
56
|
import {
|
|
@@ -67,7 +68,7 @@ import {
|
|
|
67
68
|
import type { EntityDefinition, Registry, TenantId } from "../../engine/types";
|
|
68
69
|
import type { DbRunner } from "../connection";
|
|
69
70
|
import { resolveTableName } from "../entity-table-meta";
|
|
70
|
-
import {
|
|
71
|
+
import { tableExists } from "../schema-inspection";
|
|
71
72
|
import { toSnakeCase } from "../table-builder";
|
|
72
73
|
|
|
73
74
|
const LIFECYCLE_VERBS = ["created", "updated", "deleted", "restored", "forgotten"] as const;
|
|
@@ -92,6 +93,19 @@ export type PiiBackfillResult = {
|
|
|
92
93
|
readonly failures: readonly PiiBackfillFailure[];
|
|
93
94
|
};
|
|
94
95
|
|
|
96
|
+
export type PiiBackfillScanCache = {
|
|
97
|
+
forgottenAggregates?: ReadonlySet<string>;
|
|
98
|
+
};
|
|
99
|
+
|
|
100
|
+
export type PiiBackfillBatchResult = PiiBackfillResult & {
|
|
101
|
+
// Last scanned event id; null when the batch was empty.
|
|
102
|
+
readonly lastEventId: string | null;
|
|
103
|
+
readonly scannedAll: boolean;
|
|
104
|
+
readonly firstFailedEventId: string | null;
|
|
105
|
+
readonly touchedAggregateTypes: readonly string[];
|
|
106
|
+
readonly touchedEventTypes: readonly string[];
|
|
107
|
+
};
|
|
108
|
+
|
|
95
109
|
export type PiiBackfillOptions = {
|
|
96
110
|
readonly batchSize?: number;
|
|
97
111
|
// Scan + count only, write nothing — including the subject KMS: outcomes
|
|
@@ -105,6 +119,12 @@ export type PiiBackfillOptions = {
|
|
|
105
119
|
readonly eraseUnresolvableSubjects?: boolean;
|
|
106
120
|
};
|
|
107
121
|
|
|
122
|
+
export type PiiBackfillBatchOptions = PiiBackfillOptions & {
|
|
123
|
+
readonly afterEventId?: bigint | string;
|
|
124
|
+
// Caller-held so the forgotten-set full scan is shared across batches.
|
|
125
|
+
readonly scanCache?: PiiBackfillScanCache;
|
|
126
|
+
};
|
|
127
|
+
|
|
108
128
|
type EventRow = {
|
|
109
129
|
readonly id: bigint | string;
|
|
110
130
|
readonly aggregate_id: string;
|
|
@@ -116,36 +136,94 @@ type EventRow = {
|
|
|
116
136
|
|
|
117
137
|
type FieldOutcome = "unchanged" | "encrypted" | "erased";
|
|
118
138
|
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
139
|
+
type PiiTargets = {
|
|
140
|
+
readonly entityTargets: ReadonlyMap<
|
|
141
|
+
string,
|
|
142
|
+
{ readonly entity: EntityDefinition; readonly piiFields: readonly string[] }
|
|
143
|
+
>;
|
|
144
|
+
readonly eventCatalog: ReturnType<typeof configuredEventPiiCatalog>;
|
|
145
|
+
};
|
|
146
|
+
|
|
147
|
+
function collectPiiTargets(registry: Registry): PiiTargets {
|
|
148
|
+
const entityTargets = new Map<
|
|
149
|
+
string,
|
|
150
|
+
{ readonly entity: EntityDefinition; readonly piiFields: readonly string[] }
|
|
151
|
+
>();
|
|
152
|
+
for (const [name, entity] of registry.getAllEntities()) {
|
|
153
|
+
const piiFields = collectPiiSubjectFields(entity);
|
|
154
|
+
if (piiFields.length > 0) entityTargets.set(name, { entity, piiFields });
|
|
155
|
+
}
|
|
156
|
+
return { entityTargets, eventCatalog: configuredEventPiiCatalog() };
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
function requireLocalKeySubjectKms(caller: string): LocalKeyKmsAdapter {
|
|
124
160
|
const kms = configuredPiiSubjectKms();
|
|
125
161
|
if (!kms) {
|
|
126
162
|
throw new Error(
|
|
127
|
-
|
|
163
|
+
`${caller} requires a configured subject KMS — boot with ` +
|
|
128
164
|
"runProdApp({ kms }) / configurePiiSubjectKms(adapter) before running the backfill.",
|
|
129
165
|
);
|
|
130
166
|
}
|
|
131
167
|
if (!isLocalKeyKmsAdapter(kms)) {
|
|
132
|
-
throw new Error(
|
|
168
|
+
throw new Error(`${caller} requires a local-key KMS adapter`);
|
|
133
169
|
}
|
|
134
|
-
|
|
170
|
+
return kms;
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
export async function backfillEventPiiEncryption(
|
|
174
|
+
db: DbRunner,
|
|
175
|
+
registry: Registry,
|
|
176
|
+
options: PiiBackfillOptions = {},
|
|
177
|
+
): Promise<PiiBackfillResult> {
|
|
178
|
+
requireLocalKeySubjectKms("backfillEventPiiEncryption");
|
|
179
|
+
const total = {
|
|
180
|
+
scannedEvents: 0,
|
|
181
|
+
updatedEvents: 0,
|
|
182
|
+
encryptedFields: 0,
|
|
183
|
+
erasedFields: 0,
|
|
184
|
+
ownerFromProjection: 0,
|
|
185
|
+
erasedUnresolvable: 0,
|
|
186
|
+
deletedSnapshots: 0,
|
|
187
|
+
};
|
|
188
|
+
const failures: PiiBackfillFailure[] = [];
|
|
189
|
+
const scanCache: PiiBackfillScanCache = {};
|
|
190
|
+
let afterEventId: string = "0";
|
|
191
|
+
|
|
192
|
+
for (;;) {
|
|
193
|
+
const batch = await backfillEventPiiEncryptionBatch(db, registry, {
|
|
194
|
+
...options,
|
|
195
|
+
afterEventId,
|
|
196
|
+
scanCache,
|
|
197
|
+
});
|
|
198
|
+
total.scannedEvents += batch.scannedEvents;
|
|
199
|
+
total.updatedEvents += batch.updatedEvents;
|
|
200
|
+
total.encryptedFields += batch.encryptedFields;
|
|
201
|
+
total.erasedFields += batch.erasedFields;
|
|
202
|
+
total.ownerFromProjection += batch.ownerFromProjection;
|
|
203
|
+
total.erasedUnresolvable += batch.erasedUnresolvable;
|
|
204
|
+
total.deletedSnapshots += batch.deletedSnapshots;
|
|
205
|
+
failures.push(...batch.failures);
|
|
206
|
+
if (batch.lastEventId === null || batch.scannedAll) break;
|
|
207
|
+
afterEventId = batch.lastEventId;
|
|
208
|
+
}
|
|
209
|
+
return { ...total, failures };
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
// Processes exactly one batch. Snapshots of the batch's touched aggregates are
|
|
213
|
+
// dropped inside the same call so an abort between batches never leaves a
|
|
214
|
+
// snapshot that caches plaintext for an already-encrypted stream.
|
|
215
|
+
export async function backfillEventPiiEncryptionBatch(
|
|
216
|
+
db: DbRunner,
|
|
217
|
+
registry: Registry,
|
|
218
|
+
options: PiiBackfillBatchOptions = {},
|
|
219
|
+
): Promise<PiiBackfillBatchResult> {
|
|
220
|
+
const subjectKms = requireLocalKeySubjectKms("backfillEventPiiEncryptionBatch");
|
|
135
221
|
const batchSize = options.batchSize ?? 500;
|
|
136
222
|
const raw = asRawClient(db);
|
|
137
223
|
const kmsCtx: KmsContext = { requestId: "pii-backfill" };
|
|
224
|
+
const scanCache = options.scanCache ?? {};
|
|
138
225
|
|
|
139
|
-
const entityTargets =
|
|
140
|
-
string,
|
|
141
|
-
{ readonly entity: EntityDefinition; readonly piiFields: readonly string[] }
|
|
142
|
-
>();
|
|
143
|
-
for (const [name, entity] of registry.getAllEntities()) {
|
|
144
|
-
const piiFields = collectPiiSubjectFields(entity);
|
|
145
|
-
if (piiFields.length > 0) entityTargets.set(name, { entity, piiFields });
|
|
146
|
-
}
|
|
147
|
-
const eventCatalog = configuredEventPiiCatalog();
|
|
148
|
-
|
|
226
|
+
const { entityTargets, eventCatalog } = collectPiiTargets(registry);
|
|
149
227
|
const aggregateTypes = [...entityTargets.keys()];
|
|
150
228
|
const catalogTypes = [...eventCatalog.keys()];
|
|
151
229
|
const result = {
|
|
@@ -158,65 +236,65 @@ export async function backfillEventPiiEncryption(
|
|
|
158
236
|
deletedSnapshots: 0,
|
|
159
237
|
failures: [] as PiiBackfillFailure[],
|
|
160
238
|
};
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
)
|
|
169
|
-
|
|
170
|
-
|
|
239
|
+
const emptyBatch = (): PiiBackfillBatchResult => ({
|
|
240
|
+
...result,
|
|
241
|
+
lastEventId: null,
|
|
242
|
+
scannedAll: true,
|
|
243
|
+
firstFailedEventId: null,
|
|
244
|
+
touchedAggregateTypes: [],
|
|
245
|
+
touchedEventTypes: [],
|
|
246
|
+
});
|
|
247
|
+
if (aggregateTypes.length === 0 && catalogTypes.length === 0) return emptyBatch();
|
|
248
|
+
|
|
249
|
+
const rows = (await raw.unsafe(
|
|
250
|
+
`SELECT "id", "aggregate_id", "aggregate_type", "tenant_id", "type", "payload"
|
|
251
|
+
FROM "kumiko_events"
|
|
252
|
+
WHERE ("aggregate_type" = ANY($1::text[]) OR "type" = ANY($2::text[])) AND "id" > $3::bigint
|
|
253
|
+
ORDER BY "id" ASC
|
|
254
|
+
LIMIT $4`,
|
|
255
|
+
[aggregateTypes, catalogTypes, String(options.afterEventId ?? 0), batchSize],
|
|
256
|
+
)) as ReadonlyArray<EventRow>;
|
|
257
|
+
const last = rows[rows.length - 1];
|
|
258
|
+
if (last === undefined) return emptyBatch();
|
|
259
|
+
|
|
260
|
+
const projectionOwnersByType = await loadProjectionOwners(rows);
|
|
171
261
|
const touchedAggregates = new Set<string>();
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
const outcome = await transformEvent(row, projectionOwnersByType.get(row.aggregate_type));
|
|
191
|
-
if (outcome === null) continue;
|
|
192
|
-
result.encryptedFields += outcome.encrypted;
|
|
193
|
-
result.erasedFields += outcome.erased;
|
|
194
|
-
result.ownerFromProjection += outcome.ownerFromProjection;
|
|
195
|
-
result.erasedUnresolvable += outcome.erasedUnresolvable;
|
|
196
|
-
if (!options.dryRun) {
|
|
197
|
-
await raw.unsafe(`UPDATE "kumiko_events" SET "payload" = $1::jsonb WHERE "id" = $2`, [
|
|
198
|
-
outcome.payload,
|
|
199
|
-
row.id,
|
|
200
|
-
]);
|
|
201
|
-
}
|
|
202
|
-
result.updatedEvents++;
|
|
203
|
-
touchedAggregates.add(row.aggregate_id);
|
|
204
|
-
} catch (e) {
|
|
205
|
-
const reason = e instanceof Error ? e.message : String(e);
|
|
206
|
-
result.failures.push({
|
|
207
|
-
eventId: String(row.id),
|
|
208
|
-
reason:
|
|
209
|
-
e instanceof SubjectResolutionError
|
|
210
|
-
? `${reason} (retry with { resolveOwnerFromProjection: true } to resolve the owner ` +
|
|
211
|
-
"from the entity's projection table, and/or { eraseUnresolvableSubjects: true } to " +
|
|
212
|
-
"erase fields whose subject stays unresolvable)"
|
|
213
|
-
: reason,
|
|
214
|
-
});
|
|
262
|
+
const touchedAggregateTypes = new Set<string>();
|
|
263
|
+
const touchedEventTypes = new Set<string>();
|
|
264
|
+
let firstFailedEventId: string | null = null;
|
|
265
|
+
|
|
266
|
+
for (const row of rows) {
|
|
267
|
+
result.scannedEvents++;
|
|
268
|
+
try {
|
|
269
|
+
const outcome = await transformEvent(row, projectionOwnersByType.get(row.aggregate_type));
|
|
270
|
+
if (outcome === null) continue;
|
|
271
|
+
result.encryptedFields += outcome.encrypted;
|
|
272
|
+
result.erasedFields += outcome.erased;
|
|
273
|
+
result.ownerFromProjection += outcome.ownerFromProjection;
|
|
274
|
+
result.erasedUnresolvable += outcome.erasedUnresolvable;
|
|
275
|
+
if (!options.dryRun) {
|
|
276
|
+
await raw.unsafe(`UPDATE "kumiko_events" SET "payload" = $1::jsonb WHERE "id" = $2`, [
|
|
277
|
+
outcome.payload,
|
|
278
|
+
row.id,
|
|
279
|
+
]);
|
|
215
280
|
}
|
|
281
|
+
result.updatedEvents++;
|
|
282
|
+
touchedAggregates.add(row.aggregate_id);
|
|
283
|
+
touchedAggregateTypes.add(row.aggregate_type);
|
|
284
|
+
touchedEventTypes.add(row.type);
|
|
285
|
+
} catch (e) {
|
|
286
|
+
const reason = e instanceof Error ? e.message : String(e);
|
|
287
|
+
firstFailedEventId ??= String(row.id);
|
|
288
|
+
result.failures.push({
|
|
289
|
+
eventId: String(row.id),
|
|
290
|
+
reason:
|
|
291
|
+
e instanceof SubjectResolutionError
|
|
292
|
+
? `${reason} (retry with { resolveOwnerFromProjection: true } to resolve the owner ` +
|
|
293
|
+
"from the entity's projection table, and/or { eraseUnresolvableSubjects: true } to " +
|
|
294
|
+
"erase fields whose subject stays unresolvable)"
|
|
295
|
+
: reason,
|
|
296
|
+
});
|
|
216
297
|
}
|
|
217
|
-
const last = rows[rows.length - 1];
|
|
218
|
-
if (last === undefined) break;
|
|
219
|
-
cursor = String(last.id);
|
|
220
298
|
}
|
|
221
299
|
|
|
222
300
|
// Snapshots may cache the plaintext state of touched aggregates.
|
|
@@ -228,7 +306,28 @@ export async function backfillEventPiiEncryption(
|
|
|
228
306
|
result.deletedSnapshots = deleted.length;
|
|
229
307
|
}
|
|
230
308
|
|
|
231
|
-
return
|
|
309
|
+
return {
|
|
310
|
+
...result,
|
|
311
|
+
lastEventId: String(last.id),
|
|
312
|
+
scannedAll: rows.length < batchSize,
|
|
313
|
+
firstFailedEventId,
|
|
314
|
+
touchedAggregateTypes: [...touchedAggregateTypes],
|
|
315
|
+
touchedEventTypes: [...touchedEventTypes],
|
|
316
|
+
};
|
|
317
|
+
|
|
318
|
+
// Full-scan of the store — only paid once a plaintext field actually needs
|
|
319
|
+
// encrypting, so a catch-up run over already-encrypted events never runs it.
|
|
320
|
+
async function loadForgottenAggregates(): Promise<ReadonlySet<string>> {
|
|
321
|
+
if (scanCache.forgottenAggregates) return scanCache.forgottenAggregates;
|
|
322
|
+
// Pre-KMS forgets left no key tombstone — the *.forgotten event on the
|
|
323
|
+
// stream is the only durable marker. aggregate_id doubles as the user id
|
|
324
|
+
// for user-subject lookups.
|
|
325
|
+
const forgottenRows = (await raw.unsafe(
|
|
326
|
+
`SELECT DISTINCT "aggregate_id" FROM "kumiko_events" WHERE "type" LIKE '%.forgotten'`,
|
|
327
|
+
)) as ReadonlyArray<{ aggregate_id: string }>;
|
|
328
|
+
scanCache.forgottenAggregates = new Set(forgottenRows.map((r) => r.aggregate_id));
|
|
329
|
+
return scanCache.forgottenAggregates;
|
|
330
|
+
}
|
|
232
331
|
|
|
233
332
|
async function loadProjectionOwners(
|
|
234
333
|
rows: readonly EventRow[],
|
|
@@ -249,7 +348,10 @@ export async function backfillEventPiiEncryption(
|
|
|
249
348
|
if (!target) continue;
|
|
250
349
|
const tableName = resolveTableName(aggregateType, target.entity, undefined);
|
|
251
350
|
const ownersById = new Map<string, Record<string, unknown>>();
|
|
252
|
-
|
|
351
|
+
// Existence check instead of catching undefined_table: a caught SQL error
|
|
352
|
+
// aborts the surrounding transaction (25P02). Entity never mounted/rebuilt
|
|
353
|
+
// → stage 3 still gets a chance; this batch just contributes no owners.
|
|
354
|
+
if (await tableExists(db, `public.${tableName}`)) {
|
|
253
355
|
const projectionRows = (await raw.unsafe(
|
|
254
356
|
`SELECT * FROM ${quoteIdent(tableName)} WHERE "id" = ANY($1::uuid[])`,
|
|
255
357
|
[[...ids]],
|
|
@@ -260,10 +362,6 @@ export async function backfillEventPiiEncryption(
|
|
|
260
362
|
ownersById.set(id, projectionRowToCamel(target.entity, projectionRow));
|
|
261
363
|
}
|
|
262
364
|
}
|
|
263
|
-
} catch (e) {
|
|
264
|
-
// Entity never mounted/rebuilt (no projection table yet) — stage 3
|
|
265
|
-
// still gets a chance; this batch just contributes no owners.
|
|
266
|
-
if (!isUndefinedTable(e)) throw e;
|
|
267
365
|
}
|
|
268
366
|
byType.set(aggregateType, ownersById);
|
|
269
367
|
}
|
|
@@ -367,7 +465,7 @@ export async function backfillEventPiiEncryption(
|
|
|
367
465
|
if (value === null || value === undefined) return "unchanged";
|
|
368
466
|
if (typeof value !== "string") return "unchanged";
|
|
369
467
|
if (isPiiCiphertext(value) || value === PII_ERASED_SENTINEL) return "unchanged";
|
|
370
|
-
if (isForgottenSubject(subject, row.aggregate_id)) {
|
|
468
|
+
if (await isForgottenSubject(subject, row.aggregate_id)) {
|
|
371
469
|
section[field] = PII_ERASED_SENTINEL;
|
|
372
470
|
return "erased";
|
|
373
471
|
}
|
|
@@ -400,7 +498,8 @@ export async function backfillEventPiiEncryption(
|
|
|
400
498
|
}
|
|
401
499
|
}
|
|
402
500
|
|
|
403
|
-
function isForgottenSubject(subject: SubjectId, aggregateId: string): boolean {
|
|
501
|
+
async function isForgottenSubject(subject: SubjectId, aggregateId: string): Promise<boolean> {
|
|
502
|
+
const forgottenAggregates = await loadForgottenAggregates();
|
|
404
503
|
if (forgottenAggregates.has(aggregateId)) return true;
|
|
405
504
|
return subject.kind === "user" && forgottenAggregates.has(subject.userId);
|
|
406
505
|
}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import type { AnyDb } from "../query";
|
|
2
|
+
import { asRawClient } from "../query";
|
|
3
|
+
|
|
4
|
+
export type PiiBackfillStateRow = { readonly cursor: string; readonly completed: boolean };
|
|
5
|
+
|
|
6
|
+
export async function readPiiBackfillState(
|
|
7
|
+
db: AnyDb,
|
|
8
|
+
fingerprint: string,
|
|
9
|
+
): Promise<PiiBackfillStateRow | null> {
|
|
10
|
+
const rows = (await asRawClient(db).unsafe(
|
|
11
|
+
`SELECT "cursor_event_id"::text AS "cursor", "completed_at" IS NOT NULL AS "completed"
|
|
12
|
+
FROM "kumiko_pii_backfill_state" WHERE "fingerprint" = $1`,
|
|
13
|
+
[fingerprint],
|
|
14
|
+
)) as ReadonlyArray<PiiBackfillStateRow>;
|
|
15
|
+
return rows[0] ?? null;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export async function persistPiiBackfillState(
|
|
19
|
+
db: AnyDb,
|
|
20
|
+
fingerprint: string,
|
|
21
|
+
cursor: bigint,
|
|
22
|
+
markCompleted: boolean,
|
|
23
|
+
): Promise<void> {
|
|
24
|
+
await asRawClient(db).unsafe(
|
|
25
|
+
`INSERT INTO "kumiko_pii_backfill_state" ("fingerprint", "cursor_event_id", "completed_at", "updated_at")
|
|
26
|
+
VALUES ($1, $2::bigint, CASE WHEN $3::boolean THEN now() ELSE NULL END, now())
|
|
27
|
+
ON CONFLICT ("fingerprint") DO UPDATE SET
|
|
28
|
+
"cursor_event_id" = GREATEST("kumiko_pii_backfill_state"."cursor_event_id", EXCLUDED."cursor_event_id"),
|
|
29
|
+
"completed_at" = COALESCE("kumiko_pii_backfill_state"."completed_at", EXCLUDED."completed_at"),
|
|
30
|
+
"updated_at" = now()`,
|
|
31
|
+
[fingerprint, cursor.toString(), markCompleted],
|
|
32
|
+
);
|
|
33
|
+
}
|
|
@@ -392,6 +392,32 @@ describe("buildAppSchema", () => {
|
|
|
392
392
|
expect(fields["categories"]?.["maxRows"]).toBeUndefined();
|
|
393
393
|
});
|
|
394
394
|
|
|
395
|
+
test("Select: optionTones überleben die Projection", () => {
|
|
396
|
+
const entity = {
|
|
397
|
+
fields: {
|
|
398
|
+
status: {
|
|
399
|
+
type: "select",
|
|
400
|
+
options: ["open", "closed"],
|
|
401
|
+
optionTones: { closed: "bad" },
|
|
402
|
+
},
|
|
403
|
+
area: { type: "select", options: ["a", "b"] },
|
|
404
|
+
},
|
|
405
|
+
} as unknown as EntityDefinition;
|
|
406
|
+
|
|
407
|
+
const f = defineFeature("ent", (r) => {
|
|
408
|
+
r.entity("thing", entity);
|
|
409
|
+
});
|
|
410
|
+
const app = buildAppSchema(createRegistry([f]));
|
|
411
|
+
const fields = (
|
|
412
|
+
app.features[0]!.entities["thing"] as unknown as {
|
|
413
|
+
fields: Record<string, Record<string, unknown>>;
|
|
414
|
+
}
|
|
415
|
+
).fields;
|
|
416
|
+
|
|
417
|
+
expect(fields["status"]?.["optionTones"]).toEqual({ closed: "bad" });
|
|
418
|
+
expect(fields["area"]?.["optionTones"]).toBeUndefined();
|
|
419
|
+
});
|
|
420
|
+
|
|
395
421
|
test("MultiSelectFieldDef: alle Keys sind in projectField bewusst eingeordnet", () => {
|
|
396
422
|
// Bounded completeness check, scoped to MultiSelectFieldDef's own key set
|
|
397
423
|
// (packages/types/src/fields.ts) — NOT a general FieldDefinition lockstep.
|
|
@@ -450,6 +450,8 @@ function projectField(fieldDef: FieldDefinition): FieldDefinition {
|
|
|
450
450
|
if (isJsonSafeValue(def["default"])) out["default"] = def["default"];
|
|
451
451
|
// Select: options-Liste ist plain JSON, durchschicken.
|
|
452
452
|
if (Array.isArray(def["options"])) out["options"] = def["options"];
|
|
453
|
+
if (isPlainObject(def["optionTones"]) && isJsonSafeValue(def["optionTones"]))
|
|
454
|
+
out["optionTones"] = def["optionTones"];
|
|
453
455
|
// Reference: entity-Target + labelField + multiple müssen zum Renderer.
|
|
454
456
|
// Der ReferenceInput baut die Options-Query aus refEntity/refFeature und
|
|
455
457
|
// resolved das Label über labelField — ohne diese Properties fällt das
|
package/src/event-store/index.ts
CHANGED
|
@@ -1,8 +1,12 @@
|
|
|
1
1
|
export {
|
|
2
2
|
backfillEventPiiEncryption,
|
|
3
|
+
backfillEventPiiEncryptionBatch,
|
|
4
|
+
type PiiBackfillBatchOptions,
|
|
5
|
+
type PiiBackfillBatchResult,
|
|
3
6
|
type PiiBackfillFailure,
|
|
4
7
|
type PiiBackfillOptions,
|
|
5
8
|
type PiiBackfillResult,
|
|
9
|
+
type PiiBackfillScanCache,
|
|
6
10
|
} from "../db/queries/backfill-pii";
|
|
7
11
|
export {
|
|
8
12
|
type ArchiveStreamArgs,
|
|
@@ -166,6 +166,8 @@ export function requiredKeysFromScreen(
|
|
|
166
166
|
out.add(fieldLabelKey(featureName, list.entity, normalized.field));
|
|
167
167
|
}
|
|
168
168
|
}
|
|
169
|
+
pushKey(out, list.createLabel);
|
|
170
|
+
pushKey(out, list.searchPlaceholder);
|
|
169
171
|
for (const action of list.rowActions ?? []) pushRowActionKeys(out, action);
|
|
170
172
|
for (const action of list.toolbarActions ?? []) pushToolbarActionKeys(out, action);
|
|
171
173
|
break;
|
|
@@ -177,6 +179,8 @@ export function requiredKeysFromScreen(
|
|
|
177
179
|
const normalized = normalizeListColumn(col);
|
|
178
180
|
if (normalized.label !== undefined) pushKey(out, normalized.label);
|
|
179
181
|
}
|
|
182
|
+
pushKey(out, list.createLabel);
|
|
183
|
+
pushKey(out, list.searchPlaceholder);
|
|
180
184
|
for (const action of list.rowActions ?? []) pushRowActionKeys(out, action);
|
|
181
185
|
for (const action of list.toolbarActions ?? []) pushToolbarActionKeys(out, action);
|
|
182
186
|
break;
|
|
@@ -207,6 +211,8 @@ export function requiredKeysFromScreen(
|
|
|
207
211
|
case "actionForm": {
|
|
208
212
|
const form = screen as ActionFormScreenDefinition;
|
|
209
213
|
pushKey(out, form.submitLabel);
|
|
214
|
+
pushKey(out, form.summary?.title);
|
|
215
|
+
pushKey(out, form.summary?.subtitle);
|
|
210
216
|
for (const fieldName of Object.keys(form.fields)) {
|
|
211
217
|
const override = form.fieldLabels?.[fieldName];
|
|
212
218
|
if (override !== undefined) pushKey(out, override);
|
|
@@ -318,6 +324,8 @@ export function requiredKeysFromScreen(
|
|
|
318
324
|
}
|
|
319
325
|
if (section.kind === "relatedList") {
|
|
320
326
|
pushKey(out, section.title);
|
|
327
|
+
pushKey(out, section.description);
|
|
328
|
+
pushKey(out, section.itemNoun);
|
|
321
329
|
for (const col of section.columns) {
|
|
322
330
|
const normalized = normalizeListColumn(col);
|
|
323
331
|
if (normalized.label !== undefined) pushKey(out, normalized.label);
|
|
@@ -6,7 +6,7 @@ import { afterAll, beforeAll, beforeEach, describe, expect, test } from "bun:tes
|
|
|
6
6
|
import { mkdtempSync, rmSync } from "node:fs";
|
|
7
7
|
import { tmpdir } from "node:os";
|
|
8
8
|
import { join } from "node:path";
|
|
9
|
-
import { integer, table as pgTable, uuid } from "../../db/dialect";
|
|
9
|
+
import { integer, table as pgTable, text, uuid } from "../../db/dialect";
|
|
10
10
|
import { createEventStoreExecutor } from "../../db/event-store-executor";
|
|
11
11
|
import { asRawClient, selectMany } from "../../db/query";
|
|
12
12
|
import { writeRebuildMarker } from "../../db/rebuild-marker";
|
|
@@ -21,7 +21,7 @@ import {
|
|
|
21
21
|
type ProjectionDefinition,
|
|
22
22
|
} from "../../engine";
|
|
23
23
|
import type { JobRunner } from "../../jobs/job-runner";
|
|
24
|
-
import { createProjectionStateTable } from "../../pipeline";
|
|
24
|
+
import { createEventConsumerStateTable, createProjectionStateTable } from "../../pipeline";
|
|
25
25
|
import {
|
|
26
26
|
createTestDb,
|
|
27
27
|
type TestDb,
|
|
@@ -54,6 +54,12 @@ const countsTable = pgTable("read_pending_counts", {
|
|
|
54
54
|
itemCount: integer("item_count").notNull().default(0),
|
|
55
55
|
});
|
|
56
56
|
|
|
57
|
+
const mspItemNamesTable = pgTable("read_pending_msp_names", {
|
|
58
|
+
id: uuid("id").primaryKey(),
|
|
59
|
+
tenantId: uuid("tenant_id").notNull(),
|
|
60
|
+
name: text("name"),
|
|
61
|
+
});
|
|
62
|
+
|
|
57
63
|
// Exists in the DB but has no registered projection — models an owning
|
|
58
64
|
// feature missing from the composition (distinct from a deliberately DROPped
|
|
59
65
|
// table, which is never pushed at all).
|
|
@@ -83,6 +89,20 @@ const countsProjection: ProjectionDefinition = {
|
|
|
83
89
|
const feature = defineFeature("pendingtest", (r) => {
|
|
84
90
|
r.entity("pending-item", itemEntity);
|
|
85
91
|
r.projection(countsProjection);
|
|
92
|
+
r.multiStreamProjection({
|
|
93
|
+
name: "pending-item-names",
|
|
94
|
+
table: mspItemNamesTable,
|
|
95
|
+
apply: {
|
|
96
|
+
"pending-item.created": async (event, tx) => {
|
|
97
|
+
const payload = event.payload as { name: string };
|
|
98
|
+
await asRawClient(tx).unsafe(
|
|
99
|
+
`INSERT INTO "read_pending_msp_names" (id, tenant_id, name) VALUES ($1::uuid, $2::uuid, $3)
|
|
100
|
+
ON CONFLICT (id) DO UPDATE SET name = EXCLUDED.name`,
|
|
101
|
+
[event.aggregateId, event.tenantId, payload.name],
|
|
102
|
+
);
|
|
103
|
+
},
|
|
104
|
+
},
|
|
105
|
+
});
|
|
86
106
|
});
|
|
87
107
|
|
|
88
108
|
const admin = TestUsers.admin;
|
|
@@ -97,9 +117,11 @@ beforeAll(async () => {
|
|
|
97
117
|
testDb = await createTestDb();
|
|
98
118
|
await unsafeCreateEntityTable(testDb.db, itemEntity, "pending-item");
|
|
99
119
|
await createProjectionStateTable(testDb.db);
|
|
120
|
+
await createEventConsumerStateTable(testDb.db);
|
|
100
121
|
await unsafePushTables(testDb.db, {
|
|
101
122
|
readPendingCounts: countsTable,
|
|
102
123
|
readOrphanProjection: orphanProjectionTable,
|
|
124
|
+
readPendingMspNames: mspItemNamesTable,
|
|
103
125
|
});
|
|
104
126
|
tdb = createTenantDb(testDb.db, admin.tenantId);
|
|
105
127
|
markerDir = mkdtempSync(join(tmpdir(), "pending-rebuilds-"));
|
|
@@ -113,7 +135,7 @@ afterAll(async () => {
|
|
|
113
135
|
beforeEach(async () => {
|
|
114
136
|
failApply = false;
|
|
115
137
|
await asRawClient(testDb.db).unsafe(
|
|
116
|
-
`TRUNCATE kumiko_events, read_pending_items, read_pending_counts, read_orphan_projection, kumiko_projections RESTART IDENTITY CASCADE`,
|
|
138
|
+
`TRUNCATE kumiko_events, read_pending_items, read_pending_counts, read_orphan_projection, read_pending_msp_names, kumiko_projections RESTART IDENTITY CASCADE`,
|
|
117
139
|
);
|
|
118
140
|
await asRawClient(testDb.db).unsafe(`DROP TABLE IF EXISTS kumiko_pending_rebuilds`);
|
|
119
141
|
});
|
|
@@ -160,6 +182,26 @@ describe("pending-rebuilds queue", () => {
|
|
|
160
182
|
expect(await getCount()).toBe(2);
|
|
161
183
|
});
|
|
162
184
|
|
|
185
|
+
test("a queued multi-stream projection table is rebuilt through the multi-stream path", async () => {
|
|
186
|
+
await executor.create({ groupId: GROUP, name: "a" }, admin, tdb);
|
|
187
|
+
await executor.create({ groupId: GROUP, name: "b" }, admin, tdb);
|
|
188
|
+
writeRebuildMarker(markerDir, "0003_msp.sql", ["read_pending_msp_names"]);
|
|
189
|
+
await queueRebuildsFromMarkers(testDb.db, {
|
|
190
|
+
migrationsDir: markerDir,
|
|
191
|
+
appliedIds: ["0003_msp"],
|
|
192
|
+
});
|
|
193
|
+
|
|
194
|
+
const run = await runPendingRebuilds(testDb.db, registry);
|
|
195
|
+
|
|
196
|
+
expect(run.failed).toEqual([]);
|
|
197
|
+
expect(run.rebuilt).toEqual([
|
|
198
|
+
{ projection: "pendingtest:projection:pending-item-names", eventsProcessed: 2 },
|
|
199
|
+
]);
|
|
200
|
+
expect(await listPendingRebuilds(testDb.db)).toEqual([]);
|
|
201
|
+
const rows = await selectMany(testDb.db, mspItemNamesTable);
|
|
202
|
+
expect(rows.map((r) => r.name).sort()).toEqual(["a", "b"]);
|
|
203
|
+
});
|
|
204
|
+
|
|
163
205
|
test("tables without a registered projection are drained, not stuck forever", async () => {
|
|
164
206
|
writeRebuildMarker(markerDir, "0002_unmapped.sql", ["read_some_plain_table"]);
|
|
165
207
|
await queueRebuildsFromMarkers(testDb.db, {
|