@cosmicdrift/kumiko-framework 0.118.0 → 0.120.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -2
- package/src/api/__tests__/pii-leak-guard.integration.test.ts +103 -0
- package/src/api/pii-leak-guard.ts +45 -0
- package/src/api/server.ts +5 -0
- package/src/bun-db/query.ts +27 -2
- package/src/crypto/__tests__/blind-index.test.ts +130 -0
- package/src/crypto/__tests__/event-pii.test.ts +161 -0
- package/src/crypto/__tests__/pg-kms-adapter.integration.test.ts +117 -0
- package/src/crypto/__tests__/pii-field-encryption.test.ts +216 -0
- package/src/crypto/blind-index.ts +114 -0
- package/src/crypto/event-pii.ts +69 -0
- package/src/crypto/index.ts +35 -0
- package/src/crypto/kms-adapter.ts +8 -0
- package/src/crypto/pg-kms-adapter.ts +173 -0
- package/src/crypto/pii-field-encryption.ts +195 -0
- package/src/db/__tests__/blind-index.integration.test.ts +232 -0
- package/src/db/__tests__/event-store-executor.integration.test.ts +169 -1
- package/src/db/__tests__/table-builder-meta-lockstep.test.ts +39 -0
- package/src/db/apply-entity-event.ts +8 -0
- package/src/db/blind-index-cleanup.ts +53 -0
- package/src/db/entity-table-meta.ts +47 -2
- package/src/db/event-store-executor.ts +125 -30
- package/src/db/index.ts +1 -0
- package/src/db/queries/backfill-pii.ts +276 -0
- package/src/db/table-builder.ts +98 -51
- package/src/engine/__tests__/boot-validator-pii-retention.test.ts +95 -4
- package/src/engine/__tests__/unmanaged-table.test.ts +46 -1
- package/src/engine/boot-validator/pii-retention.ts +45 -9
- package/src/engine/define-feature.ts +28 -3
- package/src/engine/registry.ts +25 -0
- package/src/engine/types/feature.ts +17 -2
- package/src/engine/types/fields.ts +6 -0
- package/src/engine/types/handlers.ts +7 -0
- package/src/engine/types/index.ts +2 -0
- package/src/event-store/__tests__/backfill-pii.integration.test.ts +279 -0
- package/src/event-store/event-store.ts +11 -6
- package/src/event-store/index.ts +6 -0
|
@@ -0,0 +1,276 @@
|
|
|
1
|
+
// One-time backfill for pre-KMS plaintext PII in kumiko_events (#799).
|
|
2
|
+
//
|
|
3
|
+
// Crypto-shredding (#724/#818) only covers NEW writes — events appended
|
|
4
|
+
// before a KMS was configured still carry plaintext (user.created email,
|
|
5
|
+
// delivery attempt recipientAddress, job payloads). This tool re-encrypts
|
|
6
|
+
// them in place, per field, under the owning subject's DEK:
|
|
7
|
+
//
|
|
8
|
+
// - entity lifecycle events (<entity>.created/updated/deleted/forgotten/
|
|
9
|
+
// restored) for every entity with PII subject annotations
|
|
10
|
+
// - custom events from the event-PII catalog (r.defineEvent piiFields)
|
|
11
|
+
//
|
|
12
|
+
// Already-forgotten subjects must NOT get a fresh key minted for their old
|
|
13
|
+
// plaintext — three erased-detection layers write [[erased]] instead:
|
|
14
|
+
// 1. KeyErasedError from the KMS (subject forgotten in the KMS era)
|
|
15
|
+
// 2. the event's own aggregate has a *.forgotten event (pre-KMS forget)
|
|
16
|
+
// 3. the resolved user subject's id has a *.forgotten event (custom
|
|
17
|
+
// events referencing a pre-KMS-forgotten user)
|
|
18
|
+
//
|
|
19
|
+
// Idempotent: ciphertext and sentinel values pass through untouched — a
|
|
20
|
+
// second run reports 0 updates. One failing event does not abort the run;
|
|
21
|
+
// failures are collected and reported (fail-loud at the caller).
|
|
22
|
+
//
|
|
23
|
+
// Snapshots of touched aggregates are dropped (they may cache plaintext);
|
|
24
|
+
// the next snapshotting load recreates them. AFTER a run, rebuild the
|
|
25
|
+
// affected projections — applyEntityEvent materializes ciphertext AND the
|
|
26
|
+
// blind-index columns, which keeps equality lookups (login by email) alive.
|
|
27
|
+
|
|
28
|
+
import { asRawClient } from "../../bun-db";
|
|
29
|
+
import { configuredEventPiiCatalog } from "../../crypto/event-pii";
|
|
30
|
+
import type { KmsContext, LocalKeyKmsAdapter, SubjectId } from "../../crypto/kms-adapter";
|
|
31
|
+
import { KeyErasedError } from "../../crypto/kms-adapter";
|
|
32
|
+
import {
|
|
33
|
+
configuredPiiSubjectKms,
|
|
34
|
+
encryptPiiValueForSubject,
|
|
35
|
+
isPiiCiphertext,
|
|
36
|
+
PII_ERASED_SENTINEL,
|
|
37
|
+
} from "../../crypto/pii-field-encryption";
|
|
38
|
+
import { collectPiiSubjectFields, resolveSubjectForField } from "../../crypto/subject-resolver";
|
|
39
|
+
import type { EntityDefinition, Registry, TenantId } from "../../engine/types";
|
|
40
|
+
import type { DbRunner } from "../connection";
|
|
41
|
+
|
|
42
|
+
const LIFECYCLE_VERBS = ["created", "updated", "deleted", "restored", "forgotten"] as const;
|
|
43
|
+
|
|
44
|
+
export type PiiBackfillFailure = {
|
|
45
|
+
readonly eventId: string;
|
|
46
|
+
readonly reason: string;
|
|
47
|
+
};
|
|
48
|
+
|
|
49
|
+
export type PiiBackfillResult = {
|
|
50
|
+
readonly scannedEvents: number;
|
|
51
|
+
readonly updatedEvents: number;
|
|
52
|
+
readonly encryptedFields: number;
|
|
53
|
+
readonly erasedFields: number;
|
|
54
|
+
readonly deletedSnapshots: number;
|
|
55
|
+
readonly failures: readonly PiiBackfillFailure[];
|
|
56
|
+
};
|
|
57
|
+
|
|
58
|
+
export type PiiBackfillOptions = {
|
|
59
|
+
readonly batchSize?: number;
|
|
60
|
+
// Scan + count only, write nothing.
|
|
61
|
+
readonly dryRun?: boolean;
|
|
62
|
+
};
|
|
63
|
+
|
|
64
|
+
type EventRow = {
|
|
65
|
+
readonly id: bigint | string;
|
|
66
|
+
readonly aggregate_id: string;
|
|
67
|
+
readonly aggregate_type: string;
|
|
68
|
+
readonly tenant_id: string;
|
|
69
|
+
readonly type: string;
|
|
70
|
+
readonly payload: Record<string, unknown>;
|
|
71
|
+
};
|
|
72
|
+
|
|
73
|
+
type FieldOutcome = "unchanged" | "encrypted" | "erased";
|
|
74
|
+
|
|
75
|
+
export async function backfillEventPiiEncryption(
|
|
76
|
+
db: DbRunner,
|
|
77
|
+
registry: Registry,
|
|
78
|
+
options: PiiBackfillOptions = {},
|
|
79
|
+
): Promise<PiiBackfillResult> {
|
|
80
|
+
const kms = configuredPiiSubjectKms();
|
|
81
|
+
if (!kms) {
|
|
82
|
+
throw new Error(
|
|
83
|
+
"backfillEventPiiEncryption requires a configured subject KMS — boot with " +
|
|
84
|
+
"runProdApp({ kms }) / configurePiiSubjectKms(adapter) before running the backfill.",
|
|
85
|
+
);
|
|
86
|
+
}
|
|
87
|
+
const batchSize = options.batchSize ?? 500;
|
|
88
|
+
const raw = asRawClient(db);
|
|
89
|
+
const kmsCtx: KmsContext = { requestId: "pii-backfill" };
|
|
90
|
+
|
|
91
|
+
const entityTargets = new Map<
|
|
92
|
+
string,
|
|
93
|
+
{ readonly entity: EntityDefinition; readonly piiFields: readonly string[] }
|
|
94
|
+
>();
|
|
95
|
+
for (const [name, entity] of registry.getAllEntities()) {
|
|
96
|
+
const piiFields = collectPiiSubjectFields(entity);
|
|
97
|
+
if (piiFields.length > 0) entityTargets.set(name, { entity, piiFields });
|
|
98
|
+
}
|
|
99
|
+
const eventCatalog = configuredEventPiiCatalog();
|
|
100
|
+
|
|
101
|
+
const aggregateTypes = [...entityTargets.keys()];
|
|
102
|
+
const catalogTypes = [...eventCatalog.keys()];
|
|
103
|
+
const result = {
|
|
104
|
+
scannedEvents: 0,
|
|
105
|
+
updatedEvents: 0,
|
|
106
|
+
encryptedFields: 0,
|
|
107
|
+
erasedFields: 0,
|
|
108
|
+
deletedSnapshots: 0,
|
|
109
|
+
failures: [] as PiiBackfillFailure[],
|
|
110
|
+
};
|
|
111
|
+
if (aggregateTypes.length === 0 && catalogTypes.length === 0) return result;
|
|
112
|
+
|
|
113
|
+
// Pre-KMS forgets left no key tombstone — the *.forgotten event on the
|
|
114
|
+
// stream is the only durable marker. Collect once; aggregate_id doubles
|
|
115
|
+
// as the user id for user-subject lookups.
|
|
116
|
+
const forgottenRows = (await raw.unsafe(
|
|
117
|
+
`SELECT DISTINCT "aggregate_id" FROM "kumiko_events" WHERE "type" LIKE '%.forgotten'`,
|
|
118
|
+
)) as ReadonlyArray<{ aggregate_id: string }>;
|
|
119
|
+
const forgottenAggregates = new Set(forgottenRows.map((r) => r.aggregate_id));
|
|
120
|
+
|
|
121
|
+
const touchedAggregates = new Set<string>();
|
|
122
|
+
let cursor = "0";
|
|
123
|
+
|
|
124
|
+
for (;;) {
|
|
125
|
+
const rows = (await raw.unsafe(
|
|
126
|
+
`SELECT "id", "aggregate_id", "aggregate_type", "tenant_id", "type", "payload"
|
|
127
|
+
FROM "kumiko_events"
|
|
128
|
+
WHERE ("aggregate_type" = ANY($1::text[]) OR "type" = ANY($2::text[])) AND "id" > $3::bigint
|
|
129
|
+
ORDER BY "id" ASC
|
|
130
|
+
LIMIT $4`,
|
|
131
|
+
[aggregateTypes, catalogTypes, cursor, batchSize],
|
|
132
|
+
)) as ReadonlyArray<EventRow>;
|
|
133
|
+
if (rows.length === 0) break;
|
|
134
|
+
|
|
135
|
+
for (const row of rows) {
|
|
136
|
+
result.scannedEvents++;
|
|
137
|
+
try {
|
|
138
|
+
const outcome = await transformEvent(row);
|
|
139
|
+
if (outcome === null) continue;
|
|
140
|
+
result.encryptedFields += outcome.encrypted;
|
|
141
|
+
result.erasedFields += outcome.erased;
|
|
142
|
+
if (!options.dryRun) {
|
|
143
|
+
await raw.unsafe(`UPDATE "kumiko_events" SET "payload" = $1::jsonb WHERE "id" = $2`, [
|
|
144
|
+
JSON.stringify(outcome.payload),
|
|
145
|
+
row.id,
|
|
146
|
+
]);
|
|
147
|
+
}
|
|
148
|
+
result.updatedEvents++;
|
|
149
|
+
touchedAggregates.add(row.aggregate_id);
|
|
150
|
+
} catch (e) {
|
|
151
|
+
result.failures.push({
|
|
152
|
+
eventId: String(row.id),
|
|
153
|
+
reason: e instanceof Error ? e.message : String(e),
|
|
154
|
+
});
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
const last = rows[rows.length - 1];
|
|
158
|
+
if (last === undefined) break;
|
|
159
|
+
cursor = String(last.id);
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
// Snapshots may cache the plaintext state of touched aggregates.
|
|
163
|
+
if (!options.dryRun && touchedAggregates.size > 0) {
|
|
164
|
+
const deleted = (await raw.unsafe(
|
|
165
|
+
`DELETE FROM "kumiko_snapshots" WHERE "aggregate_id" = ANY($1::uuid[]) RETURNING "aggregate_id"`,
|
|
166
|
+
[[...touchedAggregates]],
|
|
167
|
+
)) as ReadonlyArray<unknown>;
|
|
168
|
+
result.deletedSnapshots = deleted.length;
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
return result;
|
|
172
|
+
|
|
173
|
+
async function transformEvent(
|
|
174
|
+
row: EventRow,
|
|
175
|
+
): Promise<{ payload: Record<string, unknown>; encrypted: number; erased: number } | null> {
|
|
176
|
+
const counters = { encrypted: 0, erased: 0 };
|
|
177
|
+
const payload = structuredClone(row.payload);
|
|
178
|
+
|
|
179
|
+
const catalogFields = eventCatalog.get(row.type);
|
|
180
|
+
if (catalogFields) {
|
|
181
|
+
for (const [field, spec] of Object.entries(catalogFields)) {
|
|
182
|
+
const subjectId = payload[spec.subjectField];
|
|
183
|
+
if (typeof subjectId !== "string" || subjectId.length === 0) continue;
|
|
184
|
+
const outcome = await encryptField(payload, field, { kind: "user", userId: subjectId });
|
|
185
|
+
bump(outcome);
|
|
186
|
+
}
|
|
187
|
+
} else {
|
|
188
|
+
const target = entityTargets.get(row.aggregate_type);
|
|
189
|
+
if (!target || !isLifecycleEventOf(row.type, row.aggregate_type)) return null;
|
|
190
|
+
const sections = lifecycleSections(payload);
|
|
191
|
+
for (const section of sections) {
|
|
192
|
+
// Update-changes may carry a pii field without its owner field —
|
|
193
|
+
// resolve subjects from the merged view; aggregate_id backs the
|
|
194
|
+
// self-subject when a section lacks the id column.
|
|
195
|
+
const subjectSource: Record<string, unknown> = {
|
|
196
|
+
id: row.aggregate_id,
|
|
197
|
+
...Object.assign({}, ...sections),
|
|
198
|
+
...section,
|
|
199
|
+
};
|
|
200
|
+
for (const field of target.piiFields) {
|
|
201
|
+
const subject = resolveSubjectForField(target.entity, field, subjectSource, {
|
|
202
|
+
// @cast-boundary db-read — tenant_id column is the branded TenantId
|
|
203
|
+
tenantId: row.tenant_id as TenantId,
|
|
204
|
+
});
|
|
205
|
+
if (subject === null) continue;
|
|
206
|
+
const outcome = await encryptField(section, field, subject);
|
|
207
|
+
bump(outcome);
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
if (counters.encrypted === 0 && counters.erased === 0) return null;
|
|
213
|
+
return { payload, ...counters };
|
|
214
|
+
|
|
215
|
+
function bump(outcome: FieldOutcome): void {
|
|
216
|
+
if (outcome === "encrypted") counters.encrypted++;
|
|
217
|
+
if (outcome === "erased") counters.erased++;
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
async function encryptField(
|
|
221
|
+
section: Record<string, unknown>,
|
|
222
|
+
field: string,
|
|
223
|
+
subject: SubjectId,
|
|
224
|
+
): Promise<FieldOutcome> {
|
|
225
|
+
const value = section[field];
|
|
226
|
+
if (value === null || value === undefined) return "unchanged";
|
|
227
|
+
if (typeof value !== "string") return "unchanged";
|
|
228
|
+
if (isPiiCiphertext(value) || value === PII_ERASED_SENTINEL) return "unchanged";
|
|
229
|
+
if (isForgottenSubject(subject, row.aggregate_id)) {
|
|
230
|
+
section[field] = PII_ERASED_SENTINEL;
|
|
231
|
+
return "erased";
|
|
232
|
+
}
|
|
233
|
+
try {
|
|
234
|
+
section[field] = await encryptPiiValueForSubject(
|
|
235
|
+
kms as LocalKeyKmsAdapter,
|
|
236
|
+
subject,
|
|
237
|
+
value,
|
|
238
|
+
kmsCtx,
|
|
239
|
+
);
|
|
240
|
+
return "encrypted";
|
|
241
|
+
} catch (e) {
|
|
242
|
+
if (e instanceof KeyErasedError) {
|
|
243
|
+
section[field] = PII_ERASED_SENTINEL;
|
|
244
|
+
return "erased";
|
|
245
|
+
}
|
|
246
|
+
throw e;
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
function isForgottenSubject(subject: SubjectId, aggregateId: string): boolean {
|
|
252
|
+
if (forgottenAggregates.has(aggregateId)) return true;
|
|
253
|
+
return subject.kind === "user" && forgottenAggregates.has(subject.userId);
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
function isLifecycleEventOf(eventType: string, aggregateType: string): boolean {
|
|
258
|
+
if (!eventType.startsWith(`${aggregateType}.`)) return false;
|
|
259
|
+
const verb = eventType.slice(aggregateType.length + 1);
|
|
260
|
+
return (LIFECYCLE_VERBS as readonly string[]).includes(verb);
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
// created events carry the fields flat; updated carries { changes, previous };
|
|
264
|
+
// deleted/forgotten/restored carry { previous }. Returned sections are the
|
|
265
|
+
// mutable objects INSIDE the payload clone — encryptField writes in place.
|
|
266
|
+
function lifecycleSections(payload: Record<string, unknown>): Record<string, unknown>[] {
|
|
267
|
+
const sections: Record<string, unknown>[] = [];
|
|
268
|
+
if (isRecord(payload["changes"])) sections.push(payload["changes"]);
|
|
269
|
+
if (isRecord(payload["previous"])) sections.push(payload["previous"]);
|
|
270
|
+
if (sections.length === 0) sections.push(payload);
|
|
271
|
+
return sections;
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
275
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
276
|
+
}
|
package/src/db/table-builder.ts
CHANGED
|
@@ -69,7 +69,13 @@ function fieldToColumns(
|
|
|
69
69
|
const base = text(snakeName);
|
|
70
70
|
const withDefault =
|
|
71
71
|
"default" in field && field.default !== undefined ? base.default(field.default) : base;
|
|
72
|
-
|
|
72
|
+
const mainCol = { [name]: field.required ? withDefault.notNull() : withDefault };
|
|
73
|
+
// lookupable → nullable HMAC-Blind-Index-Spalte (lock-step mit
|
|
74
|
+
// fieldToColumnMeta in entity-table-meta.ts).
|
|
75
|
+
if (field.type === "text" && field.lookupable === true) {
|
|
76
|
+
return { ...mainCol, [`${name}Bidx`]: text(`${snakeName}_bidx`) };
|
|
77
|
+
}
|
|
78
|
+
return mainCol;
|
|
73
79
|
}
|
|
74
80
|
case "boolean":
|
|
75
81
|
return {
|
|
@@ -272,63 +278,72 @@ type NullCol<_T> = ColumnHandle & { readonly __notNull: false };
|
|
|
272
278
|
// columns (resolved via FileRef table). `notNull` propagiert von
|
|
273
279
|
// `field.required` (literal preserved by createXField generics).
|
|
274
280
|
type ColumnsForField<K extends string, F extends FieldDefinition> = F extends {
|
|
275
|
-
type: "text"
|
|
281
|
+
type: "text";
|
|
282
|
+
lookupable: true;
|
|
276
283
|
}
|
|
277
|
-
? F extends { required: true }
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
?
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
:
|
|
284
|
+
? (F extends { required: true }
|
|
285
|
+
? { readonly [P in K]: Col<string> }
|
|
286
|
+
: { readonly [P in K]: NullCol<string> }) & {
|
|
287
|
+
readonly [P in `${K}Bidx`]: NullCol<string>;
|
|
288
|
+
}
|
|
289
|
+
: F extends {
|
|
290
|
+
type: "text" | "select" | "tz";
|
|
291
|
+
}
|
|
292
|
+
? F extends { required: true }
|
|
293
|
+
? { readonly [P in K]: Col<string> }
|
|
294
|
+
: { readonly [P in K]: NullCol<string> }
|
|
295
|
+
: F extends { type: "boolean" }
|
|
296
|
+
? // boolean default OR required → notNull (DB has DEFAULT, structurally never-null)
|
|
297
|
+
F extends { default: boolean } | { required: true }
|
|
298
|
+
? { readonly [P in K]: Col<boolean> }
|
|
299
|
+
: { readonly [P in K]: NullCol<boolean> }
|
|
300
|
+
: F extends { type: "multiSelect" }
|
|
301
|
+
? // jsonb default `[]`, immer notNull
|
|
302
|
+
{ readonly [P in K]: Col<readonly string[]> }
|
|
303
|
+
: F extends { type: "number" }
|
|
293
304
|
? F extends { required: true }
|
|
294
305
|
? { readonly [P in K]: Col<number> }
|
|
295
306
|
: { readonly [P in K]: NullCol<number> }
|
|
296
|
-
: F extends { type: "
|
|
307
|
+
: F extends { type: "decimal" }
|
|
297
308
|
? F extends { required: true }
|
|
298
|
-
? { readonly [P in K]: Col<number> }
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
: F extends { type: "
|
|
311
|
-
?
|
|
312
|
-
{ readonly [P in K]: Col<
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
: F extends { type: "
|
|
309
|
+
? { readonly [P in K]: Col<number> }
|
|
310
|
+
: { readonly [P in K]: NullCol<number> }
|
|
311
|
+
: F extends { type: "money" }
|
|
312
|
+
? F extends { required: true }
|
|
313
|
+
? { readonly [P in K]: Col<number> } & {
|
|
314
|
+
readonly [P in `${K}Currency`]: Col<string>;
|
|
315
|
+
}
|
|
316
|
+
: { readonly [P in K]: NullCol<number> } & {
|
|
317
|
+
readonly [P in `${K}Currency`]: Col<string>;
|
|
318
|
+
}
|
|
319
|
+
: F extends { type: "reference"; multiple: true }
|
|
320
|
+
? { readonly [P in K]: Col<readonly string[]> }
|
|
321
|
+
: F extends { type: "reference" }
|
|
322
|
+
? F extends { required: true }
|
|
323
|
+
? { readonly [P in K]: Col<string> }
|
|
324
|
+
: { readonly [P in K]: NullCol<string> }
|
|
325
|
+
: F extends { type: "embedded" }
|
|
326
|
+
? // jsonb default `{}`, immer notNull
|
|
327
|
+
{ readonly [P in K]: Col<Readonly<Record<string, unknown>>> }
|
|
328
|
+
: F extends { type: "date" | "timestamp" }
|
|
318
329
|
? F extends { required: true }
|
|
319
|
-
? { readonly [P in
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
: { readonly [P in `${K}Utc`]: NullCol<Temporal.Instant> } & {
|
|
323
|
-
readonly [P in `${K}Tz`]: NullCol<string>;
|
|
324
|
-
}
|
|
325
|
-
: F extends { type: "file" | "image" }
|
|
330
|
+
? { readonly [P in K]: Col<Temporal.Instant> }
|
|
331
|
+
: { readonly [P in K]: NullCol<Temporal.Instant> }
|
|
332
|
+
: F extends { type: "locatedTimestamp" }
|
|
326
333
|
? F extends { required: true }
|
|
327
|
-
? { readonly [P in K]: Col<
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
334
|
+
? { readonly [P in `${K}Utc`]: Col<Temporal.Instant> } & {
|
|
335
|
+
readonly [P in `${K}Tz`]: Col<string>;
|
|
336
|
+
}
|
|
337
|
+
: { readonly [P in `${K}Utc`]: NullCol<Temporal.Instant> } & {
|
|
338
|
+
readonly [P in `${K}Tz`]: NullCol<string>;
|
|
339
|
+
}
|
|
340
|
+
: F extends { type: "file" | "image" }
|
|
341
|
+
? F extends { required: true }
|
|
342
|
+
? { readonly [P in K]: Col<string> }
|
|
343
|
+
: { readonly [P in K]: NullCol<string> }
|
|
344
|
+
: F extends { type: "files" | "images" }
|
|
345
|
+
? Record<never, never>
|
|
346
|
+
: never;
|
|
332
347
|
|
|
333
348
|
type UnionToIntersection<U> = (U extends unknown ? (k: U) => void : never) extends (
|
|
334
349
|
k: infer I,
|
|
@@ -526,6 +541,18 @@ export function buildEntityTable<E extends EntityDefinition>(
|
|
|
526
541
|
).on(column);
|
|
527
542
|
}
|
|
528
543
|
}
|
|
544
|
+
// lookupable-Felder: Index auf der bidx-Spalte (lock-step mit
|
|
545
|
+
// buildEntityTableMeta).
|
|
546
|
+
const bidxFieldByField = new Map<string, string>();
|
|
547
|
+
for (const [name, field] of Object.entries(entity.fields)) {
|
|
548
|
+
if (field.type !== "text" || field.lookupable !== true) continue;
|
|
549
|
+
bidxFieldByField.set(name, `${name}Bidx`);
|
|
550
|
+
const column = tHandle[`${name}Bidx`];
|
|
551
|
+
if (column) {
|
|
552
|
+
const idxName = `${tableName}_${toSnakeCase(name)}_bidx_idx`;
|
|
553
|
+
indexes[idxName] = index(idxName).on(column);
|
|
554
|
+
}
|
|
555
|
+
}
|
|
529
556
|
// entity.indexes = composite/unique-Indices die der Author explizit
|
|
530
557
|
// deklariert hat. Spalten werden via field-name (camelCase) angesprochen,
|
|
531
558
|
// der Index-Name folgt der Convention <table>_<col1>_<col2>_<unique|idx>
|
|
@@ -546,6 +573,26 @@ export function buildEntityTable<E extends EntityDefinition>(
|
|
|
546
573
|
chain = chain.where(def.where as SqlExpression);
|
|
547
574
|
}
|
|
548
575
|
indexes[indexName] = chain;
|
|
576
|
+
// Partielles bidx-Pendant für unique-Indices über lookupable-Spalten
|
|
577
|
+
// (lock-step mit buildEntityTableMeta).
|
|
578
|
+
if (def.unique === true && def.where === undefined) {
|
|
579
|
+
const bidxFieldNames = def.columns.map((c) => bidxFieldByField.get(c) ?? c);
|
|
580
|
+
if (bidxFieldNames.some((c, i) => c !== def.columns[i])) {
|
|
581
|
+
const bidxCols = bidxFieldNames
|
|
582
|
+
.map((c) => tHandle[c])
|
|
583
|
+
.filter((col): col is ColumnHandle => col !== undefined);
|
|
584
|
+
if (bidxCols.length === bidxFieldNames.length) {
|
|
585
|
+
const whereText = bidxFieldNames
|
|
586
|
+
.filter((c, i) => c !== def.columns[i])
|
|
587
|
+
.map((c) => `"${toSnakeCase(c)}" IS NOT NULL`)
|
|
588
|
+
.join(" AND ");
|
|
589
|
+
const partialWhere: SqlExpression = { kind: "sql-expr", text: whereText, params: [] };
|
|
590
|
+
indexes[`${indexName}_bidx`] = uniqueIndex(`${indexName}_bidx`)
|
|
591
|
+
.on(...bidxCols)
|
|
592
|
+
.where(partialWhere);
|
|
593
|
+
}
|
|
594
|
+
}
|
|
595
|
+
}
|
|
549
596
|
}
|
|
550
597
|
return indexes;
|
|
551
598
|
},
|
|
@@ -133,19 +133,36 @@ describe("validateBoot — PII annotations", () => {
|
|
|
133
133
|
);
|
|
134
134
|
});
|
|
135
135
|
|
|
136
|
-
test("userOwned.ownerField
|
|
136
|
+
test("userOwned.ownerField on a text field passes (ES userId-by-convention)", () => {
|
|
137
137
|
const feature = defineFeature("test", (r) => {
|
|
138
138
|
r.entity(
|
|
139
139
|
"comment",
|
|
140
140
|
createEntity({
|
|
141
141
|
fields: {
|
|
142
|
-
body: createLongTextField({ userOwned: { ownerField: "
|
|
143
|
-
|
|
142
|
+
body: createLongTextField({ userOwned: { ownerField: "authorId" } }),
|
|
143
|
+
authorId: createTextField(),
|
|
144
|
+
},
|
|
145
|
+
}),
|
|
146
|
+
);
|
|
147
|
+
});
|
|
148
|
+
expect(() => validateBoot([feature])).not.toThrow();
|
|
149
|
+
});
|
|
150
|
+
|
|
151
|
+
test("userOwned.ownerField pointing to a non-id-capable field throws", () => {
|
|
152
|
+
const feature = defineFeature("test", (r) => {
|
|
153
|
+
r.entity(
|
|
154
|
+
"comment",
|
|
155
|
+
createEntity({
|
|
156
|
+
fields: {
|
|
157
|
+
body: createLongTextField({ userOwned: { ownerField: "isPublic" } }),
|
|
158
|
+
isPublic: { type: "boolean" },
|
|
144
159
|
},
|
|
145
160
|
}),
|
|
146
161
|
);
|
|
147
162
|
});
|
|
148
|
-
expect(() => validateBoot([feature])).toThrow(
|
|
163
|
+
expect(() => validateBoot([feature])).toThrow(
|
|
164
|
+
/must be a reference or text \(userId\) field, got type "boolean"/,
|
|
165
|
+
);
|
|
149
166
|
});
|
|
150
167
|
|
|
151
168
|
test("userOwned.ownerField referencing non-user entity warns", () => {
|
|
@@ -568,3 +585,77 @@ describe("validateBoot — retention", () => {
|
|
|
568
585
|
expect(matchingWarn).toBeUndefined();
|
|
569
586
|
});
|
|
570
587
|
});
|
|
588
|
+
|
|
589
|
+
describe("validateBoot — lookupable / blind-index (#818)", () => {
|
|
590
|
+
test("lookupable on subject-annotated text field passes", () => {
|
|
591
|
+
const feature = defineFeature("test", (r) => {
|
|
592
|
+
r.entity(
|
|
593
|
+
"user",
|
|
594
|
+
createEntity({
|
|
595
|
+
fields: {
|
|
596
|
+
email: createTextField({ pii: true, lookupable: true }),
|
|
597
|
+
},
|
|
598
|
+
}),
|
|
599
|
+
);
|
|
600
|
+
});
|
|
601
|
+
expect(() => validateBoot([feature])).not.toThrow();
|
|
602
|
+
});
|
|
603
|
+
|
|
604
|
+
test("lookupable without subject annotation throws", () => {
|
|
605
|
+
const feature = defineFeature("test", (r) => {
|
|
606
|
+
r.entity(
|
|
607
|
+
"doc",
|
|
608
|
+
createEntity({
|
|
609
|
+
fields: {
|
|
610
|
+
slug: createTextField({ lookupable: true }),
|
|
611
|
+
},
|
|
612
|
+
}),
|
|
613
|
+
);
|
|
614
|
+
});
|
|
615
|
+
expect(() => validateBoot([feature])).toThrow(/lookupable.*without a subject annotation/);
|
|
616
|
+
});
|
|
617
|
+
|
|
618
|
+
test("lookupable on non-text field throws", () => {
|
|
619
|
+
const feature = defineFeature("test", (r) => {
|
|
620
|
+
r.entity(
|
|
621
|
+
"person",
|
|
622
|
+
createEntity({
|
|
623
|
+
fields: {
|
|
624
|
+
// longText carries PiiAnnotations too — lookupable must still be rejected.
|
|
625
|
+
bio: createLongTextField({ userOwned: { ownerField: "ownerId" }, lookupable: true }),
|
|
626
|
+
ownerId: createTextField({ required: true }),
|
|
627
|
+
},
|
|
628
|
+
}),
|
|
629
|
+
);
|
|
630
|
+
});
|
|
631
|
+
expect(() => validateBoot([feature])).toThrow(/only apply to text fields/);
|
|
632
|
+
});
|
|
633
|
+
|
|
634
|
+
test("searchable combined with a subject annotation throws", () => {
|
|
635
|
+
const feature = defineFeature("test", (r) => {
|
|
636
|
+
r.entity(
|
|
637
|
+
"user",
|
|
638
|
+
createEntity({
|
|
639
|
+
fields: {
|
|
640
|
+
displayName: createTextField({ pii: true, searchable: true }),
|
|
641
|
+
},
|
|
642
|
+
}),
|
|
643
|
+
);
|
|
644
|
+
});
|
|
645
|
+
expect(() => validateBoot([feature])).toThrow(/searchable.*cannot work/);
|
|
646
|
+
});
|
|
647
|
+
|
|
648
|
+
test("sortable combined with a subject annotation throws", () => {
|
|
649
|
+
const feature = defineFeature("test", (r) => {
|
|
650
|
+
r.entity(
|
|
651
|
+
"user",
|
|
652
|
+
createEntity({
|
|
653
|
+
fields: {
|
|
654
|
+
email: createTextField({ pii: true, sortable: true }),
|
|
655
|
+
},
|
|
656
|
+
}),
|
|
657
|
+
);
|
|
658
|
+
});
|
|
659
|
+
expect(() => validateBoot([feature])).toThrow(/sortable.*cannot work/);
|
|
660
|
+
});
|
|
661
|
+
});
|
|
@@ -3,7 +3,11 @@
|
|
|
3
3
|
// drizzle migrate-runner). See define-feature.ts / DX-4.
|
|
4
4
|
|
|
5
5
|
import { describe, expect, test } from "bun:test";
|
|
6
|
-
import {
|
|
6
|
+
import {
|
|
7
|
+
buildEntityTableMeta,
|
|
8
|
+
defineUnmanagedTable,
|
|
9
|
+
resolveTableName,
|
|
10
|
+
} from "../../db/entity-table-meta";
|
|
7
11
|
import { defineFeature } from "../define-feature";
|
|
8
12
|
import { createEntity, createTextField } from "../index";
|
|
9
13
|
import { createRegistry } from "../registry";
|
|
@@ -125,3 +129,44 @@ describe("createRegistry — unmanagedTable aggregation", () => {
|
|
|
125
129
|
);
|
|
126
130
|
});
|
|
127
131
|
});
|
|
132
|
+
|
|
133
|
+
describe("createRegistry — unmanaged tables with PII-annotated fields (#820)", () => {
|
|
134
|
+
const piiEntity = createEntity({
|
|
135
|
+
table: "ut_pii_probe",
|
|
136
|
+
fields: {
|
|
137
|
+
userId: createTextField({ required: true }),
|
|
138
|
+
ip: createTextField({ userOwned: { ownerField: "userId" } }),
|
|
139
|
+
},
|
|
140
|
+
});
|
|
141
|
+
const piiMeta = buildEntityTableMeta("ut-pii-probe", piiEntity);
|
|
142
|
+
|
|
143
|
+
test("buildEntityTableMeta records the subject-annotated field names", () => {
|
|
144
|
+
expect(piiMeta.piiSubjectFields).toEqual(["ip"]);
|
|
145
|
+
});
|
|
146
|
+
|
|
147
|
+
test("rejects a PII-carrying unmanaged table without piiEncryptedOnWrite", () => {
|
|
148
|
+
const feat = defineFeature("probe", (r) => {
|
|
149
|
+
r.unmanagedTable(piiMeta, { reason: "direct-write store" });
|
|
150
|
+
});
|
|
151
|
+
expect(() => createRegistry([feat])).toThrow(
|
|
152
|
+
/has PII-annotated fields \(ip\) but direct writes bypass the executor's PII encryption/,
|
|
153
|
+
);
|
|
154
|
+
});
|
|
155
|
+
|
|
156
|
+
test("accepts it once the feature declares piiEncryptedOnWrite", () => {
|
|
157
|
+
const feat = defineFeature("probe", (r) => {
|
|
158
|
+
r.unmanagedTable(piiMeta, {
|
|
159
|
+
reason: "direct-write store",
|
|
160
|
+
piiEncryptedOnWrite: true,
|
|
161
|
+
});
|
|
162
|
+
});
|
|
163
|
+
expect(() => createRegistry([feat])).not.toThrow();
|
|
164
|
+
});
|
|
165
|
+
|
|
166
|
+
test("a PII-free unmanaged table needs no declaration", () => {
|
|
167
|
+
const feat = defineFeature("probe", (r) => {
|
|
168
|
+
r.unmanagedTable(probeMeta, { reason: "plain store" });
|
|
169
|
+
});
|
|
170
|
+
expect(() => createRegistry([feat])).not.toThrow();
|
|
171
|
+
});
|
|
172
|
+
});
|