@cosmicdrift/kumiko-framework 0.118.0 → 0.120.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -2
- package/src/api/__tests__/pii-leak-guard.integration.test.ts +103 -0
- package/src/api/pii-leak-guard.ts +45 -0
- package/src/api/server.ts +5 -0
- package/src/bun-db/query.ts +27 -2
- package/src/crypto/__tests__/blind-index.test.ts +130 -0
- package/src/crypto/__tests__/event-pii.test.ts +161 -0
- package/src/crypto/__tests__/pg-kms-adapter.integration.test.ts +117 -0
- package/src/crypto/__tests__/pii-field-encryption.test.ts +216 -0
- package/src/crypto/blind-index.ts +114 -0
- package/src/crypto/event-pii.ts +69 -0
- package/src/crypto/index.ts +35 -0
- package/src/crypto/kms-adapter.ts +8 -0
- package/src/crypto/pg-kms-adapter.ts +173 -0
- package/src/crypto/pii-field-encryption.ts +195 -0
- package/src/db/__tests__/blind-index.integration.test.ts +232 -0
- package/src/db/__tests__/event-store-executor.integration.test.ts +169 -1
- package/src/db/__tests__/table-builder-meta-lockstep.test.ts +39 -0
- package/src/db/apply-entity-event.ts +8 -0
- package/src/db/blind-index-cleanup.ts +53 -0
- package/src/db/entity-table-meta.ts +47 -2
- package/src/db/event-store-executor.ts +125 -30
- package/src/db/index.ts +1 -0
- package/src/db/queries/backfill-pii.ts +276 -0
- package/src/db/table-builder.ts +98 -51
- package/src/engine/__tests__/boot-validator-pii-retention.test.ts +95 -4
- package/src/engine/__tests__/unmanaged-table.test.ts +46 -1
- package/src/engine/boot-validator/pii-retention.ts +45 -9
- package/src/engine/define-feature.ts +28 -3
- package/src/engine/registry.ts +25 -0
- package/src/engine/types/feature.ts +17 -2
- package/src/engine/types/fields.ts +6 -0
- package/src/engine/types/handlers.ts +7 -0
- package/src/engine/types/index.ts +2 -0
- package/src/event-store/__tests__/backfill-pii.integration.test.ts +279 -0
- package/src/event-store/event-store.ts +11 -6
- package/src/event-store/index.ts +6 -0
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
import { describe, expect, test } from "bun:test";
|
|
2
|
+
import { createEntity, createTextField } from "../../engine/factories";
|
|
3
|
+
import { InMemoryKmsAdapter } from "../in-memory-kms-adapter";
|
|
4
|
+
import { KeyNotFoundError, type KmsContext, subjectIdFromKey } from "../kms-adapter";
|
|
5
|
+
import {
|
|
6
|
+
decryptPiiFieldValues,
|
|
7
|
+
encryptPiiFieldValues,
|
|
8
|
+
isPiiCiphertext,
|
|
9
|
+
PII_ERASED_SENTINEL,
|
|
10
|
+
} from "../pii-field-encryption";
|
|
11
|
+
import { collectPiiSubjectFields } from "../subject-resolver";
|
|
12
|
+
|
|
13
|
+
const UUID_A = "6b2f4a0e-1c9d-4f3a-9d2e-00000000000a";
|
|
14
|
+
const UUID_B = "6b2f4a0e-1c9d-4f3a-9d2e-00000000000b";
|
|
15
|
+
const KMS_CTX: KmsContext = { requestId: "test" };
|
|
16
|
+
|
|
17
|
+
const userLikeEntity = createEntity({
|
|
18
|
+
fields: {
|
|
19
|
+
email: createTextField({ required: true, pii: true }),
|
|
20
|
+
role: createTextField(),
|
|
21
|
+
},
|
|
22
|
+
table: "pii_users",
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
const commentEntity = createEntity({
|
|
26
|
+
fields: {
|
|
27
|
+
body: createTextField({ userOwned: { ownerField: "authorId" } }),
|
|
28
|
+
authorId: createTextField({ required: true }),
|
|
29
|
+
},
|
|
30
|
+
table: "pii_comments",
|
|
31
|
+
});
|
|
32
|
+
|
|
33
|
+
const brandingEntity = createEntity({
|
|
34
|
+
fields: {
|
|
35
|
+
brandColor: createTextField({ tenantOwned: true }),
|
|
36
|
+
},
|
|
37
|
+
table: "pii_branding",
|
|
38
|
+
});
|
|
39
|
+
|
|
40
|
+
describe("subjectIdFromKey", () => {
|
|
41
|
+
test("round-trips user and tenant keys, rejects garbage", () => {
|
|
42
|
+
expect(subjectIdFromKey(`user:${UUID_A}`)).toEqual({ kind: "user", userId: UUID_A });
|
|
43
|
+
expect(subjectIdFromKey(`tenant:${UUID_B}`)).toEqual({ kind: "tenant", tenantId: UUID_B });
|
|
44
|
+
expect(() => subjectIdFromKey("widget:x")).toThrow(/Invalid subject key/);
|
|
45
|
+
});
|
|
46
|
+
});
|
|
47
|
+
|
|
48
|
+
describe("encryptPiiFieldValues / decryptPiiFieldValues", () => {
|
|
49
|
+
test("pii self-subject: DB value is sniffable ciphertext, decrypt restores plaintext", async () => {
|
|
50
|
+
const kms = new InMemoryKmsAdapter();
|
|
51
|
+
const fields = collectPiiSubjectFields(userLikeEntity);
|
|
52
|
+
const row = { id: UUID_A, email: "marc@example.com", role: "admin" };
|
|
53
|
+
|
|
54
|
+
const stored = await encryptPiiFieldValues(row, userLikeEntity, fields, kms, KMS_CTX);
|
|
55
|
+
expect(isPiiCiphertext(stored["email"])).toBe(true);
|
|
56
|
+
expect(String(stored["email"])).toStartWith(`kumiko-pii:v1:user:${UUID_A}:`);
|
|
57
|
+
expect(stored["role"]).toBe("admin");
|
|
58
|
+
expect(row["email"]).toBe("marc@example.com");
|
|
59
|
+
|
|
60
|
+
const read = await decryptPiiFieldValues(stored, fields, kms, KMS_CTX);
|
|
61
|
+
expect(read["email"]).toBe("marc@example.com");
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
test("first write auto-creates the subject key", async () => {
|
|
65
|
+
const kms = new InMemoryKmsAdapter();
|
|
66
|
+
await expect(kms.getKey({ kind: "user", userId: UUID_A })).rejects.toBeInstanceOf(
|
|
67
|
+
KeyNotFoundError,
|
|
68
|
+
);
|
|
69
|
+
await encryptPiiFieldValues(
|
|
70
|
+
{ id: UUID_A, email: "a@b.c" },
|
|
71
|
+
userLikeEntity,
|
|
72
|
+
["email"],
|
|
73
|
+
kms,
|
|
74
|
+
KMS_CTX,
|
|
75
|
+
);
|
|
76
|
+
const dek = await kms.getKey({ kind: "user", userId: UUID_A });
|
|
77
|
+
expect(dek.length).toBe(32);
|
|
78
|
+
});
|
|
79
|
+
|
|
80
|
+
test("userOwned: ciphertext names the owner subject, not the row", async () => {
|
|
81
|
+
const kms = new InMemoryKmsAdapter();
|
|
82
|
+
const stored = await encryptPiiFieldValues(
|
|
83
|
+
{ id: UUID_A, body: "secret note", authorId: UUID_B },
|
|
84
|
+
commentEntity,
|
|
85
|
+
["body"],
|
|
86
|
+
kms,
|
|
87
|
+
KMS_CTX,
|
|
88
|
+
);
|
|
89
|
+
expect(String(stored["body"])).toStartWith(`kumiko-pii:v1:user:${UUID_B}:`);
|
|
90
|
+
});
|
|
91
|
+
|
|
92
|
+
test("tenantOwned without tenantId column falls back to write-time tenant", async () => {
|
|
93
|
+
const kms = new InMemoryKmsAdapter();
|
|
94
|
+
const stored = await encryptPiiFieldValues(
|
|
95
|
+
{ id: UUID_A, brandColor: "#ff0000" },
|
|
96
|
+
brandingEntity,
|
|
97
|
+
["brandColor"],
|
|
98
|
+
kms,
|
|
99
|
+
KMS_CTX,
|
|
100
|
+
{ tenantId: UUID_B },
|
|
101
|
+
);
|
|
102
|
+
expect(String(stored["brandColor"])).toStartWith(`kumiko-pii:v1:tenant:${UUID_B}:`);
|
|
103
|
+
});
|
|
104
|
+
|
|
105
|
+
test("subjectSource resolves the owner when the partial row lacks it (update changes)", async () => {
|
|
106
|
+
const kms = new InMemoryKmsAdapter();
|
|
107
|
+
const changes = { body: "edited" };
|
|
108
|
+
const stored = await encryptPiiFieldValues(changes, commentEntity, ["body"], kms, KMS_CTX, {
|
|
109
|
+
onlyKeys: ["body"],
|
|
110
|
+
subjectSource: { id: UUID_A, body: "edited", authorId: UUID_B },
|
|
111
|
+
});
|
|
112
|
+
expect(String(stored["body"])).toStartWith(`kumiko-pii:v1:user:${UUID_B}:`);
|
|
113
|
+
});
|
|
114
|
+
|
|
115
|
+
test("erased subject: decrypt yields the sentinel, re-encrypt passes it through", async () => {
|
|
116
|
+
const kms = new InMemoryKmsAdapter();
|
|
117
|
+
const fields = ["email"] as const;
|
|
118
|
+
const stored = await encryptPiiFieldValues(
|
|
119
|
+
{ id: UUID_A, email: "gone@example.com" },
|
|
120
|
+
userLikeEntity,
|
|
121
|
+
fields,
|
|
122
|
+
kms,
|
|
123
|
+
KMS_CTX,
|
|
124
|
+
);
|
|
125
|
+
await kms.eraseKey({ kind: "user", userId: UUID_A });
|
|
126
|
+
|
|
127
|
+
const read = await decryptPiiFieldValues(stored, fields, kms, KMS_CTX);
|
|
128
|
+
expect(read["email"]).toBe(PII_ERASED_SENTINEL);
|
|
129
|
+
|
|
130
|
+
const reStored = await encryptPiiFieldValues(read, userLikeEntity, fields, kms, KMS_CTX);
|
|
131
|
+
expect(reStored["email"]).toBe(PII_ERASED_SENTINEL);
|
|
132
|
+
});
|
|
133
|
+
|
|
134
|
+
test("write to an erased subject fails (forget is permanent)", async () => {
|
|
135
|
+
const kms = new InMemoryKmsAdapter();
|
|
136
|
+
await kms.createKey({ kind: "user", userId: UUID_A });
|
|
137
|
+
await kms.eraseKey({ kind: "user", userId: UUID_A });
|
|
138
|
+
await expect(
|
|
139
|
+
encryptPiiFieldValues(
|
|
140
|
+
{ id: UUID_A, email: "new@b.c" },
|
|
141
|
+
userLikeEntity,
|
|
142
|
+
["email"],
|
|
143
|
+
kms,
|
|
144
|
+
KMS_CTX,
|
|
145
|
+
),
|
|
146
|
+
).rejects.toThrow(/erased/);
|
|
147
|
+
});
|
|
148
|
+
|
|
149
|
+
test("legacy plaintext rows pass through decrypt unchanged", async () => {
|
|
150
|
+
const kms = new InMemoryKmsAdapter();
|
|
151
|
+
const read = await decryptPiiFieldValues(
|
|
152
|
+
{ id: UUID_A, email: "legacy@example.com" },
|
|
153
|
+
["email"],
|
|
154
|
+
kms,
|
|
155
|
+
KMS_CTX,
|
|
156
|
+
);
|
|
157
|
+
expect(read["email"]).toBe("legacy@example.com");
|
|
158
|
+
});
|
|
159
|
+
|
|
160
|
+
test("already-ciphertext values are not double-encrypted (re-encrypt paths)", async () => {
|
|
161
|
+
const kms = new InMemoryKmsAdapter();
|
|
162
|
+
const once = await encryptPiiFieldValues(
|
|
163
|
+
{ id: UUID_A, email: "a@b.c" },
|
|
164
|
+
userLikeEntity,
|
|
165
|
+
["email"],
|
|
166
|
+
kms,
|
|
167
|
+
KMS_CTX,
|
|
168
|
+
);
|
|
169
|
+
const twice = await encryptPiiFieldValues(once, userLikeEntity, ["email"], kms, KMS_CTX);
|
|
170
|
+
expect(twice["email"]).toBe(once["email"]);
|
|
171
|
+
});
|
|
172
|
+
|
|
173
|
+
test("null values and fields outside onlyKeys are skipped", async () => {
|
|
174
|
+
const kms = new InMemoryKmsAdapter();
|
|
175
|
+
const stored = await encryptPiiFieldValues(
|
|
176
|
+
{ id: UUID_A, email: null, role: "x" },
|
|
177
|
+
userLikeEntity,
|
|
178
|
+
["email"],
|
|
179
|
+
kms,
|
|
180
|
+
KMS_CTX,
|
|
181
|
+
);
|
|
182
|
+
expect(stored["email"]).toBeNull();
|
|
183
|
+
|
|
184
|
+
const skipped = await encryptPiiFieldValues(
|
|
185
|
+
{ id: UUID_A, email: "a@b.c" },
|
|
186
|
+
userLikeEntity,
|
|
187
|
+
["email"],
|
|
188
|
+
kms,
|
|
189
|
+
KMS_CTX,
|
|
190
|
+
{ onlyKeys: ["role"] },
|
|
191
|
+
);
|
|
192
|
+
expect(skipped["email"]).toBe("a@b.c");
|
|
193
|
+
});
|
|
194
|
+
|
|
195
|
+
test("non-string pii value throws", async () => {
|
|
196
|
+
const kms = new InMemoryKmsAdapter();
|
|
197
|
+
await expect(
|
|
198
|
+
encryptPiiFieldValues({ id: UUID_A, email: 42 }, userLikeEntity, ["email"], kms, KMS_CTX),
|
|
199
|
+
).rejects.toThrow(/must be a string/);
|
|
200
|
+
});
|
|
201
|
+
|
|
202
|
+
test("ciphertext without a key row fails loud (wrong key store, not shredded)", async () => {
|
|
203
|
+
const kmsA = new InMemoryKmsAdapter();
|
|
204
|
+
const stored = await encryptPiiFieldValues(
|
|
205
|
+
{ id: UUID_A, email: "a@b.c" },
|
|
206
|
+
userLikeEntity,
|
|
207
|
+
["email"],
|
|
208
|
+
kmsA,
|
|
209
|
+
KMS_CTX,
|
|
210
|
+
);
|
|
211
|
+
const kmsB = new InMemoryKmsAdapter();
|
|
212
|
+
await expect(decryptPiiFieldValues(stored, ["email"], kmsB, KMS_CTX)).rejects.toBeInstanceOf(
|
|
213
|
+
KeyNotFoundError,
|
|
214
|
+
);
|
|
215
|
+
});
|
|
216
|
+
});
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
// Blind-Index für PII-Equality-Lookups (kumiko-framework#818).
|
|
2
|
+
// `lookupable: true` auf einem Subject-annotierten text-Feld erzeugt eine
|
|
3
|
+
// generierte Spalte `<snake>_bidx` mit deterministischem HMAC über den
|
|
4
|
+
// KLARTEXT-Wert — Equality-Queries matchen `(col = $1 OR col_bidx = $2)`
|
|
5
|
+
// und funktionieren damit für Klartext-Alt-Rows UND verschlüsselte Rows.
|
|
6
|
+
//
|
|
7
|
+
// Wertformat: kumiko-bidx:v1:<base64url(HMAC-SHA256(key, value))>
|
|
8
|
+
//
|
|
9
|
+
// HMAC ist byte-exact ohne Normalisierung — repliziert exakt die heutige
|
|
10
|
+
// case-sensitive Equality-Semantik (Lowercasing wäre eine Verhaltensänderung).
|
|
11
|
+
//
|
|
12
|
+
// Der Key ist bewusst ein EIGENER 32-Byte-Key (env: KUMIKO_BLIND_INDEX_KEY),
|
|
13
|
+
// nicht vom PLATFORM_KEK abgeleitet und nicht Teil des KmsAdapter-Contracts —
|
|
14
|
+
// BYOK-Adapter dürfen ihn nie sehen. Rotation = Recompute aller bidx-Spalten
|
|
15
|
+
// (Rollout-Fahrplan Schritt 7).
|
|
16
|
+
//
|
|
17
|
+
// bidx-Werte leben NUR in der Projektion, nie in Event-Payloads: ein
|
|
18
|
+
// deterministischer HMAC im immutablen Event wäre permanente Linkage und
|
|
19
|
+
// bräche Forget. applyEntityEvent berechnet sie beim Apply (live + rebuild),
|
|
20
|
+
// nach Key-Erase recomputet der Rebuild sie zu NULL.
|
|
21
|
+
|
|
22
|
+
import { createHmac } from "node:crypto";
|
|
23
|
+
import { requestContext } from "../api/request-context";
|
|
24
|
+
import type { EntityDefinition } from "../engine/types/fields";
|
|
25
|
+
import {
|
|
26
|
+
configuredPiiSubjectKms,
|
|
27
|
+
decryptPiiFieldValues,
|
|
28
|
+
isPiiCiphertext,
|
|
29
|
+
PII_ERASED_SENTINEL,
|
|
30
|
+
} from "./pii-field-encryption";
|
|
31
|
+
|
|
32
|
+
const BLIND_INDEX_PREFIX = "kumiko-bidx:v1:";
|
|
33
|
+
const BLIND_INDEX_KEY_LENGTH = 32;
|
|
34
|
+
|
|
35
|
+
export function computeBlindIndex(key: Uint8Array, plaintext: string): string {
|
|
36
|
+
const mac = createHmac("sha256", key).update(plaintext, "utf8").digest("base64url");
|
|
37
|
+
return `${BLIND_INDEX_PREFIX}${mac}`;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export function decodeBlindIndexKey(base64Key: string): Uint8Array {
|
|
41
|
+
const decoded = Buffer.from(base64Key, "base64");
|
|
42
|
+
if (decoded.length !== BLIND_INDEX_KEY_LENGTH) {
|
|
43
|
+
throw new Error(
|
|
44
|
+
`Blind-index key must be ${BLIND_INDEX_KEY_LENGTH} bytes base64-encoded, got ${decoded.length} bytes. ` +
|
|
45
|
+
`Generate one: openssl rand -base64 32`,
|
|
46
|
+
);
|
|
47
|
+
}
|
|
48
|
+
return decoded;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
// Boot-injected app-wide key, mirroring configurePiiSubjectKms: run{Prod,Dev}App
|
|
52
|
+
// call configureBlindIndexKey once; query compilers + applyEntityEvent resolve
|
|
53
|
+
// lazily. Absent key = blind-indexing off (bidx columns stay NULL, equality
|
|
54
|
+
// lookups match plaintext rows only — pre-#818 behavior).
|
|
55
|
+
let injectedKey: Uint8Array | undefined;
|
|
56
|
+
|
|
57
|
+
export function configureBlindIndexKey(base64Key: string | undefined): void {
|
|
58
|
+
injectedKey = base64Key === undefined ? undefined : decodeBlindIndexKey(base64Key);
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
export function configuredBlindIndexKey(): Uint8Array | undefined {
|
|
62
|
+
return injectedKey;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/** @internal test-only */
|
|
66
|
+
export function resetBlindIndexKeyForTests(): void {
|
|
67
|
+
injectedKey = undefined;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
export function blindIndexFieldName(fieldName: string): string {
|
|
71
|
+
return `${fieldName}Bidx`;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
// The field names that carry a blind index — precomputed per apply like
|
|
75
|
+
// collectPiiSubjectFields. Only text fields qualify (boot-validated).
|
|
76
|
+
export function collectLookupableFields(entity: EntityDefinition): readonly string[] {
|
|
77
|
+
return Object.entries(entity.fields)
|
|
78
|
+
.filter(([, field]) => field.type === "text" && field.lookupable === true)
|
|
79
|
+
.map(([name]) => name);
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
// bidx values for every lookupable field PRESENT in `values` (create payload
|
|
83
|
+
// or update changes — absent fields stay untouched). Ciphertext values are
|
|
84
|
+
// decrypted first so the HMAC always covers the plaintext; an erased subject
|
|
85
|
+
// (or the erased sentinel) yields NULL — the lookup stops matching, which is
|
|
86
|
+
// exactly the forget semantics.
|
|
87
|
+
export async function computeBlindIndexValues(
|
|
88
|
+
values: Record<string, unknown>,
|
|
89
|
+
lookupableFields: readonly string[],
|
|
90
|
+
): Promise<Record<string, unknown>> {
|
|
91
|
+
const key = configuredBlindIndexKey();
|
|
92
|
+
if (key === undefined || lookupableFields.length === 0) return {};
|
|
93
|
+
const out: Record<string, unknown> = {};
|
|
94
|
+
for (const name of lookupableFields) {
|
|
95
|
+
if (!(name in values)) continue;
|
|
96
|
+
out[blindIndexFieldName(name)] = await blindIndexForValue(key, values[name]);
|
|
97
|
+
}
|
|
98
|
+
return out;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
async function blindIndexForValue(key: Uint8Array, value: unknown): Promise<string | null> {
|
|
102
|
+
if (typeof value !== "string" || value === PII_ERASED_SENTINEL) return null;
|
|
103
|
+
if (!isPiiCiphertext(value)) return computeBlindIndex(key, value);
|
|
104
|
+
const kms = configuredPiiSubjectKms();
|
|
105
|
+
// Ciphertext without a KMS can't be decrypted here; the same read would
|
|
106
|
+
// also surface raw ciphertext — misconfiguration is caught at boot.
|
|
107
|
+
if (kms === undefined) return null;
|
|
108
|
+
const decrypted = await decryptPiiFieldValues({ value }, ["value"], kms, {
|
|
109
|
+
requestId: requestContext.get()?.requestId ?? "blind-index",
|
|
110
|
+
});
|
|
111
|
+
const plain = decrypted["value"];
|
|
112
|
+
if (typeof plain !== "string" || plain === PII_ERASED_SENTINEL) return null;
|
|
113
|
+
return computeBlindIndex(key, plain);
|
|
114
|
+
}
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
// PII on custom-event payloads (#799). Entity CRUD events get their PII
|
|
2
|
+
// encrypted by the executor; events written via ctx.appendEvent / MSP-apply /
|
|
3
|
+
// low-level append() (delivery attempt-log, jobs run-logger) had no encrypt
|
|
4
|
+
// path at all. `r.defineEvent(name, schema, { piiFields })` declares which
|
|
5
|
+
// payload fields are PII and which payload field names the owning user;
|
|
6
|
+
// createRegistry publishes the catalog and append() — the single write funnel
|
|
7
|
+
// into kumiko_events — encrypts every catalogued field. No caller can forget.
|
|
8
|
+
|
|
9
|
+
import { requestContext } from "../api/request-context";
|
|
10
|
+
import type { EventPiiFields } from "../engine/types/handlers";
|
|
11
|
+
import { configuredPiiSubjectKms, encryptPiiValueForSubject } from "./pii-field-encryption";
|
|
12
|
+
|
|
13
|
+
export type EventPiiCatalog = ReadonlyMap<string, EventPiiFields>;
|
|
14
|
+
|
|
15
|
+
// Boot-injected like configurePiiSubjectKms — createRegistry calls this with
|
|
16
|
+
// the catalog collected from all defineEvent registrations.
|
|
17
|
+
let catalog: EventPiiCatalog = new Map();
|
|
18
|
+
|
|
19
|
+
export function configureEventPiiCatalog(next: EventPiiCatalog): void {
|
|
20
|
+
catalog = next;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export function configuredEventPiiCatalog(): EventPiiCatalog {
|
|
24
|
+
return catalog;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/** @internal test-only */
|
|
28
|
+
export function resetEventPiiCatalogForTests(): void {
|
|
29
|
+
catalog = new Map();
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
// Encrypts catalogued payload fields under the owning user's DEK. No-op when
|
|
33
|
+
// the event type is uncatalogued or no subject KMS is configured (plaintext
|
|
34
|
+
// rollout mode — the hard boot gate governs whether that is acceptable).
|
|
35
|
+
// A null/absent subject field (system cron runs, recipient-less skip
|
|
36
|
+
// attempts) leaves the value plaintext: there is no user key to shred.
|
|
37
|
+
export async function encryptEventPayloadPii(
|
|
38
|
+
eventType: string,
|
|
39
|
+
payload: Record<string, unknown>,
|
|
40
|
+
): Promise<Record<string, unknown>> {
|
|
41
|
+
const piiFields = catalog.get(eventType);
|
|
42
|
+
if (!piiFields) return payload;
|
|
43
|
+
const kms = configuredPiiSubjectKms();
|
|
44
|
+
if (!kms) return payload;
|
|
45
|
+
|
|
46
|
+
let out: Record<string, unknown> | undefined;
|
|
47
|
+
for (const [field, spec] of Object.entries(piiFields)) {
|
|
48
|
+
const value = payload[field];
|
|
49
|
+
if (value === null || value === undefined) continue;
|
|
50
|
+
if (typeof value !== "string") {
|
|
51
|
+
throw new Error(
|
|
52
|
+
`Event "${eventType}" piiFields."${field}" must be a string payload field, got ${typeof value}`,
|
|
53
|
+
);
|
|
54
|
+
}
|
|
55
|
+
const subjectId = payload[spec.subjectField];
|
|
56
|
+
if (typeof subjectId !== "string" || subjectId.length === 0) continue;
|
|
57
|
+
const encrypted = await encryptPiiValueForSubject(
|
|
58
|
+
kms,
|
|
59
|
+
{ kind: "user", userId: subjectId },
|
|
60
|
+
value,
|
|
61
|
+
{ requestId: requestContext.get()?.requestId ?? "append-event" },
|
|
62
|
+
);
|
|
63
|
+
if (encrypted !== value) {
|
|
64
|
+
out ??= { ...payload };
|
|
65
|
+
out[field] = encrypted;
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
return out ?? payload;
|
|
69
|
+
}
|
package/src/crypto/index.ts
CHANGED
|
@@ -1,3 +1,20 @@
|
|
|
1
|
+
export {
|
|
2
|
+
blindIndexFieldName,
|
|
3
|
+
collectLookupableFields,
|
|
4
|
+
computeBlindIndex,
|
|
5
|
+
computeBlindIndexValues,
|
|
6
|
+
configureBlindIndexKey,
|
|
7
|
+
configuredBlindIndexKey,
|
|
8
|
+
decodeBlindIndexKey,
|
|
9
|
+
resetBlindIndexKeyForTests,
|
|
10
|
+
} from "./blind-index";
|
|
11
|
+
export {
|
|
12
|
+
configuredEventPiiCatalog,
|
|
13
|
+
configureEventPiiCatalog,
|
|
14
|
+
type EventPiiCatalog,
|
|
15
|
+
encryptEventPayloadPii,
|
|
16
|
+
resetEventPiiCatalogForTests,
|
|
17
|
+
} from "./event-pii";
|
|
1
18
|
export { InMemoryKmsAdapter } from "./in-memory-kms-adapter";
|
|
2
19
|
export {
|
|
3
20
|
isLocalKeyKmsAdapter,
|
|
@@ -12,10 +29,28 @@ export {
|
|
|
12
29
|
type SubjectDek,
|
|
13
30
|
type SubjectId,
|
|
14
31
|
type SubjectKey,
|
|
32
|
+
subjectIdFromKey,
|
|
15
33
|
subjectIdToKey,
|
|
16
34
|
subjectKeyForTenant,
|
|
17
35
|
subjectKeyForUser,
|
|
18
36
|
} from "./kms-adapter";
|
|
37
|
+
export {
|
|
38
|
+
createPgKmsAdapter,
|
|
39
|
+
PgKmsAdapter,
|
|
40
|
+
type PgKmsAdapterOptions,
|
|
41
|
+
} from "./pg-kms-adapter";
|
|
42
|
+
export {
|
|
43
|
+
configuredPiiSubjectKms,
|
|
44
|
+
configurePiiSubjectKms,
|
|
45
|
+
decryptPiiFieldValues,
|
|
46
|
+
type EncryptPiiOptions,
|
|
47
|
+
encryptPiiFieldValues,
|
|
48
|
+
encryptPiiValueForSubject,
|
|
49
|
+
isPiiCiphertext,
|
|
50
|
+
PII_CIPHERTEXT_PREFIX,
|
|
51
|
+
PII_ERASED_SENTINEL,
|
|
52
|
+
resetPiiSubjectKmsForTests,
|
|
53
|
+
} from "./pii-field-encryption";
|
|
19
54
|
export {
|
|
20
55
|
createRequestKmsCache,
|
|
21
56
|
type RequestKmsCache,
|
|
@@ -24,6 +24,14 @@ export function subjectIdToKey(subject: SubjectId): SubjectKey {
|
|
|
24
24
|
: subjectKeyForTenant(subject.tenantId);
|
|
25
25
|
}
|
|
26
26
|
|
|
27
|
+
export function subjectIdFromKey(key: SubjectKey): SubjectId {
|
|
28
|
+
if (key.startsWith("user:")) return { kind: "user", userId: key.slice("user:".length) };
|
|
29
|
+
if (key.startsWith("tenant:")) {
|
|
30
|
+
return { kind: "tenant", tenantId: key.slice("tenant:".length) as TenantId }; // @cast-boundary parse of a key this module minted
|
|
31
|
+
}
|
|
32
|
+
throw new Error(`Invalid subject key: ${key}`);
|
|
33
|
+
}
|
|
34
|
+
|
|
27
35
|
export interface KmsContext {
|
|
28
36
|
readonly tenantId?: TenantId;
|
|
29
37
|
readonly requestId: string;
|
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
import { createCipheriv, createDecipheriv, randomBytes } from "node:crypto";
|
|
2
|
+
import postgres from "postgres";
|
|
3
|
+
import {
|
|
4
|
+
KeyAlreadyExistsError,
|
|
5
|
+
KeyErasedError,
|
|
6
|
+
KeyNotFoundError,
|
|
7
|
+
type KmsContext,
|
|
8
|
+
type KmsHealth,
|
|
9
|
+
type LocalKeyKmsAdapter,
|
|
10
|
+
type SubjectDek,
|
|
11
|
+
type SubjectId,
|
|
12
|
+
subjectIdToKey,
|
|
13
|
+
} from "./kms-adapter";
|
|
14
|
+
|
|
15
|
+
const IV_LENGTH = 12;
|
|
16
|
+
const TAG_LENGTH = 16;
|
|
17
|
+
|
|
18
|
+
// Envelope layout [iv:12][tag:16][ct] — the platform KEK wraps each subject
|
|
19
|
+
// DEK at rest, so a subject-keys-DB dump alone reveals no key material.
|
|
20
|
+
function wrapDek(kek: Buffer, dek: Buffer): Buffer {
|
|
21
|
+
const iv = randomBytes(IV_LENGTH);
|
|
22
|
+
const cipher = createCipheriv("aes-256-gcm", kek, iv);
|
|
23
|
+
const ciphertext = Buffer.concat([cipher.update(dek), cipher.final()]);
|
|
24
|
+
return Buffer.concat([iv, cipher.getAuthTag(), ciphertext]);
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
function unwrapDek(kek: Buffer, wrapped: Uint8Array): SubjectDek {
|
|
28
|
+
const envelope = Buffer.from(wrapped);
|
|
29
|
+
const iv = envelope.subarray(0, IV_LENGTH);
|
|
30
|
+
const tag = envelope.subarray(IV_LENGTH, IV_LENGTH + TAG_LENGTH);
|
|
31
|
+
const ciphertext = envelope.subarray(IV_LENGTH + TAG_LENGTH);
|
|
32
|
+
const decipher = createDecipheriv("aes-256-gcm", kek, iv);
|
|
33
|
+
decipher.setAuthTag(tag);
|
|
34
|
+
return Buffer.concat([decipher.update(ciphertext), decipher.final()]);
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
function decodePlatformKek(base64: string): Buffer {
|
|
38
|
+
const kek = Buffer.from(base64, "base64");
|
|
39
|
+
if (kek.length !== 32) {
|
|
40
|
+
throw new Error(
|
|
41
|
+
`PgKmsAdapter: platformKek must decode to 32 bytes, got ${kek.length} — expected a base64-encoded AES-256 key (openssl rand -base64 32)`,
|
|
42
|
+
);
|
|
43
|
+
}
|
|
44
|
+
return kek;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
const PG_UNIQUE_VIOLATION = "23505";
|
|
48
|
+
|
|
49
|
+
function isPgUniqueViolation(error: unknown): boolean {
|
|
50
|
+
return (
|
|
51
|
+
typeof error === "object" &&
|
|
52
|
+
error !== null &&
|
|
53
|
+
"code" in error &&
|
|
54
|
+
error.code === PG_UNIQUE_VIOLATION
|
|
55
|
+
);
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
interface SubjectKeyRow {
|
|
59
|
+
cipher_key: Uint8Array | null;
|
|
60
|
+
erased: boolean;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
export interface PgKmsAdapterOptions {
|
|
64
|
+
/** Connection string of the DEDICATED subject-keys cluster — never the app DB (its backup retention must stay shorter, see kms-adapter.md). */
|
|
65
|
+
readonly databaseUrl: string;
|
|
66
|
+
/** Base64-encoded 32-byte platform KEK (PLATFORM_KEK env var). */
|
|
67
|
+
readonly platformKek: string;
|
|
68
|
+
readonly maxConnections?: number;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
// Default production adapter: DEKs live in a separate Postgres cluster,
|
|
72
|
+
// KEK-wrapped at rest. Schema is created lazily on first use — the
|
|
73
|
+
// subject-keys cluster has no drizzle migration pipeline of its own.
|
|
74
|
+
export class PgKmsAdapter implements LocalKeyKmsAdapter {
|
|
75
|
+
readonly capabilities = { mode: "local-key" } as const;
|
|
76
|
+
|
|
77
|
+
private readonly sql: ReturnType<typeof postgres>;
|
|
78
|
+
private readonly kek: Buffer;
|
|
79
|
+
private schemaReady: Promise<void> | undefined;
|
|
80
|
+
|
|
81
|
+
constructor(options: PgKmsAdapterOptions) {
|
|
82
|
+
this.kek = decodePlatformKek(options.platformKek);
|
|
83
|
+
this.sql = postgres(options.databaseUrl, {
|
|
84
|
+
max: options.maxConnections ?? 4,
|
|
85
|
+
// The connection is exclusive to this adapter and its only DDL is
|
|
86
|
+
// idempotent IF NOT EXISTS — notices are pure boot-log noise.
|
|
87
|
+
onnotice: () => {},
|
|
88
|
+
});
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
async createKey(subject: SubjectId, ctx: KmsContext): Promise<void> {
|
|
92
|
+
await this.ensureSchema();
|
|
93
|
+
const wrapped = wrapDek(this.kek, randomBytes(32));
|
|
94
|
+
try {
|
|
95
|
+
await this.sql`
|
|
96
|
+
INSERT INTO kumiko_subject_keys (subject_id, cipher_key, created_by)
|
|
97
|
+
VALUES (${subjectIdToKey(subject)}, ${wrapped}, ${ctx.userId ?? null})`;
|
|
98
|
+
} catch (error) {
|
|
99
|
+
if (isPgUniqueViolation(error)) throw new KeyAlreadyExistsError(subject);
|
|
100
|
+
throw error;
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
async getKey(subject: SubjectId, _ctx: KmsContext): Promise<SubjectDek> {
|
|
105
|
+
await this.ensureSchema();
|
|
106
|
+
const rows = await this.sql<SubjectKeyRow[]>`
|
|
107
|
+
SELECT cipher_key, (erased_at IS NOT NULL) AS erased
|
|
108
|
+
FROM kumiko_subject_keys
|
|
109
|
+
WHERE subject_id = ${subjectIdToKey(subject)}`;
|
|
110
|
+
const row = rows[0];
|
|
111
|
+
if (!row) throw new KeyNotFoundError(subject);
|
|
112
|
+
if (row.erased || row.cipher_key === null) throw new KeyErasedError(subject);
|
|
113
|
+
return unwrapDek(this.kek, row.cipher_key);
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
async eraseKey(subject: SubjectId, ctx: KmsContext): Promise<void> {
|
|
117
|
+
await this.ensureSchema();
|
|
118
|
+
// Guard on erased_at IS NULL keeps repeat erases from overwriting the
|
|
119
|
+
// original tombstone audit fields; unknown subjects update zero rows.
|
|
120
|
+
await this.sql`
|
|
121
|
+
UPDATE kumiko_subject_keys
|
|
122
|
+
SET cipher_key = NULL,
|
|
123
|
+
erased_at = now(),
|
|
124
|
+
erased_by = ${ctx.userId ?? null},
|
|
125
|
+
erase_reason = ${ctx.eraseReason ?? null}
|
|
126
|
+
WHERE subject_id = ${subjectIdToKey(subject)} AND erased_at IS NULL`;
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
async health(): Promise<KmsHealth> {
|
|
130
|
+
const start = Date.now();
|
|
131
|
+
await this.ensureSchema();
|
|
132
|
+
await this.sql`SELECT 1`;
|
|
133
|
+
return { ok: true, latencyMs: Date.now() - start };
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
async close(): Promise<void> {
|
|
137
|
+
await this.sql.end();
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
private ensureSchema(): Promise<void> {
|
|
141
|
+
// A failed attempt resets the memo so a transient DB outage does not
|
|
142
|
+
// poison the adapter for the rest of the process lifetime.
|
|
143
|
+
this.schemaReady ??= this.createSchema().catch((error) => {
|
|
144
|
+
this.schemaReady = undefined;
|
|
145
|
+
throw error;
|
|
146
|
+
});
|
|
147
|
+
return this.schemaReady;
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
private async createSchema(): Promise<void> {
|
|
151
|
+
await this.sql`
|
|
152
|
+
CREATE TABLE IF NOT EXISTS kumiko_subject_keys (
|
|
153
|
+
subject_id TEXT PRIMARY KEY,
|
|
154
|
+
cipher_key BYTEA,
|
|
155
|
+
kek_version INTEGER NOT NULL DEFAULT 1,
|
|
156
|
+
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
|
157
|
+
created_by TEXT,
|
|
158
|
+
erased_at TIMESTAMPTZ,
|
|
159
|
+
erased_by TEXT,
|
|
160
|
+
erase_reason TEXT
|
|
161
|
+
)`;
|
|
162
|
+
await this.sql`
|
|
163
|
+
CREATE INDEX IF NOT EXISTS kumiko_subject_keys_erased_idx
|
|
164
|
+
ON kumiko_subject_keys (erased_at) WHERE erased_at IS NOT NULL`;
|
|
165
|
+
await this.sql`
|
|
166
|
+
CREATE INDEX IF NOT EXISTS kumiko_subject_keys_audit_idx
|
|
167
|
+
ON kumiko_subject_keys (created_at, erased_at)`;
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
export function createPgKmsAdapter(options: PgKmsAdapterOptions): PgKmsAdapter {
|
|
172
|
+
return new PgKmsAdapter(options);
|
|
173
|
+
}
|