@cosmicdrift/kumiko-framework 0.157.2 → 0.159.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +7 -2
- package/src/__tests__/consumer-cli.integration.test.ts +110 -0
- package/src/api/__tests__/auth-routes-cookie.test.ts +16 -1
- package/src/api/__tests__/csrf-constants-sync.test.ts +20 -0
- package/src/api/__tests__/jwt.test.ts +150 -1
- package/src/api/__tests__/redis-login-rate-limiter.integration.test.ts +66 -0
- package/src/api/__tests__/server-jwt-ttl.test.ts +58 -0
- package/src/api/api-constants.ts +4 -0
- package/src/api/auth-middleware.ts +48 -59
- package/src/api/auth-routes.ts +83 -17
- package/src/api/index.ts +8 -4
- package/src/api/jwt.ts +148 -7
- package/src/api/pii-leak-guard.ts +5 -2
- package/src/api/server.ts +19 -5
- package/src/bun-db/__tests__/select-many-retry.test.ts +79 -0
- package/src/bun-db/query.ts +34 -2
- package/src/consumer-cli.ts +87 -0
- package/src/crypto/__tests__/pii-field-encryption.test.ts +69 -13
- package/src/crypto/blind-index.ts +8 -4
- package/src/crypto/event-pii.ts +1 -0
- package/src/crypto/pii-field-encryption.ts +49 -15
- package/src/db/__tests__/event-store-executor-context.pii-roundtrip.test.ts +67 -0
- package/src/db/__tests__/event-store-executor-write-verbs.integration.test.ts +305 -0
- package/src/db/__tests__/event-store-executor.integration.test.ts +5 -5
- package/src/db/blind-index-cleanup.ts +3 -1
- package/src/db/connection.ts +3 -11
- package/src/db/encryption.ts +2 -3
- package/src/db/entity-table-meta-types.ts +92 -0
- package/src/db/entity-table-meta.ts +16 -90
- package/src/db/queries/backfill-pii.ts +1 -0
- package/src/db/queries/event-consumer.ts +35 -2
- package/src/engine/__tests__/boot-validator-boot-check.test.ts +99 -0
- package/src/engine/__tests__/boot-validator-gdpr-storage.test.ts +7 -233
- package/src/engine/__tests__/define-roles.test.ts +21 -0
- package/src/engine/__tests__/event-type-map-augmentation.test.ts +24 -0
- package/src/engine/__tests__/store-table.test.ts +12 -0
- package/src/engine/boot-validator/action-wiring.ts +1 -1
- package/src/engine/boot-validator/boot-check.ts +21 -0
- package/src/engine/boot-validator/entity-list-screens.ts +1 -1
- package/src/engine/boot-validator/gdpr-storage.ts +0 -112
- package/src/engine/boot-validator/index.ts +3 -9
- package/src/engine/boot-validator/screens.ts +1 -1
- package/src/engine/define-feature.ts +1 -0
- package/src/engine/define-handler.ts +10 -91
- package/src/engine/entity-handlers.ts +15 -27
- package/src/engine/feature-builder-state.ts +3 -0
- package/src/engine/feature-config-events-jobs.ts +1 -1
- package/src/engine/feature-entity-handlers.ts +1 -1
- package/src/engine/feature-ui-extensions.ts +5 -1
- package/src/engine/field-helpers.ts +31 -0
- package/src/engine/handler-helpers.ts +26 -0
- package/src/engine/hook-helpers.ts +14 -0
- package/src/engine/index.ts +2 -2
- package/src/engine/ownership.ts +22 -76
- package/src/engine/registry-validate.ts +1 -1
- package/src/engine/screen-helpers.ts +54 -0
- package/src/engine/tier-resolver-extension.ts +3 -2
- package/src/engine/types/define-handler.ts +94 -0
- package/src/engine/types/entity-handlers.ts +30 -0
- package/src/engine/types/event-type-map.ts +1 -37
- package/src/engine/types/feature.ts +45 -0
- package/src/engine/types/fields.ts +19 -31
- package/src/engine/types/handlers.ts +7 -26
- package/src/engine/types/hooks.ts +1 -15
- package/src/engine/types/http-route.ts +1 -72
- package/src/engine/types/identifiers.ts +1 -47
- package/src/engine/types/index.ts +34 -9
- package/src/engine/types/ownership.ts +83 -0
- package/src/engine/types/relations.ts +1 -51
- package/src/engine/types/screen.ts +0 -46
- package/src/engine/types/target-ref.ts +1 -21
- package/src/engine/types/tree-node.ts +1 -129
- package/src/entrypoint/index.ts +2 -2
- package/src/event-store/__tests__/event-store.integration.test.ts +31 -0
- package/src/event-store/__tests__/unscoped-stream-primitives.guard.test.ts +43 -0
- package/src/event-store/event-store.ts +28 -32
- package/src/event-store/events-schema.ts +1 -10
- package/src/event-store/index.ts +3 -2
- package/src/event-store/types.ts +22 -0
- package/src/files/__tests__/in-memory-provider.contract.test.ts +4 -0
- package/src/files/file-handle.ts +2 -19
- package/src/i18n/required-surface-keys.ts +1 -1
- package/src/logging/types.ts +1 -7
- package/src/observability/types/index.ts +1 -29
- package/src/observability/types/metric.ts +1 -56
- package/src/observability/types/provider.ts +1 -32
- package/src/observability/types/span.ts +1 -58
- package/src/pipeline/__tests__/dispatcher.test.ts +38 -1
- package/src/pipeline/__tests__/event-dispatcher-delivery-max-attempts.test.ts +126 -0
- package/src/pipeline/__tests__/event-dispatcher-rearm.integration.test.ts +180 -0
- package/src/pipeline/dispatch-shared.ts +12 -2
- package/src/pipeline/entity-cache.ts +2 -33
- package/src/pipeline/event-consumer-state.ts +28 -3
- package/src/pipeline/event-dispatcher-admin.ts +4 -0
- package/src/pipeline/event-dispatcher-delivery.ts +29 -3
- package/src/pipeline/event-dispatcher.ts +27 -1
- package/src/pipeline/system-hooks.ts +7 -0
- package/src/search/types.ts +1 -39
- package/src/secrets/__tests__/envelope-cipher.test.ts +2 -30
- package/src/secrets/__tests__/envelope.test.ts +1 -1
- package/src/secrets/envelope-cipher.ts +13 -39
- package/src/stack/__tests__/event-collector.test.ts +42 -0
- package/src/testing/__tests__/late-bound.test.ts +25 -0
- package/src/testing/__tests__/wait-for.test.ts +53 -0
- package/src/testing/boot-validator-fixture.ts +1 -1
- package/src/testing/file-provider-contract.ts +84 -0
- package/src/testing/handler-context.ts +1 -1
- package/src/testing/index.ts +1 -0
- package/src/time/geo-tz.ts +1 -32
- package/src/ui-types/index.ts +7 -7
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
// #1163: Bun.SQL can hand out a closed connection under load (AbortError
|
|
2
|
+
// "The connection was closed."). Pure reads retry exactly once on a fresh
|
|
3
|
+
// pool checkout; tx handles, non-matching errors, and genuine user aborts
|
|
4
|
+
// must NOT retry.
|
|
5
|
+
|
|
6
|
+
import { describe, expect, test } from "bun:test";
|
|
7
|
+
import { buildEntityTable } from "../../db/table-builder";
|
|
8
|
+
import { selectMany } from "../query";
|
|
9
|
+
|
|
10
|
+
function closedConnectionError(): Error {
|
|
11
|
+
return Object.assign(new Error("The connection was closed."), { name: "AbortError" });
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
type FakeClient = {
|
|
15
|
+
unsafe: (sql: string, params?: readonly unknown[]) => Promise<readonly unknown[]>;
|
|
16
|
+
begin?: () => never;
|
|
17
|
+
calls: number;
|
|
18
|
+
};
|
|
19
|
+
|
|
20
|
+
function fakeClient(failures: Error[], opts: { tx?: boolean } = {}): FakeClient {
|
|
21
|
+
const remaining = [...failures];
|
|
22
|
+
const client: FakeClient = {
|
|
23
|
+
calls: 0,
|
|
24
|
+
unsafe: async () => {
|
|
25
|
+
client.calls++;
|
|
26
|
+
const err = remaining.shift();
|
|
27
|
+
if (err) throw err;
|
|
28
|
+
return [{ id: "r1", title: "ok", tenant_id: "t1", inserted_at: null, updated_at: null }];
|
|
29
|
+
},
|
|
30
|
+
};
|
|
31
|
+
// A top-level pool client has begin(); a transaction handle does not.
|
|
32
|
+
if (!opts.tx)
|
|
33
|
+
client.begin = () => {
|
|
34
|
+
throw new Error("not used in test");
|
|
35
|
+
};
|
|
36
|
+
return client;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
const table = buildEntityTable("note", {
|
|
40
|
+
fields: { title: { type: "text", required: true } },
|
|
41
|
+
});
|
|
42
|
+
|
|
43
|
+
describe("selectMany — closed-connection retry (#1163)", () => {
|
|
44
|
+
test("retries once on AbortError 'connection was closed' and returns rows", async () => {
|
|
45
|
+
const db = fakeClient([closedConnectionError()]);
|
|
46
|
+
const rows = await selectMany(db, table);
|
|
47
|
+
expect(rows).toHaveLength(1);
|
|
48
|
+
expect(rows[0]?.title).toBe("ok");
|
|
49
|
+
expect(db.calls).toBe(2);
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
test("gives up after the single retry when the connection stays closed", async () => {
|
|
53
|
+
const db = fakeClient([closedConnectionError(), closedConnectionError()]);
|
|
54
|
+
await expect(selectMany(db, table)).rejects.toThrow("connection was closed");
|
|
55
|
+
expect(db.calls).toBe(2);
|
|
56
|
+
});
|
|
57
|
+
|
|
58
|
+
test("never retries on a transaction handle (no begin)", async () => {
|
|
59
|
+
const db = fakeClient([closedConnectionError()], { tx: true });
|
|
60
|
+
await expect(selectMany(db, table)).rejects.toThrow("connection was closed");
|
|
61
|
+
expect(db.calls).toBe(1);
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
test("does not retry a genuine user abort (different message)", async () => {
|
|
65
|
+
const userAbort = Object.assign(new Error("The operation was aborted."), {
|
|
66
|
+
name: "AbortError",
|
|
67
|
+
});
|
|
68
|
+
const db = fakeClient([userAbort]);
|
|
69
|
+
await expect(selectMany(db, table)).rejects.toThrow("operation was aborted");
|
|
70
|
+
expect(db.calls).toBe(1);
|
|
71
|
+
});
|
|
72
|
+
|
|
73
|
+
test("does not retry generic query errors", async () => {
|
|
74
|
+
const syntax = Object.assign(new Error("syntax error at or near"), { name: "PostgresError" });
|
|
75
|
+
const db = fakeClient([syntax]);
|
|
76
|
+
await expect(selectMany(db, table)).rejects.toThrow("syntax error");
|
|
77
|
+
expect(db.calls).toBe(1);
|
|
78
|
+
});
|
|
79
|
+
});
|
package/src/bun-db/query.ts
CHANGED
|
@@ -562,6 +562,38 @@ function buildWhereClause(
|
|
|
562
562
|
return { sqlText: conditions.join(" AND "), values };
|
|
563
563
|
}
|
|
564
564
|
|
|
565
|
+
// #1163: under load the Bun.SQL pool can hand out a connection the server
|
|
566
|
+
// already closed ("The connection was closed.", AbortError code 20) — no
|
|
567
|
+
// validate-on-checkout exists. One retry re-checks out a fresh connection;
|
|
568
|
+
// safe for pure reads. Tx handles are never retried: their transaction is
|
|
569
|
+
// dead once the connection dropped, and a retry would run on the same dead
|
|
570
|
+
// handle. Detection is name+message (not name alone) so a genuine user
|
|
571
|
+
// abort (AbortSignal cancel) is NOT retried.
|
|
572
|
+
function isClosedConnectionError(err: unknown): boolean {
|
|
573
|
+
if (err === null || typeof err !== "object") return false;
|
|
574
|
+
const e = err as { name?: unknown; message?: unknown };
|
|
575
|
+
return (
|
|
576
|
+
e.name === "AbortError" &&
|
|
577
|
+
typeof e.message === "string" &&
|
|
578
|
+
/connection was closed/i.test(e.message)
|
|
579
|
+
);
|
|
580
|
+
}
|
|
581
|
+
|
|
582
|
+
async function unsafeRead<TRow>(
|
|
583
|
+
db: AnyDb,
|
|
584
|
+
sqlText: string,
|
|
585
|
+
params: readonly unknown[],
|
|
586
|
+
): Promise<readonly TRow[]> {
|
|
587
|
+
const raw = asRawClient(db);
|
|
588
|
+
try {
|
|
589
|
+
return (await raw.unsafe(sqlText, params)) as readonly TRow[];
|
|
590
|
+
} catch (err) {
|
|
591
|
+
// TransactionSql has savepoint(), only a top-level pool client has begin().
|
|
592
|
+
if (typeof raw.begin !== "function" || !isClosedConnectionError(err)) throw err;
|
|
593
|
+
return (await raw.unsafe(sqlText, params)) as readonly TRow[];
|
|
594
|
+
}
|
|
595
|
+
}
|
|
596
|
+
|
|
565
597
|
// biome-ignore lint/suspicious/noExplicitAny: opt-in default loosens row type for unannotated test fixtures
|
|
566
598
|
export async function selectMany<TRow = any>(
|
|
567
599
|
db: AnyDb,
|
|
@@ -598,7 +630,7 @@ export async function selectMany<TRow = any>(
|
|
|
598
630
|
}
|
|
599
631
|
sqlText += ` LIMIT ${options.limit}`;
|
|
600
632
|
}
|
|
601
|
-
const raw = (await
|
|
633
|
+
const raw = (await unsafeRead(db, sqlText, values)) as readonly Record<string, unknown>[];
|
|
602
634
|
return coerceRows(raw, info) as readonly TRow[];
|
|
603
635
|
}
|
|
604
636
|
|
|
@@ -832,7 +864,7 @@ export async function countWhere(
|
|
|
832
864
|
sqlText += ` WHERE ${w.sqlText}`;
|
|
833
865
|
values = w.values;
|
|
834
866
|
}
|
|
835
|
-
const rows = (await
|
|
867
|
+
const rows = (await unsafeRead(db, sqlText, values)) as readonly { count: number }[];
|
|
836
868
|
return rows[0]?.count ?? 0;
|
|
837
869
|
}
|
|
838
870
|
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
// Shared core for the standalone consumer-ops CLI (status | restart).
|
|
2
|
+
//
|
|
3
|
+
// A dead event consumer (halt-on-poison after maxAttempts) previously had no
|
|
4
|
+
// recovery surface in the standalone prod bundle — only raw SQL against
|
|
5
|
+
// kumiko_event_consumers. Mirrors schema-cli.ts's shape (single runXCli(argv,
|
|
6
|
+
// out) entry point, own DB connection) so `kumiko-consumer` ships the same
|
|
7
|
+
// way `kumiko-schema` does.
|
|
8
|
+
|
|
9
|
+
import { createDbConnection } from "./db";
|
|
10
|
+
import { getConsumerState, restartConsumer } from "./pipeline";
|
|
11
|
+
import { ensureTemporalPolyfill } from "./time";
|
|
12
|
+
|
|
13
|
+
export type ConsumerCliOut = {
|
|
14
|
+
readonly log: (line: string) => void;
|
|
15
|
+
readonly err: (line: string) => void;
|
|
16
|
+
};
|
|
17
|
+
|
|
18
|
+
function parseInstanceIdFlag(argv: readonly string[]): string | undefined {
|
|
19
|
+
const i = argv.indexOf("--instance-id");
|
|
20
|
+
return i === -1 ? undefined : argv[i + 1];
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export async function runConsumerCli(
|
|
24
|
+
argv: readonly string[],
|
|
25
|
+
out: ConsumerCliOut,
|
|
26
|
+
): Promise<number> {
|
|
27
|
+
// The standalone bundle never runs runProdApp/runDevApp's boot, which is
|
|
28
|
+
// where Temporal normally gets installed — ConsumerRecoveryState.updatedAt
|
|
29
|
+
// is a Temporal.Instant, so without this every subcommand throws "Temporal
|
|
30
|
+
// is not defined" (same failure mode as schema-cli, see its polyfill test).
|
|
31
|
+
await ensureTemporalPolyfill();
|
|
32
|
+
const sub = argv[0];
|
|
33
|
+
|
|
34
|
+
if (sub !== "status" && sub !== "restart") {
|
|
35
|
+
out.log("");
|
|
36
|
+
out.log(" Subcommands:");
|
|
37
|
+
out.log(" status <name> [--instance-id <id>] Zeigt Status + Cursor eines Consumers");
|
|
38
|
+
out.log(" restart <name> [--instance-id <id>] Reaktiviert einen dead-Consumer (idle)");
|
|
39
|
+
out.log("");
|
|
40
|
+
return sub === undefined ? 0 : 1;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
const name = argv[1];
|
|
44
|
+
if (!name) {
|
|
45
|
+
out.err(` Usage: consumer ${sub} <name> [--instance-id <id>]`);
|
|
46
|
+
return 1;
|
|
47
|
+
}
|
|
48
|
+
const instanceId = parseInstanceIdFlag(argv);
|
|
49
|
+
|
|
50
|
+
const dbUrl = process.env["DATABASE_URL"];
|
|
51
|
+
if (!dbUrl) {
|
|
52
|
+
out.err(" DATABASE_URL not set.");
|
|
53
|
+
return 1;
|
|
54
|
+
}
|
|
55
|
+
const { db, close } = createDbConnection(dbUrl);
|
|
56
|
+
try {
|
|
57
|
+
if (sub === "status") {
|
|
58
|
+
const state = await getConsumerState(db, name, instanceId);
|
|
59
|
+
if (!state) {
|
|
60
|
+
out.err(` Consumer "${name}" (instance_id="${instanceId ?? "__shared__"}") not found.`);
|
|
61
|
+
return 1;
|
|
62
|
+
}
|
|
63
|
+
out.log("");
|
|
64
|
+
out.log(` ${state.name} (instance_id="${state.instanceId}")`);
|
|
65
|
+
out.log(` status: ${state.status}`);
|
|
66
|
+
out.log(` cursor: ${state.lastProcessedEventId}`);
|
|
67
|
+
out.log(` attempts: ${state.attempts}`);
|
|
68
|
+
out.log(` rearmCount: ${state.rearmCount}`);
|
|
69
|
+
out.log(` lastError: ${state.lastError ?? "-"}`);
|
|
70
|
+
out.log(` updatedAt: ${state.updatedAt.toString()}`);
|
|
71
|
+
out.log("");
|
|
72
|
+
return 0;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
// restart
|
|
76
|
+
const result = await restartConsumer(db, name, instanceId);
|
|
77
|
+
out.log("");
|
|
78
|
+
out.log(` ✓ ${result.name} (instance_id="${result.instanceId}") → ${result.status}`);
|
|
79
|
+
out.log("");
|
|
80
|
+
return 0;
|
|
81
|
+
} catch (e) {
|
|
82
|
+
out.err(` ✗ ${e instanceof Error ? e.message : String(e)}`);
|
|
83
|
+
return 1;
|
|
84
|
+
} finally {
|
|
85
|
+
await close();
|
|
86
|
+
}
|
|
87
|
+
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { describe, expect, test } from "bun:test";
|
|
2
|
+
import { createCipheriv, randomBytes } from "node:crypto";
|
|
2
3
|
import { createEntity, createTextField } from "../../engine/factories";
|
|
3
4
|
import { InMemoryKmsAdapter } from "../in-memory-kms-adapter";
|
|
4
5
|
import {
|
|
@@ -62,7 +63,7 @@ describe("encryptPiiFieldValues / decryptPiiFieldValues", () => {
|
|
|
62
63
|
|
|
63
64
|
const stored = await encryptPiiFieldValues(row, userLikeEntity, fields, kms, KMS_CTX);
|
|
64
65
|
expect(isPiiCiphertext(stored["email"])).toBe(true);
|
|
65
|
-
expect(String(stored["email"])).toStartWith(
|
|
66
|
+
expect(String(stored["email"])).toStartWith(`${PII_CIPHERTEXT_PREFIX}user:${UUID_A}:`);
|
|
66
67
|
expect(stored["role"]).toBe("admin");
|
|
67
68
|
expect(row["email"]).toBe("marc@example.com");
|
|
68
69
|
|
|
@@ -95,7 +96,7 @@ describe("encryptPiiFieldValues / decryptPiiFieldValues", () => {
|
|
|
95
96
|
kms,
|
|
96
97
|
KMS_CTX,
|
|
97
98
|
);
|
|
98
|
-
expect(String(stored["body"])).toStartWith(
|
|
99
|
+
expect(String(stored["body"])).toStartWith(`${PII_CIPHERTEXT_PREFIX}user:${UUID_B}:`);
|
|
99
100
|
});
|
|
100
101
|
|
|
101
102
|
test("tenantOwned without tenantId column falls back to write-time tenant", async () => {
|
|
@@ -108,7 +109,7 @@ describe("encryptPiiFieldValues / decryptPiiFieldValues", () => {
|
|
|
108
109
|
KMS_CTX,
|
|
109
110
|
{ tenantId: UUID_B },
|
|
110
111
|
);
|
|
111
|
-
expect(String(stored["brandColor"])).toStartWith(
|
|
112
|
+
expect(String(stored["brandColor"])).toStartWith(`${PII_CIPHERTEXT_PREFIX}tenant:${UUID_B}:`);
|
|
112
113
|
});
|
|
113
114
|
|
|
114
115
|
test("subjectSource resolves the owner when the partial row lacks it (update changes)", async () => {
|
|
@@ -118,7 +119,7 @@ describe("encryptPiiFieldValues / decryptPiiFieldValues", () => {
|
|
|
118
119
|
onlyKeys: ["body"],
|
|
119
120
|
subjectSource: { id: UUID_A, body: "edited", authorId: UUID_B },
|
|
120
121
|
});
|
|
121
|
-
expect(String(stored["body"])).toStartWith(
|
|
122
|
+
expect(String(stored["body"])).toStartWith(`${PII_CIPHERTEXT_PREFIX}user:${UUID_B}:`);
|
|
122
123
|
});
|
|
123
124
|
|
|
124
125
|
test("erased subject: decrypt yields the sentinel, re-encrypt passes it through", async () => {
|
|
@@ -243,7 +244,7 @@ describe("piiEncrypted alias (kumiko-platform#457)", () => {
|
|
|
243
244
|
const row = { id: UUID_A, tenantId: UUID_B, iban: "DE89370400440532013000" };
|
|
244
245
|
const stored = await encryptPiiFieldValues(row, brandingWithAccess, fields, kms, KMS_CTX);
|
|
245
246
|
expect(isPiiCiphertext(stored["iban"])).toBe(true);
|
|
246
|
-
expect(String(stored["iban"])).toStartWith(
|
|
247
|
+
expect(String(stored["iban"])).toStartWith(`${PII_CIPHERTEXT_PREFIX}tenant:${UUID_B}:`);
|
|
247
248
|
|
|
248
249
|
const read = await decryptPiiFieldValues(stored, fields, kms, KMS_CTX);
|
|
249
250
|
expect(read["iban"]).toBe("DE89370400440532013000");
|
|
@@ -274,27 +275,39 @@ describe("encryptPiiValueForSubject / decryptPiiValueForSubject (kumiko-platform
|
|
|
274
275
|
test("round-trips a single value for a tenant subject", async () => {
|
|
275
276
|
const kms = new InMemoryKmsAdapter();
|
|
276
277
|
const subject = { kind: "tenant" as const, tenantId: UUID_B };
|
|
277
|
-
const stored = await encryptPiiValueForSubject(
|
|
278
|
+
const stored = await encryptPiiValueForSubject(
|
|
279
|
+
kms,
|
|
280
|
+
subject,
|
|
281
|
+
"DE89370400440532013000",
|
|
282
|
+
KMS_CTX,
|
|
283
|
+
"iban",
|
|
284
|
+
);
|
|
278
285
|
expect(isPiiCiphertext(stored)).toBe(true);
|
|
279
|
-
expect(stored).toStartWith(
|
|
286
|
+
expect(stored).toStartWith(`${PII_CIPHERTEXT_PREFIX}tenant:${UUID_B}:`);
|
|
280
287
|
|
|
281
|
-
const read = await decryptPiiValueForSubject(kms, stored, KMS_CTX);
|
|
288
|
+
const read = await decryptPiiValueForSubject(kms, stored, KMS_CTX, "iban");
|
|
282
289
|
expect(read).toBe("DE89370400440532013000");
|
|
283
290
|
});
|
|
284
291
|
|
|
285
292
|
test("erased subject: decrypt yields the sentinel", async () => {
|
|
286
293
|
const kms = new InMemoryKmsAdapter();
|
|
287
294
|
const subject = { kind: "user" as const, userId: UUID_A };
|
|
288
|
-
const stored = await encryptPiiValueForSubject(
|
|
295
|
+
const stored = await encryptPiiValueForSubject(
|
|
296
|
+
kms,
|
|
297
|
+
subject,
|
|
298
|
+
"+49 151 00000000",
|
|
299
|
+
KMS_CTX,
|
|
300
|
+
"phone",
|
|
301
|
+
);
|
|
289
302
|
await kms.eraseKey(subject);
|
|
290
303
|
|
|
291
|
-
const read = await decryptPiiValueForSubject(kms, stored, KMS_CTX);
|
|
304
|
+
const read = await decryptPiiValueForSubject(kms, stored, KMS_CTX, "phone");
|
|
292
305
|
expect(read).toBe(PII_ERASED_SENTINEL);
|
|
293
306
|
});
|
|
294
307
|
|
|
295
308
|
test("plaintext passes through decrypt unchanged (pre-engine rows)", async () => {
|
|
296
309
|
const kms = new InMemoryKmsAdapter();
|
|
297
|
-
const read = await decryptPiiValueForSubject(kms, "plain-value", KMS_CTX);
|
|
310
|
+
const read = await decryptPiiValueForSubject(kms, "plain-value", KMS_CTX, "value");
|
|
298
311
|
expect(read).toBe("plain-value");
|
|
299
312
|
});
|
|
300
313
|
});
|
|
@@ -308,13 +321,56 @@ describe("cross-subject decrypt leak (kumiko-framework#1190)", () => {
|
|
|
308
321
|
// existing "ciphertext without a key row fails loud" (KeyNotFoundError) case.
|
|
309
322
|
await kms.createKey(subjectA);
|
|
310
323
|
|
|
311
|
-
const storedForB = await encryptPiiValueForSubject(
|
|
324
|
+
const storedForB = await encryptPiiValueForSubject(
|
|
325
|
+
kms,
|
|
326
|
+
subjectB,
|
|
327
|
+
"tenant-b-secret",
|
|
328
|
+
KMS_CTX,
|
|
329
|
+
"secret",
|
|
330
|
+
);
|
|
312
331
|
const blob = storedForB.slice(storedForB.lastIndexOf(":") + 1);
|
|
313
332
|
const forgedForA = `${PII_CIPHERTEXT_PREFIX}${subjectIdToKey(subjectA)}:${blob}`;
|
|
314
333
|
|
|
315
|
-
const attempt = decryptPiiValueForSubject(kms, forgedForA, KMS_CTX);
|
|
334
|
+
const attempt = decryptPiiValueForSubject(kms, forgedForA, KMS_CTX, "secret");
|
|
316
335
|
await expect(attempt).rejects.not.toBeInstanceOf(KeyNotFoundError);
|
|
317
336
|
await expect(attempt).rejects.not.toBeInstanceOf(KeyErasedError);
|
|
318
337
|
await expect(attempt).rejects.toThrow();
|
|
319
338
|
});
|
|
320
339
|
});
|
|
340
|
+
|
|
341
|
+
describe("cross-field decrypt leak (kumiko-framework#1263)", () => {
|
|
342
|
+
test("ciphertext cut-and-pasted into a different field of the SAME subject fails GCM auth", async () => {
|
|
343
|
+
const kms = new InMemoryKmsAdapter();
|
|
344
|
+
const subject = { kind: "user" as const, userId: UUID_A };
|
|
345
|
+
const storedAsEmail = await encryptPiiValueForSubject(
|
|
346
|
+
kms,
|
|
347
|
+
subject,
|
|
348
|
+
"secret@example.com",
|
|
349
|
+
KMS_CTX,
|
|
350
|
+
"email",
|
|
351
|
+
);
|
|
352
|
+
|
|
353
|
+
// Same subject, same DEK — only the declared field differs. Key
|
|
354
|
+
// selection alone can't catch this; AAD must.
|
|
355
|
+
const attempt = decryptPiiValueForSubject(kms, storedAsEmail, KMS_CTX, "phone");
|
|
356
|
+
await expect(attempt).rejects.toThrow();
|
|
357
|
+
});
|
|
358
|
+
|
|
359
|
+
test("legacy v1 ciphertext (no AAD) still decrypts under the matching subject/field", async () => {
|
|
360
|
+
const kms = new InMemoryKmsAdapter();
|
|
361
|
+
const subject = { kind: "user" as const, userId: UUID_A };
|
|
362
|
+
await kms.createKey(subject);
|
|
363
|
+
const dek = await kms.getKey(subject);
|
|
364
|
+
|
|
365
|
+
// Hand-roll a pre-#1263 v1 blob (no setAAD) — real pre-migration rows
|
|
366
|
+
// look exactly like this.
|
|
367
|
+
const iv = randomBytes(12);
|
|
368
|
+
const cipher = createCipheriv("aes-256-gcm", dek, iv);
|
|
369
|
+
const ciphertext = Buffer.concat([cipher.update("legacy-value", "utf8"), cipher.final()]);
|
|
370
|
+
const blob = Buffer.concat([iv, cipher.getAuthTag(), ciphertext]);
|
|
371
|
+
const v1 = `kumiko-pii:v1:${subjectIdToKey(subject)}:${blob.toString("base64")}`;
|
|
372
|
+
|
|
373
|
+
const read = await decryptPiiValueForSubject(kms, v1, KMS_CTX, "email");
|
|
374
|
+
expect(read).toBe("legacy-value");
|
|
375
|
+
});
|
|
376
|
+
});
|
|
@@ -93,22 +93,26 @@ export async function computeBlindIndexValues(
|
|
|
93
93
|
const out: Record<string, unknown> = {};
|
|
94
94
|
for (const name of lookupableFields) {
|
|
95
95
|
if (!(name in values)) continue;
|
|
96
|
-
out[blindIndexFieldName(name)] = await blindIndexForValue(key, values[name]);
|
|
96
|
+
out[blindIndexFieldName(name)] = await blindIndexForValue(key, name, values[name]);
|
|
97
97
|
}
|
|
98
98
|
return out;
|
|
99
99
|
}
|
|
100
100
|
|
|
101
|
-
async function blindIndexForValue(
|
|
101
|
+
async function blindIndexForValue(
|
|
102
|
+
key: Uint8Array,
|
|
103
|
+
field: string,
|
|
104
|
+
value: unknown,
|
|
105
|
+
): Promise<string | null> {
|
|
102
106
|
if (typeof value !== "string" || value === PII_ERASED_SENTINEL) return null;
|
|
103
107
|
if (!isPiiCiphertext(value)) return computeBlindIndex(key, value);
|
|
104
108
|
const kms = configuredPiiSubjectKms();
|
|
105
109
|
// Ciphertext without a KMS can't be decrypted here; the same read would
|
|
106
110
|
// also surface raw ciphertext — misconfiguration is caught at boot.
|
|
107
111
|
if (kms === undefined) return null;
|
|
108
|
-
const decrypted = await decryptPiiFieldValues({ value }, [
|
|
112
|
+
const decrypted = await decryptPiiFieldValues({ [field]: value }, [field], kms, {
|
|
109
113
|
requestId: requestContext.get()?.requestId ?? "blind-index",
|
|
110
114
|
});
|
|
111
|
-
const plain = decrypted[
|
|
115
|
+
const plain = decrypted[field];
|
|
112
116
|
if (typeof plain !== "string" || plain === PII_ERASED_SENTINEL) return null;
|
|
113
117
|
return computeBlindIndex(key, plain);
|
|
114
118
|
}
|
package/src/crypto/event-pii.ts
CHANGED
|
@@ -3,7 +3,12 @@
|
|
|
3
3
|
// erase subject — kms.eraseKey(subject) makes every value unreadable at once.
|
|
4
4
|
// Storage format is a sniffable string that fits existing text columns and
|
|
5
5
|
// names its subject inline, so decrypt needs no schema change and no resolver:
|
|
6
|
-
// kumiko-pii:
|
|
6
|
+
// kumiko-pii:v2:<subjectKey>:<base64(iv|tag|ciphertext)>
|
|
7
|
+
// v2 GCM-binds the ciphertext to `subjectKey|field` as AAD (#1263) — subject
|
|
8
|
+
// selects the DEK, field stops a cut-and-paste between two fields of the
|
|
9
|
+
// SAME subject (same DEK, so key selection alone can't catch that) from
|
|
10
|
+
// decrypting silently. v1 (no AAD) stays decrypt-only for pre-#1263 rows;
|
|
11
|
+
// every new write emits v2.
|
|
7
12
|
|
|
8
13
|
import { createCipheriv, createDecipheriv, randomBytes } from "node:crypto";
|
|
9
14
|
import type { EntityDefinition } from "../engine/types/fields";
|
|
@@ -26,24 +31,40 @@ import { resolveSubjectForField } from "./subject-resolver";
|
|
|
26
31
|
// Spec value (crypto-shredding.md) — renderers show it verbatim.
|
|
27
32
|
export const PII_ERASED_SENTINEL = "[[erased]]";
|
|
28
33
|
|
|
29
|
-
|
|
34
|
+
const PII_CIPHERTEXT_PREFIX_V1 = "kumiko-pii:v1:";
|
|
35
|
+
export const PII_CIPHERTEXT_PREFIX = "kumiko-pii:v2:";
|
|
30
36
|
const IV_LENGTH = 12;
|
|
31
37
|
const AUTH_TAG_LENGTH = 16;
|
|
32
38
|
|
|
33
39
|
export function isPiiCiphertext(value: unknown): value is string {
|
|
34
|
-
return
|
|
40
|
+
return (
|
|
41
|
+
typeof value === "string" &&
|
|
42
|
+
(value.startsWith(PII_CIPHERTEXT_PREFIX) || value.startsWith(PII_CIPHERTEXT_PREFIX_V1))
|
|
43
|
+
);
|
|
35
44
|
}
|
|
36
45
|
|
|
37
|
-
function
|
|
46
|
+
function buildAad(subject: SubjectId, field: string): Buffer {
|
|
47
|
+
return Buffer.from(`${subjectIdToKey(subject)}|${field}`, "utf8");
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function encryptValue(
|
|
51
|
+
subject: SubjectId,
|
|
52
|
+
dek: SubjectDek,
|
|
53
|
+
plaintext: string,
|
|
54
|
+
field: string,
|
|
55
|
+
): string {
|
|
38
56
|
const iv = randomBytes(IV_LENGTH);
|
|
39
57
|
const cipher = createCipheriv("aes-256-gcm", dek, iv);
|
|
58
|
+
cipher.setAAD(buildAad(subject, field));
|
|
40
59
|
const ciphertext = Buffer.concat([cipher.update(plaintext, "utf8"), cipher.final()]);
|
|
41
60
|
const blob = Buffer.concat([iv, cipher.getAuthTag(), ciphertext]);
|
|
42
61
|
return `${PII_CIPHERTEXT_PREFIX}${subjectIdToKey(subject)}:${blob.toString("base64")}`;
|
|
43
62
|
}
|
|
44
63
|
|
|
45
|
-
function parseCiphertext(value: string): { subject: SubjectId; blob: Buffer } {
|
|
46
|
-
const
|
|
64
|
+
function parseCiphertext(value: string): { subject: SubjectId; blob: Buffer; hasAad: boolean } {
|
|
65
|
+
const hasAad = value.startsWith(PII_CIPHERTEXT_PREFIX);
|
|
66
|
+
const prefix = hasAad ? PII_CIPHERTEXT_PREFIX : PII_CIPHERTEXT_PREFIX_V1;
|
|
67
|
+
const rest = value.slice(prefix.length);
|
|
47
68
|
// subjectKey itself contains ":" ("user:<id>") — base64 never does, so the
|
|
48
69
|
// last ":" is always the key/blob separator.
|
|
49
70
|
const sep = rest.lastIndexOf(":");
|
|
@@ -51,14 +72,22 @@ function parseCiphertext(value: string): { subject: SubjectId; blob: Buffer } {
|
|
|
51
72
|
return {
|
|
52
73
|
subject: subjectIdFromKey(rest.slice(0, sep)),
|
|
53
74
|
blob: Buffer.from(rest.slice(sep + 1), "base64"),
|
|
75
|
+
hasAad,
|
|
54
76
|
};
|
|
55
77
|
}
|
|
56
78
|
|
|
57
|
-
function decryptValue(
|
|
79
|
+
function decryptValue(
|
|
80
|
+
dek: SubjectDek,
|
|
81
|
+
blob: Buffer,
|
|
82
|
+
subject: SubjectId,
|
|
83
|
+
field: string,
|
|
84
|
+
hasAad: boolean,
|
|
85
|
+
): string {
|
|
58
86
|
const iv = blob.subarray(0, IV_LENGTH);
|
|
59
87
|
const tag = blob.subarray(IV_LENGTH, IV_LENGTH + AUTH_TAG_LENGTH);
|
|
60
88
|
const ciphertext = blob.subarray(IV_LENGTH + AUTH_TAG_LENGTH);
|
|
61
89
|
const decipher = createDecipheriv("aes-256-gcm", dek, iv);
|
|
90
|
+
if (hasAad) decipher.setAAD(buildAad(subject, field));
|
|
62
91
|
decipher.setAuthTag(tag);
|
|
63
92
|
return Buffer.concat([decipher.update(ciphertext), decipher.final()]).toString("utf8");
|
|
64
93
|
}
|
|
@@ -87,31 +116,36 @@ async function getOrCreateDek(
|
|
|
87
116
|
|
|
88
117
|
// Single-value encrypt for callers that resolve the subject themselves
|
|
89
118
|
// (event-pii catalog, backfill). Ciphertext/sentinel inputs pass through —
|
|
90
|
-
// idempotent like the field-map variant.
|
|
119
|
+
// idempotent like the field-map variant. `field` names the value's storage
|
|
120
|
+
// slot (payload field / config key) — part of the AAD, so it must match the
|
|
121
|
+
// `field` passed to decryptPiiValueForSubject for the same value.
|
|
91
122
|
export async function encryptPiiValueForSubject(
|
|
92
123
|
kms: LocalKeyKmsAdapter,
|
|
93
124
|
subject: SubjectId,
|
|
94
125
|
value: string,
|
|
95
126
|
kmsCtx: KmsContext,
|
|
127
|
+
field: string,
|
|
96
128
|
): Promise<string> {
|
|
97
129
|
if (isPiiCiphertext(value) || value === PII_ERASED_SENTINEL) return value;
|
|
98
130
|
const dek = await getOrCreateDek(kms, subject, kmsCtx);
|
|
99
|
-
return encryptValue(subject, dek, value);
|
|
131
|
+
return encryptValue(subject, dek, value, field);
|
|
100
132
|
}
|
|
101
133
|
|
|
102
134
|
// Single-value decrypt for callers that don't have an entity/field-map to
|
|
103
135
|
// pass through decryptPiiFieldValues (config values). The subject lives
|
|
104
136
|
// inside the ciphertext itself, so no subject/scope resolution is needed
|
|
105
|
-
// on this side — only the encrypt direction has to pick one.
|
|
137
|
+
// on this side — only the encrypt direction has to pick one. `field` must
|
|
138
|
+
// match the one used at encrypt time (AAD) or decrypt fails loud.
|
|
106
139
|
export async function decryptPiiValueForSubject(
|
|
107
140
|
kms: LocalKeyKmsAdapter,
|
|
108
141
|
value: string,
|
|
109
142
|
kmsCtx: KmsContext,
|
|
143
|
+
field: string,
|
|
110
144
|
): Promise<string> {
|
|
111
145
|
if (!isPiiCiphertext(value)) return value;
|
|
112
|
-
const { subject, blob } = parseCiphertext(value);
|
|
146
|
+
const { subject, blob, hasAad } = parseCiphertext(value);
|
|
113
147
|
try {
|
|
114
|
-
return decryptValue(await kms.getKey(subject, kmsCtx), blob);
|
|
148
|
+
return decryptValue(await kms.getKey(subject, kmsCtx), blob, subject, field, hasAad);
|
|
115
149
|
} catch (e) {
|
|
116
150
|
if (!(e instanceof KeyErasedError)) throw e;
|
|
117
151
|
return PII_ERASED_SENTINEL;
|
|
@@ -156,7 +190,7 @@ export async function encryptPiiFieldValues(
|
|
|
156
190
|
// skip: collectPiiSubjectFields only yields annotated fields — null is unreachable, kept as a type guard
|
|
157
191
|
if (subject === null) continue;
|
|
158
192
|
const dek = await getOrCreateDek(kms, subject, kmsCtx);
|
|
159
|
-
out[name] = encryptValue(subject, dek, value);
|
|
193
|
+
out[name] = encryptValue(subject, dek, value, name);
|
|
160
194
|
}
|
|
161
195
|
return out;
|
|
162
196
|
}
|
|
@@ -174,9 +208,9 @@ export async function decryptPiiFieldValues(
|
|
|
174
208
|
// Pre-engine plaintext rows pass through unchanged (mixed-state reads
|
|
175
209
|
// work during rollout; backfill is tracked in kumiko-framework#799).
|
|
176
210
|
if (!isPiiCiphertext(value)) continue;
|
|
177
|
-
const { subject, blob } = parseCiphertext(value);
|
|
211
|
+
const { subject, blob, hasAad } = parseCiphertext(value);
|
|
178
212
|
try {
|
|
179
|
-
out[name] = decryptValue(await kms.getKey(subject, kmsCtx), blob);
|
|
213
|
+
out[name] = decryptValue(await kms.getKey(subject, kmsCtx), blob, subject, name, hasAad);
|
|
180
214
|
} catch (e) {
|
|
181
215
|
// KeyNotFound deliberately propagates: ciphertext without a key row
|
|
182
216
|
// means the key store is wrong (not shredded) — fail loud.
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import { describe, expect, test } from "bun:test";
|
|
2
|
+
import {
|
|
3
|
+
decryptPiiFieldValues,
|
|
4
|
+
InMemoryKmsAdapter,
|
|
5
|
+
isPiiCiphertext,
|
|
6
|
+
type KmsContext,
|
|
7
|
+
} from "../../crypto";
|
|
8
|
+
import { createEntity, createTextField } from "../../engine";
|
|
9
|
+
import { createTestUser, testUserId } from "../../stack/test-users";
|
|
10
|
+
import { createTestEnvelopeCipher } from "../../testing";
|
|
11
|
+
import { buildExecutorContext, type Table } from "../event-store-executor-context";
|
|
12
|
+
|
|
13
|
+
const TEST_KEY = Buffer.from("a]bJm#kP9xQ2@wN!vL$hR5yT8eU0iO3f").toString("base64");
|
|
14
|
+
|
|
15
|
+
describe("event-store-executor-context — encryptForStorage/decryptForRead layering", () => {
|
|
16
|
+
const entity = createEntity({
|
|
17
|
+
table: "pii_roundtrip_test",
|
|
18
|
+
fields: {
|
|
19
|
+
userId: createTextField({ required: true }),
|
|
20
|
+
// Both markers at once — the auth-mfa.totpSecret/recoveryCodes shape
|
|
21
|
+
// that first surfaced the ordering bug (pii-subject-encryption
|
|
22
|
+
// integration test).
|
|
23
|
+
secretNote: createTextField({ encrypted: true, userOwned: { ownerField: "userId" } }),
|
|
24
|
+
},
|
|
25
|
+
});
|
|
26
|
+
const encryption = createTestEnvelopeCipher(TEST_KEY);
|
|
27
|
+
const kms = new InMemoryKmsAdapter();
|
|
28
|
+
const kmsCtx: KmsContext = { requestId: "test" };
|
|
29
|
+
const context = buildExecutorContext({} as Table, entity, {
|
|
30
|
+
entityName: "piiRoundtripTest",
|
|
31
|
+
encryption,
|
|
32
|
+
kms,
|
|
33
|
+
});
|
|
34
|
+
const user = createTestUser({ id: testUserId(1) });
|
|
35
|
+
|
|
36
|
+
test("round-trips through both layers", async () => {
|
|
37
|
+
const row = { userId: user.id, secretNote: "the actual secret" };
|
|
38
|
+
const stored = await context.encryptForStorage(row, user);
|
|
39
|
+
expect(stored["secretNote"]).not.toBe("the actual secret");
|
|
40
|
+
|
|
41
|
+
const read = await context.decryptForRead(stored);
|
|
42
|
+
expect(read["secretNote"]).toBe("the actual secret");
|
|
43
|
+
});
|
|
44
|
+
|
|
45
|
+
test("stores PII(envelope(plaintext)) — PII is the outer layer, not envelope", async () => {
|
|
46
|
+
const row = { userId: user.id, secretNote: "the actual secret" };
|
|
47
|
+
const stored = await context.encryptForStorage(row, user);
|
|
48
|
+
const storedNote = stored["secretNote"];
|
|
49
|
+
|
|
50
|
+
// Outer layer is PII ciphertext.
|
|
51
|
+
expect(isPiiCiphertext(storedNote)).toBe(true);
|
|
52
|
+
|
|
53
|
+
// Peeling only the outer (PII) layer must leave an envelope-ciphertext
|
|
54
|
+
// string underneath, not the plaintext directly — proves encryptForStorage
|
|
55
|
+
// wrapped the envelope-ciphertext with PII, not the raw plaintext.
|
|
56
|
+
const piiPeeled = await decryptPiiFieldValues(stored, ["secretNote"], kms, kmsCtx);
|
|
57
|
+
const innerValue = piiPeeled["secretNote"];
|
|
58
|
+
expect(innerValue).not.toBe("the actual secret");
|
|
59
|
+
expect(isPiiCiphertext(innerValue)).toBe(false);
|
|
60
|
+
await expect(encryption.decrypt(innerValue as string)).resolves.toBe("the actual secret");
|
|
61
|
+
|
|
62
|
+
// decryptForRead must peel PII first: feeding the stored value straight
|
|
63
|
+
// to the envelope cipher (skipping the PII unwrap) has to fail — this is
|
|
64
|
+
// the ordering bug the roundtrip guards against.
|
|
65
|
+
await expect(encryption.decrypt(storedNote as string)).rejects.toThrow();
|
|
66
|
+
});
|
|
67
|
+
});
|