@cosmicdrift/kumiko-framework 0.173.1 → 0.174.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/package.json +3 -3
  2. package/src/api/__tests__/api.test.ts +11 -2
  3. package/src/api/__tests__/request-id-middleware.test.ts +73 -0
  4. package/src/api/request-id-middleware.ts +13 -2
  5. package/src/api/sse-broker.ts +7 -3
  6. package/src/crypto/__tests__/kms-wiring.test.ts +6 -0
  7. package/src/crypto/ciphertext-pattern.ts +19 -0
  8. package/src/crypto/kms-wiring.ts +5 -0
  9. package/src/db/__tests__/eagerload.integration.test.ts +119 -1
  10. package/src/db/__tests__/event-store-executor-write-verbs.integration.test.ts +45 -0
  11. package/src/db/__tests__/migrate-runner.test.ts +16 -0
  12. package/src/db/blind-index-cleanup.ts +15 -24
  13. package/src/db/eagerload.ts +75 -9
  14. package/src/db/entity-table-meta.ts +6 -1
  15. package/src/db/event-store-executor-write.ts +38 -3
  16. package/src/db/migrate-runner.ts +5 -0
  17. package/src/engine/__tests__/boot-validator-pii-retention.test.ts +19 -0
  18. package/src/engine/__tests__/boot-validator.test.ts +51 -1
  19. package/src/engine/__tests__/ownership.test.ts +23 -0
  20. package/src/engine/__tests__/schema-builder.test.ts +76 -6
  21. package/src/engine/boot-validator/access-roles.ts +63 -21
  22. package/src/engine/boot-validator/entity-handler.ts +4 -0
  23. package/src/engine/boot-validator/index.ts +17 -2
  24. package/src/engine/boot-validator/pii-retention.ts +17 -10
  25. package/src/engine/boot-validator/screens.ts +10 -2
  26. package/src/engine/boot-validator.ts +1 -0
  27. package/src/engine/create-app.ts +4 -2
  28. package/src/engine/field-access.ts +17 -3
  29. package/src/engine/index.ts +2 -0
  30. package/src/engine/ownership.ts +19 -0
  31. package/src/engine/schema-builder.ts +46 -22
  32. package/src/entrypoint/index.ts +2 -5
  33. package/src/jobs/__tests__/scheduler-id.test.ts +13 -1
  34. package/src/jobs/job-runner.ts +7 -1
  35. package/src/pipeline/dispatch-shared.ts +11 -3
  36. package/src/pipeline/dispatch-stream.ts +3 -6
  37. package/src/pipeline/system-hooks.ts +20 -7
  38. package/src/schema-cli.ts +7 -5
  39. package/src/search/__tests__/reindex-entity.integration.test.ts +97 -1
  40. package/src/search/purge-subject.ts +2 -9
  41. package/src/search/reindex-entity.ts +15 -2
  42. package/src/secrets/derive-purpose-secret.ts +6 -16
  43. package/src/testing/__tests__/e2e-generator.test.ts +7 -0
  44. package/src/testing/__tests__/wait-for.test.ts +2 -2
  45. package/src/testing/e2e-generator.ts +5 -0
  46. package/src/testing/shared-entities.ts +3 -3
@@ -158,6 +158,12 @@ export function userCanReadFieldRow(
158
158
  for (const role of user.roles) {
159
159
  const rule = accessMap[role];
160
160
  if (!rule) continue;
161
+ // where-rules are entity-level SQL predicates (buildOwnershipClause);
162
+ // matchesRule can't evaluate them in-memory and throws. Field-level
163
+ // access is boot-validator-rejected for where-rules, but this function
164
+ // is also reachable from hand-rolled entity-level reads.
165
+ // skip: where-rules are SQL-layer only — fail closed instead of throwing.
166
+ if (rule !== "all" && rule.kind === "where") continue;
161
167
  if (matchesRule(rule, user, row)) return true;
162
168
  }
163
169
  return false;
@@ -180,6 +186,8 @@ export function userCanWriteFieldRow(
180
186
  const rule = accessMap[role];
181
187
  if (!rule) continue;
182
188
  if (rule === "all") return true;
189
+ // skip: where-rules are SQL-layer only — fail closed instead of throwing.
190
+ if (rule.kind === "where") continue;
183
191
  if (matchesRule(rule, user, oldRow) && matchesRule(rule, user, newRow)) return true;
184
192
  }
185
193
  return false;
@@ -272,6 +280,17 @@ export function shiftParams(fragment: SqlFragment, shift: number): SqlFragment {
272
280
  // SQL names via the kumiko:schema:Columns symbol. Unknown column on a from-rule
273
281
  // is a boot-time misconfiguration; at request time we treat it as empty
274
282
  // (safe default) rather than passing silently.
283
+ //
284
+ // Caller obligations (fw#1700) — this function returns ONLY the ownership
285
+ // fragment, not the full row-access contract. A raw-SQL caller (not going
286
+ // through `ctx.db`, which already applies all three) must additionally:
287
+ // 1. Pass `paramStart` as `params.length + 1` for its own already-bound
288
+ // params, or `$N` placeholders in the returned fragment silently splice
289
+ // into the wrong query params.
290
+ // 2. Apply tenant + soft-delete scoping itself (event-store-executor-read.ts
291
+ // does this outside `buildOwnershipClause`) — this function does not.
292
+ // 3. Treat `kind: "empty"` (see `OwnershipClause`) as a hard DENY, and
293
+ // `kind: "pass"` as an explicit bypass — not as "no additional filter".
275
294
  export function buildOwnershipClause(
276
295
  user: SessionUser,
277
296
  accessMap: OwnershipMap | undefined,
@@ -55,7 +55,17 @@ function embeddedSubFieldToZod(subField: EmbeddedSubFieldDef): z.ZodTypeAny {
55
55
  }
56
56
  }
57
57
 
58
- export function fieldToZod(field: FieldDefinition, currencies: readonly string[]): z.ZodTypeAny {
58
+ export function fieldToZod(
59
+ field: FieldDefinition,
60
+ currencies: readonly string[],
61
+ opts: { readonly applyDefaults?: boolean } = {},
62
+ ): z.ZodTypeAny {
63
+ // Insert callers want `.default(...)` applied so an omitted field falls
64
+ // back to it; buildUpdateSchema passes applyDefaults: false so an omitted
65
+ // field on update stays omitted (a `{ title }` patch must not clobber
66
+ // other columns with their defaults) while a field's own default value is
67
+ // still known here for "" → default mapping (select case below).
68
+ const applyDefaults = opts.applyDefaults ?? true;
59
69
  switch (field.type) {
60
70
  case "text": {
61
71
  let schema = z.string();
@@ -63,7 +73,7 @@ export function fieldToZod(field: FieldDefinition, currencies: readonly string[]
63
73
  if (field.format === "email") schema = schema.email();
64
74
  if (field.format === "url") schema = schema.url();
65
75
  if (field.required) schema = schema.min(1);
66
- return field.default !== undefined ? schema.default(field.default) : schema;
76
+ return field.default !== undefined && applyDefaults ? schema.default(field.default) : schema;
67
77
  }
68
78
  case "longText": {
69
79
  // longText hat keine `format`-Variante (per type-design). Nur
@@ -71,24 +81,32 @@ export function fieldToZod(field: FieldDefinition, currencies: readonly string[]
71
81
  let schema = z.string();
72
82
  if (field.maxLength) schema = schema.max(field.maxLength);
73
83
  if (field.required) schema = schema.min(1);
74
- return field.default !== undefined ? schema.default(field.default) : schema;
84
+ return field.default !== undefined && applyDefaults ? schema.default(field.default) : schema;
75
85
  }
76
86
  case "boolean": {
77
87
  const schema = z.boolean();
78
- return field.default !== undefined ? schema.default(field.default) : schema;
88
+ return field.default !== undefined && applyDefaults ? schema.default(field.default) : schema;
79
89
  }
80
90
  case "select": {
81
91
  const [first, ...rest] = field.options;
82
92
  if (!first) return z.string();
83
93
  const enumSchema = z.enum([first, ...rest]);
84
- if (field.default !== undefined)
94
+ if (field.default !== undefined) {
85
95
  // Untouched <select> sends "" too; with a default that maps to the
86
96
  // default (same semantics as undefined) instead of the invalid-value
87
- // rejection from #1702. A field with a default is never "unset".
88
- return z.preprocess(
89
- (value) => (value === "" ? field.default : value),
90
- enumSchema.default(field.default),
97
+ // rejection from #1702. A field with a default is never "unset"
98
+ // true on both insert AND update, so this branch (and its "" → default
99
+ // mapping) fires regardless of applyDefaults; only the `.default(...)`
100
+ // schema-level fallback for OMITTED input is update-gated below.
101
+ // `null` maps the same way: the no-default branch below normalizes
102
+ // an untouched select to null, and a client that reuses that value
103
+ // against a since-defaulted field must not get rejected either.
104
+ const mapped = z.preprocess(
105
+ (value) => (value === "" || value === null ? field.default : value),
106
+ enumSchema,
91
107
  );
108
+ return applyDefaults ? mapped.default(field.default) : mapped;
109
+ }
92
110
  if (field.required) return enumSchema;
93
111
  // Optional select without a default: an untouched HTML <select> submits
94
112
  // "" for its placeholder option. Treat that as "unset" (null) instead of
@@ -105,14 +123,19 @@ export function fieldToZod(field: FieldDefinition, currencies: readonly string[]
105
123
  // in buildInsertSchema kümmert sich um „darf fehlen".
106
124
  let schema = z.array(z.enum([first, ...rest]));
107
125
  if (field.required) schema = schema.min(1);
108
- return field.default !== undefined ? schema.default([...field.default]) : schema;
126
+ return field.default !== undefined && applyDefaults
127
+ ? schema.default([...field.default])
128
+ : schema;
109
129
  }
110
130
  case "number": {
111
131
  let schema = z.number();
112
- if (field.integer) schema = schema.int();
132
+ // `integer: true` maps to a Postgres int4 column (entity-table-meta.ts)
133
+ // — bound it here so an out-of-range write fails loud (400) at the
134
+ // schema boundary instead of dying in Postgres (22003 → 500).
135
+ if (field.integer) schema = schema.int().min(-2147483648).max(2147483647);
113
136
  if (field.min !== undefined) schema = schema.min(field.min);
114
137
  if (field.max !== undefined) schema = schema.max(field.max);
115
- return field.default !== undefined ? schema.default(field.default) : schema;
138
+ return field.default !== undefined && applyDefaults ? schema.default(field.default) : schema;
116
139
  }
117
140
  case "decimal": {
118
141
  // Stored as numeric(precision, scale), surfaced as JS number. Bound the
@@ -126,7 +149,7 @@ export function fieldToZod(field: FieldDefinition, currencies: readonly string[]
126
149
  .refine((n) => isRepresentableAtScale(n, field.scale), {
127
150
  message: `at most ${field.scale} decimal places`,
128
151
  });
129
- return field.default !== undefined ? schema.default(field.default) : schema;
152
+ return field.default !== undefined && applyDefaults ? schema.default(field.default) : schema;
130
153
  }
131
154
  case "bigInt": {
132
155
  // JS-`number`-Round-trip via mode:"number"; sicher bis 2^53.
@@ -134,7 +157,7 @@ export function fieldToZod(field: FieldDefinition, currencies: readonly string[]
134
157
  // Float reinwirft (z.B. parseFloat-Bug), beim Insert sofort
135
158
  // failed statt silent-Truncation zu kassieren.
136
159
  const schema = z.number().int().safe();
137
- return field.default !== undefined ? schema.default(field.default) : schema;
160
+ return field.default !== undefined && applyDefaults ? schema.default(field.default) : schema;
138
161
  }
139
162
  case "money": {
140
163
  const [first, ...rest] = currencies;
@@ -242,14 +265,15 @@ export function buildUpdateSchema(
242
265
  const shape: Record<string, z.ZodTypeAny> = {};
243
266
 
244
267
  for (const [name, field] of Object.entries(entity.fields)) {
245
- // Update schemas never apply defaults — a user that sends only
246
- // `{ title }` means "only change title"; zod defaults would silently
247
- // inject default values for every omitted field and clobber existing
248
- // data via the event-store-executor's `changes` payload.
249
- // Cast widens the discriminated union so destructure works for variants
250
- // without a `default` field; remainder is structurally a FieldDefinition.
251
- const { default: _default, ...stripped } = field as FieldDefinition & { default?: unknown }; // @cast-boundary schema-walk
252
- shape[name] = fieldToZod(stripped as FieldDefinition, currencies).optional(); // @cast-boundary schema-walk
268
+ // Update schemas never apply defaults for OMITTED fields — a user that
269
+ // sends only `{ title }` means "only change title"; zod defaults would
270
+ // silently inject default values for every omitted field and clobber
271
+ // existing data via the event-store-executor's `changes` payload.
272
+ // The field is passed through un-stripped (unlike before fw#1703) so
273
+ // fieldToZod still knows the default for its "" default mapping
274
+ // (e.g. select) applyDefaults: false only suppresses the schema-level
275
+ // `.default(...)` fallback for a genuinely omitted key.
276
+ shape[name] = fieldToZod(field, currencies, { applyDefaults: false }).optional();
253
277
  }
254
278
 
255
279
  return z.object(shape);
@@ -142,11 +142,8 @@ export type WorkerEntrypoint = {
142
142
  readonly eventDispatcher: EventDispatcher;
143
143
  readonly jobRunner: JobRunner;
144
144
  readonly observability: ObservabilityProvider;
145
- // Same dispatcher the API process exposes a worker builds the identical
146
- // server, only without routes. App-wired components that run in the worker
147
- // and must persist their result need it: JobContext has no write/query
148
- // (handlers.ts JobContext), so writing goes through dispatchSystemWrite,
149
- // the pattern inbound-mail-foundation/watch-supervisor.ts established.
145
+ // Same dispatcher the API process exposes. Background components in the
146
+ // worker persist through the write-path JobContext has no write/query.
150
147
  readonly dispatcher: Dispatcher;
151
148
  readonly mode: "worker";
152
149
  // Starts event-dispatcher poll + BullMQ worker. SIGTERM triggers
@@ -1,5 +1,5 @@
1
1
  import { describe, expect, test } from "bun:test";
2
- import { schedulerIdForJobName } from "../job-runner";
2
+ import { bootJobIdForJobName, schedulerIdForJobName } from "../job-runner";
3
3
 
4
4
  describe("schedulerIdForJobName", () => {
5
5
  test("strips dots and colons so BullMQ job ids stay under the 5-segment legacy heuristic", () => {
@@ -16,3 +16,15 @@ describe("schedulerIdForJobName", () => {
16
16
  expect(schedulerIdForJobName("app.job.tick")).toBe("scheduler-app-job-tick");
17
17
  });
18
18
  });
19
+
20
+ describe("bootJobIdForJobName", () => {
21
+ test("strips colons, same hazard as schedulerIdForJobName (fw#1604)", () => {
22
+ const id = bootJobIdForJobName("publicstatus:job:uptime-probe");
23
+ expect(id).toBe("boot-publicstatus-job-uptime-probe");
24
+ expect(id.includes(":")).toBe(false);
25
+ });
26
+
27
+ test("still collapses dotted QNs", () => {
28
+ expect(bootJobIdForJobName("app.job.tick")).toBe("boot-app-job-tick");
29
+ });
30
+ });
@@ -45,6 +45,12 @@ export function schedulerIdForJobName(jobName: string): string {
45
45
  return `scheduler-${jobName.replace(/[.:]/g, "-")}`;
46
46
  }
47
47
 
48
+ // Same colon-in-BullMQ-id hazard as schedulerIdForJobName (fw#1603/#1604) —
49
+ // a QN like "publicstatus:job:uptime-probe" must not leave ":" in the id.
50
+ export function bootJobIdForJobName(jobName: string): string {
51
+ return `boot-${jobName.replace(/[.:]/g, "-")}`;
52
+ }
53
+
48
54
  // ponytail: migration shim, remove after fw#1603 deploy is everywhere.
49
55
  function legacySchedulerIdForJobName(jobName: string): string {
50
56
  return `scheduler-${jobName.replace(/\./g, "-")}`;
@@ -518,7 +524,7 @@ export function createJobRunner(options: JobRunnerOptions): JobRunner {
518
524
  if (laneForJob(jobDef) !== consumerLane) continue;
519
525
  if (jobDef.runOnBoot) {
520
526
  const bootName = jobDef.perTenant ? `_perTenant:${name}` : name;
521
- await consumerQueue.add(bootName, {}, { jobId: `boot-${name.replace(/\./g, "-")}` });
527
+ await consumerQueue.add(bootName, {}, { jobId: bootJobIdForJobName(name) });
522
528
  }
523
529
  }
524
530
 
@@ -60,7 +60,7 @@ import {
60
60
  observabilityContext,
61
61
  } from "../observability";
62
62
  import { buildBucketKey } from "../rate-limit";
63
- import { createTzContext } from "../time";
63
+ import { createTzContext, isValidIanaTimeZone } from "../time";
64
64
  import { appendDomainEventCore } from "./append-event-core";
65
65
  import { resolveAuthClaims as runAuthClaimsResolver } from "./auth-claims-resolver";
66
66
  import { executeQuery } from "./dispatch-query";
@@ -523,10 +523,18 @@ export async function buildHandlerContext(
523
523
  // tenant (createTzContext's own default). An app-injected GeoTzProvider
524
524
  // (context.geoTzProvider) feeds ctx.tz.fromCoordinates / fromAddress.
525
525
  const tenantTz = config !== undefined ? await config("tenant:config:timezone") : undefined;
526
+ // Guarded against garbage: an unvalidated string here (free-form config
527
+ // key, legacy JWT claim predating validation) blows up every ctx.tz call
528
+ // for the whole tenant with a RangeError. Fall back to UTC/tenant instead
529
+ // of trusting the raw value.
530
+ const safeTenantTz =
531
+ typeof tenantTz === "string" && isValidIanaTimeZone(tenantTz) ? tenantTz : "UTC";
532
+ const safeUserTz =
533
+ user.timezone !== undefined && isValidIanaTimeZone(user.timezone) ? user.timezone : undefined;
526
534
  const tz = createTzContext({
527
535
  ...(context.geoTzProvider !== undefined ? { geoTz: context.geoTzProvider } : {}),
528
- tenant: typeof tenantTz === "string" ? tenantTz : "UTC",
529
- ...(user.timezone !== undefined && { user: user.timezone }),
536
+ tenant: safeTenantTz,
537
+ ...(safeUserTz !== undefined && { user: safeUserTz }),
530
538
  });
531
539
 
532
540
  return {
@@ -59,12 +59,9 @@ async function* executeStreamInner(
59
59
  const invalidated = new Promise<void>((resolve) => {
60
60
  resolveInvalidated = resolve;
61
61
  });
62
- const unsubscribeAccessInvalidation = ctx.sseBroker?.subscribeAccessInvalidation?.(
63
- user.id,
64
- () => {
65
- resolveInvalidated?.();
66
- },
67
- );
62
+ const unsubscribeAccessInvalidation = ctx.sseBroker?.subscribeAccessInvalidation(user.id, () => {
63
+ resolveInvalidated?.();
64
+ });
68
65
 
69
66
  let iterator: AsyncIterator<unknown> | undefined;
70
67
  // When access is revoked mid-pull, `iterator.next()` is still in flight.
@@ -94,7 +94,7 @@ export function createSearchEventConsumer(
94
94
  // #1610 — subject-annotated searchable fields are ciphertext in the event
95
95
  // payload; decrypt into the derived index only. No KMS → omit ciphertext
96
96
  // values rather than indexing blobs.
97
- async function decryptSearchableSubjectFields(
97
+ export async function decryptSearchableSubjectFields(
98
98
  entityName: string,
99
99
  state: Record<string, unknown>,
100
100
  registry: Registry,
@@ -111,12 +111,25 @@ async function decryptSearchableSubjectFields(
111
111
  }
112
112
  return out;
113
113
  }
114
- return decryptPiiFieldValues(state, fields, kms, {
115
- requestId: "system:consumer:search",
116
- });
114
+ try {
115
+ return await decryptPiiFieldValues(state, fields, kms, {
116
+ requestId: "system:consumer:search",
117
+ });
118
+ } catch (err) {
119
+ console.warn(
120
+ `[kumiko:search] decryptSearchableSubjectFields failed for "${entityName}" — ` +
121
+ `dropping ciphertext fields for this document instead of wedging the consumer.`,
122
+ err,
123
+ );
124
+ const out = { ...state };
125
+ for (const name of fields) {
126
+ if (isPiiCiphertext(out[name])) delete out[name];
127
+ }
128
+ return out;
129
+ }
117
130
  }
118
131
 
119
- function hasErasedSearchableSubjectField(
132
+ export function hasErasedSearchableSubjectField(
120
133
  entityName: string,
121
134
  state: Record<string, unknown>,
122
135
  registry: Registry,
@@ -416,7 +429,7 @@ export function createAccessInvalidationEventConsumer(sseBroker: SseBroker): Eve
416
429
  // poison would otherwise permanently stop access-invalidation for
417
430
  // every user behind one bad row).
418
431
  if (typeof userId !== "string" || userId.length === 0) return;
419
- sseBroker.publishAccessInvalidation?.(userId);
432
+ sseBroker.publishAccessInvalidation(userId);
420
433
  }
421
434
 
422
435
  if (
@@ -427,7 +440,7 @@ export function createAccessInvalidationEventConsumer(sseBroker: SseBroker): Eve
427
440
  // skip: previous snapshot missing/malformed userId — same fail-open
428
441
  // reasoning as above.
429
442
  if (userId === undefined) return;
430
- sseBroker.publishAccessInvalidation?.(userId);
443
+ sseBroker.publishAccessInvalidation(userId);
431
444
  }
432
445
  },
433
446
  };
package/src/schema-cli.ts CHANGED
@@ -289,11 +289,13 @@ export async function runSchemaCli(
289
289
  }
290
290
  if (mismatches.some((m) => m.kind === "unexpected-table")) {
291
291
  out.err(
292
- " Fix (unexpected-table): register the raw/hand-written table via `table()` " +
293
- "(or `defineUnmanagedTable()`) from `@cosmicdrift/kumiko-framework/db`, then " +
294
- "`r.storeTable(meta, { reason: ... })` inside a feature — this adds it to " +
295
- "ENTITY_METAS and the snapshot without going through r.entity(). See the " +
296
- "bundled `jobs` feature's job-run-log store table for the pattern.",
292
+ " Fix (unexpected-table): build a meta via `defineUnmanagedTable()` " +
293
+ "from `@cosmicdrift/kumiko-framework/db`, then `r.storeTable(meta, { reason: ... })` " +
294
+ "inside a feature — this adds it to ENTITY_METAS immediately; the .snapshot.json " +
295
+ "only picks it up on the NEXT `kumiko-schema generate` run, which you still need " +
296
+ "to run and commit. `table()` returns a query handle, not a storeTable()-compatible " +
297
+ "meta — don't pass its result to storeTable(). See the bundled `jobs` feature's " +
298
+ "job-run-log store table for the pattern.",
297
299
  );
298
300
  }
299
301
  if (mismatches.some((m) => m.kind !== "unexpected-table")) {
@@ -5,6 +5,11 @@
5
5
  // nothing.
6
6
 
7
7
  import { afterAll, beforeAll, describe, expect, test } from "bun:test";
8
+ import {
9
+ configurePiiSubjectKms,
10
+ InMemoryKmsAdapter,
11
+ isPiiCiphertext,
12
+ } from "@cosmicdrift/kumiko-framework/crypto";
8
13
  import {
9
14
  asRawClient,
10
15
  buildEntityTable,
@@ -20,6 +25,7 @@ import {
20
25
  TestUsers,
21
26
  unsafeCreateEntityTable,
22
27
  } from "@cosmicdrift/kumiko-framework/stack";
28
+ import { resetPiiSubjectKmsForTests } from "@cosmicdrift/kumiko-framework/testing";
23
29
 
24
30
  const widgetEntity = createEntity({
25
31
  table: "read_reindex_widgets",
@@ -35,16 +41,38 @@ const widgetFeature = defineFeature("reindex-test", (r) => {
35
41
  r.entity("widget", widgetEntity);
36
42
  });
37
43
 
44
+ // fw#1611: pii + searchable — the read-table column holds ciphertext, a
45
+ // naive backfill would index blobs (or resurrect a crypto-shredded row).
46
+ const contactEntity = createEntity({
47
+ table: "read_reindex_contacts",
48
+ fields: {
49
+ label: createTextField({ required: true, maxLength: 100, pii: true, searchable: true }),
50
+ },
51
+ });
52
+ const contactTable = buildEntityTable("contact", contactEntity);
53
+ const contactFeature = defineFeature("reindex-pii-test", (r) => {
54
+ r.entity("contact", contactEntity);
55
+ });
56
+
38
57
  let stack: TestStack;
58
+ let kms: InMemoryKmsAdapter;
39
59
  const admin = TestUsers.admin;
40
60
 
41
61
  beforeAll(async () => {
42
- stack = await setupTestStack({ features: [widgetFeature] });
62
+ stack = await setupTestStack({ features: [widgetFeature, contactFeature] });
43
63
  await unsafeCreateEntityTable(stack.db, widgetEntity);
64
+ await unsafeCreateEntityTable(stack.db, contactEntity, "contact");
44
65
  await createEventsTable(stack.db);
66
+ // Shared across both pii tests below (not per-test) — reindexEntity scans
67
+ // the whole tenant table regardless of which test created which row, so a
68
+ // fresh KMS instance per test would make earlier rows' subject keys
69
+ // unresolvable (KeyNotFoundError) instead of exercising the erased path.
70
+ kms = new InMemoryKmsAdapter();
71
+ configurePiiSubjectKms(kms);
45
72
  });
46
73
 
47
74
  afterAll(async () => {
75
+ resetPiiSubjectKmsForTests();
48
76
  await stack.cleanup();
49
77
  });
50
78
 
@@ -141,4 +169,72 @@ describe("reindexEntity", () => {
141
169
  );
142
170
  }
143
171
  });
172
+
173
+ // fw#1611: reindexEntity read the read-table row (ciphertext for pii+
174
+ // searchable fields) straight into the search document, skipping the
175
+ // decrypt step createSearchEventConsumer applies on the live path.
176
+ test("decrypts pii+searchable fields before indexing — backfill is findable by plaintext, not ciphertext", async () => {
177
+ const plain = "UniqueReindexPiiLabel1611";
178
+ const executor = createEventStoreExecutor(contactTable, contactEntity, {
179
+ entityName: "contact",
180
+ });
181
+ const created = await executor.create(
182
+ { label: plain },
183
+ admin,
184
+ createTenantDb(stack.db, admin.tenantId, "system"),
185
+ );
186
+ if (!created.isSuccess) throw new Error("seed failed");
187
+
188
+ // No dispatcher run — row exists only on the read-table, ciphertext.
189
+ const row = (
190
+ await asRawClient(stack.db).unsafe(
191
+ `SELECT label FROM "read_reindex_contacts" WHERE id = $1`,
192
+ [created.data.id],
193
+ )
194
+ )[0] as { label: string };
195
+ expect(isPiiCiphertext(row.label)).toBe(true);
196
+
197
+ const result = await reindexEntity(
198
+ stack.db,
199
+ stack.registry,
200
+ stack.search,
201
+ "contact",
202
+ admin.tenantId,
203
+ );
204
+ expect(result.indexedRows).toBe(1);
205
+ expect(result.failures).toHaveLength(0);
206
+
207
+ const hits = await stack.search.search(admin.tenantId, plain, { filterType: "contact" });
208
+ expect(hits.some((h) => String(h.entityId) === String(created.data.id))).toBe(true);
209
+ });
210
+
211
+ test("skips a row whose subject key was already erased — reindex must not resurrect a crypto-shredded row", async () => {
212
+ const plain = "ErasedReindexPiiLabel1611";
213
+ const executor = createEventStoreExecutor(contactTable, contactEntity, {
214
+ entityName: "contact",
215
+ });
216
+ const created = await executor.create(
217
+ { label: plain },
218
+ admin,
219
+ createTenantDb(stack.db, admin.tenantId, "system"),
220
+ );
221
+ if (!created.isSuccess) throw new Error("seed failed");
222
+
223
+ // pii: true → subject key is the entity id itself.
224
+ await kms.eraseKey({ kind: "user", userId: String(created.data.id) });
225
+
226
+ const result = await reindexEntity(
227
+ stack.db,
228
+ stack.registry,
229
+ stack.search,
230
+ "contact",
231
+ admin.tenantId,
232
+ );
233
+ // Shares the tenant/table with the preceding test, so other rows may
234
+ // also index here — what matters is that THIS erased row didn't.
235
+ expect(result.failures).toHaveLength(0);
236
+
237
+ const hits = await stack.search.search(admin.tenantId, plain, { filterType: "contact" });
238
+ expect(hits.some((h) => String(h.entityId) === String(created.data.id))).toBe(false);
239
+ });
144
240
  });
@@ -8,6 +8,7 @@
8
8
  // 2. Ciphertext LIKE prefix (same as nullBlindIndexesForSubject) for rows
9
9
  // that still carry the subject key in encrypted columns.
10
10
 
11
+ import { quoteIdent, subjectCiphertextLikePattern } from "../crypto/ciphertext-pattern";
11
12
  import type { SubjectId } from "../crypto/kms-adapter";
12
13
  import { collectSearchableSubjectFields } from "../crypto/subject-resolver";
13
14
  import type { DbRunner } from "../db/connection";
@@ -19,14 +20,6 @@ import type { EntityId, TenantId } from "../engine/types/identifiers";
19
20
  import { toSnakeCase } from "../utils/case";
20
21
  import type { SearchAdapter } from "./types";
21
22
 
22
- function quoteIdent(name: string): string {
23
- return `"${name.replace(/"/g, '""')}"`;
24
- }
25
-
26
- function escapeLikePattern(value: string): string {
27
- return value.replace(/[\\%_]/g, (m) => `\\${m}`);
28
- }
29
-
30
23
  /** Build OR predicates for rows owned by `subject` (id / ownerField / tenant_id). */
31
24
  function ownershipPredicates(
32
25
  entity: EntityDefinition,
@@ -76,7 +69,7 @@ export async function purgeSearchDocumentsForSubject(
76
69
  /** When set, also match rows by ownership — needed after anonymize rewrites ciphertext. */
77
70
  subject?: SubjectId,
78
71
  ): Promise<void> {
79
- const likePattern = `kumiko-pii:v%:${escapeLikePattern(subjectKey)}:%`;
72
+ const likePattern = subjectCiphertextLikePattern(subjectKey);
80
73
  const byTenant = new Map<string, { entityType: string; entityId: EntityId }[]>();
81
74
  const seen = new Set<string>();
82
75
 
@@ -9,7 +9,11 @@ import type { DbRunner } from "../db/connection";
9
9
  import { resolveTableName } from "../db/entity-table-meta";
10
10
  import { executeRawQuery } from "../db/queries/raw-sql";
11
11
  import type { Registry, TenantId } from "../engine/types";
12
- import { buildSearchDocument } from "../pipeline/system-hooks";
12
+ import {
13
+ buildSearchDocument,
14
+ decryptSearchableSubjectFields,
15
+ hasErasedSearchableSubjectField,
16
+ } from "../pipeline/system-hooks";
13
17
  import { toSnakeCase } from "../utils/case";
14
18
  import type { SearchAdapter, SearchDocument } from "./types";
15
19
 
@@ -135,7 +139,16 @@ export async function reindexEntity(
135
139
  result.scannedRows++;
136
140
  const entityId = String(row["id"]);
137
141
  try {
138
- const state = rowToState(row, fieldNames);
142
+ const rawState = rowToState(row, fieldNames);
143
+ // `pii + searchable` fields hold ciphertext on the read-table row —
144
+ // decrypt the same way the live write-path consumer does. An
145
+ // erased subject key doesn't throw, it swaps the value for
146
+ // PII_ERASED_SENTINEL (see decryptSearchableSubjectFields) — check
147
+ // for that AFTER decrypting, mirroring the softDelete filter above:
148
+ // resurrecting a crypto-shredded row here would undo
149
+ // purgeSearchDocumentsForSubject and make it findable again.
150
+ const state = await decryptSearchableSubjectFields(entityName, rawState, registry);
151
+ if (hasErasedSearchableSubjectField(entityName, state, registry)) continue;
139
152
  const doc = await buildSearchDocument(entityName, entityId, state, registry);
140
153
  if (doc) docs.push({ entityId, doc });
141
154
  } catch (e) {
@@ -1,21 +1,11 @@
1
1
  import { hkdfSync } from "node:crypto";
2
2
 
3
- // One master secret, many purposes: HKDF turns `JWT_SECRET` into an
4
- // independent secret per trust boundary, so a token-signing key for MFA setup
5
- // cannot be used to forge a deletion token and neither can be walked back to
6
- // the master. Rotating the master rotates every purpose with it.
7
- //
8
- // The alternative is an env var per purpose. That is not more secure (same
9
- // blast radius if the deploy is compromised), it is just more operations, and
10
- // in practice one of them ends up unset in some environment.
11
- //
12
- // The purpose string is a domain separator and part of the contract: change it
13
- // and every previously issued token for that purpose stops verifying. Version
14
- // them ("mfa-setup-token-v1") so a single purpose can be rotated deliberately
15
- // without touching the master or the other purposes.
16
- //
17
- // Lived copy-pasted in four apps before fw#1623 (money-horse, kumiko-studio,
18
- // publicstatus, plus a stale worktree) — identical bodies, drifting comments.
3
+ // HKDF turns one master secret into an independent secret per purpose, so
4
+ // a token-signing key for MFA setup can't forge a deletion token and
5
+ // neither can be walked back to the master. `purpose` is a domain
6
+ // separator, not a label changing it invalidates every previously
7
+ // issued token for that purpose; version it ("mfa-setup-token-v1") to
8
+ // rotate one purpose deliberately.
19
9
  export function derivePurposeSecret(masterSecret: string, purpose: string): string {
20
10
  if (!masterSecret) {
21
11
  throw new Error("derivePurposeSecret: masterSecret must not be empty.");
@@ -259,4 +259,11 @@ describe("generateZodFixture", () => {
259
259
  expect(() => generateZodFixture(z.object({}))).toThrow(/not supported yet/);
260
260
  expect(() => generateZodFixture(z.array(z.string()))).toThrow(/not supported yet/);
261
261
  });
262
+
263
+ test("pipe (select-with-default, same shape schema-builder's z.preprocess produces) unwraps to the underlying type's fixture (fw#1712)", () => {
264
+ const enumSchema = z.enum(["a", "b"]);
265
+ const pipe = z.preprocess((value) => (value === "" ? "a" : value), enumSchema);
266
+ expect(pipe._def.type).toBe("pipe");
267
+ expect(generateZodFixture(pipe)).toBe("a");
268
+ });
262
269
  });
@@ -9,11 +9,11 @@ describe("waitFor", () => {
9
9
  () => {
10
10
  calls++;
11
11
  },
12
- { delays: [200, 200, 200] },
12
+ { delays: [2000] },
13
13
  );
14
14
  expect(calls).toBe(1);
15
15
  // try-first: must not burn the first delay when the condition already holds
16
- expect(Date.now() - started).toBeLessThan(100);
16
+ expect(Date.now() - started).toBeLessThan(500);
17
17
  });
18
18
 
19
19
  test("retries on failure and succeeds once fn passes", async () => {
@@ -355,6 +355,7 @@ type ZodInternals = {
355
355
  readonly format?: string;
356
356
  readonly innerType?: z.ZodTypeAny;
357
357
  readonly entries?: Record<string, string>;
358
+ readonly out?: z.ZodTypeAny;
358
359
  };
359
360
 
360
361
  function readZodInternals(schema: z.ZodTypeAny): ZodInternals | undefined {
@@ -384,6 +385,10 @@ export function generateZodFixture(schema: z.ZodTypeAny): unknown {
384
385
  }
385
386
  case "date":
386
387
  return new Date("2026-01-01T00:00:00Z");
388
+ case "pipe": {
389
+ if (!def?.out) throw new Error("zod pipe without out");
390
+ return generateZodFixture(def.out);
391
+ }
387
392
  default:
388
393
  throw new Error(`generateZodFixture: not supported yet: ${typeName ?? "<unknown>"}`);
389
394
  }