@cosmicdrift/kumiko-framework 0.119.0 → 0.121.0

package/src/db/queries/backfill-pii.ts

@@ -0,0 +1,276 @@
+// One-time backfill for pre-KMS plaintext PII in kumiko_events (#799).
+//
+// Crypto-shredding (#724/#818) only covers NEW writes — events appended
+// before a KMS was configured still carry plaintext (user.created email,
+// delivery attempt recipientAddress, job payloads). This tool re-encrypts
+// them in place, per field, under the owning subject's DEK:
+//
+//   - entity lifecycle events (<entity>.created/updated/deleted/forgotten/
+//     restored) for every entity with PII subject annotations
+//   - custom events from the event-PII catalog (r.defineEvent piiFields)
+//
+// Already-forgotten subjects must NOT get a fresh key minted for their old
+// plaintext — three erased-detection layers write [[erased]] instead:
+//   1. KeyErasedError from the KMS (subject forgotten in the KMS era)
+//   2. the event's own aggregate has a *.forgotten event (pre-KMS forget)
+//   3. the resolved user subject's id has a *.forgotten event (custom
+//      events referencing a pre-KMS-forgotten user)
+//
+// Idempotent: ciphertext and sentinel values pass through untouched — a
+// second run reports 0 updates. One failing event does not abort the run;
+// failures are collected and reported (fail-loud at the caller).
+//
+// Snapshots of touched aggregates are dropped (they may cache plaintext);
+// the next snapshotting load recreates them. AFTER a run, rebuild the
+// affected projections — applyEntityEvent materializes ciphertext AND the
+// blind-index columns, which keeps equality lookups (login by email) alive.
+
+import { asRawClient } from "../../bun-db";
+import { configuredEventPiiCatalog } from "../../crypto/event-pii";
+import type { KmsContext, LocalKeyKmsAdapter, SubjectId } from "../../crypto/kms-adapter";
+import { KeyErasedError } from "../../crypto/kms-adapter";
+import {
+  configuredPiiSubjectKms,
+  encryptPiiValueForSubject,
+  isPiiCiphertext,
+  PII_ERASED_SENTINEL,
+} from "../../crypto/pii-field-encryption";
+import { collectPiiSubjectFields, resolveSubjectForField } from "../../crypto/subject-resolver";
+import type { EntityDefinition, Registry, TenantId } from "../../engine/types";
+import type { DbRunner } from "../connection";
+
+const LIFECYCLE_VERBS = ["created", "updated", "deleted", "restored", "forgotten"] as const;
+
+export type PiiBackfillFailure = {
+  readonly eventId: string;
+  readonly reason: string;
+};
+
+export type PiiBackfillResult = {
+  readonly scannedEvents: number;
+  readonly updatedEvents: number;
+  readonly encryptedFields: number;
+  readonly erasedFields: number;
+  readonly deletedSnapshots: number;
+  readonly failures: readonly PiiBackfillFailure[];
+};
+
+export type PiiBackfillOptions = {
+  readonly batchSize?: number;
+  // Scan + count only, write nothing.
+  readonly dryRun?: boolean;
+};
+
+type EventRow = {
+  readonly id: bigint | string;
+  readonly aggregate_id: string;
+  readonly aggregate_type: string;
+  readonly tenant_id: string;
+  readonly type: string;
+  readonly payload: Record<string, unknown>;
+};
+
+type FieldOutcome = "unchanged" | "encrypted" | "erased";
+
+export async function backfillEventPiiEncryption(
+  db: DbRunner,
+  registry: Registry,
+  options: PiiBackfillOptions = {},
+): Promise<PiiBackfillResult> {
+  const kms = configuredPiiSubjectKms();
+  if (!kms) {
+    throw new Error(
+      "backfillEventPiiEncryption requires a configured subject KMS — boot with " +
+        "runProdApp({ kms }) / configurePiiSubjectKms(adapter) before running the backfill.",
+    );
+  }
+  const batchSize = options.batchSize ?? 500;
+  const raw = asRawClient(db);
+  const kmsCtx: KmsContext = { requestId: "pii-backfill" };
+
+  const entityTargets = new Map<
+    string,
+    { readonly entity: EntityDefinition; readonly piiFields: readonly string[] }
+  >();
+  for (const [name, entity] of registry.getAllEntities()) {
+    const piiFields = collectPiiSubjectFields(entity);
+    if (piiFields.length > 0) entityTargets.set(name, { entity, piiFields });
+  }
+  const eventCatalog = configuredEventPiiCatalog();
+
+  const aggregateTypes = [...entityTargets.keys()];
+  const catalogTypes = [...eventCatalog.keys()];
+  const result = {
+    scannedEvents: 0,
+    updatedEvents: 0,
+    encryptedFields: 0,
+    erasedFields: 0,
+    deletedSnapshots: 0,
+    failures: [] as PiiBackfillFailure[],
+  };
+  if (aggregateTypes.length === 0 && catalogTypes.length === 0) return result;
+
+  // Pre-KMS forgets left no key tombstone — the *.forgotten event on the
+  // stream is the only durable marker. Collect once; aggregate_id doubles
+  // as the user id for user-subject lookups.
+  const forgottenRows = (await raw.unsafe(
+    `SELECT DISTINCT "aggregate_id" FROM "kumiko_events" WHERE "type" LIKE '%.forgotten'`,
+  )) as ReadonlyArray<{ aggregate_id: string }>;
+  const forgottenAggregates = new Set(forgottenRows.map((r) => r.aggregate_id));
+
+  const touchedAggregates = new Set<string>();
+  let cursor = "0";
+
+  for (;;) {
+    const rows = (await raw.unsafe(
+      `SELECT "id", "aggregate_id", "aggregate_type", "tenant_id", "type", "payload"
+         FROM "kumiko_events"
+        WHERE ("aggregate_type" = ANY($1::text[]) OR "type" = ANY($2::text[])) AND "id" > $3::bigint
+        ORDER BY "id" ASC
+        LIMIT $4`,
+      [aggregateTypes, catalogTypes, cursor, batchSize],
+    )) as ReadonlyArray<EventRow>;
+    if (rows.length === 0) break;
+
+    for (const row of rows) {
+      result.scannedEvents++;
+      try {
+        const outcome = await transformEvent(row);
+        if (outcome === null) continue;
+        result.encryptedFields += outcome.encrypted;
+        result.erasedFields += outcome.erased;
+        if (!options.dryRun) {
+          await raw.unsafe(`UPDATE "kumiko_events" SET "payload" = $1::jsonb WHERE "id" = $2`, [
+            JSON.stringify(outcome.payload),
+            row.id,
+          ]);
+        }
+        result.updatedEvents++;
+        touchedAggregates.add(row.aggregate_id);
+      } catch (e) {
+        result.failures.push({
+          eventId: String(row.id),
+          reason: e instanceof Error ? e.message : String(e),
+        });
+      }
+    }
+    const last = rows[rows.length - 1];
+    if (last === undefined) break;
+    cursor = String(last.id);
+  }
+
+  // Snapshots may cache the plaintext state of touched aggregates.
+  if (!options.dryRun && touchedAggregates.size > 0) {
+    const deleted = (await raw.unsafe(
+      `DELETE FROM "kumiko_snapshots" WHERE "aggregate_id" = ANY($1::uuid[]) RETURNING "aggregate_id"`,
+      [[...touchedAggregates]],
+    )) as ReadonlyArray<unknown>;
+    result.deletedSnapshots = deleted.length;
+  }
+
+  return result;
+
+  async function transformEvent(
+    row: EventRow,
+  ): Promise<{ payload: Record<string, unknown>; encrypted: number; erased: number } | null> {
+    const counters = { encrypted: 0, erased: 0 };
+    const payload = structuredClone(row.payload);
+
+    const catalogFields = eventCatalog.get(row.type);
+    if (catalogFields) {
+      for (const [field, spec] of Object.entries(catalogFields)) {
+        const subjectId = payload[spec.subjectField];
+        if (typeof subjectId !== "string" || subjectId.length === 0) continue;
+        const outcome = await encryptField(payload, field, { kind: "user", userId: subjectId });
+        bump(outcome);
+      }
+    } else {
+      const target = entityTargets.get(row.aggregate_type);
+      if (!target || !isLifecycleEventOf(row.type, row.aggregate_type)) return null;
+      const sections = lifecycleSections(payload);
+      for (const section of sections) {
+        // Update-changes may carry a pii field without its owner field —
+        // resolve subjects from the merged view; aggregate_id backs the
+        // self-subject when a section lacks the id column.
+        const subjectSource: Record<string, unknown> = {
+          id: row.aggregate_id,
+          ...Object.assign({}, ...sections),
+          ...section,
+        };
+        for (const field of target.piiFields) {
+          const subject = resolveSubjectForField(target.entity, field, subjectSource, {
+            // @cast-boundary db-read — tenant_id column is the branded TenantId
+            tenantId: row.tenant_id as TenantId,
+          });
+          if (subject === null) continue;
+          const outcome = await encryptField(section, field, subject);
+          bump(outcome);
+        }
+      }
+    }
+
+    if (counters.encrypted === 0 && counters.erased === 0) return null;
+    return { payload, ...counters };
+
+    function bump(outcome: FieldOutcome): void {
+      if (outcome === "encrypted") counters.encrypted++;
+      if (outcome === "erased") counters.erased++;
+    }
+
+    async function encryptField(
+      section: Record<string, unknown>,
+      field: string,
+      subject: SubjectId,
+    ): Promise<FieldOutcome> {
+      const value = section[field];
+      if (value === null || value === undefined) return "unchanged";
+      if (typeof value !== "string") return "unchanged";
+      if (isPiiCiphertext(value) || value === PII_ERASED_SENTINEL) return "unchanged";
+      if (isForgottenSubject(subject, row.aggregate_id)) {
+        section[field] = PII_ERASED_SENTINEL;
+        return "erased";
+      }
+      try {
+        section[field] = await encryptPiiValueForSubject(
+          kms as LocalKeyKmsAdapter,
+          subject,
+          value,
+          kmsCtx,
+        );
+        return "encrypted";
+      } catch (e) {
+        if (e instanceof KeyErasedError) {
+          section[field] = PII_ERASED_SENTINEL;
+          return "erased";
+        }
+        throw e;
+      }
+    }
+  }
+
+  function isForgottenSubject(subject: SubjectId, aggregateId: string): boolean {
+    if (forgottenAggregates.has(aggregateId)) return true;
+    return subject.kind === "user" && forgottenAggregates.has(subject.userId);
+  }
+}
+
+function isLifecycleEventOf(eventType: string, aggregateType: string): boolean {
+  if (!eventType.startsWith(`${aggregateType}.`)) return false;
+  const verb = eventType.slice(aggregateType.length + 1);
+  return (LIFECYCLE_VERBS as readonly string[]).includes(verb);
+}
+
+// created events carry the fields flat; updated carries { changes, previous };
+// deleted/forgotten/restored carry { previous }. Returned sections are the
+// mutable objects INSIDE the payload clone — encryptField writes in place.
+function lifecycleSections(payload: Record<string, unknown>): Record<string, unknown>[] {
+  const sections: Record<string, unknown>[] = [];
+  if (isRecord(payload["changes"])) sections.push(payload["changes"]);
+  if (isRecord(payload["previous"])) sections.push(payload["previous"]);
+  if (sections.length === 0) sections.push(payload);
+  return sections;
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}

package/src/engine/define-feature.ts

@@ -1,4 +1,4 @@
-import type { ZodType, z } from "zod";
+import { ZodObject, type ZodType, type z } from "zod";
 import type { EntityTableMeta } from "../db/entity-table-meta";
 import { toTableName } from "../db/table-builder";
 import { LifecycleHookTypes } from "./constants";
@@ -19,6 +19,7 @@ import type {
   EntityRef,
   EventDef,
   EventMigrationDef,
+  EventPiiFields,
   EventUpcastFn,
   ExtensionSelectorDef,
   FeatureDefinition,
@@ -511,7 +512,7 @@ export function defineFeature<const TName extends string, TExports = undefined>(
     defineEvent: <const TInner extends string, TPayload>(
       eventName: TInner,
       schema: ZodType<TPayload>,
-      options?: { readonly version?: number },
+      options?: { readonly version?: number; readonly piiFields?: EventPiiFields },
     ): EventDef<TPayload, QualifiedEventName<TName, TInner>> => {
       // Return the fully-qualified event name so callers can pass it
       // straight to ctx.appendEvent without hand-building the
@@ -530,6 +531,27 @@ export function defineFeature<const TName extends string, TExports = undefined>(
           `[Feature ${name}] defineEvent("${eventName}"): version must be a positive integer, got ${String(version)}`,
         );
       }
+      // piiFields misconfiguration is a boot-time error, not a silent
+      // plaintext leak: both the pii field and its subjectField must exist
+      // on the payload schema (checkable when the schema is a ZodObject).
+      const piiFields = options?.piiFields;
+      if (piiFields) {
+        const shape = schema instanceof ZodObject ? schema.shape : undefined;
+        for (const [field, spec] of Object.entries(piiFields)) {
+          if (field === spec.subjectField) {
+            throw new Error(
+              `[Feature ${name}] defineEvent("${eventName}"): piiFields."${field}" cannot use itself as subjectField — the subject id is a plaintext pseudonymous fk, the pii field is the value it owns.`,
+            );
+          }
+          for (const required of [field, spec.subjectField]) {
+            if (shape && !(required in shape)) {
+              throw new Error(
+                `[Feature ${name}] defineEvent("${eventName}"): piiFields references "${required}" which is not a field of the payload schema.`,
+              );
+            }
+          }
+        }
+      }
       // @cast-boundary engine-bridge — runtime-string mirrors the
       // template-literal-type via QualifiedEventName + toKebab. Both
       // sides are tested (CamelToKebab type-tests + scan-events kebab
@@ -538,6 +560,7 @@ export function defineFeature<const TName extends string, TExports = undefined>(
         name: qualified as QualifiedEventName<TName, TInner>,
         schema,
         version,
+        ...(piiFields !== undefined && { piiFields }),
       };
       events[eventName] = def;
       return def;

package/src/engine/registry.ts

@@ -1,3 +1,4 @@
+import { configureEventPiiCatalog } from "../crypto/event-pii";
 import { applyEntityEvent } from "../db/apply-entity-event";
 import {
   assertBackingTableSuperset,
@@ -26,6 +27,7 @@ import type {
   EntityProjectionExtension,
   EntityRelations,
   EventDef,
+  EventPiiFields,
   EventUpcastFn,
   FeatureDefinition,
   FeatureMetricDef,
@@ -1363,6 +1365,16 @@ export function createRegistry(features: readonly FeatureDefinition[]): Registry
     return false;
   })();
 
+  // Publish the event-PII catalog (#799): append() — the single write funnel
+  // into kumiko_events — encrypts catalogued payload fields regardless of
+  // which path produced the event (ctx.appendEvent, MSP-apply, low-level
+  // append in delivery/jobs loggers).
+  const eventPiiCatalog = new Map<string, EventPiiFields>();
+  for (const [qualified, def] of eventMap) {
+    if (def.piiFields) eventPiiCatalog.set(qualified, def.piiFields);
+  }
+  configureEventPiiCatalog(eventPiiCatalog);
+
   // Auto-wire the soft-delete cleanup cron + its grace-days config key when ANY
   // entity opts into softDelete — the framework owns this machinery, no feature
   // declares it (mirrors the auto restore-handler). Job-runner reads getAllJobs
@@ -1395,6 +1407,10 @@ export function createRegistry(features: readonly FeatureDefinition[]): Registry
       return entityMap.get(name);
     },
 
+    getAllEntities(): ReadonlyMap<string, EntityDefinition> {
+      return entityMap;
+    },
+
     getWriteHandler(name: string): WriteHandlerDef | undefined {
       return writeHandlerMap.get(name);
     },

package/src/engine/types/feature.ts

@@ -36,6 +36,7 @@ import type {
   EntityRef,
   EventDef,
   EventMigrationDef,
+  EventPiiFields,
   EventUpcastFn,
   HandlerRef,
   NameOrRef,
@@ -529,10 +530,14 @@ export type FeatureRegistrar<TFeature extends string = string> = {
   // on first registration. When you bump the payload shape, raise version
   // AND register r.eventMigration(shortName, N, N+1, transform) — the
   // framework refuses to boot if the chain from 1 → version has gaps.
+  //
+  // `options.piiFields` declares PII payload fields encrypted under the DEK
+  // of the user named by `subjectField` (crypto-shredding, #799). append()
+  // enforces the catalog on every write path.
   defineEvent<const TInner extends string, TPayload>(
     name: TInner,
     schema: ZodType<TPayload>,
-    options?: { readonly version?: number },
+    options?: { readonly version?: number; readonly piiFields?: EventPiiFields },
   ): EventDef<TPayload, QualifiedEventName<TFeature, TInner>>;
 
   // Register a step-wise payload transform for event-schema evolution.
@@ -789,6 +794,7 @@ export type Registry = {
 
   getFeature(name: string): FeatureDefinition | undefined;
   getEntity(name: string): EntityDefinition | undefined;
+  getAllEntities(): ReadonlyMap<string, EntityDefinition>;
   getWriteHandler(name: string): WriteHandlerDef | undefined;
   getQueryHandler(name: string): QueryHandlerDef | undefined;
   getSearchableFields(entityName: string): readonly string[];

package/src/engine/types/handlers.ts

@@ -584,6 +584,12 @@ export type QualifiedEventName<
   TInner extends string,
 > = `${CamelToKebab<TFeature>}:event:${CamelToKebab<TInner>}`;
 
+// PII payload fields on a custom event (#799): `field` is encrypted under
+// the DEK of the user named by the payload's `subjectField` (crypto-
+// shredding). A null subject field leaves the value plaintext — there is
+// no user key to shred for system-triggered events.
+export type EventPiiFields = Readonly<Record<string, { readonly subjectField: string }>>;
+
 export type EventDef<TPayload = unknown, TName extends string = string> = {
   readonly name: TName;
   readonly schema: ZodType<TPayload>;
@@ -592,6 +598,7 @@ export type EventDef<TPayload = unknown, TName extends string = string> = {
   // upcasts older stored events. Reads consult this to decide if upcasters
   // need to run before the payload hits consumer code.
   readonly version: number;
+  readonly piiFields?: EventPiiFields;
 };
 
 // Args for ctx.appendEvent — explicit aggregate target, Marten-style.

package/src/engine/types/index.ts

@@ -136,6 +136,7 @@ export type {
   EntityRef,
   EventDef,
   EventMigrationDef,
+  EventPiiFields,
   EventUpcastCtx,
   EventUpcastFn,
   FetchForWritingArgs,