@cosmicdrift/kumiko-framework 0.209.1 → 0.210.0

package/src/db/queries/backfill-pii.ts

@@ -20,12 +20,25 @@
 // second run reports 0 updates. One failing event does not abort the run;
 // failures are collected and reported (fail-loud at the caller).
 //
+// Owner resolution for entity-lifecycle events (backfill-only — the live
+// write path in event-store-executor-write.ts / subject-resolver.ts stays
+// fail-closed and is untouched by this):
+//   1. payload — the event section itself names the owner field.
+//   2. projection ({@link PiiBackfillOptions.resolveOwnerFromProjection}) —
+//      pre-owner-field-addition events don't carry it; read it from the
+//      entity's current projection row instead.
+//   3. erase ({@link PiiBackfillOptions.eraseUnresolvableSubjects}) — still
+//      unresolvable (hard-deleted aggregate, no projection) → PII_ERASED_SENTINEL.
+//   Both stages are opt-in and off by default; without them an unresolvable
+//   subject still fails loud into `failures`, as before.
+//
 // Snapshots of touched aggregates are dropped (they may cache plaintext);
 // the next snapshotting load recreates them. AFTER a run, rebuild the
 // affected projections — applyEntityEvent materializes ciphertext AND the
 // blind-index columns, which keeps equality lookups (login by email) alive.
 
 import { asRawClient } from "../../bun-db";
+import { quoteIdent } from "../../crypto/ciphertext-pattern";
 import { configuredEventPiiCatalog } from "../../crypto/event-pii";
 import type { KmsContext, LocalKeyKmsAdapter, SubjectId } from "../../crypto/kms-adapter";
 import { KeyErasedError } from "../../crypto/kms-adapter";
@@ -35,9 +48,16 @@ import {
   isPiiCiphertext,
   PII_ERASED_SENTINEL,
 } from "../../crypto/pii-field-encryption";
-import { collectPiiSubjectFields, resolveSubjectForField } from "../../crypto/subject-resolver";
+import {
+  collectPiiSubjectFields,
+  resolveSubjectForField,
+  SubjectResolutionError,
+} from "../../crypto/subject-resolver";
 import type { EntityDefinition, Registry, TenantId } from "../../engine/types";
 import type { DbRunner } from "../connection";
+import { resolveTableName } from "../entity-table-meta";
+import { isUndefinedTable } from "../pg-error";
+import { toSnakeCase } from "../table-builder";
 
 const LIFECYCLE_VERBS = ["created", "updated", "deleted", "restored", "forgotten"] as const;
 
@@ -51,6 +71,12 @@ export type PiiBackfillResult = {
   readonly updatedEvents: number;
   readonly encryptedFields: number;
   readonly erasedFields: number;
+  // Subset of encryptedFields + erasedFields whose owner came from stage 2
+  // (the payload itself lacked it).
+  readonly ownerFromProjection: number;
+  // Subset of erasedFields written because the subject stayed unresolvable
+  // through stage 3, not because the subject was already forgotten.
+  readonly erasedUnresolvable: number;
   readonly deletedSnapshots: number;
   readonly failures: readonly PiiBackfillFailure[];
 };
@@ -59,6 +85,12 @@ export type PiiBackfillOptions = {
   readonly batchSize?: number;
   // Scan + count only, write nothing.
   readonly dryRun?: boolean;
+  // Stage 2: fall back to the entity's projection row (by aggregate_id)
+  // when a lifecycle event's payload doesn't name the owner field.
+  readonly resolveOwnerFromProjection?: boolean;
+  // Stage 3: write PII_ERASED_SENTINEL for subjects still unresolvable
+  // after stage 1+2, instead of failing the event into `failures`.
+  readonly eraseUnresolvableSubjects?: boolean;
 };
 
 type EventRow = {
@@ -105,6 +137,8 @@ export async function backfillEventPiiEncryption(
     updatedEvents: 0,
     encryptedFields: 0,
     erasedFields: 0,
+    ownerFromProjection: 0,
+    erasedUnresolvable: 0,
     deletedSnapshots: 0,
     failures: [] as PiiBackfillFailure[],
   };
@@ -132,13 +166,17 @@ export async function backfillEventPiiEncryption(
     )) as ReadonlyArray<EventRow>;
     if (rows.length === 0) break;
 
+    const projectionOwnersByType = await loadProjectionOwners(rows);
+
     for (const row of rows) {
       result.scannedEvents++;
       try {
-        const outcome = await transformEvent(row);
+        const outcome = await transformEvent(row, projectionOwnersByType.get(row.aggregate_type));
         if (outcome === null) continue;
         result.encryptedFields += outcome.encrypted;
         result.erasedFields += outcome.erased;
+        result.ownerFromProjection += outcome.ownerFromProjection;
+        result.erasedUnresolvable += outcome.erasedUnresolvable;
         if (!options.dryRun) {
           await raw.unsafe(`UPDATE "kumiko_events" SET "payload" = $1::jsonb WHERE "id" = $2`, [
             outcome.payload,
@@ -148,9 +186,15 @@ export async function backfillEventPiiEncryption(
         result.updatedEvents++;
         touchedAggregates.add(row.aggregate_id);
       } catch (e) {
+        const reason = e instanceof Error ? e.message : String(e);
         result.failures.push({
           eventId: String(row.id),
-          reason: e instanceof Error ? e.message : String(e),
+          reason:
+            e instanceof SubjectResolutionError
+              ? `${reason} (retry with { resolveOwnerFromProjection: true } to resolve the owner ` +
+                "from the entity's projection table, and/or { eraseUnresolvableSubjects: true } to " +
+                "erase fields whose subject stays unresolvable)"
+              : reason,
         });
       }
     }
@@ -170,10 +214,57 @@ export async function backfillEventPiiEncryption(
 
   return result;
 
+  async function loadProjectionOwners(
+    rows: readonly EventRow[],
+  ): Promise<ReadonlyMap<string, ReadonlyMap<string, Record<string, unknown>>>> {
+    const byType = new Map<string, ReadonlyMap<string, Record<string, unknown>>>();
+    if (!options.resolveOwnerFromProjection) return byType;
+
+    const idsByType = new Map<string, Set<string>>();
+    for (const row of rows) {
+      if (!entityTargets.has(row.aggregate_type)) continue;
+      const ids = idsByType.get(row.aggregate_type) ?? new Set<string>();
+      ids.add(row.aggregate_id);
+      idsByType.set(row.aggregate_type, ids);
+    }
+
+    for (const [aggregateType, ids] of idsByType) {
+      const target = entityTargets.get(aggregateType);
+      if (!target) continue;
+      const tableName = resolveTableName(aggregateType, target.entity, undefined);
+      const ownersById = new Map<string, Record<string, unknown>>();
+      try {
+        const projectionRows = (await raw.unsafe(
+          `SELECT * FROM ${quoteIdent(tableName)} WHERE "id" = ANY($1::uuid[])`,
+          [[...ids]],
+        )) as ReadonlyArray<Record<string, unknown>>;
+        for (const projectionRow of projectionRows) {
+          const id = projectionRow["id"];
+          if (typeof id === "string") {
+            ownersById.set(id, projectionRowToCamel(target.entity, projectionRow));
+          }
+        }
+      } catch (e) {
+        // Entity never mounted/rebuilt (no projection table yet) — stage 3
+        // still gets a chance; this batch just contributes no owners.
+        if (!isUndefinedTable(e)) throw e;
+      }
+      byType.set(aggregateType, ownersById);
+    }
+    return byType;
+  }
+
   async function transformEvent(
     row: EventRow,
-  ): Promise<{ payload: Record<string, unknown>; encrypted: number; erased: number } | null> {
-    const counters = { encrypted: 0, erased: 0 };
+    projectionOwners: ReadonlyMap<string, Record<string, unknown>> | undefined,
+  ): Promise<{
+    payload: Record<string, unknown>;
+    encrypted: number;
+    erased: number;
+    ownerFromProjection: number;
+    erasedUnresolvable: number;
+  } | null> {
+    const counters = { encrypted: 0, erased: 0, ownerFromProjection: 0, erasedUnresolvable: 0 };
     const payload = structuredClone(row.payload);
 
     const catalogFields = eventCatalog.get(row.type);
@@ -187,6 +278,21 @@ export async function backfillEventPiiEncryption(
     } else {
       const target = entityTargets.get(row.aggregate_type);
       if (!target || !isLifecycleEventOf(row.type, row.aggregate_type)) return null;
+      await applyEntityLifecycleFields(target, projectionOwners?.get(row.aggregate_id));
+    }
+
+    if (counters.encrypted === 0 && counters.erased === 0) return null;
+    return { payload, ...counters };
+
+    function bump(outcome: FieldOutcome): void {
+      if (outcome === "encrypted") counters.encrypted++;
+      if (outcome === "erased") counters.erased++;
+    }
+
+    async function applyEntityLifecycleFields(
+      target: { readonly entity: EntityDefinition; readonly piiFields: readonly string[] },
+      projectionOwnerRow: Record<string, unknown> | undefined,
+    ): Promise<void> {
       const sections = lifecycleSections(payload);
       for (const section of sections) {
         // Update-changes may carry a pii field without its owner field —
@@ -198,25 +304,39 @@ export async function backfillEventPiiEncryption(
           ...section,
         };
         for (const field of target.piiFields) {
-          const subject = resolveSubjectForField(target.entity, field, subjectSource, {
+          const value = section[field];
+          if (value === null || value === undefined) continue;
+          if (typeof value !== "string") continue;
+          // Already-ciphertext/sentinel fields must never enter owner
+          // resolution — an unresolvable owner on an already-handled field
+          // must stay untouched, not get erased by stage 3.
+          if (isPiiCiphertext(value) || value === PII_ERASED_SENTINEL) continue;
+
+          const resolution = resolveOwnerSubject(
+            target.entity,
+            field,
+            subjectSource,
+            projectionOwnerRow,
             // @cast-boundary db-read — tenant_id column is the branded TenantId
-            tenantId: row.tenant_id as TenantId,
-          });
-          if (subject === null) continue;
-          const outcome = await encryptField(section, field, subject);
+            row.tenant_id as TenantId,
+          );
+          if (resolution.kind === "unannotated") continue;
+          if (resolution.kind === "unresolved") {
+            if (options.eraseUnresolvableSubjects) {
+              section[field] = PII_ERASED_SENTINEL;
+              counters.erased++;
+              counters.erasedUnresolvable++;
+              continue;
+            }
+            throw resolution.error;
+          }
+          if (resolution.viaProjection) counters.ownerFromProjection++;
+          const outcome = await encryptField(section, field, resolution.subject);
           bump(outcome);
         }
       }
     }
 
-    if (counters.encrypted === 0 && counters.erased === 0) return null;
-    return { payload, ...counters };
-
-    function bump(outcome: FieldOutcome): void {
-      if (outcome === "encrypted") counters.encrypted++;
-      if (outcome === "erased") counters.erased++;
-    }
-
     async function encryptField(
       section: Record<string, unknown>,
       field: string,
@@ -255,6 +375,56 @@ export async function backfillEventPiiEncryption(
   }
 }
 
+type OwnerSubjectResolution =
+  | { readonly kind: "unannotated" }
+  | { readonly kind: "unresolved"; readonly error: SubjectResolutionError }
+  | { readonly kind: "resolved"; readonly subject: SubjectId; readonly viaProjection: boolean };
+
+// Stage 1 (payload) then stage 2 (projection row, when supplied) — stage 3
+// (erase) is the caller's call, made from the "unresolved" outcome.
+function resolveOwnerSubject(
+  entity: EntityDefinition,
+  field: string,
+  subjectSource: Record<string, unknown>,
+  projectionOwnerRow: Record<string, unknown> | undefined,
+  tenantId: TenantId,
+): OwnerSubjectResolution {
+  try {
+    const subject = resolveSubjectForField(entity, field, subjectSource, { tenantId });
+    return subject === null
+      ? { kind: "unannotated" }
+      : { kind: "resolved", subject, viaProjection: false };
+  } catch (e) {
+    if (!(e instanceof SubjectResolutionError)) throw e;
+    if (projectionOwnerRow) {
+      try {
+        const augmented = { ...subjectSource, ...projectionOwnerRow };
+        const subject = resolveSubjectForField(entity, field, augmented, { tenantId });
+        if (subject !== null) return { kind: "resolved", subject, viaProjection: true };
+      } catch (e2) {
+        if (!(e2 instanceof SubjectResolutionError)) throw e2;
+      }
+    }
+    return { kind: "unresolved", error: e };
+  }
+}
+
+// Projection columns are snake_case (author_id); event-payload/subject
+// fields are camelCase (authorId) — forward-map from entity.fields like
+// reindexEntity's rowToState, so any owner field resolves generically
+// instead of hand-listing column names per entity.
+function projectionRowToCamel(
+  entity: EntityDefinition,
+  row: Record<string, unknown>,
+): Record<string, unknown> {
+  const camel: Record<string, unknown> = {};
+  for (const fieldName of Object.keys(entity.fields)) {
+    const snake = toSnakeCase(fieldName);
+    if (Object.hasOwn(row, snake)) camel[fieldName] = row[snake];
+  }
+  return camel;
+}
+
 function isLifecycleEventOf(eventType: string, aggregateType: string): boolean {
   if (!eventType.startsWith(`${aggregateType}.`)) return false;
   const verb = eventType.slice(aggregateType.length + 1);

package/src/engine/__tests__/boot-validator-boot-check.test.ts

@@ -25,7 +25,12 @@ const requiresUserDataHook = (features: readonly { readonly name: string }[]) =>
 
 const promptStore = () =>
   defineFeature("prompt-store", (r) => {
-    const promptFields = { text: createTextField({ pii: true }) };
+    const promptFields = {
+      text: createTextField({
+        personal: "self",
+        find: "none",
+      }),
+    };
     r.entity("prompt", createEntity({ fields: promptFields }));
     r.bootCheck(({ features }) => {
       // Conditional on this feature's own shape (has a pii field), closed