@cosmicdrift/kumiko-framework 0.118.0 → 0.120.0

package/src/db/blind-index-cleanup.ts

@@ -0,0 +1,53 @@
+// Sofortiges Blind-Index-Nulling nach einem Subject-Erase (#818).
+//
+// Nach kms.eraseKey ist der Ciphertext unlesbar, aber die deterministische
+// bidx-Spalte bliebe bis zum nächsten Write/Rebuild matchbar — ein
+// Linkage-Fenster ("hat irgendeine Row den Wert X"). Dieser Sweep schließt
+// es sofort: der Ciphertext nennt sein Subject inline
+// (kumiko-pii:v1:<subjectKey>:...), also findet ein LIKE-Prefix-Match exakt
+// die Rows des erased Subjects — pro lookupable-Feld ein UPDATE.
+//
+// Rows, die der Forget-Lauf ohnehin via Executor löscht/anonymisiert,
+// bekommen ihren bidx dort automatisch neu berechnet; dieser Sweep deckt
+// die liegen bleibenden Rows ab (fremde Entities mit userOwned-Feldern).
+
+import { collectLookupableFields } from "../crypto/blind-index";
+import type { FeatureDefinition } from "../engine/types";
+import { toSnakeCase } from "../utils/case";
+import type { DbRunner } from "./connection";
+import { resolveTableName } from "./entity-table-meta";
+import { executeRawQuery } from "./queries/raw-sql";
+
+function quoteIdent(name: string): string {
+  return `"${name.replace(/"/g, '""')}"`;
+}
+
+function escapeLikePattern(value: string): string {
+  return value.replace(/[\\%_]/g, (m) => `\\${m}`);
+}
+
+export async function nullBlindIndexesForSubject(
+  db: DbRunner,
+  features: ReadonlyMap<string, FeatureDefinition>,
+  subjectKey: string,
+): Promise<void> {
+  const likePattern = `kumiko-pii:v1:${escapeLikePattern(subjectKey)}:%`;
+  for (const feature of features.values()) {
+    for (const [entityName, entity] of Object.entries(feature.entities ?? {})) {
+      const lookupable = collectLookupableFields(entity);
+      if (lookupable.length === 0) continue;
+      // Kein featureName-Prefix — der Dispatcher baut Entity-Tables ohne
+      // (buildEntityTable ohne featureName-Option), der Sweep muss dieselben
+      // Namen treffen.
+      const tableName = resolveTableName(entityName, entity, undefined);
+      for (const fieldName of lookupable) {
+        const snake = toSnakeCase(fieldName);
+        await executeRawQuery(
+          db,
+          `UPDATE ${quoteIdent(tableName)} SET ${quoteIdent(`${snake}_bidx`)} = NULL WHERE ${quoteIdent(snake)} LIKE $1`,
+          [likePattern],
+        );
+      }
+    }
+  }
+}

package/src/db/entity-table-meta.ts

@@ -18,6 +18,7 @@
 //      auf Sondercases beschränken (child-projection-tables ohne tenant,
 //      append-only-logs mit serial PK, aggregate-ID ohne DEFAULT, …).
 
+import { collectPiiSubjectFields } from "../crypto";
 import type {
   EntityDefinition,
   EntityIndexDef,
@@ -92,6 +93,10 @@ export type EntityTableMeta = {
   // trägt Verantwortung. Migration-Generator + Tooling können den
   // discriminator nutzen um Warnungen zu rendern ("X tables are unmanaged").
   readonly source: "managed" | "unmanaged";
+  // PII-Subject-annotated field names (pii/userOwned/tenantOwned). Set by
+  // buildEntityTableMeta so the registry can reject r.unmanagedTable stores
+  // whose direct writes would skip the executor's encryption (#820).
+  readonly piiSubjectFields?: readonly string[];
 };
 
 // Standard base-columns für event-sourced Read-Model-Tabellen. Spiegelt
@@ -153,7 +158,7 @@ function fieldToColumnMeta(
     case "text":
     case "longText": {
       const def = fieldDefaultLiteral(field);
-      return [
+      const cols: ColumnMeta[] = [
         {
           name: snake,
           pgType: "text",
@@ -161,6 +166,13 @@ function fieldToColumnMeta(
           ...(def !== undefined && { defaultSql: def }),
         },
       ];
+      // lookupable → HMAC-Blind-Index-Pendant. Nullable ist Pflicht:
+      // managedChangeRequiresRecreate würde eine NOT-NULL-Spalte ohne
+      // Default auf Bestandstabellen als DROP+Rebuild diffen.
+      if (field.type === "text" && field.lookupable === true) {
+        cols.push({ name: `${snake}_bidx`, pgType: "text", notNull: false });
+      }
+      return cols;
     }
     case "boolean": {
       const def = fieldDefaultLiteral(field);
@@ -305,10 +317,17 @@ export function buildEntityTableMeta(
   for (const c of baseCols) colByName.set(c.name, c);
 
   const fieldNameToSnake = new Map<string, string>();
+  const bidxSnakeByFieldSnake = new Map<string, string>();
   for (const [name, field] of Object.entries(entity.fields)) {
     const fieldCols = fieldToColumnMeta(name, field, entity);
     for (const c of fieldCols) colByName.set(c.name, c);
-    if (fieldCols.length === 1 && fieldCols[0]) fieldNameToSnake.set(name, fieldCols[0].name);
+    // Multi-column fields map to their primary column when its name IS the
+    // field's snake (text+bidx, money+currency) — matches the toSnakeCase
+    // fallback below, so explicit indexes keep resolving.
+    const primary = fieldCols[0];
+    if (primary && primary.name === toSnakeCase(name)) fieldNameToSnake.set(name, primary.name);
+    const bidxCol = fieldCols.find((c) => c.name.endsWith("_bidx"));
+    if (primary && bidxCol) bidxSnakeByFieldSnake.set(primary.name, bidxCol.name);
   }
 
   // Preserve base-col order, then any new user-col-names in fields-order.
@@ -347,6 +366,12 @@ export function buildEntityTableMeta(
     indexes.push({ name: `${tableName}_${snake}_idx`, columns: [snake] });
   }
 
+  // lookupable-Felder: Index auf der bidx-Spalte — der OR-Rewrite der
+  // Query-Compiler trifft sie bei jedem Equality-Lookup.
+  for (const bidxSnake of bidxSnakeByFieldSnake.values()) {
+    indexes.push({ name: `${tableName}_${bidxSnake}_idx`, columns: [bidxSnake] });
+  }
+
   // Explizit deklarierte indexes (EntityIndexDef). `def.where` ist ein
   // SqlExpression (`sql\`…\`` aus @cosmicdrift/kumiko-framework/db) —
   // renderbar via `.text`. Unbekannte where-Shapes bleiben needsManualWhere.
@@ -364,13 +389,33 @@ export function buildEntityTableMeta(
       ...(whereSql !== undefined && { whereSql }),
       ...(def.where !== undefined && whereSql === undefined && { needsManualWhere: true }),
     });
+    // Unique-Index über lookupable-Spalten: partielles bidx-Pendant, damit
+    // Uniqueness auch für verschlüsselte Rows greift. Das Original bleibt
+    // für Klartext-Alt-Rows; partial (IS NOT NULL) weil bidx bei erased/
+    // key-losen Rows NULL ist.
+    if (def.unique === true && def.where === undefined) {
+      const bidxCols = cols.map((c) => bidxSnakeByFieldSnake.get(c) ?? c);
+      if (bidxCols.some((c, i) => c !== cols[i])) {
+        const notNullParts = bidxCols
+          .filter((c, i) => c !== cols[i])
+          .map((c) => `"${c}" IS NOT NULL`);
+        indexes.push({
+          name: `${indexName}_bidx`,
+          columns: bidxCols,
+          unique: true,
+          whereSql: notNullParts.join(" AND "),
+        });
+      }
+    }
   }
 
+  const piiSubjectFields = collectPiiSubjectFields(entity);
   return {
     tableName,
     columns,
     indexes,
     source: "managed",
+    ...(piiSubjectFields.length > 0 && { piiSubjectFields }),
   };
 }
 

package/src/db/event-store-executor.ts

@@ -1,4 +1,14 @@
 import { requestContext } from "../api/request-context";
+import {
+  collectPiiSubjectFields,
+  computeBlindIndex,
+  configuredBlindIndexKey,
+  configuredPiiSubjectKms,
+  decryptPiiFieldValues,
+  encryptPiiFieldValues,
+  type KmsContext,
+  type LocalKeyKmsAdapter,
+} from "../crypto";
 import { executeRawQuery } from "../db/queries/raw-sql";
 import type { WhereObject } from "../db/query";
 import { coerceRow, extractTableInfo, selectMany } from "../db/query";
@@ -121,6 +131,8 @@ export type EventStoreExecutorOptions = {
   entityCache?: EntityCache;
   /** Override the boot-injected cipher for fields marked `encrypted: true`. */
   encryption?: EnvelopeCipher;
+  /** Override the boot-injected subject KMS for pii-annotated fields. */
+  kms?: LocalKeyKmsAdapter;
 };
 
 // F8 helper: PG-23505 (unique-violation) catched aus applyEntityEvent
@@ -307,28 +319,64 @@ export function createEventStoreExecutor(
   const encryptedFields = collectEncryptedFieldNames(entity);
   const hasEncryptedFields = encryptedFields.size > 0;
 
+  const piiSubjectFields = collectPiiSubjectFields(entity);
+  const hasPiiFields = piiSubjectFields.length > 0;
+
   function fieldCipher(): EnvelopeCipher {
     if (options.encryption) return options.encryption;
     return resolveEntityFieldEncryption();
   }
 
+  // No adapter configured = crypto-shredding off; pii fields stay plaintext
+  // (pre-#724 behavior). The hard boot gate ships with the prod-grade
+  // PgKmsAdapter (phase E).
+  function piiKms(): LocalKeyKmsAdapter | undefined {
+    return options.kms ?? configuredPiiSubjectKms();
+  }
+
+  function kmsContextFor(user?: SessionUser): KmsContext {
+    return {
+      requestId: requestContext.get()?.requestId ?? "event-store-executor",
+      ...(user && { tenantId: user.tenantId, userId: String(user.id) }),
+    };
+  }
+
   // Async on purpose: the envelope cipher wraps/unwraps DEKs via the
   // MasterKeyProvider. Callers MUST await — a missed await here writes
   // "[object Promise]" into the projection, which the Promise return
   // types turn into a compile error at every call site.
   async function encryptForStorage(
     row: Record<string, unknown>,
-    onlyKeys?: Iterable<string>,
+    user: SessionUser,
+    opts?: { onlyKeys?: Iterable<string>; subjectSource?: Record<string, unknown> },
   ): Promise<Record<string, unknown>> {
-    if (!hasEncryptedFields) return row;
-    return encryptEntityFieldValues(row, encryptedFields, fieldCipher(), {
-      ...(onlyKeys !== undefined && { onlyKeys }),
-    });
+    let out = row;
+    if (hasEncryptedFields) {
+      out = await encryptEntityFieldValues(out, encryptedFields, fieldCipher(), {
+        ...(opts?.onlyKeys !== undefined && { onlyKeys: opts.onlyKeys }),
+      });
+    }
+    const kms = piiKms();
+    if (hasPiiFields && kms) {
+      out = await encryptPiiFieldValues(out, entity, piiSubjectFields, kms, kmsContextFor(user), {
+        tenantId: user.tenantId,
+        ...(opts?.onlyKeys !== undefined && { onlyKeys: opts.onlyKeys }),
+        ...(opts?.subjectSource !== undefined && { subjectSource: opts.subjectSource }),
+      });
+    }
+    return out;
   }
 
   async function decryptForRead(row: Record<string, unknown>): Promise<Record<string, unknown>> {
-    if (!hasEncryptedFields) return row;
-    return decryptEntityFieldValues(row, encryptedFields, fieldCipher());
+    let out = row;
+    if (hasEncryptedFields) {
+      out = await decryptEntityFieldValues(out, encryptedFields, fieldCipher());
+    }
+    const kms = piiKms();
+    if (hasPiiFields && kms) {
+      out = await decryptPiiFieldValues(out, piiSubjectFields, kms, kmsContextFor());
+    }
+    return out;
   }
 
   function applyDefaults(payload: Record<string, unknown>): Record<string, unknown> {
@@ -469,7 +517,12 @@ export function createEventStoreExecutor(
       // Alle Compound-Types (locatedTimestamp, money, ...) gehen durch
       // dieselbe Pipeline. Caller schickt combined API-Form, Framework
       // speichert flat DB-Form. Siehe db/compound-types.ts.
-      const flatData = await encryptForStorage(flattenCompoundTypes(data, entity));
+      // subjectSource carries the freshly minted aggregateId: the create
+      // payload has no id column, but a pii:true self-subject resolves from it.
+      const flatCreateData = flattenCompoundTypes(data, entity);
+      const flatData = await encryptForStorage(flatCreateData, user, {
+        subjectSource: { ...flatCreateData, id: aggregateId },
+      });
 
       // 1. Append event (same TX as the projection write — both must succeed
       //    or both roll back; the dispatcher wraps both in one transaction).
@@ -563,7 +616,9 @@ export function createEventStoreExecutor(
           previous: {},
           isNew: true,
           entityName,
-          event,
+          // Persisted event carries ciphertext by design — the caller-facing
+          // echo must be plaintext like every other response field (#820).
+          event: { ...event, payload: stripSensitive(flatCreateData) },
         },
       };
     },
@@ -650,10 +705,13 @@ export function createEventStoreExecutor(
 
       try {
         // Compound-Types Auto-Convert (alle in einem Pass).
-        const flatChanges = await encryptForStorage(
-          flattenCompoundTypes(payload.changes, entity),
-          Object.keys(payload.changes),
-        );
+        // subjectSource: partial changes may carry a pii field without its
+        // ownerField — the merged row still names the subject.
+        const flatChangesPlain = flattenCompoundTypes(payload.changes, entity);
+        const flatChanges = await encryptForStorage(flatChangesPlain, user, {
+          onlyKeys: Object.keys(payload.changes),
+          subjectSource: mergedNew,
+        });
 
         // The event payload carries BOTH `changes` (what the user asked for) AND
         // `previous` (the pre-update row). Cross-aggregate projections need the
@@ -673,7 +731,7 @@ export function createEventStoreExecutor(
           type: entityEventName(entityName, "updated"),
           payload: {
             changes: stripSensitive(flatChanges),
-            previous: stripSensitive(await encryptForStorage(previous)),
+            previous: stripSensitive(await encryptForStorage(previous, user)),
           },
           metadata: buildEventMetadata(user),
         });
@@ -717,7 +775,13 @@ export function createEventStoreExecutor(
             previous,
             isNew: false,
             entityName,
-            event,
+            event: {
+              ...event,
+              payload: {
+                changes: stripSensitive(flatChangesPlain),
+                previous: stripSensitive(previous),
+              },
+            },
           },
         };
       } catch (e) {
@@ -783,7 +847,7 @@ export function createEventStoreExecutor(
         tenantId: streamTenantFor(user),
         expectedVersion: currentVersion,
         type: entityEventName(entityName, "deleted"),
-        payload: { previous: stripSensitive(await encryptForStorage(existing)) },
+        payload: { previous: stripSensitive(await encryptForStorage(existing, user)) },
         metadata: buildEventMetadata(user),
       });
 
@@ -805,7 +869,13 @@ export function createEventStoreExecutor(
 
       return {
         isSuccess: true,
-        data: { kind: "delete", id: payload.id, data: existing, entityName, event },
+        data: {
+          kind: "delete",
+          id: payload.id,
+          data: existing,
+          entityName,
+          event: { ...event, payload: { previous: stripSensitive(existing) } },
+        },
       };
     },
 
@@ -846,7 +916,9 @@ export function createEventStoreExecutor(
         tenantId: streamTenantFor(user),
         expectedVersion: currentVersion,
         type: entityEventName(entityName, "forgotten"),
-        payload: { previous: stripSensitive(existing) },
+        // Re-encrypt like delete(): `existing` came decrypted from loadById —
+        // plaintext must not land in the immutable log, least of all on forget.
+        payload: { previous: stripSensitive(await encryptForStorage(existing, user)) },
         metadata: buildEventMetadata(user),
       });
 
@@ -863,7 +935,13 @@ export function createEventStoreExecutor(
 
       return {
         isSuccess: true,
-        data: { kind: "delete", id: payload.id, data: existing, entityName, event },
+        data: {
+          kind: "delete",
+          id: payload.id,
+          data: existing,
+          entityName,
+          event: { ...event, payload: { previous: stripSensitive(existing) } },
+        },
       };
     },
 
@@ -947,6 +1025,7 @@ export function createEventStoreExecutor(
         rehydrateCompoundTypes(restored as DbRow, entity) as DbRow,
       );
 
+      const previousPlain = await decryptForRead(data);
       return {
         isSuccess: true,
         data: {
@@ -954,10 +1033,10 @@ export function createEventStoreExecutor(
           id: payload.id,
           data: restoredHydrated,
           changes: { isDeleted: false },
-          previous: await decryptForRead(data),
+          previous: previousPlain,
           isNew: false,
           entityName,
-          event,
+          event: { ...event, payload: { previous: stripSensitive(previousPlain) } },
         },
       };
     },
@@ -1067,8 +1146,19 @@ export function createEventStoreExecutor(
               whereSql.push(`${colSql(field)} ${opSym} $${params.length}`);
             }
           } else {
-            params.push(value);
-            whereSql.push(`${colSql(field)} = $${params.length}`);
+            // Blind-Index-OR-Rewrite (#818), lock-step mit buildWhereClause
+            // in bun-db/query.ts — Equality auf lookupable-Feldern matcht
+            // Klartext-Arm ODER HMAC-Arm.
+            const bidxKey = configuredBlindIndexKey();
+            if (bidxKey !== undefined && typeof value === "string" && table[`${field}Bidx`]) {
+              params.push(value, computeBlindIndex(bidxKey, value));
+              whereSql.push(
+                `(${colSql(field)} = $${params.length - 1} OR ${colSql(`${field}Bidx`)} = $${params.length})`,
+              );
+            } else {
+              params.push(value);
+              whereSql.push(`${colSql(field)} = $${params.length}`);
+            }
           }
         }
       };
@@ -1086,21 +1176,26 @@ export function createEventStoreExecutor(
       const listSql = `SELECT * FROM "${tableName}"${whereClauseSqlText}${orderByClause} LIMIT ${limit}${offsetClause}`;
 
       const rawRows = await executeRawQuery<Record<string, unknown>>(db.raw, listSql, params);
-      // Read-Side rehydrate pro Row + snake→camel coercion für driver-agnostic Feldnamen
+      // Read-Side rehydrate pro Row + snake→camel coercion für driver-agnostic Feldnamen.
+      // Coerce BEFORE decrypt: the raw SELECT * rows carry snake_case column
+      // names, while the encrypted/pii field lists are camelCase — decrypting
+      // first silently skipped every multi-word field (ciphertext leaked to
+      // the caller).
       const tableInfo = extractTableInfo(table);
-      const rows = await Promise.all(
-        rawRows.map(async (r) =>
-          coerceRow(await decryptForRead(rehydrateCompoundTypes(r, entity)), tableInfo),
-        ),
+      const encryptedRows = rawRows.map((r) =>
+        coerceRow(rehydrateCompoundTypes(r, entity), tableInfo),
       );
+      const rows = await Promise.all(encryptedRows.map((r) => decryptForRead(r)));
 
       // list rows carry the READ-ROW version (display-only), never an optimistic-lock
       // base — edit flows reload via detail(), which reconciles the stream version.
+      // Cache the still-encrypted form: same at-rest guarantee as detail()'s
+      // encryptForStorage round-trip, without paying a re-encrypt.
       if (entityCache && entityName && rows.length > 0) {
         await entityCache.mset(
           user.tenantId,
           entityName,
-          rows.map((r) => ({ id: r["id"] as EntityId, data: r })), // @cast-boundary engine-payload
+          encryptedRows.map((r) => ({ id: r["id"] as EntityId, data: r })), // @cast-boundary engine-payload
         );
       }
 
@@ -1180,7 +1275,7 @@ export function createEventStoreExecutor(
           user.tenantId,
           entityName,
           payload.id,
-          await encryptForStorage(coerced),
+          await encryptForStorage(coerced, user),
         );
       }
 

package/src/db/index.ts

@@ -1,4 +1,5 @@
 export { assertExistsIn } from "./assert-exists-in";
+export { nullBlindIndexesForSubject } from "./blind-index-cleanup";
 export { collectTableMetas } from "./collect-table-metas";
 export { flattenCompoundTypes, rehydrateCompoundTypes } from "./compound-types";
 export { seedConfigValues } from "./config-seed";