@cosmicdrift/kumiko-framework 0.155.1 → 0.156.0

package/src/db/collect-table-metas.ts

@@ -1,10 +1,9 @@
 // collectTableMetas — kanonische ENTITY_METAS-Quelle für `kumiko schema
 // generate`. Erfasst dieselben Tabellen-Quellen wie der setupTestStack-
-// auto-push (entities, unmanagedTables, projections, multiStreamProjections,
-// rawTables) — die frühere Template-Variante sammelte nur entities +
-// unmanagedTables, wodurch projection-only-Tabellen (z.B. billing-foundation
-// read_subscriptions) nie in Migrations landeten und der erste Prod-Write
-// crashte (#255).
+// auto-push (entities, rawTables, projections, multiStreamProjections) —
+// die frühere Template-Variante sammelte nur entities + rawTables, wodurch
+// projection-only-Tabellen (z.B. billing-foundation read_subscriptions) nie
+// in Migrations landeten und der erste Prod-Write crashte (#255).
 
 import type { FeatureDefinition } from "../engine/types";
 import { compareByCodepoint } from "../utils";
@@ -60,11 +59,11 @@ export function collectTableMetas(
       metas.push(meta);
       byName.set(meta.tableName, { meta, origin: `entity "${name}" (${feature.name})` });
     }
-    for (const entry of Object.values(feature.unmanagedTables)) {
+    for (const entry of Object.values(feature.rawTables)) {
       metas.push(entry.meta);
       byName.set(entry.meta.tableName, {
         meta: entry.meta,
-        origin: `unmanagedTable "${entry.name}" (${feature.name})`,
+        origin: `rawTable "${entry.name}" (${feature.name})`,
       });
     }
   }

package/src/db/entity-table-meta.ts

@@ -95,7 +95,7 @@ export type EntityTableMeta = {
   // discriminator nutzen um Warnungen zu rendern ("X tables are unmanaged").
   readonly source: "managed" | "unmanaged";
   // PII-Subject-annotated field names (pii/userOwned/tenantOwned). Set by
-  // buildEntityTableMeta so the registry can reject r.unmanagedTable stores
+  // buildEntityTableMeta so the registry can reject r.rawTable stores
   // whose direct writes would skip the executor's encryption (#820).
   readonly piiSubjectFields?: readonly string[];
 };

package/src/db/feature-table-sources.ts

@@ -1,8 +1,8 @@
 // Single enumeration of every table-bearing registration on a feature
-// (r.projection, r.multiStreamProjection with table, r.rawTable,
-// r.unmanagedTable). Consumed by BOTH the setupTestStack auto-push and
-// collectTableMetas — one list, so test-DB-push and `kumiko schema generate`
-// cannot drift apart again (#255).
+// (r.projection, r.multiStreamProjection with table, r.rawTable).
+// Consumed by BOTH the setupTestStack auto-push and collectTableMetas —
+// one list, so test-DB-push and `kumiko schema generate` cannot drift
+// apart again (#255).
 // A new table-bearing registrar must be added HERE, not in the consumers.
 
 import type { FeatureDefinition } from "../engine/types";
@@ -30,13 +30,7 @@ export function enumerateFeatureTableSources(
     });
   }
   for (const [name, raw] of Object.entries(feature.rawTables)) {
-    sources.push({ table: raw.table, origin: `rawTable "${name}" (${feature.name})` });
-  }
-  for (const entry of Object.values(feature.unmanagedTables)) {
-    sources.push({
-      table: entry.meta,
-      origin: `unmanagedTable "${entry.name}" (${feature.name})`,
-    });
+    sources.push({ table: raw.meta, origin: `rawTable "${name}" (${feature.name})` });
   }
   return sources;
 }

package/src/db/migrate-generator.ts

@@ -202,13 +202,40 @@ export function diffSnapshots(prev: Snapshot | null, next: Snapshot): SchemaDiff
 }
 
 // Managed projections are event-stream derivatives: in-place-unsafe changes (NOT NULL w/o default, UNIQUE, SET NOT NULL, type change, dropped col) → DROP+CREATE + replay; additive-safe changes stay cheap ALTERs.
+function describeRecreateReasons(td: TableDiff): readonly string[] {
+  const reasons: string[] = [];
+  if (td.droppedColumns.length > 0) {
+    reasons.push(`dropped column(s): ${td.droppedColumns.join(", ")}`);
+  }
+  const notNullNoDefault = td.newColumns
+    .filter((c) => c.notNull && c.defaultSql === undefined)
+    .map((c) => c.name);
+  if (notNullNoDefault.length > 0) {
+    reasons.push(`new NOT NULL column(s) without default: ${notNullNoDefault.join(", ")}`);
+  }
+  const newUniqueIndexes = td.newIndexes
+    .filter((idx) => idx.unique === true)
+    .map((idx) => idx.name);
+  if (newUniqueIndexes.length > 0) {
+    reasons.push(`new UNIQUE index(es): ${newUniqueIndexes.join(", ")}`);
+  }
+  const madeNotNull = td.changedColumns
+    .filter((c) => c.nullabilityChanged?.to === true)
+    .map((c) => c.name);
+  if (madeNotNull.length > 0) {
+    reasons.push(`column(s) set NOT NULL: ${madeNotNull.join(", ")}`);
+  }
+  const typeChanged = td.changedColumns
+    .filter((c) => c.typeChanged !== undefined)
+    .map((c) => c.name);
+  if (typeChanged.length > 0) {
+    reasons.push(`column type change: ${typeChanged.join(", ")}`);
+  }
+  return reasons;
+}
+
 export function managedChangeRequiresRecreate(td: TableDiff): boolean {
-  if (td.droppedColumns.length > 0) return true;
-  if (td.newColumns.some((c) => c.notNull && c.defaultSql === undefined)) return true;
-  if (td.newIndexes.some((idx) => idx.unique === true)) return true;
-  return td.changedColumns.some(
-    (c) => c.nullabilityChanged?.to === true || c.typeChanged !== undefined,
-  );
+  return describeRecreateReasons(td).length > 0;
 }
 
 // --- SQL-Render ---------------------------------------------------------
@@ -296,6 +323,12 @@ export function renderMigrationSql(
       lines.push(`-- ${td.tableName}`);
       if (td.nextMeta.source === "managed" && managedChangeRequiresRecreate(td)) {
         lines.push("-- managed projection — recreated + rebuilt from events (see .rebuild.json)");
+        lines.push(
+          `-- WARN: destructive change (${describeRecreateReasons(td).join("; ")}) forces DROP+CREATE + full event replay.`,
+        );
+        lines.push(
+          "-- Consider an Expand/Contract split across two releases to avoid the rebuild — see docs/guides/expand-contract-managed-projections.md.",
+        );
         lines.push(`DROP TABLE IF EXISTS ${quoteIdent(td.tableName)};`);
         lines.push(...renderTableDdl(td.nextMeta));
         lines.push("");

package/src/db/queries/shadow-swap.ts

@@ -173,8 +173,8 @@ const UNREACHABLE_SAMPLE_LIMIT = 20;
 //     by the #494 backfill-then-rebuild flow.
 // Checking event existence INCLUDING archived streams leaves every one of them
 // alone: those rows all have a real event, so they are not ghosts. Column-level
-// drift detection (fail-hard vs. graceful repair) is the open question deferred
-// from #722.
+// drift is a SEPARATE, non-blocking check — see countColumnDrift below (#916,
+// resolves the #722 open question: observe, don't block).
 //
 // Implicit projections only (caller-gated). aggregate_id and the entity id are
 // both uuid, so the anti-join probes the events index without a cast.
@@ -207,12 +207,94 @@ export async function assertNoUnreachableLiveRows(
     `projection-rebuild "${projectionName}": ${countLabel} live rows in "${tableName}" have no ` +
       `event in the projection's source streams and cannot be reconstructed by replay — the swap ` +
       `would silently drop them (ids: ${ids.join(", ")}). A handler direct-inserted these rows ` +
-      `without emitting a .created event. Fix: register the table with r.unmanagedTable(meta, ` +
+      `without emitting a .created event. Fix: register the table with r.rawTable(meta, ` +
       `{ reason }) to opt out of rebuild, or emit the missing events. See ` +
       `docs/reference/entity-write-patterns.md. Rebuild aborted; live table untouched.`,
   );
 }
 
+// Columns ignored by countColumnDrift — the one PROVABLY legitimate class of
+// live-vs-shadow divergence. A blind-index column (`<field>_bidx`) is
+// recomputed to NULL on GDPR key-shredding; the NULL is the intended end
+// state, not drift. Everything else that legitimately diverges (archived
+// streams, #494 backfill) either never reaches this comparison (archived rows
+// are absent from the shadow entirely, see swapShadowIntoLive) or IS real
+// column drift that the #494 backfill-then-rebuild flow relies on replay to
+// heal — reporting it (without blocking) is correct, not a false positive.
+const COLUMN_DRIFT_SAMPLE_LIMIT = 20;
+
+export type ColumnDriftResult = {
+  readonly rowCount: number;
+  // Capped sample of "<id>.<column>" pairs for log/ops triage.
+  readonly sample: readonly string[];
+};
+
+// Runs INSIDE the rebuild tx, in the same slot as assertNoUnreachableLiveRows
+// (after replay settles, before swapShadowIntoLive). Non-blocking counterpart
+// to the ghost-row guard: reports live rows whose column values differ from
+// the freshly-replayed shadow, WITHOUT aborting the swap (#916, resolves the
+// #722 open question in favor of observe-not-block).
+//
+// Why non-blocking: a legacy column direct-written before its handler emitted
+// events (#494) diverges from replay by design — that divergence is exactly
+// what the backfill-then-rebuild flow relies on replay to heal. Failing hard
+// here would make rebuild mutually exclusive with that shipped healing path.
+// There is no reliable metadata to distinguish "#494 healing in progress" from
+// "someone else corrupted this row" short of an open-ended per-column policy
+// blocklist — wrong-by-default whenever a class is missed. So: surface it,
+// don't police it. The caller logs the result; ops decides.
+//
+// Caveat: sensitive CUSTOM fields still diverge until #972 (Subject-DEK
+// design) — regular sensitive fields carry event-payload ciphertext parity
+// post-#973 and don't drift. Both are reported the same as any other column
+// drift; this is deliberate (see module comment above), not an oversight.
+//
+// Relies on assertLiveColumnsMatchMeta having already run: live/shadow/meta
+// column sets are known to match, so the diff can walk meta.columns directly.
+export async function countColumnDrift(
+  tx: AnyDb,
+  tableName: string,
+  meta: EntityTableMeta,
+): Promise<ColumnDriftResult> {
+  const comparable = meta.columns.filter((c) => c.primaryKey !== true && !c.name.endsWith("_bidx"));
+  // skip: nothing to compare (id-only or all-bidx table) — no drift is possible
+  if (comparable.length === 0) return { rowCount: 0, sample: [] };
+  const t = quoteTableIdent(tableName);
+  const raw = asRawClient(tx);
+  const driftCte = `WITH drifted AS (
+     SELECT l."id" AS id, string_agg(diff.col, ',') AS drifted_columns
+     FROM public.${t} l
+     JOIN ${SCHEMA_IDENT}.${t} s ON s."id" = l."id"
+     CROSS JOIN LATERAL (
+       VALUES ${comparable
+         .map(
+           (c) =>
+             `('${c.name}', l.${quoteTableIdent(c.name)} IS DISTINCT FROM s.${quoteTableIdent(c.name)})`,
+         )
+         .join(", ")}
+     ) AS diff(col, differs)
+     WHERE diff.differs
+     GROUP BY l."id"
+   )`;
+  // Two passes over `drifted`: an unbounded COUNT for the true total (replay
+  // already scanned every row this run, so a second scan here is cheap by
+  // comparison) plus a capped sample for the log. rowCount must NEVER be
+  // min(actual, LIMIT) — that would silently understate severity to ops.
+  const totalRows = await raw.unsafe<{ total: string }>(
+    `${driftCte} SELECT count(*)::text AS total FROM drifted`,
+  );
+  const rowCount = Number(totalRows[0]?.total ?? "0");
+  // skip: no drift — nothing to sample
+  if (rowCount === 0) return { rowCount: 0, sample: [] };
+  const rows = await raw.unsafe<{ id: unknown; drifted_columns: string }>(
+    `${driftCte} SELECT id, drifted_columns FROM drifted LIMIT ${COLUMN_DRIFT_SAMPLE_LIMIT}`,
+  );
+  const sample = rows.flatMap((r) =>
+    r.drifted_columns.split(",").map((col) => `${String(r.id)}.${col}`),
+  );
+  return { rowCount, sample };
+}
+
 // Atomic swap, INSIDE the rebuild tx, AFTER replay. Schema-qualified so the
 // active shadow search_path can't redirect them. DROP without CASCADE: if any
 // object depends on the live table the swap fails loud and the whole rebuild

package/src/db/tenant-db.ts

@@ -1,4 +1,5 @@
 import {
+  asEntityTableMeta,
   asRawClient,
   deleteMany as bunDeleteMany,
   fetchOne as bunFetchOne,
@@ -79,7 +80,14 @@ function tableNameOf(table: Table): string {
   return typeof sym === "string" ? sym : "<unknown>";
 }
 
+// Checks the canonical EntityTableMeta (branded EntityTable's KUMIKO_META_SYMBOL
+// or a plain buildEntityTableMeta/defineUnmanagedTable result), not a direct
+// `table.tenantId` property read — the latter only exists on branded EntityTables
+// and silently returned false (no tenant filter!) for plain EntityTableMeta
+// tables like unmanaged direct-write stores, e.g. userSessionTable.
 function hasTenantColumn(table: Table): boolean {
+  const meta = asEntityTableMeta(table);
+  if (meta) return meta.columns.some((c) => c.name === "tenant_id");
   return (table as Record<string, unknown>)["tenantId"] !== undefined;
 }
 

package/src/engine/__tests__/boot-validator-pii-retention.test.ts

@@ -708,3 +708,105 @@ describe("validateBoot — lookupable / blind-index (#818)", () => {
     expect(() => validateBoot([feature])).toThrow(/sortable.*cannot work/);
   });
 });
+
+describe("validateBoot — piiEncrypted (kumiko-platform#231/#456)", () => {
+  test("piiEncrypted on a plain text field passes", () => {
+    const feature = defineFeature("test", (r) => {
+      r.entity(
+        "tenant",
+        createEntity({
+          fields: {
+            iban: createTextField({
+              piiEncrypted: true,
+              tenantOwned: true,
+              access: { read: ["TenantAdmin"] },
+            }),
+          },
+        }),
+      );
+    });
+    expect(() => validateBoot([feature])).not.toThrow();
+  });
+
+  test("piiEncrypted on a non-text field throws", () => {
+    const feature = defineFeature("test", (r) => {
+      r.entity(
+        "tenant",
+        createEntity({
+          fields: {
+            amount: { ...createNumberField(), piiEncrypted: true } as unknown as ReturnType<
+              typeof createNumberField
+            >,
+          },
+        }),
+      );
+    });
+    expect(() => validateBoot([feature])).toThrow(/piiEncrypted.*only applies to text fields/);
+  });
+
+  test("piiEncrypted without a subject annotation throws (kumiko-platform#457)", () => {
+    const feature = defineFeature("test", (r) => {
+      r.entity(
+        "tenant",
+        createEntity({
+          fields: {
+            iban: createTextField({ piiEncrypted: true }),
+          },
+        }),
+      );
+    });
+    expect(() => validateBoot([feature])).toThrow(/piiEncrypted.*without a subject annotation/);
+  });
+
+  test("piiEncrypted combined with searchable throws", () => {
+    const feature = defineFeature("test", (r) => {
+      r.entity(
+        "tenant",
+        createEntity({
+          fields: {
+            iban: createTextField({
+              piiEncrypted: true,
+              tenantOwned: true,
+              access: { read: ["TenantAdmin"] },
+              searchable: true,
+            }),
+          },
+        }),
+      );
+    });
+    expect(() => validateBoot([feature])).toThrow(/piiEncrypted.*searchable.*cannot work/);
+  });
+
+  test("piiEncrypted combined with sortable throws", () => {
+    const feature = defineFeature("test", (r) => {
+      r.entity(
+        "tenant",
+        createEntity({
+          fields: {
+            iban: createTextField({
+              piiEncrypted: true,
+              tenantOwned: true,
+              access: { read: ["TenantAdmin"] },
+              sortable: true,
+            }),
+          },
+        }),
+      );
+    });
+    expect(() => validateBoot([feature])).toThrow(/piiEncrypted.*sortable.*cannot work/);
+  });
+
+  test("piiEncrypted without access.read throws (kumiko-platform#460)", () => {
+    const feature = defineFeature("test", (r) => {
+      r.entity(
+        "tenant",
+        createEntity({
+          fields: {
+            iban: createTextField({ piiEncrypted: true, tenantOwned: true }),
+          },
+        }),
+      );
+    });
+    expect(() => validateBoot([feature])).toThrow(/piiEncrypted.*without.*access.*read/);
+  });
+});

package/src/engine/__tests__/build-config-feature-schema.test.ts

@@ -435,4 +435,20 @@ describe("buildConfigFeatureSchema — cross-feature group", () => {
     });
     expect(() => buildConfigFeatureSchema(createRegistry([feature]))).toThrow(/kebab-case/);
   });
+
+  test("invalid (non-kebab) group value throws even on a maskless key", () => {
+    // Maskless keys never appear in the hub — group is a no-op for them at
+    // runtime — but the type contract still promises kebab-case is
+    // boot-validated regardless of `mask`. collectMaskedKeys used to check
+    // the mask-gate before the group check, so a maskless key with a bad
+    // group silently passed boot; the check now runs before the mask-gate.
+    const feature = defineFeature("badgroupmaskless", (r) => {
+      r.config({
+        keys: {
+          flag: createTenantConfig("boolean", { group: "Not Kebab!" }),
+        },
+      });
+    });
+    expect(() => buildConfigFeatureSchema(createRegistry([feature]))).toThrow(/kebab-case/);
+  });
 });

package/src/engine/__tests__/feature-crud-shorthand.test.ts

@@ -0,0 +1,28 @@
+import { describe, expect, test } from "bun:test";
+import { defineFeature } from "../define-feature";
+import { createEntity, createTextField } from "../factories";
+
+const taskEntity = createEntity({
+  table: "crud_shorthand_tasks",
+  fields: { title: createTextField({ required: true }) },
+  softDelete: true,
+});
+
+describe("r.crud", () => {
+  test("registers the entity + full CRUD handler set, same as registerEntityCrud", () => {
+    const write = { access: { roles: ["Admin"] } } as const;
+    const read = { access: { openToAll: true } } as const;
+
+    const feature = defineFeature("via-crud", (r) => {
+      r.crud("task", taskEntity, { write, read });
+    });
+
+    expect(Object.keys(feature.entities ?? {})).toEqual(["task"]);
+    expect(Object.keys(feature.writeHandlers ?? {}).sort()).toEqual(
+      ["task:create", "task:delete", "task:restore", "task:update"].sort(),
+    );
+    expect(Object.keys(feature.queryHandlers ?? {}).sort()).toEqual(
+      ["task:detail", "task:list"].sort(),
+    );
+  });
+});

package/src/engine/__tests__/field-access.test.ts

@@ -1,5 +1,5 @@
 import { describe, expect, test } from "bun:test";
-import { checkWriteFieldRoles, filterReadFields } from "../field-access";
+import { checkWriteFieldRoles, filterReadFields, PII_MASKED_VALUE } from "../field-access";
 import type { EntityDefinition } from "../types";
 
 const entity: EntityDefinition = {
@@ -25,6 +25,28 @@ describe("filterReadFields", () => {
     const filtered = filterReadFields(entity, row, admin);
     expect(filtered["secret"]).toBe("visible");
   });
+
+  test("masks piiEncrypted fields instead of stripping them (kumiko-platform#463)", () => {
+    const entityWithPii: EntityDefinition = {
+      fields: {
+        ...entity.fields,
+        iban: {
+          type: "text",
+          piiEncrypted: true,
+          tenantOwned: true,
+          access: { read: { admin: "all" } },
+        },
+      },
+    };
+    const row = { id: 1, title: "Hello", secret: "hidden", iban: "DE89370400440532013000" };
+
+    const filteredForEditor = filterReadFields(entityWithPii, row, editor);
+    expect(filteredForEditor["secret"]).toBeUndefined();
+    expect(filteredForEditor["iban"]).toBe(PII_MASKED_VALUE);
+
+    const filteredForAdmin = filterReadFields(entityWithPii, row, admin);
+    expect(filteredForAdmin["iban"]).toBe("DE89370400440532013000");
+  });
 });
 
 describe("checkWriteFieldRoles", () => {