@cosmicdrift/kumiko-framework 0.209.1 → 0.211.0

package/src/db/event-store-executor-read.ts

@@ -8,7 +8,7 @@ import { SYSTEM_TENANT_ID } from "../engine/types/identifiers";
 import { UnprocessableError } from "../errors";
 import { getStreamVersion } from "../event-store";
 import { rehydrateCompoundTypes } from "./compound-types";
-import { decodeCursor, encodeCursor } from "./cursor";
+import { decodeKeysetCursor, encodeCursor, encodeKeysetCursor } from "./cursor";
 import type { EventStoreExecutor } from "./event-store-executor";
 import { buildFilterWhere, type ExecutorContext } from "./event-store-executor-context";
 import { toSnakeCase } from "./table-builder";
@@ -43,6 +43,54 @@ export function resolveListPagination(payload: {
   return { limit: Math.min(limit, MAX_LIST_LIMIT), offset };
 }
 
+// Cursor sort values travel as text and bind as plain string params — Postgres
+// infers the parameter type from the column they are compared against, the same
+// way prepareValue binds a timestamptz as an ISO string. undefined means the
+// driver value has no faithful text form, which downgrades the page to the
+// legacy id-only boundary instead of emitting a wrong one.
+function toCursorSortText(value: unknown): string | null | undefined {
+  if (value === undefined) return undefined;
+  if (value === null) return null;
+  if (typeof value === "string") return value;
+  if (typeof value === "number" || typeof value === "bigint" || typeof value === "boolean") {
+    return String(value);
+  }
+  if (value instanceof Date) return value.toISOString();
+  if (typeof value === "object") {
+    const tag = (value as { [Symbol.toStringTag]?: unknown })[Symbol.toStringTag];
+    if (typeof tag === "string" && tag.startsWith("Temporal.")) return String(value);
+    return JSON.stringify(value);
+  }
+  return undefined;
+}
+
+// Keyset boundary for `ORDER BY <sort> <dir>, id ASC` under Postgres' DEFAULT
+// null ordering (ASC → NULLS LAST, DESC → NULLS FIRST). An explicit NULLS clause
+// would change the visible order and cost the sort its index.
+function keysetBoundarySql(
+  sortCol: string,
+  idCol: string,
+  cursor: { readonly id: string; readonly sortValue: string | null },
+  descending: boolean,
+  params: unknown[],
+): string {
+  params.push(cursor.id);
+  const afterId = `${idCol} > $${params.length}`;
+  const nullsComeFirst = descending;
+  if (cursor.sortValue === null) {
+    return nullsComeFirst
+      ? `(${sortCol} IS NOT NULL OR ${afterId})`
+      : `(${sortCol} IS NULL AND ${afterId})`;
+  }
+  params.push(cursor.sortValue);
+  const sortParam = `$${params.length}`;
+  const beyond = `${sortCol} ${descending ? "<" : ">"} ${sortParam}`;
+  const tie = `(${sortCol} = ${sortParam} AND ${afterId})`;
+  return nullsComeFirst
+    ? `(${sortCol} IS NOT NULL AND (${beyond} OR ${tie}))`
+    : `(${sortCol} IS NULL OR ${beyond} OR ${tie})`;
+}
+
 export function createReadVerbs(ctx: ExecutorContext): Pick<EventStoreExecutor, "list" | "detail"> {
   const {
     table,
@@ -105,8 +153,11 @@ export function createReadVerbs(ctx: ExecutorContext): Pick<EventStoreExecutor,
       const tableName = String((table as unknown as Record<symbol, unknown>)[KUMIKO_NAME_SYMBOL]);
       const whereSql: string[] = [];
       const params: unknown[] = [];
-      const colSql = (field: string): string =>
-        `"${(table[field] as { name?: string } | undefined)?.name ?? toSnakeCase(field)}"`;
+      const physicalCol = (field: string): string =>
+        (table[field] as { name?: string } | undefined)?.name ?? toSnakeCase(field);
+      const colSql = (field: string): string => `"${physicalCol(field)}"`;
+      const sortField = payload.sort && table[payload.sort] ? payload.sort : undefined;
+      const sortDescending = payload.sortDirection === "desc";
 
       // Tenant-Filter (replicates TenantDb's readWhere semantics).
       if (table["tenantId"] !== undefined && db.mode === "tenant") {
@@ -117,8 +168,21 @@ export function createReadVerbs(ctx: ExecutorContext): Pick<EventStoreExecutor,
         whereSql.push(`${colSql("isDeleted")} = FALSE`);
       }
       if (payload.cursor) {
-        params.push(decodeCursor(payload.cursor));
-        whereSql.push(`${colSql("id")} > $${params.length}`);
+        const cursor = decodeKeysetCursor(payload.cursor);
+        if (sortField === undefined || cursor.sortValue === undefined) {
+          params.push(cursor.id);
+          whereSql.push(`${colSql("id")} > $${params.length}`);
+        } else {
+          whereSql.push(
+            keysetBoundarySql(
+              colSql(sortField),
+              colSql("id"),
+              { id: cursor.id, sortValue: cursor.sortValue },
+              sortDescending,
+              params,
+            ),
+          );
+        }
       }
       if (filterIds) {
         const placeholders = filterIds.map((id) => {
@@ -199,8 +263,8 @@ export function createReadVerbs(ctx: ExecutorContext): Pick<EventStoreExecutor,
       if (payload.filters !== undefined) for (const f of payload.filters) applyFilter(f);
 
       const orderByClause =
-        payload.sort && table[payload.sort]
-          ? ` ORDER BY ${colSql(payload.sort)} ${payload.sortDirection === "desc" ? "DESC" : "ASC"}, ${colSql("id")} ASC`
+        sortField !== undefined
+          ? ` ORDER BY ${colSql(sortField)} ${sortDescending ? "DESC" : "ASC"}, ${colSql("id")} ASC`
           : ` ORDER BY ${colSql("id")} ASC`;
       const useOffset = !payload.cursor && offset > 0;
       const offsetClause = useOffset ? ` OFFSET ${offset}` : "";
@@ -234,8 +298,15 @@ export function createReadVerbs(ctx: ExecutorContext): Pick<EventStoreExecutor,
       }
 
       const lastRow = rows[rows.length - 1];
-      const nextCursor =
-        rows.length === limit && lastRow ? encodeCursor(lastRow["id"] as string) : null; // @cast-boundary engine-payload
+      const lastRaw = rawRows[rawRows.length - 1];
+      let nextCursor: string | null = null;
+      if (rows.length === limit && lastRow && lastRaw) {
+        const cursorId = lastRow["id"] as string; // @cast-boundary engine-payload
+        const sortText =
+          sortField === undefined ? undefined : toCursorSortText(lastRaw[physicalCol(sortField)]);
+        nextCursor =
+          sortText === undefined ? encodeCursor(cursorId) : encodeKeysetCursor(sortText, cursorId);
+      }
 
       // total: extra COUNT(*) — nur wenn explizit angefordert (Pager-UI).
       // Postgres-Cost ist O(table-scan) ohne Filter, mit Filter so teuer

package/src/db/index.ts

@@ -12,8 +12,8 @@ export type {
   DbTx,
 } from "./connection";
 export { createDbConnection, dbConnectionOptionsFromEnv } from "./connection";
-export type { CursorQueryOptions, CursorResult } from "./cursor";
-export { decodeCursor, encodeCursor } from "./cursor";
+export type { CursorQueryOptions, CursorResult, DecodedKeysetCursor } from "./cursor";
+export { decodeCursor, decodeKeysetCursor, encodeCursor, encodeKeysetCursor } from "./cursor";
 export type { SchemaTable, SelectQuery, TableColumns } from "./dialect";
 export {
   bigint,

package/src/db/pg-error.ts

@@ -49,6 +49,13 @@ export function isLockNotAvailable(e: unknown): boolean {
   return extractPgError(e)?.code === "55P03";
 }
 
+// PG SQLSTATE 42P01 — "undefined table". A projection table for an entity
+// that was never mounted/rebuilt (or was dropped since) — callers treat
+// this as "no row found", not as a fatal error.
+export function isUndefinedTable(e: unknown): boolean {
+  return extractPgError(e)?.code === "42P01";
+}
+
 export function constraintOf(e: unknown): string | undefined {
   return extractPgError(e)?.constraint_name;
 }

package/src/db/queries/backfill-pii.ts

@@ -20,12 +20,25 @@
 // second run reports 0 updates. One failing event does not abort the run;
 // failures are collected and reported (fail-loud at the caller).
 //
+// Owner resolution for entity-lifecycle events (backfill-only — the live
+// write path in event-store-executor-write.ts / subject-resolver.ts stays
+// fail-closed and is untouched by this):
+//   1. payload — the event section itself names the owner field.
+//   2. projection ({@link PiiBackfillOptions.resolveOwnerFromProjection}) —
+//      pre-owner-field-addition events don't carry it; read it from the
+//      entity's current projection row instead.
+//   3. erase ({@link PiiBackfillOptions.eraseUnresolvableSubjects}) — still
+//      unresolvable (hard-deleted aggregate, no projection) → PII_ERASED_SENTINEL.
+//   Both stages are opt-in and off by default; without them an unresolvable
+//   subject still fails loud into `failures`, as before.
+//
 // Snapshots of touched aggregates are dropped (they may cache plaintext);
 // the next snapshotting load recreates them. AFTER a run, rebuild the
 // affected projections — applyEntityEvent materializes ciphertext AND the
 // blind-index columns, which keeps equality lookups (login by email) alive.
 
 import { asRawClient } from "../../bun-db";
+import { quoteIdent } from "../../crypto/ciphertext-pattern";
 import { configuredEventPiiCatalog } from "../../crypto/event-pii";
 import type { KmsContext, LocalKeyKmsAdapter, SubjectId } from "../../crypto/kms-adapter";
 import { KeyErasedError } from "../../crypto/kms-adapter";
@@ -35,9 +48,16 @@ import {
   isPiiCiphertext,
   PII_ERASED_SENTINEL,
 } from "../../crypto/pii-field-encryption";
-import { collectPiiSubjectFields, resolveSubjectForField } from "../../crypto/subject-resolver";
+import {
+  collectPiiSubjectFields,
+  resolveSubjectForField,
+  SubjectResolutionError,
+} from "../../crypto/subject-resolver";
 import type { EntityDefinition, Registry, TenantId } from "../../engine/types";
 import type { DbRunner } from "../connection";
+import { resolveTableName } from "../entity-table-meta";
+import { isUndefinedTable } from "../pg-error";
+import { toSnakeCase } from "../table-builder";
 
 const LIFECYCLE_VERBS = ["created", "updated", "deleted", "restored", "forgotten"] as const;
 
@@ -51,6 +71,12 @@ export type PiiBackfillResult = {
   readonly updatedEvents: number;
   readonly encryptedFields: number;
   readonly erasedFields: number;
+  // Subset of encryptedFields + erasedFields whose owner came from stage 2
+  // (the payload itself lacked it).
+  readonly ownerFromProjection: number;
+  // Subset of erasedFields written because the subject stayed unresolvable
+  // through stage 3, not because the subject was already forgotten.
+  readonly erasedUnresolvable: number;
   readonly deletedSnapshots: number;
   readonly failures: readonly PiiBackfillFailure[];
 };
@@ -59,6 +85,12 @@ export type PiiBackfillOptions = {
   readonly batchSize?: number;
   // Scan + count only, write nothing.
   readonly dryRun?: boolean;
+  // Stage 2: fall back to the entity's projection row (by aggregate_id)
+  // when a lifecycle event's payload doesn't name the owner field.
+  readonly resolveOwnerFromProjection?: boolean;
+  // Stage 3: write PII_ERASED_SENTINEL for subjects still unresolvable
+  // after stage 1+2, instead of failing the event into `failures`.
+  readonly eraseUnresolvableSubjects?: boolean;
 };
 
 type EventRow = {
@@ -105,6 +137,8 @@ export async function backfillEventPiiEncryption(
     updatedEvents: 0,
     encryptedFields: 0,
     erasedFields: 0,
+    ownerFromProjection: 0,
+    erasedUnresolvable: 0,
     deletedSnapshots: 0,
     failures: [] as PiiBackfillFailure[],
   };
@@ -132,13 +166,17 @@ export async function backfillEventPiiEncryption(
     )) as ReadonlyArray<EventRow>;
     if (rows.length === 0) break;
 
+    const projectionOwnersByType = await loadProjectionOwners(rows);
+
     for (const row of rows) {
       result.scannedEvents++;
       try {
-        const outcome = await transformEvent(row);
+        const outcome = await transformEvent(row, projectionOwnersByType.get(row.aggregate_type));
         if (outcome === null) continue;
         result.encryptedFields += outcome.encrypted;
         result.erasedFields += outcome.erased;
+        result.ownerFromProjection += outcome.ownerFromProjection;
+        result.erasedUnresolvable += outcome.erasedUnresolvable;
         if (!options.dryRun) {
           await raw.unsafe(`UPDATE "kumiko_events" SET "payload" = $1::jsonb WHERE "id" = $2`, [
             outcome.payload,
@@ -148,9 +186,15 @@ export async function backfillEventPiiEncryption(
         result.updatedEvents++;
         touchedAggregates.add(row.aggregate_id);
       } catch (e) {
+        const reason = e instanceof Error ? e.message : String(e);
         result.failures.push({
           eventId: String(row.id),
-          reason: e instanceof Error ? e.message : String(e),
+          reason:
+            e instanceof SubjectResolutionError
+              ? `${reason} (retry with { resolveOwnerFromProjection: true } to resolve the owner ` +
+                "from the entity's projection table, and/or { eraseUnresolvableSubjects: true } to " +
+                "erase fields whose subject stays unresolvable)"
+              : reason,
         });
       }
     }
@@ -170,10 +214,57 @@ export async function backfillEventPiiEncryption(
 
   return result;
 
+  async function loadProjectionOwners(
+    rows: readonly EventRow[],
+  ): Promise<ReadonlyMap<string, ReadonlyMap<string, Record<string, unknown>>>> {
+    const byType = new Map<string, ReadonlyMap<string, Record<string, unknown>>>();
+    if (!options.resolveOwnerFromProjection) return byType;
+
+    const idsByType = new Map<string, Set<string>>();
+    for (const row of rows) {
+      if (!entityTargets.has(row.aggregate_type)) continue;
+      const ids = idsByType.get(row.aggregate_type) ?? new Set<string>();
+      ids.add(row.aggregate_id);
+      idsByType.set(row.aggregate_type, ids);
+    }
+
+    for (const [aggregateType, ids] of idsByType) {
+      const target = entityTargets.get(aggregateType);
+      if (!target) continue;
+      const tableName = resolveTableName(aggregateType, target.entity, undefined);
+      const ownersById = new Map<string, Record<string, unknown>>();
+      try {
+        const projectionRows = (await raw.unsafe(
+          `SELECT * FROM ${quoteIdent(tableName)} WHERE "id" = ANY($1::uuid[])`,
+          [[...ids]],
+        )) as ReadonlyArray<Record<string, unknown>>;
+        for (const projectionRow of projectionRows) {
+          const id = projectionRow["id"];
+          if (typeof id === "string") {
+            ownersById.set(id, projectionRowToCamel(target.entity, projectionRow));
+          }
+        }
+      } catch (e) {
+        // Entity never mounted/rebuilt (no projection table yet) — stage 3
+        // still gets a chance; this batch just contributes no owners.
+        if (!isUndefinedTable(e)) throw e;
+      }
+      byType.set(aggregateType, ownersById);
+    }
+    return byType;
+  }
+
   async function transformEvent(
     row: EventRow,
-  ): Promise<{ payload: Record<string, unknown>; encrypted: number; erased: number } | null> {
-    const counters = { encrypted: 0, erased: 0 };
+    projectionOwners: ReadonlyMap<string, Record<string, unknown>> | undefined,
+  ): Promise<{
+    payload: Record<string, unknown>;
+    encrypted: number;
+    erased: number;
+    ownerFromProjection: number;
+    erasedUnresolvable: number;
+  } | null> {
+    const counters = { encrypted: 0, erased: 0, ownerFromProjection: 0, erasedUnresolvable: 0 };
     const payload = structuredClone(row.payload);
 
     const catalogFields = eventCatalog.get(row.type);
@@ -187,6 +278,21 @@ export async function backfillEventPiiEncryption(
     } else {
       const target = entityTargets.get(row.aggregate_type);
       if (!target || !isLifecycleEventOf(row.type, row.aggregate_type)) return null;
+      await applyEntityLifecycleFields(target, projectionOwners?.get(row.aggregate_id));
+    }
+
+    if (counters.encrypted === 0 && counters.erased === 0) return null;
+    return { payload, ...counters };
+
+    function bump(outcome: FieldOutcome): void {
+      if (outcome === "encrypted") counters.encrypted++;
+      if (outcome === "erased") counters.erased++;
+    }
+
+    async function applyEntityLifecycleFields(
+      target: { readonly entity: EntityDefinition; readonly piiFields: readonly string[] },
+      projectionOwnerRow: Record<string, unknown> | undefined,
+    ): Promise<void> {
       const sections = lifecycleSections(payload);
       for (const section of sections) {
         // Update-changes may carry a pii field without its owner field —
@@ -198,25 +304,39 @@ export async function backfillEventPiiEncryption(
           ...section,
         };
         for (const field of target.piiFields) {
-          const subject = resolveSubjectForField(target.entity, field, subjectSource, {
+          const value = section[field];
+          if (value === null || value === undefined) continue;
+          if (typeof value !== "string") continue;
+          // Already-ciphertext/sentinel fields must never enter owner
+          // resolution — an unresolvable owner on an already-handled field
+          // must stay untouched, not get erased by stage 3.
+          if (isPiiCiphertext(value) || value === PII_ERASED_SENTINEL) continue;
+
+          const resolution = resolveOwnerSubject(
+            target.entity,
+            field,
+            subjectSource,
+            projectionOwnerRow,
             // @cast-boundary db-read — tenant_id column is the branded TenantId
-            tenantId: row.tenant_id as TenantId,
-          });
-          if (subject === null) continue;
-          const outcome = await encryptField(section, field, subject);
+            row.tenant_id as TenantId,
+          );
+          if (resolution.kind === "unannotated") continue;
+          if (resolution.kind === "unresolved") {
+            if (options.eraseUnresolvableSubjects) {
+              section[field] = PII_ERASED_SENTINEL;
+              counters.erased++;
+              counters.erasedUnresolvable++;
+              continue;
+            }
+            throw resolution.error;
+          }
+          if (resolution.viaProjection) counters.ownerFromProjection++;
+          const outcome = await encryptField(section, field, resolution.subject);
           bump(outcome);
         }
       }
     }
 
-    if (counters.encrypted === 0 && counters.erased === 0) return null;
-    return { payload, ...counters };
-
-    function bump(outcome: FieldOutcome): void {
-      if (outcome === "encrypted") counters.encrypted++;
-      if (outcome === "erased") counters.erased++;
-    }
-
     async function encryptField(
       section: Record<string, unknown>,
       field: string,
@@ -255,6 +375,56 @@ export async function backfillEventPiiEncryption(
   }
 }
 
+type OwnerSubjectResolution =
+  | { readonly kind: "unannotated" }
+  | { readonly kind: "unresolved"; readonly error: SubjectResolutionError }
+  | { readonly kind: "resolved"; readonly subject: SubjectId; readonly viaProjection: boolean };
+
+// Stage 1 (payload) then stage 2 (projection row, when supplied) — stage 3
+// (erase) is the caller's call, made from the "unresolved" outcome.
+function resolveOwnerSubject(
+  entity: EntityDefinition,
+  field: string,
+  subjectSource: Record<string, unknown>,
+  projectionOwnerRow: Record<string, unknown> | undefined,
+  tenantId: TenantId,
+): OwnerSubjectResolution {
+  try {
+    const subject = resolveSubjectForField(entity, field, subjectSource, { tenantId });
+    return subject === null
+      ? { kind: "unannotated" }
+      : { kind: "resolved", subject, viaProjection: false };
+  } catch (e) {
+    if (!(e instanceof SubjectResolutionError)) throw e;
+    if (projectionOwnerRow) {
+      try {
+        const augmented = { ...subjectSource, ...projectionOwnerRow };
+        const subject = resolveSubjectForField(entity, field, augmented, { tenantId });
+        if (subject !== null) return { kind: "resolved", subject, viaProjection: true };
+      } catch (e2) {
+        if (!(e2 instanceof SubjectResolutionError)) throw e2;
+      }
+    }
+    return { kind: "unresolved", error: e };
+  }
+}
+
+// Projection columns are snake_case (author_id); event-payload/subject
+// fields are camelCase (authorId) — forward-map from entity.fields like
+// reindexEntity's rowToState, so any owner field resolves generically
+// instead of hand-listing column names per entity.
+function projectionRowToCamel(
+  entity: EntityDefinition,
+  row: Record<string, unknown>,
+): Record<string, unknown> {
+  const camel: Record<string, unknown> = {};
+  for (const fieldName of Object.keys(entity.fields)) {
+    const snake = toSnakeCase(fieldName);
+    if (Object.hasOwn(row, snake)) camel[fieldName] = row[snake];
+  }
+  return camel;
+}
+
 function isLifecycleEventOf(eventType: string, aggregateType: string): boolean {
   if (!eventType.startsWith(`${aggregateType}.`)) return false;
   const verb = eventType.slice(aggregateType.length + 1);

package/src/engine/__tests__/boot-validator-boot-check.test.ts

@@ -25,7 +25,12 @@ const requiresUserDataHook = (features: readonly { readonly name: string }[]) =>
 
 const promptStore = () =>
   defineFeature("prompt-store", (r) => {
-    const promptFields = { text: createTextField({ pii: true }) };
+    const promptFields = {
+      text: createTextField({
+        personal: "self",
+        find: "none",
+      }),
+    };
     r.entity("prompt", createEntity({ fields: promptFields }));
     r.bootCheck(({ features }) => {
       // Conditional on this feature's own shape (has a pii field), closed