@cosmicdrift/kumiko-framework 0.396.0 → 0.397.0

package/dist/crypto/index.d.ts

@@ -7,5 +7,6 @@ export { KeyAlreadyExistsError, KeyErasedError, type KmsAdapter, type KmsContext
 export { ENCRYPTION_UNAVAILABLE_CODE, EncryptionUnavailableError } from "./kms-outage.ts";
 export { type KmsWiring, requireKmsWiring, requireKmsWiringAsync, resolveKmsWiring, resolveKmsWiringAsync, } from "./kms-wiring.ts";
 export { createPgKmsAdapter, rewrapSubjectKeys, } from "./pg-kms-adapter.ts";
+export { PII_CIPHERTEXT_CORRUPT_CODE, PiiCiphertextCorruptError, } from "./pii-ciphertext-corrupt.ts";
 export { configuredPiiSubjectKms, configurePiiSubjectKms, decryptPiiFieldValues, decryptPiiValueForSubject, encryptPiiFieldValues, encryptPiiJsonValueForSubject, encryptPiiValueForSubject, isPiiCiphertext, PII_CIPHERTEXT_PREFIX, PII_ERASED_SENTINEL, } from "./pii-field-encryption.ts";
 export { collectPiiEncryptedFields, collectPiiSubjectFields, isRecordKeyedField, resolveSubjectForField, SubjectResolutionError, } from "./subject-resolver.ts";

package/dist/crypto/index.js

@@ -7,5 +7,6 @@ export { KeyAlreadyExistsError, KeyErasedError, subjectIdSchema, subjectIdToKey,
 export { ENCRYPTION_UNAVAILABLE_CODE, EncryptionUnavailableError } from "./kms-outage.js";
 export { requireKmsWiring, requireKmsWiringAsync, resolveKmsWiring, resolveKmsWiringAsync, } from "./kms-wiring.js";
 export { createPgKmsAdapter, rewrapSubjectKeys, } from "./pg-kms-adapter.js";
+export { PII_CIPHERTEXT_CORRUPT_CODE, PiiCiphertextCorruptError, } from "./pii-ciphertext-corrupt.js";
 export { configuredPiiSubjectKms, configurePiiSubjectKms, decryptPiiFieldValues, decryptPiiValueForSubject, encryptPiiFieldValues, encryptPiiJsonValueForSubject, encryptPiiValueForSubject, isPiiCiphertext, PII_CIPHERTEXT_PREFIX, PII_ERASED_SENTINEL, } from "./pii-field-encryption.js";
 export { collectPiiEncryptedFields, collectPiiSubjectFields, isRecordKeyedField, resolveSubjectForField, SubjectResolutionError, } from "./subject-resolver.js";

package/dist/crypto/pii-ciphertext-corrupt.d.ts

@@ -0,0 +1,8 @@
+import { KumikoError } from "../errors/kumiko-error.ts";
+export declare const PII_CIPHERTEXT_CORRUPT_CODE = "pii_ciphertext_corrupt";
+export declare class PiiCiphertextCorruptError extends KumikoError {
+    readonly field: string;
+    readonly code = "pii_ciphertext_corrupt";
+    readonly httpStatus = 500;
+    constructor(field: string, cause?: Error);
+}

package/dist/crypto/pii-ciphertext-corrupt.js

@@ -0,0 +1,18 @@
+import { KumikoError } from "../errors/kumiko-error.js";
+export const PII_CIPHERTEXT_CORRUPT_CODE = "pii_ciphertext_corrupt";
+// The stored value does not decrypt under its subject's key although the key store answered:
+// damaged bytes, a value copied into another field or subject (the AAD binds both), or a value
+// written under another key. A retry fails the same way, unlike EncryptionUnavailableError.
+export class PiiCiphertextCorruptError extends KumikoError {
+    field;
+    code = PII_CIPHERTEXT_CORRUPT_CODE;
+    httpStatus = 500;
+    constructor(field, cause) {
+        super({
+            message: `the stored PII value of field "${field}" does not decrypt under its subject key`,
+            i18nKey: "errors.internal",
+            ...(cause && { cause }),
+        });
+        this.field = field;
+    }
+}

package/dist/crypto/pii-field-encryption.js

@@ -18,6 +18,7 @@ import { createCipheriv, createDecipheriv, randomBytes } from "node:crypto";
 import { assertIrreversibleOperationAllowed } from "../pipeline/irreversible-operation-gate.js";
 import { isLocalKeyKmsAdapter, KeyAlreadyExistsError, KeyErasedError, KeyNotFoundError, PII_ERASED_SENTINEL, subjectIdFromKey, subjectIdToKey, } from "./kms-adapter.js";
 import { classifyKmsOutage } from "./kms-outage.js";
+import { PiiCiphertextCorruptError } from "./pii-ciphertext-corrupt.js";
 import { isPiiCiphertext, PII_CIPHERTEXT_PREFIX, PII_CIPHERTEXT_PREFIX_JSON, PII_CIPHERTEXT_PREFIX_V1, } from "./pii-ciphertext-format.js";
 import { resolveSubjectForField } from "./subject-resolver.js";
 // Spec value (crypto-shredding.md) — renderers show it verbatim.
@@ -58,14 +59,35 @@ function parseCiphertext(value) {
     };
 }
 function decryptValue(dek, blob, subject, field, hasAad, isJson) {
-    const iv = blob.subarray(0, IV_LENGTH);
-    const tag = blob.subarray(IV_LENGTH, IV_LENGTH + AUTH_TAG_LENGTH);
-    const ciphertext = blob.subarray(IV_LENGTH + AUTH_TAG_LENGTH);
-    const decipher = createDecipheriv("aes-256-gcm", dek, iv);
-    if (hasAad)
-        decipher.setAAD(buildAad(subject, field, isJson));
-    decipher.setAuthTag(tag);
-    return Buffer.concat([decipher.update(ciphertext), decipher.final()]).toString("utf8");
+    try {
+        const iv = blob.subarray(0, IV_LENGTH);
+        const tag = blob.subarray(IV_LENGTH, IV_LENGTH + AUTH_TAG_LENGTH);
+        const ciphertext = blob.subarray(IV_LENGTH + AUTH_TAG_LENGTH);
+        const decipher = createDecipheriv("aes-256-gcm", dek, iv);
+        if (hasAad)
+            decipher.setAAD(buildAad(subject, field, isJson));
+        decipher.setAuthTag(tag);
+        return Buffer.concat([decipher.update(ciphertext), decipher.final()]).toString("utf8");
+    }
+    catch (e) {
+        throw new PiiCiphertextCorruptError(field, e instanceof Error ? e : undefined);
+    }
+}
+function parseCiphertextOf(value, field) {
+    try {
+        return parseCiphertext(value);
+    }
+    catch (e) {
+        throw new PiiCiphertextCorruptError(field, e instanceof Error ? e : undefined);
+    }
+}
+function parseDecryptedJson(plain, field) {
+    try {
+        return JSON.parse(plain);
+    }
+    catch (e) {
+        throw new PiiCiphertextCorruptError(field, e instanceof Error ? e : undefined);
+    }
 }
 // First write to a PII field of a new subject creates the key implicitly
 // (user-signup / tenant-create need no extra hook). The AlreadyExists catch
@@ -118,7 +140,7 @@ export async function encryptPiiJsonValueForSubject(kms, subject, value, kmsCtx,
 export async function decryptPiiValueForSubject(kms, value, kmsCtx, field) {
     if (!isPiiCiphertext(value))
         return value;
-    const { subject, blob, hasAad, isJson } = parseCiphertext(value);
+    const { subject, blob, hasAad, isJson } = parseCiphertextOf(value, field);
     if (isJson) {
         throw new Error(`PII field "${field}" holds a jsonb ciphertext — decrypt it via decryptPiiFieldValues`);
     }
@@ -193,11 +215,11 @@ export async function decryptPiiFieldValues(row, piiFields, kms, kmsCtx) {
         // work during rollout).
         if (!isPiiCiphertext(value))
             continue;
-        const { subject, blob, hasAad, isJson } = parseCiphertext(value);
+        const { subject, blob, hasAad, isJson } = parseCiphertextOf(value, name);
         try {
             const key = await kms.getKey(subject, kmsCtx);
             const plain = decryptValue(key, blob, subject, name, hasAad, isJson);
-            const decoded = isJson ? JSON.parse(plain) : plain;
+            const decoded = isJson ? parseDecryptedJson(plain, name) : plain;
             out[name] = decoded;
         }
         catch (e) {

package/dist/db/entity-column-meta.js

@@ -90,7 +90,7 @@ const COLUMN_META_EMITTERS = {
     reference: referenceColumnMeta,
     money: moneyColumnMeta,
     embedded: (snake, field) => jsonbColumn(snake, field.multiple === true ? EMPTY_JSONB_ARRAY : EMPTY_JSONB_OBJECT),
-    jsonb: (snake) => jsonbColumn(snake, EMPTY_JSONB_OBJECT),
+    jsonb: (snake, field) => jsonbColumn(snake, field.emptyValue === "array" ? EMPTY_JSONB_ARRAY : EMPTY_JSONB_OBJECT),
     // Real PG `date` — Temporal.PlainDate, no time-of-day/timezone component.
     date: (snake, field) => requiredColumn(snake, "date", field),
     timestamp: (snake, field) => requiredColumn(snake, "timestamptz", field),

package/dist/db/event-store-executor-read-paging.js

@@ -1,6 +1,6 @@
 import { UnprocessableError } from "../errors/index.js";
 import { decodeKeysetCursor, encodeCursor, encodeKeysetCursor } from "./cursor.js";
-import { renderSortSql } from "./event-store-executor-read-sort.js";
+import { renderSortSql, sortsNullsFirst } from "./event-store-executor-read-sort.js";
 import { executeRawQueryRead } from "./queries/raw-sql.js";
 import { physicalColumnName } from "./table-builder.js";
 // Defense-in-depth pagination guard. The handler boundary (entityListSchema)
@@ -48,13 +48,13 @@ function toCursorSortText(value) {
     }
     return undefined;
 }
-// Keyset boundary for `ORDER BY <sort> <dir>, id ASC` under Postgres' DEFAULT
-// null ordering (ASC → NULLS LAST, DESC → NULLS FIRST). An explicit NULLS clause
-// would change the visible order and cost the sort its index.
-function keysetBoundarySql(sortCol, idCol, cursor, descending, params) {
+// Keyset boundary for `ORDER BY <sort> <dir> [NULLS FIRST|LAST], id ASC`. Without a declared
+// `sortNulls` Postgres' default applies (ASC → NULLS LAST, DESC → NULLS FIRST), which keeps the
+// sort on a plain index; a declared one may need an index with the same NULLS order.
+function keysetBoundarySql(sortCol, idCol, cursor, order, params) {
     params.push(cursor.id);
     const afterId = `${idCol} > $${params.length}`;
-    const nullsComeFirst = descending;
+    const { descending, nullsComeFirst } = order;
     if (cursor.sortValue === null) {
         return nullsComeFirst
             ? `(${sortCol} IS NOT NULL OR ${afterId})`
@@ -75,7 +75,7 @@ export function appendCursorBoundary(where, encodedCursor, sort) {
         where.whereSql.push(`${where.colSql("id")} > $${where.params.length}`);
     }
     else {
-        where.whereSql.push(keysetBoundarySql(renderSortSql(sort, where), where.colSql("id"), { id: cursor.id, sortValue: cursor.sortValue }, sort.descending, where.params));
+        where.whereSql.push(keysetBoundarySql(renderSortSql(sort, where), where.colSql("id"), { id: cursor.id, sortValue: cursor.sortValue }, { descending: sort.descending, nullsComeFirst: sortsNullsFirst(sort) }, where.params));
     }
 }
 export function encodeNextCursor(cursorId, sort, lastRaw, lastSortLabel, table) {

package/dist/db/event-store-executor-read-sort.d.ts

@@ -1,4 +1,4 @@
-import type { EntityDefinition } from "@cosmicdrift/kumiko-types/fields";
+import type { EntityDefinition, SortNulls } from "@cosmicdrift/kumiko-types/fields";
 import { type OwnershipClause } from "../engine/ownership.ts";
 import type { SessionUser } from "../engine/types/index.ts";
 import type { EventStoreExecutor } from "./event-store-executor.ts";
@@ -13,15 +13,18 @@ export type ListSort = {
     readonly kind: "column";
     readonly field: string;
     readonly descending: boolean;
+    readonly nulls: SortNulls | undefined;
 } | {
     readonly kind: "referenceLabel";
     readonly field: string;
     readonly descending: boolean;
+    readonly nulls: SortNulls | undefined;
     readonly labelSql: (params: unknown[]) => string;
 };
 export type SortedListSort = Exclude<ListSort, {
     readonly kind: "unsorted";
 }>;
+export declare function sortsNullsFirst(sort: SortedListSort): boolean;
 export declare function resolveListSort(request: {
     readonly payload: Pick<ListPayload, "sort" | "sortDirection">;
     readonly table: Table;

package/dist/db/event-store-executor-read-sort.js

@@ -7,6 +7,19 @@ import { schemaTableName } from "./table-introspection.js";
 // Projected alongside the row when a reference sort is active, so the keyset
 // cursor carries the label that was ordered by instead of the FK's UUID.
 const SORT_LABEL_ALIAS = "__kumiko_sort_label";
+function declaredSortNulls(entity, field) {
+    const def = entity.fields[field];
+    return def !== undefined && "sortNulls" in def ? def.sortNulls : undefined;
+}
+// Postgres' own null order without an explicit NULLS clause: last ascending, first descending.
+export function sortsNullsFirst(sort) {
+    return sort.nulls === undefined ? sort.descending : sort.nulls === "first";
+}
+function nullsClause(sort) {
+    if (sort.nulls === undefined)
+        return "";
+    return sort.nulls === "first" ? " NULLS FIRST" : " NULLS LAST";
+}
 // Resolve a `sortable: true` reference field's label inline for
 // ORDER BY. A correlated scalar subquery, not a join: the result set keeps one
 // row per entity row, the read model stays untouched, and the lookup rides the
@@ -80,8 +93,10 @@ export function resolveListSort(request) {
     if (field === undefined)
         return { kind: "unsorted" };
     const descending = payload.sortDirection === "desc";
-    if (entity.fields[field]?.type !== "reference")
-        return { kind: "column", field, descending };
+    const nulls = declaredSortNulls(entity, field);
+    if (entity.fields[field]?.type !== "reference") {
+        return { kind: "column", field, descending, nulls };
+    }
     // A reference column stores a UUID, so it is never sorted by
     // directly: either the field opted into `sortable` and we order by the
     // target row's label, or the sort is dropped to plain id order.
@@ -91,7 +106,7 @@ export function resolveListSort(request) {
         : undefined;
     return labelSql === undefined
         ? { kind: "unsorted" }
-        : { kind: "referenceLabel", field, descending, labelSql };
+        : { kind: "referenceLabel", field, descending, nulls, labelSql };
 }
 export function renderSortSql(sort, where) {
     return sort.kind === "referenceLabel" ? sort.labelSql(where.params) : where.colSql(sort.field);
@@ -102,7 +117,7 @@ export function renderListOrderBy(sort, where) {
         return { orderByClause: ` ORDER BY ${idAscending}`, sortLabelProjection: "" };
     }
     const sortSql = renderSortSql(sort, where);
-    const direction = sort.descending ? "DESC" : "ASC";
+    const direction = `${sort.descending ? "DESC" : "ASC"}${nullsClause(sort)}`;
     if (sort.kind === "column") {
         return {
             orderByClause: ` ORDER BY ${sortSql} ${direction}, ${idAscending}`,

package/dist/db/queries/backfill-pii.d.ts

@@ -40,3 +40,5 @@ export type PiiBackfillBatchOptions = PiiBackfillOptions & {
 };
 export declare function backfillEventPiiEncryption(db: DbRunner, registry: Registry, options?: PiiBackfillOptions): Promise<PiiBackfillResult>;
 export declare function backfillEventPiiEncryptionBatch(db: DbRunner, registry: Registry, options?: PiiBackfillBatchOptions): Promise<PiiBackfillBatchResult>;
+export declare function isLifecycleEventOf(eventType: string, aggregateType: string): boolean;
+export declare function lifecycleSections(payload: Record<string, unknown>): Record<string, unknown>[];

package/dist/db/queries/backfill-pii.js

@@ -611,7 +611,7 @@ function projectionRowToCamel(entity, row) {
     }
     return camel;
 }
-function isLifecycleEventOf(eventType, aggregateType) {
+export function isLifecycleEventOf(eventType, aggregateType) {
     if (!eventType.startsWith(`${aggregateType}.`))
         return false;
     const verb = eventType.slice(aggregateType.length + 1);
@@ -620,7 +620,7 @@ function isLifecycleEventOf(eventType, aggregateType) {
 // created events carry the fields flat; updated carries { changes, previous };
 // deleted/forgotten/restored carry { previous }. Returned sections are the
 // mutable objects INSIDE the payload clone — encryptField writes in place.
-function lifecycleSections(payload) {
+export function lifecycleSections(payload) {
     const sections = [];
     if (isRecord(payload["changes"]))
         sections.push(payload["changes"]);

package/dist/db/queries/event-store-payload-scrub.d.ts

@@ -0,0 +1,12 @@
+import type { DbRunner } from "../connection.ts";
+export type ScrubEventRow = {
+    readonly id: bigint | string;
+    readonly aggregate_id: string;
+    readonly aggregate_type: string;
+    readonly tenant_id: string;
+    readonly type: string;
+    readonly payload: Record<string, unknown>;
+};
+export declare function selectEventsOfTypeAfter(db: DbRunner, eventType: string, afterEventId: string, limit: number): Promise<readonly ScrubEventRow[]>;
+export declare function replaceEventPayload(db: DbRunner, eventId: bigint | string, payload: Record<string, unknown>): Promise<void>;
+export declare function deleteSnapshotsOfAggregates(db: DbRunner, aggregateIds: readonly string[]): Promise<number>;

package/dist/db/queries/event-store-payload-scrub.js

@@ -0,0 +1,20 @@
+import { asRawClient } from "../query.js";
+import { toPostgresArrayLiteral } from "./postgres-array-literal.js";
+export function selectEventsOfTypeAfter(db, eventType, afterEventId, limit) {
+    return asRawClient(db).unsafe(`SELECT "id", "aggregate_id", "aggregate_type", "tenant_id", "type", "payload"
+       FROM "kumiko_events"
+      WHERE "type" = $1 AND "id" > $2::bigint
+      ORDER BY "id" ASC
+      LIMIT $3`, [eventType, afterEventId, limit]);
+}
+export async function replaceEventPayload(db, eventId, payload) {
+    await asRawClient(db).unsafe(`UPDATE "kumiko_events" SET "payload" = $1::jsonb WHERE "id" = $2`, [
+        payload,
+        eventId,
+    ]);
+}
+// Snapshots may cache the replaced values of the streams; the next snapshotting load recreates them.
+export async function deleteSnapshotsOfAggregates(db, aggregateIds) {
+    const deleted = await asRawClient(db).unsafe(`DELETE FROM "kumiko_snapshots" WHERE "aggregate_id" = ANY($1::uuid[]) RETURNING "aggregate_id"`, [toPostgresArrayLiteral([...aggregateIds])]);
+    return deleted.length;
+}

package/dist/engine/factories.js

@@ -277,7 +277,7 @@ export function createEmbeddedListField(schema, overrides) {
     };
 }
 // Free-form jsonb column — see the `JsonbFieldDef` documentation. Schema-less,
-// defaults to `{}`, and is NOT NULL. Primarily used by the custom-fields
+// defaults to `{}` (`[]` with `emptyValue: "array"`), and is NOT NULL. Primarily used by the custom-fields
 // bundle for the host entity's `customFields` column; other valid uses include
 // tenant configuration blobs, AI-inferred metadata, and future tag arrays.
 export function createJsonbField(overrides) {

package/dist/engine/schema-builder.js

@@ -315,8 +315,10 @@ export function fieldToZod(field, currencies, opts = {}) {
             return embeddedFieldToZod(field);
         case "jsonb":
             // Free-form jsonb — keys are tenant-/runtime-defined. Validation
-            // passes through any plain object.
-            return z.record(z.string(), z.unknown());
+            // passes through any plain object, or any array for emptyValue "array".
+            return field.emptyValue === "array"
+                ? z.array(z.unknown())
+                : z.record(z.string(), z.unknown());
         case "date":
             return withDateBounds(z.string().date(), field.min, field.max);
         case "timestamp":

package/dist/files/feature.d.ts

@@ -7,6 +7,7 @@ export type FilesFeatureOptions = {
     readonly unclaimedUploadReaders?: readonly string[];
     readonly maxUploadSize?: string;
     readonly uploadSweepGraceHours?: number;
+    readonly pendingUploadGraceHours?: number;
 };
 export declare const FILES_FEATURE: "files";
 export declare function createFilesFeature(options?: FilesFeatureOptions): FeatureDefinition;

package/dist/files/feature.js

@@ -2,6 +2,7 @@ import { defineFeature } from "../engine/index.js";
 import { bindReferencedFilesWrite } from "./bind-referenced-files.write.js";
 import { FILE_REF_ENTITY_NAME, fileRefEntity } from "./file-ref-entity.js";
 import { DEFAULT_UPLOAD_SWEEP_GRACE_HOURS, MIN_UPLOAD_SWEEP_GRACE_HOURS, sweepOrphanedUploads, } from "./sweep-orphaned-uploads.js";
+import { sweepStalePendingFileRefs } from "./sweep-stale-pending-file-refs.js";
 import { fileUploadLedgerTable } from "./upload-ledger.js";
 export { fileRefEntity } from "./file-ref-entity.js";
 // `fileRef` is a regular ES entity. Uploads and deletes go through the standard
@@ -15,14 +16,18 @@ export { fileRefEntity } from "./file-ref-entity.js";
 // rebuildProjection. It lives in the framework next to file-routes and
 // fileRefsTable; bundled-features/files only re-exports it.
 export const FILES_FEATURE = "files";
-function assertValidSweepGraceHours(hours) {
+function assertValidSweepGraceHours(option, hours) {
     if (!Number.isInteger(hours) || hours < MIN_UPLOAD_SWEEP_GRACE_HOURS) {
-        throw new Error(`createFilesFeature: uploadSweepGraceHours must be a whole number of hours >= ${MIN_UPLOAD_SWEEP_GRACE_HOURS}, got ${String(hours)}`);
+        throw new Error(`createFilesFeature: ${option} must be a whole number of hours >= ${MIN_UPLOAD_SWEEP_GRACE_HOURS}, got ${String(hours)}`);
     }
 }
 export function createFilesFeature(options = {}) {
     const uploadSweepGraceHours = options.uploadSweepGraceHours ?? DEFAULT_UPLOAD_SWEEP_GRACE_HOURS;
-    assertValidSweepGraceHours(uploadSweepGraceHours);
+    assertValidSweepGraceHours("uploadSweepGraceHours", uploadSweepGraceHours);
+    const { pendingUploadGraceHours } = options;
+    if (pendingUploadGraceHours !== undefined) {
+        assertValidSweepGraceHours("pendingUploadGraceHours", pendingUploadGraceHours);
+    }
     return defineFeature(FILES_FEATURE, (r) => {
         r.describe("Exposes the `fileRef` entity from the framework core so that uploaded files — tracked in the `file_refs` table by `createFileRoutes` — participate in cross-feature hooks: `user-data-rights-defaults` automatically includes file blobs in GDPR exports and forget flows, and tenant-lifecycle cleanup deletes all refs on tenant destroy. Upload/download routes are registered by the server bootstrap when a `file-provider-*` feature is mounted; this feature carries their access/size policy via `createFilesFeature(opts?)`.");
         r.uiHints({
@@ -46,6 +51,14 @@ export function createFilesFeature(options = {}) {
                     graceHours: uploadSweepGraceHours,
                     log: ctx.obs.log,
                 });
+                if (pendingUploadGraceHours !== undefined) {
+                    await sweepStalePendingFileRefs({
+                        db: ctx.db,
+                        now: ctx.tz.now(),
+                        graceHours: pendingUploadGraceHours,
+                        log: ctx.obs.log,
+                    });
+                }
             },
         });
         return { routeOptions: options };

package/dist/files/file-ref-entity.d.ts

@@ -14,6 +14,7 @@ export declare const fileRefEntity: import("@cosmicdrift/kumiko-types/fields").E
         readonly description?: string;
         readonly required?: boolean;
         readonly sortable?: boolean;
+        readonly sortNulls?: import("@cosmicdrift/kumiko-types/fields").SortNulls;
         readonly filterable?: boolean;
         readonly sensitive?: boolean;
         readonly default?: number;

package/dist/files/file-ref-table.d.ts

@@ -13,6 +13,7 @@ export declare const fileRefsTable: import("../db/table-builder.ts").EntityTable
         readonly description?: string;
         readonly required?: boolean;
         readonly sortable?: boolean;
+        readonly sortNulls?: import("@cosmicdrift/kumiko-types/fields").SortNulls;
         readonly filterable?: boolean;
         readonly sensitive?: boolean;
         readonly default?: number;

package/dist/files/sweep-stale-pending-file-refs.d.ts

@@ -0,0 +1,10 @@
+import type { Logger } from "@cosmicdrift/kumiko-types/logger";
+import type { TenantDb } from "../db/tenant-db.ts";
+export declare const STALE_PENDING_FILE_REF_SWEEP_BATCH = 500;
+export declare function sweepStalePendingFileRefs(args: {
+    readonly db: TenantDb;
+    readonly now: Temporal.Instant;
+    readonly graceHours: number;
+    readonly log: Pick<Logger, "warn"> | undefined;
+    readonly pageSize?: number;
+}): Promise<number>;

package/dist/files/sweep-stale-pending-file-refs.js

@@ -0,0 +1,46 @@
+import { selectMany } from "../db/query.js";
+import { createSystemUser, SYSTEM_USER_ID } from "../engine/system-user.js";
+import { getFileRefExecutor } from "./create-file-ref.js";
+import { fileRefsTable } from "./file-ref-table.js";
+export const STALE_PENDING_FILE_REF_SWEEP_BATCH = 500;
+// A pending upload names its target field (entityType set) but no record yet. Uploads without a
+// target field (inbox, dropzone) and files the system generated are kept on purpose.
+function isStalePendingUpload(row) {
+    return row.entityType !== null && row.insertedById !== SYSTEM_USER_ID;
+}
+// Soft-deletes pending uploads whose record never came, through the fileRef executor, so the
+// soft-delete cleanup later removes their bytes like those of any deleted file.
+export async function sweepStalePendingFileRefs(args) {
+    const { db, now, graceHours, log } = args;
+    const pageSize = args.pageSize ?? STALE_PENDING_FILE_REF_SWEEP_BATCH;
+    const cutoff = now.subtract({ hours: graceHours });
+    const executor = getFileRefExecutor();
+    const user = createSystemUser(db.tenantId);
+    let swept = 0;
+    let cursor;
+    for (;;) {
+        const rows = await selectMany(db, fileRefsTable, {
+            entityId: null,
+            isDeleted: false,
+            insertedAt: { lt: cutoff },
+            ...(cursor !== undefined && { id: { gt: cursor } }),
+        }, {
+            orderBy: { col: "id", direction: "asc" },
+            fields: ["id", "entityType", "insertedById"],
+            limit: pageSize,
+        });
+        const lastRow = rows[rows.length - 1];
+        // skip: an empty page means every due row has been visited
+        if (lastRow === undefined)
+            return swept;
+        cursor = lastRow.id;
+        for (const row of rows.filter(isStalePendingUpload)) {
+            const removed = await executor.delete({ id: row.id }, user, db);
+            if (removed.isSuccess) {
+                swept++;
+                continue;
+            }
+            log?.warn(`[pending-upload-sweep] pending fileRef ${row.id} not deleted, retried next run: ${removed.error.code}`);
+        }
+    }
+}

package/dist/migrations/event-payload-scrub.d.ts

@@ -0,0 +1,31 @@
+import type { DbConnection } from "../db/connection.ts";
+import type { Registry } from "../engine/types/index.ts";
+import type { Logger } from "../logging/types.ts";
+import { type PiiRebuildRun } from "./pii-event-backfill.ts";
+export type EventScrubCandidate = {
+    readonly eventId: string;
+    readonly eventType: string;
+    readonly aggregateId: string;
+    readonly aggregateType: string;
+    readonly tenantId: string;
+    readonly field: string;
+    readonly payload: Readonly<Record<string, unknown>>;
+};
+export type ScrubEventPayloadOptions = {
+    readonly eventType: string;
+    readonly fields: readonly string[];
+    readonly reason: string;
+    readonly shouldScrub?: (value: unknown, candidate: EventScrubCandidate) => boolean | Promise<boolean>;
+    readonly dryRun?: boolean;
+    readonly batchSize?: number;
+    readonly logger?: Pick<Logger, "error"> & Partial<Pick<Logger, "warn" | "debug">>;
+};
+export type ScrubEventPayloadResult = {
+    readonly scannedEvents: number;
+    readonly scrubbedEvents: number;
+    readonly scrubbedFields: number;
+    readonly deletedSnapshots: number;
+    readonly queuedTables: readonly string[];
+    readonly rebuild: PiiRebuildRun | null;
+};
+export declare function scrubEventPayloadFields(db: DbConnection, registry: Registry, options: ScrubEventPayloadOptions): Promise<ScrubEventPayloadResult>;

package/dist/migrations/event-payload-scrub.js

@@ -0,0 +1,132 @@
+// Replaces chosen payload fields of stored events with PII_ERASED_SENTINEL and rebuilds the
+// projections fed by them. For personal data that crypto-shredding cannot reach: plaintext from
+// before the field was declared personal without a subject left to erase, or a ciphertext that no
+// longer decrypts. A read-model UPDATE alone would come back with the next rebuild.
+import { PII_ERASED_SENTINEL } from "../crypto/pii-field-encryption.js";
+import { isLifecycleEventOf, lifecycleSections } from "../db/queries/backfill-pii.js";
+import { deleteSnapshotsOfAggregates, replaceEventPayload, selectEventsOfTypeAfter, } from "../db/queries/event-store-payload-scrub.js";
+import { runInSystemRowLevelSecurityScope } from "../db/row-level-security-scope.js";
+import { withoutStatementTimeout } from "../db/statement-timeout.js";
+import { assertReason } from "../engine/escape-hatch-reason.js";
+import { createFallbackLogger } from "../logging/utils.js";
+import { createPendingRebuildsTable } from "./pending-rebuilds.js";
+import { affectedProjectionTables, ensureInfraTable, queueRebuildTables, rebuildQueuedTables, } from "./pii-event-backfill.js";
+const EVENT_SCRUB_MIGRATION_PREFIX = "event-scrub:";
+const DEFAULT_BATCH_SIZE = 500;
+function hasValue(value) {
+    return value !== undefined && value !== null && value !== PII_ERASED_SENTINEL;
+}
+function payloadSections(row, payload) {
+    return isLifecycleEventOf(row.type, row.aggregate_type) ? lifecycleSections(payload) : [payload];
+}
+async function scrubPayload(row, options) {
+    const payload = structuredClone(row.payload);
+    let scrubbed = 0;
+    for (const section of payloadSections(row, payload)) {
+        for (const field of options.fields) {
+            const value = section[field];
+            if (!hasValue(value))
+                continue;
+            const candidate = {
+                eventId: String(row.id),
+                eventType: row.type,
+                aggregateId: row.aggregate_id,
+                aggregateType: row.aggregate_type,
+                tenantId: row.tenant_id,
+                field,
+                payload: row.payload,
+            };
+            if (options.shouldScrub && !(await options.shouldScrub(value, candidate)))
+                continue;
+            section[field] = PII_ERASED_SENTINEL;
+            scrubbed++;
+        }
+    }
+    return { payload, scrubbed };
+}
+// One transaction per batch: payloads, the snapshots that may cache the old values and the
+// rebuild queue move together, so an abort leaves no scrubbed stream with a stale snapshot.
+async function scrubOneBatch(db, registry, args) {
+    const { options, totals } = args;
+    return db.begin(async (tx) => {
+        const rows = await selectEventsOfTypeAfter(tx, options.eventType, args.afterEventId, args.batchSize);
+        const last = rows[rows.length - 1];
+        if (last === undefined)
+            return { lastEventId: null, scannedAll: true };
+        const touchedAggregates = new Set();
+        const touchedAggregateTypes = new Set();
+        for (const row of rows) {
+            totals.scannedEvents++;
+            const { payload, scrubbed } = await scrubPayload(row, options);
+            if (scrubbed === 0)
+                continue;
+            totals.scrubbedEvents++;
+            totals.scrubbedFields += scrubbed;
+            if (options.dryRun)
+                continue;
+            await replaceEventPayload(tx, row.id, payload);
+            touchedAggregates.add(row.aggregate_id);
+            touchedAggregateTypes.add(row.aggregate_type);
+        }
+        if (touchedAggregates.size > 0) {
+            totals.deletedSnapshots += await deleteSnapshotsOfAggregates(tx, [...touchedAggregates]);
+            const tables = affectedProjectionTables(registry, touchedAggregateTypes, new Set([options.eventType]));
+            await queueRebuildTables(tx, tables, args.migrationId);
+            for (const table of tables)
+                args.queuedTables.add(table);
+        }
+        return { lastEventId: String(last.id), scannedAll: rows.length < args.batchSize };
+    });
+}
+async function scrubAllBatches(db, registry, options) {
+    const log = createFallbackLogger("event-payload-scrub", options.logger);
+    if (!options.dryRun) {
+        await ensureInfraTable(db, "kumiko_pending_rebuilds", createPendingRebuildsTable);
+    }
+    const migrationId = `${EVENT_SCRUB_MIGRATION_PREFIX}${crypto.randomUUID()}`;
+    const totals = {
+        scannedEvents: 0,
+        scrubbedEvents: 0,
+        scrubbedFields: 0,
+        deletedSnapshots: 0,
+    };
+    const queuedTables = new Set();
+    let afterEventId = "0";
+    for (;;) {
+        const batch = await scrubOneBatch(db, registry, {
+            options,
+            afterEventId,
+            batchSize: options.batchSize ?? DEFAULT_BATCH_SIZE,
+            migrationId,
+            totals,
+            queuedTables,
+        });
+        if (batch.lastEventId === null)
+            break;
+        afterEventId = batch.lastEventId;
+        if (batch.scannedAll)
+            break;
+    }
+    const rebuild = queuedTables.size === 0
+        ? null
+        : await rebuildQueuedTables(db, registry, {
+            migrationIdPrefix: migrationId,
+            signal: undefined,
+            log,
+            failureMessage: "Projection rebuild after an event payload scrub failed; the tables stay queued in kumiko_pending_rebuilds",
+        });
+    log.debug?.("Event payload scrub finished", {
+        eventType: options.eventType,
+        fields: options.fields,
+        reason: options.reason,
+        ...totals,
+    });
+    return { ...totals, queuedTables: [...queuedTables], rebuild };
+}
+export function scrubEventPayloadFields(db, registry, options) {
+    assertReason("scrubEventPayloadFields", options.reason);
+    if (options.fields.length === 0) {
+        throw new Error("scrubEventPayloadFields: name at least one payload field to scrub");
+    }
+    return runInSystemRowLevelSecurityScope("event payload scrub rewrites one event type across all tenants", () => withoutStatementTimeout(db, (unboundedDb) => scrubAllBatches(unboundedDb, registry, options)));
+}

package/dist/migrations/index.d.ts

@@ -1,3 +1,4 @@
+export { type EventScrubCandidate, type ScrubEventPayloadOptions, type ScrubEventPayloadResult, scrubEventPayloadFields, } from "./event-payload-scrub.ts";
 export { assertKumikoSchemaCurrent, SchemaDriftError, } from "./kumiko-drift.ts";
 export { enqueueProjectionRebuild, listPendingRebuilds, PROJECTION_REBUILD_JOB, queueRebuildsFromMarkers, rebuildProjectionOrMultiStream, reportPendingRebuildRun, runPendingRebuilds, } from "./pending-rebuilds.ts";
 export { runPiiEventBackfill } from "./pii-event-backfill.ts";

package/dist/migrations/index.js

@@ -1,4 +1,6 @@
 // Drizzle-free gate (kumiko/migrations system) — the canonical boot-gate.
+// Erases chosen payload fields of stored events and rebuilds the projections fed by them.
+export { scrubEventPayloadFields, } from "./event-payload-scrub.js";
 export { assertKumikoSchemaCurrent, SchemaDriftError, } from "./kumiko-drift.js";
 // Persistente Pending-Rebuild-Queue (survives Rebuild-Failures + Crashes).
 export { enqueueProjectionRebuild, listPendingRebuilds, PROJECTION_REBUILD_JOB, queueRebuildsFromMarkers, rebuildProjectionOrMultiStream, reportPendingRebuildRun, runPendingRebuilds, } from "./pending-rebuilds.js";

package/dist/migrations/pii-event-backfill.d.ts

@@ -1,9 +1,10 @@
-import type { DbConnection } from "../db/connection.ts";
+import type { DbConnection, DbRunner } from "../db/connection.ts";
 import { type PiiBackfillFailure } from "../db/queries/backfill-pii.ts";
 import type { Registry } from "../engine/types/index.ts";
 import type { Logger } from "../logging/types.ts";
 export declare const piiBackfillStateTable: import("@cosmicdrift/kumiko-types/executor-brand").WritableSchemaTable;
 export declare function createPiiBackfillStateTable(db: DbConnection): Promise<void>;
+export declare function ensureInfraTable(db: DbConnection, tableName: string, create: (db: DbConnection) => Promise<void>): Promise<void>;
 export declare function piiAnnotationEntries(registry: Registry): readonly string[];
 export declare function piiAnnotationFingerprint(registry: Registry): string;
 export type PiiEventBackfillOptions = {
@@ -38,4 +39,12 @@ export type PiiEventBackfillResult = {
     readonly queuedTables: readonly string[];
     readonly rebuild: PiiRebuildRun | null;
 };
+export declare function queueRebuildTables(db: DbRunner, tableNames: readonly string[], migrationId: string): Promise<void>;
+export declare function affectedProjectionTables(registry: Registry, touchedAggregateTypes: ReadonlySet<string>, touchedEventTypes: ReadonlySet<string>): readonly string[];
 export declare function runPiiEventBackfill(db: DbConnection, registry: Registry, options?: PiiEventBackfillOptions): Promise<PiiEventBackfillResult>;
+export declare function rebuildQueuedTables(db: DbConnection, registry: Registry, options: {
+    readonly migrationIdPrefix: string;
+    readonly signal: AbortSignal | undefined;
+    readonly log: Pick<Logger, "error" | "warn">;
+    readonly failureMessage: string;
+}): Promise<PiiRebuildRun | null>;

package/dist/migrations/pii-event-backfill.js

@@ -56,7 +56,7 @@ export async function createPiiBackfillStateTable(db) {
 }
 // Replicas boot concurrently: two of them can both pass the tableExists gate and
 // one CREATE then fails. The table existing afterwards means the other won.
-async function ensureInfraTable(db, tableName, create) {
+export async function ensureInfraTable(db, tableName, create) {
     try {
         await create(db);
     }
@@ -86,12 +86,12 @@ export function piiAnnotationFingerprint(registry) {
         .update([...piiAnnotationEntries(registry), PII_BACKFILL_TRANSFORM_VERSION].join("\n"))
         .digest("hex");
 }
-async function queueRebuildTables(db, tableNames, migrationId) {
+export async function queueRebuildTables(db, tableNames, migrationId) {
     for (const tableName of tableNames) {
         await upsertOnConflict(db, pendingRebuildsTable, { tableName, migrationId }, { conflictKeys: ["tableName"], update: { migrationId } });
     }
 }
-function affectedProjectionTables(registry, touchedAggregateTypes, touchedEventTypes) {
+export function affectedProjectionTables(registry, touchedAggregateTypes, touchedEventTypes) {
     const tables = new Set();
     for (const [name, def] of registry.getAllProjections()) {
         const sources = [
@@ -253,7 +253,14 @@ async function backfillPiiEvents(db, registry, options) {
     };
     const aborted = await scanAllBatches(ctx, progress, options.signal, initialState?.failedEventIds ?? []);
     reportRunOutcome(log, progress);
-    const rebuild = aborted ? null : await rebuildQueuedTables(db, registry, options.signal, log);
+    const rebuild = aborted
+        ? null
+        : await rebuildQueuedTables(db, registry, {
+            migrationIdPrefix: PII_BACKFILL_MIGRATION_PREFIX,
+            signal: options.signal,
+            log,
+            failureMessage: "Projection rebuild after PII backfill failed; tables stay queued and are retried on the next boot",
+        });
     const queuedTables = [...progress.queuedTables];
     log.debug("PII event backfill finished", {
         mode,
@@ -281,8 +288,11 @@ function maxBigInt(a, b) {
 // Only our own queue rows: runPendingRebuilds would drain the whole queue and
 // silently clear entries this process cannot map (a worker with a narrower
 // feature set would swallow rebuilds queued by the API pod or by schema apply).
-async function rebuildQueuedTables(db, registry, signal, log) {
-    const ownRows = (await listPendingRebuildRows(db)).filter((row) => row.migrationId.startsWith(PII_BACKFILL_MIGRATION_PREFIX));
+// Rebuilds the projections of the tables queued under the run's migration id prefix; a failed one
+// stays queued.
+export async function rebuildQueuedTables(db, registry, options) {
+    const { signal, log } = options;
+    const ownRows = (await listPendingRebuildRows(db)).filter((row) => row.migrationId.startsWith(options.migrationIdPrefix));
     if (ownRows.length === 0)
         return null;
     const tableToProjection = buildProjectionTableIndex(registry);
@@ -317,8 +327,7 @@ async function rebuildQueuedTables(db, registry, signal, log) {
             failed.push({ projection, error: e instanceof Error ? e.message : String(e) });
         }
     }
-    if (failed.length > 0) {
-        log.error("Projection rebuild after PII backfill failed; tables stay queued and are retried on the next boot", { failed });
-    }
+    if (failed.length > 0)
+        log.error(options.failureMessage, { failed });
     return { rebuilt, failed, skippedTables };
 }

package/dist/pipeline/dispatch-write.js

@@ -465,6 +465,9 @@ async function guardFieldTransition(ctx, args) {
     }
     const currentValue = row[fieldName] ??
         row[toSnakeCase(fieldName)]; // @cast-boundary engine-bridge
+    // skip: an unchanged value is no transition (an edit form resends the current status)
+    if (currentValue === newValue)
+        return;
     guardTransition(getTransitions(ctx, { entityName, fieldName, map: transitionMap }), currentValue, newValue);
 }
 // The handler itself plus the lifecycle pipeline run under the same

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cosmicdrift/kumiko-framework",
-  "version": "0.396.0",
+  "version": "0.397.0",
   "description": "Framework core — engine, pipeline, API, DB, and every other bit that makes Kumiko go.",
   "license": "MIT",
   "author": "Marc Frost <marc@cosmicdriftgamestudio.com>",
@@ -203,7 +203,7 @@
     "./package.json": "./package.json"
   },
   "dependencies": {
-    "@cosmicdrift/kumiko-types": "0.396.0",
+    "@cosmicdrift/kumiko-types": "0.397.0",
     "bullmq": "^5.76.7",
     "hono": "^4.13.1",
     "i18next": "^26.1.0",
@@ -225,9 +225,9 @@
     }
   },
   "devDependencies": {
-    "@cosmicdrift/kumiko-testing": "0.396.0",
+    "@cosmicdrift/kumiko-testing": "0.397.0",
     "bun-types": "^1.4.4",
-    "fflate": "^0.8.2",
+    "fflate": "^0.8.3",
     "meilisearch": "^0.58.0",
     "pino-pretty": "^13.1.3"
   },

package/src/changes.json

@@ -1,4 +1,40 @@
 [
+  {
+    "version": "0.397.0",
+    "type": "improvement",
+    "title": "Scrub primitive for event payload fields",
+    "detail": "Plaintext written before a field was declared personal without a subject to erase, or a ciphertext that no longer decrypts, stayed in `kumiko_events`, and the only way out was a raw `UPDATE` on the event store, which the event-store-writes guard rejects. The scrub walks the events of one type in batches, reaches the `changes` and `previous` sections of entity lifecycle events, lets `shouldScrub(value, candidate)` narrow the values (for example to those that throw `PiiCiphertextCorruptError`), drops the snapshots of touched streams and queues the affected projection and multi-stream projection tables in `kumiko_pending_rebuilds` in the same transaction, then rebuilds them; a failed rebuild stays queued. `reason` is required and placeholders are refused; `dryRun` only counts."
+  },
+  {
+    "version": "0.397.0",
+    "type": "improvement",
+    "title": "jsonb fields can default to an empty array",
+    "detail": "A jsonb field that stores a list had the fixed column default `{}`, so a row that never wrote it read an object while every other row read an array, and the generated write schema only accepted plain objects. `emptyValue: \"array\"` switches the column default to `[]` and the write schema to an array; the default stays `\"object\"`. The schema migration renders `ALTER COLUMN ... SET DEFAULT '[]'::jsonb`; existing rows that never wrote the field get `[]` with the next rebuild of the projection."
+  },
+  {
+    "version": "0.397.0",
+    "type": "improvement",
+    "title": "Abandoned form uploads can expire",
+    "detail": "An upload sent with entityType and fieldName but without a record stays pending until a save binds it. When the form was abandoned, the fileRef row and its bytes stayed forever without an owner that deletes them. With `pendingUploadGraceHours` the `files:job:sweep-orphaned-uploads` job soft-deletes such rows through the fileRef executor once they are older than the grace period. Uploads without a target field (inbox, dropzone) and files the system user created are kept. It is opt-in because an app that stores fileRef ids without binding them would lose those files."
+  },
+  {
+    "version": "0.397.0",
+    "type": "improvement",
+    "title": "Corrupt PII ciphertext has its own error type",
+    "detail": "`decryptPiiFieldValues`, `decryptPiiValueForSubject` and every read path built on them threw Node's generic \"unable to authenticate data\" error for a failed GCM auth tag, the same shape as any other failure. Now a failed auth tag, an AAD mismatch (value copied into another field or subject), a malformed ciphertext and an undecodable jsonb plaintext throw `PiiCiphertextCorruptError` with the field name; it is a KumikoError, so it keeps its code through the dispatcher. An export can blank one such row and still fail loud on an outage, which keeps its own `EncryptionUnavailableError`."
+  },
+  {
+    "version": "0.397.0",
+    "type": "improvement",
+    "title": "sortNulls places rows without a value in list sorting",
+    "detail": "Without the option the list keeps Postgres' order (nulls last ascending, first descending), which a plain index serves. With `sortNulls` the list query adds `NULLS FIRST` or `NULLS LAST` to the order and the keyset cursor follows it, so paging across the rows without a value stays complete. A sort that must stay on an index needs an index with the same NULLS order."
+  },
+  {
+    "version": "0.397.0",
+    "type": "fix",
+    "title": "Transition guard ignores an unchanged state value",
+    "detail": "An update whose `changes` carry the state field with the value the row already has is no transition, so the guard skips it instead of rejecting it as a self-transition that the map does not list. Before, a rename on a `sold` or `finished` row failed as soon as the client sent the current status along."
+  },
   {
     "version": "0.396.0",
     "type": "breaking",