@cosmicdrift/kumiko-bundled-features 0.165.0 → 2.0.0

package/src/config/__tests__/reencrypt-job-kek-rotation.integration.test.ts

@@ -57,9 +57,12 @@ const keyDef = createSystemConfig("text", {
   write: access.systemAdmin,
 });
 
+const GARBAGE_KEY = "kek-rot:config:garbage-value";
+const GARBAGE_VALUE = "not-json-and-not-an-envelope";
+
 const rotFeature = defineFeature("kek-rot", (r) => {
   r.requires("config");
-  r.config({ keys: { "secret-pass": keyDef } });
+  r.config({ keys: { "secret-pass": keyDef, "garbage-value": keyDef } });
 });
 
 const executor = createEventStoreExecutor(configValuesTable, configValueEntity, {
@@ -80,10 +83,21 @@ async function seedV1Row(): Promise<void> {
   if (!result.isSuccess) throw new Error(`seed failed: ${result.error.code}`);
 }
 
+async function seedGarbageRow(): Promise<void> {
+  const systemUser = createSystemUser(SYSTEM_TENANT_ID);
+  const tdb = createTenantDb(stack.db, SYSTEM_TENANT_ID, "system");
+  const result = await executor.create(
+    { key: GARBAGE_KEY, value: GARBAGE_VALUE, tenantId: SYSTEM_TENANT_ID, userId: null },
+    systemUser,
+    tdb,
+  );
+  if (!result.isSuccess) throw new Error(`seed failed: ${result.error.code}`);
+}
+
 type RawConfigRow = { value: string };
 
-async function readRawValue(): Promise<RawConfigRow> {
-  const rows = await selectMany<RawConfigRow>(stack.db, configValuesTable, { key: KEY });
+async function readRawValue(key: string = KEY): Promise<RawConfigRow> {
+  const rows = await selectMany<RawConfigRow>(stack.db, configValuesTable, { key });
   const row = rows[0];
   if (!row) throw new Error("no config row");
   return row;
@@ -97,13 +111,35 @@ const noopLog = {
   child: () => noopLog,
 };
 
-function jobCtx(): Parameters<typeof reencryptJob>[1] {
+type CapturedLog = { info: string[]; warn: string[] };
+
+type TestJobLog = {
+  info: (msg: string) => void;
+  warn: (msg: string) => void;
+  error: () => void;
+  debug: () => void;
+  child: () => TestJobLog;
+};
+
+function capturingLog(captured: CapturedLog): TestJobLog {
+  const log: TestJobLog = {
+    info: (msg) => captured.info.push(msg),
+    warn: (msg) => captured.warn.push(msg),
+    error: () => {},
+    debug: () => {},
+    child: () => log,
+  };
+  return log;
+}
+
+function jobCtx(captured?: CapturedLog): Parameters<typeof reencryptJob>[1] {
+  const log = captured ? capturingLog(captured) : noopLog;
   return {
     db: stack.db,
     registry: stack.registry,
     masterKeyProvider: mutableProvider,
     configEncryption: cipher,
-    log: noopLog,
+    log,
   } as unknown as Parameters<typeof reencryptJob>[1]; // @cast-boundary test-seam — job only reads db/registry/masterKeyProvider/configEncryption/log
 }
 
@@ -115,12 +151,49 @@ beforeAll(async () => {
   });
   await unsafePushTables(stack.db, { configValuesTable });
   await seedV1Row();
+  await seedGarbageRow();
 });
 
 afterAll(async () => {
   await stack.cleanup();
 });
 
+describe("config KEK-rotation job — unrecognized values are not silently skipped (kumiko-framework#1513)", () => {
+  test("a non-envelope row is counted failed, never alreadyCurrent, and is left untouched", async () => {
+    const beforeRow = await readRawValue(GARBAGE_KEY);
+    expect(beforeRow.value).toBe(GARBAGE_VALUE);
+
+    // Runs before the #1187 rotation test below flips the master-key
+    // version, so counts are deterministic: the V1 row is current, the
+    // garbage row is unrecognized, nothing needs rotating yet.
+    const captured: CapturedLog = { info: [], warn: [] };
+    await reencryptJob({}, jobCtx(captured));
+
+    const completeLine = captured.info.find((line) =>
+      line.includes("[config:reencrypt] complete:"),
+    );
+    if (!completeLine) throw new Error("job did not log a completion summary");
+    const result = JSON.parse(completeLine.slice(completeLine.indexOf("{"))) as {
+      migrated: number;
+      failed: number;
+      alreadyCurrent: number;
+    };
+    expect(result.alreadyCurrent).toBe(1);
+    expect(result.migrated).toBe(0);
+    expect(result.failed).toBe(1);
+
+    const rejectedWarning = captured.warn.find((line) =>
+      line.includes("not a current-cipher envelope"),
+    );
+    expect(rejectedWarning).toBeDefined();
+
+    // Never written: an unreadable row must not be rewritten as if it
+    // decrypted cleanly, and must not be reported as already current.
+    const afterRow = await readRawValue(GARBAGE_KEY);
+    expect(afterRow.value).toBe(GARBAGE_VALUE);
+  });
+});
+
 describe("config KEK-rotation job — version-N to version-N+1 (kumiko-framework#1187)", () => {
   test("rotation rewraps the value onto the current KEK, and a full projection rebuild still lands on it", async () => {
     const beforeRow = await readRawValue();

package/src/config/db/queries/resolver.ts

@@ -1,5 +1,5 @@
 import { asRawClient } from "@cosmicdrift/kumiko-framework/bun-db";
-import type { DbRunner } from "@cosmicdrift/kumiko-framework/db";
+import type { DbRunner, TenantDb } from "@cosmicdrift/kumiko-framework/db";
 import type { TenantId } from "@cosmicdrift/kumiko-framework/engine";
 
 export type ConfigRow = {
@@ -11,7 +11,7 @@ export type ConfigRow = {
 };
 
 export async function selectConfigRowsForScope(
-  db: DbRunner,
+  db: DbRunner | TenantDb,
   systemTenantId: TenantId,
   tenantId: TenantId,
   userId: string,
@@ -27,7 +27,7 @@ export async function selectConfigRowsForScope(
 }
 
 export async function selectConfigRowsForKeys(
-  db: DbRunner,
+  db: DbRunner | TenantDb,
   keys: readonly string[],
   systemTenantId: TenantId,
   tenantId: TenantId,

package/src/config/handlers/reencrypt.job.ts

@@ -1,19 +1,20 @@
-// Re-encrypt job for `encrypted: true` config values. Two jobs in one,
-// because format detection makes them the same loop:
-//   - MIGRATION: legacy CONFIG_ENCRYPTION_KEY values (base64 blob, no key
-//     id) → envelope format under the current master key. After a clean
-//     run the legacy key can be dropped from the environment.
-//   - KEK-ROTATION: envelope values wrapped under an older kekVersion →
-//     re-encrypted under provider.currentVersion(). Config has no
-//     kek_version column (values live in a TEXT column), so unlike the
-//     secrets rotate job the version check parses the stored JSON.
+// KEK-rotation job for `encrypted: true` config values: envelope values
+// wrapped under an older kekVersion get re-encrypted under
+// provider.currentVersion(). Config has no kek_version column (values live
+// in a TEXT column), so unlike the secrets rotate job the version check
+// parses the stored JSON.
+//
+// A row whose value isn't a current-cipher envelope (malformed JSON, or
+// any pre-envelope format) isn't a supported input here — cipher.decrypt
+// rejects it, so it's counted `failed` rather than silently treated as
+// already current. There is no legacy single-key decrypt path anymore.
 //
 // Idempotent: a re-run skips rows already on the current version. Every
 // write goes through the event-store executor (config values are
 // entity-backed — raw UPDATEs would be wiped by a projection rebuild),
-// so each migration appends a normal `.updated` event whose payload
-// carries the NEW envelope: after a full run even a from-scratch rebuild
-// no longer needs the legacy key for the final state.
+// so each rotation appends a normal `.updated` event whose payload carries
+// the NEW envelope: after a full run even a from-scratch rebuild only
+// ever sees the current-KEK envelope.
 
 import { selectMany } from "@cosmicdrift/kumiko-framework/bun-db";
 import {
@@ -48,17 +49,19 @@ export type ReencryptJobResult = {
   readonly stoppedReason: ChunkedMigrationStopReason;
 };
 
-function needsReencrypt(value: string, targetVersion: number): boolean {
-  // legacy single-key format (base64 — can never start with "{")
-  if (!value.startsWith("{")) return true;
+type RowClassification = "rotate" | "current" | "unrecognized";
+
+function classifyRow(value: string, targetVersion: number): RowClassification {
+  let parsed: unknown;
   try {
-    const parsed: unknown = JSON.parse(value);
-    if (!isStoredEnvelope(parsed)) return true;
-    return parsed.kekVersion !== targetVersion;
+    parsed = JSON.parse(value);
   } catch {
-    // malformed JSON — let the decrypt attempt surface the real error
-    return true;
+    // not JSON at all — no supported format decrypts this; let
+    // cipher.decrypt reject it and count the row as failed.
+    return "unrecognized";
   }
+  if (!isStoredEnvelope(parsed)) return "unrecognized";
+  return parsed.kekVersion === targetVersion ? "current" : "rotate";
 }
 
 export const reencryptJob: JobHandlerFn = async (rawPayload, ctx): Promise<void> => {
@@ -145,14 +148,23 @@ export const reencryptJob: JobHandlerFn = async (rawPayload, ctx): Promise<void>
 
   async function migrateRow(row: ConfigRow): Promise<"migrated" | "skipped" | "failed"> {
     if (row.value === null || row.value === undefined) return "skipped";
-    if (!needsReencrypt(row.value, targetVersion)) {
+    const classification = classifyRow(row.value, targetVersion);
+    if (classification === "current") {
       alreadyCurrent++;
       return "skipped";
     }
 
-    // decrypt failure (missing legacy key, unknown kekVersion, tamper)
-    // throws → counted as failed via onRowError; the row stays untouched —
-    // never write anything we couldn't read.
+    if (classification === "unrecognized") {
+      // Not a current-cipher envelope (malformed JSON, or any pre-envelope
+      // format) — never a supported re-encrypt input, so fail loudly
+      // instead of relying on cipher.decrypt to reject it. Log only the
+      // row id, never the value (ciphertext-adjacent).
+      ctx.log?.warn?.(
+        `[config:reencrypt] row ${row.id} is not a current-cipher envelope, not re-encryptable`,
+      );
+      return "failed";
+    }
+
     const tenantId = row.tenantId as TenantId; // @cast-boundary db-row
     const plaintext = await cipher.decrypt(row.value, { tenantId });
     const reencrypted = await cipher.encrypt(plaintext, { tenantId });

package/src/custom-fields/__tests__/custom-fields.integration.test.ts

@@ -518,6 +518,20 @@ describe("custom-fields integration — no PII: `sensitive` rejected, every valu
   });
 });
 
+describe("custom-fields integration — registry facade: getMultiStreamProjectionFeature", () => {
+  test("resolves the owning feature name for a registered multi-stream projection", () => {
+    const qualifiedName = "property-test:projection:custom-fields-property-projection";
+
+    expect(stack.registry.getAllMultiStreamProjections().has(qualifiedName)).toBe(true);
+    expect(stack.registry.getMultiStreamProjectionFeature(qualifiedName)).toBe("property-test");
+    // Missing MSP → undefined (not a fallback string). server.ts only spreads
+    // featureName onto the EventConsumer when this returns a defined value.
+    expect(
+      stack.registry.getMultiStreamProjectionFeature("does-not-exist:projection:nope"),
+    ).toBeUndefined();
+  });
+});
+
 describe("custom-fields integration — value validation (Builder-Reuse)", () => {
   async function setErr(entityId: string, fieldKey: string, value: unknown) {
     return stack.http.writeErr(

package/src/document-ingest-foundation/__tests__/feature.test.ts

@@ -5,6 +5,7 @@
 // by feature.integration.test.ts.
 
 import { describe, expect, test } from "bun:test";
+import { documentExtractEntity } from "../entity";
 import { DOCUMENT_INGEST_REQUESTED_EVENT_QN } from "../events";
 import { documentIngestFoundationFeature } from "../feature";
 
@@ -23,6 +24,27 @@ describe("documentIngestFoundationFeature — shape", () => {
     ]);
   });
 
+  test("documentExtract entity pins table name, field set, and pages encryption (#1501)", () => {
+    expect(documentExtractEntity.table).toBe("read_document_extracts");
+    expect(Object.keys(documentExtractEntity.fields)).toEqual([
+      "fileRefId",
+      "storageKey",
+      "pages",
+      "meta",
+    ]);
+    expect(documentExtractEntity.fields.pages).toMatchObject({
+      type: "longText",
+      encrypted: true,
+    });
+    expect(documentExtractEntity.fields.meta).toMatchObject({ type: "jsonb" });
+  });
+
+  test("exports writeIngestPages / readIngestPages for the encrypted pages wire format (#1549)", async () => {
+    const { readIngestPages, writeIngestPages } = await import("../pages");
+    const pages = [{ pageNumber: 1, text: "x" }];
+    expect(readIngestPages(writeIngestPages(pages))).toEqual(pages);
+  });
+
   test("registers NO write/query handlers — only entity, config, event, and MSP", () => {
     expect(Object.keys(documentIngestFoundationFeature.writeHandlers)).toHaveLength(0);
     expect(Object.keys(documentIngestFoundationFeature.queryHandlers)).toHaveLength(0);
@@ -64,4 +86,20 @@ describe("documentIngestFoundationFeature.exports — typed config handles", ()
     expect(key.name).toBe("document-ingest-foundation:config:max-pages-per-file");
     expect(documentIngestFoundationFeature.configKeys["maxPagesPerFile"]?.default).toBe(50);
   });
+
+  test("ocrLanguage rejects a value that isn't a Tesseract -l argument (#1501)", () => {
+    const pattern = documentIngestFoundationFeature.configKeys["ocrLanguage"]?.pattern;
+    expect(pattern).toBeDefined();
+    const re = new RegExp(pattern!.regex, pattern!.flags);
+    expect(re.test("deu+eng")).toBe(true);
+    expect(re.test("eng")).toBe(true);
+    expect(re.test("")).toBe(false);
+    expect(re.test("EU")).toBe(false);
+    expect(re.test("rm -rf /")).toBe(false);
+  });
+
+  test("maxPagesPerFile is bounded so a tenant can't disable or unbound the cap (#1501)", () => {
+    const bounds = documentIngestFoundationFeature.configKeys["maxPagesPerFile"]?.bounds;
+    expect(bounds).toEqual({ min: 1, max: 500 });
+  });
 });

package/src/document-ingest-foundation/__tests__/pages.test.ts

@@ -0,0 +1,37 @@
+import { describe, expect, test } from "bun:test";
+import type { IngestPage } from "../entity";
+import { readIngestPages, writeIngestPages } from "../pages";
+
+const SAMPLE: readonly IngestPage[] = [
+  { pageNumber: 1, text: "Hello" },
+  { pageNumber: 2, text: "World" },
+];
+
+describe("writeIngestPages / readIngestPages", () => {
+  test("round-trips an IngestPage[] through the longText wire format", () => {
+    const raw = writeIngestPages(SAMPLE);
+    expect(typeof raw).toBe("string");
+    expect(readIngestPages(raw)).toEqual([...SAMPLE]);
+  });
+
+  test("writeIngestPages emits JSON an encrypted longText field can store", () => {
+    expect(writeIngestPages([])).toBe("[]");
+    expect(JSON.parse(writeIngestPages(SAMPLE))).toEqual([...SAMPLE]);
+  });
+
+  test("readIngestPages accepts an already-parsed array (legacy jsonb / raw SQL)", () => {
+    expect(readIngestPages([...SAMPLE])).toEqual([...SAMPLE]);
+  });
+
+  test("readIngestPages drops malformed entries and tolerates null/garbage", () => {
+    expect(readIngestPages(null)).toEqual([]);
+    expect(readIngestPages(undefined)).toEqual([]);
+    expect(readIngestPages("{not-json")).toEqual([]);
+    expect(readIngestPages({ pageNumber: 1 })).toEqual([]);
+    expect(
+      readIngestPages(
+        JSON.stringify([{ pageNumber: 1, text: "ok" }, { pageNumber: "x" }, null, "nope"]),
+      ),
+    ).toEqual([{ pageNumber: 1, text: "ok" }]);
+  });
+});

package/src/document-ingest-foundation/entity.ts

@@ -1,12 +1,17 @@
 import {
   createEntity,
   createJsonbField,
+  createLongTextField,
   createTextField,
 } from "@cosmicdrift/kumiko-framework/engine";
 
 // Phase-1 (MVP) page shape — LiteParse fills this in (kumiko-enterprise#273).
-// Kept minimal on purpose: downstream themes (provider, recipe) extend it,
-// this skeleton only needs a stable name for the jsonb `pages` column.
+// Kept minimal on purpose: downstream themes (provider, recipe) extend it.
+// Stored as JSON.stringify(IngestPage[]) in the encrypted `pages` column —
+// jsonb has no encryption support in the engine, and this column holds the
+// full extracted text of ingested documents (invoices, IDs, contracts).
+// Writers/readers MUST use writeIngestPages / readIngestPages — do not pass
+// a raw IngestPage[] into executor.create (encrypted longText requires string).
 export type IngestPage = {
   readonly pageNumber: number;
   readonly text: string;
@@ -30,7 +35,9 @@ export const documentExtractEntity = createEntity({
   fields: {
     fileRefId: createTextField({ required: true }),
     storageKey: createTextField({ required: true }),
-    pages: createJsonbField(),
+    // Encrypted — holds the full extracted document text (PII). meta is
+    // provider telemetry only and stays plaintext jsonb.
+    pages: createLongTextField({ encrypted: true }),
     meta: createJsonbField(),
   },
 });

package/src/document-ingest-foundation/feature.ts

@@ -13,6 +13,7 @@
 
 import { entityEventName } from "@cosmicdrift/kumiko-framework/db";
 import { access, createTenantConfig, defineFeature } from "@cosmicdrift/kumiko-framework/engine";
+import { z } from "zod";
 import { documentExtractEntity } from "./entity";
 import {
   DOCUMENT_INGEST_AGGREGATE_TYPE,
@@ -38,6 +39,17 @@ const ALLOWED_MIME_TYPES = new Set(["application/pdf", "image/png", "image/jpeg"
 // default (10mb); tests raise maxUploadSize instead of shrinking this.
 const MAX_FILE_BYTES = 25 * 1024 * 1024;
 
+// Narrow fileRef.created's generic entity payload at the MSP boundary.
+// Missing/invalid fields → skip (return), never cast — a bad size would
+// otherwise pass `undefined > MAX_FILE_BYTES` and a later appendDomainEvent
+// schema failure would dead the request-ingest consumer for all tenants.
+const fileRefCreatedPayloadSchema = z.object({
+  storageKey: z.string().min(1),
+  fileName: z.string().min(1),
+  mimeType: z.string().min(1),
+  size: z.number().int().min(0),
+});
+
 export const documentIngestFoundationFeature = defineFeature(FEATURE_NAME, (r) => {
   r.describe(
     "Shared PDF/Scan/Image → normalized-text ingest primitive. Owns the `documentExtract` entity (fileRefId, storageKey, per-page text + metadata) as an implicit entity-projection, the per-tenant `ocrLanguage`/`maxPagesPerFile` config keys, and a fileRef.created trigger that validates mime/size and requests ingest via `documentIngest.requested`. LiteParse provider wiring lands in follow-up features.",
@@ -55,6 +67,10 @@ export const documentIngestFoundationFeature = defineFeature(FEATURE_NAME, (r) =
     "ocrLanguage",
     createTenantConfig("text", {
       default: "deu+eng",
+      // Tesseract -l argument syntax: one or more 3-letter language codes
+      // joined by "+" (e.g. "deu+eng"). Unconstrained free text here would
+      // reach the OCR provider call unvalidated (#1501).
+      pattern: { regex: "^[a-z]{3}(\\+[a-z]{3})*$" },
       write: access.roles("TenantAdmin", "SystemAdmin"),
       read: access.roles("TenantAdmin", "SystemAdmin", "User"),
     }),
@@ -63,6 +79,10 @@ export const documentIngestFoundationFeature = defineFeature(FEATURE_NAME, (r) =
     "maxPagesPerFile",
     createTenantConfig("number", {
       default: 50,
+      // Without a bound a tenant could set 0 (silently kills ingest) or an
+      // unbounded value (OCR worker runs per-file for as long as the PDF
+      // has pages — job-queue starvation) (#1501).
+      bounds: { min: 1, max: 500 },
       write: access.roles("TenantAdmin", "SystemAdmin"),
       read: access.roles("TenantAdmin", "SystemAdmin", "User"),
     }),
@@ -74,14 +94,12 @@ export const documentIngestFoundationFeature = defineFeature(FEATURE_NAME, (r) =
     name: "request-ingest",
     apply: {
       [FILE_REF_CREATED]: async (event, _tx, ctx) => {
-        // entity-event payloads are generic Record<string, unknown> — narrow
-        // at the MSP boundary (files-post-processing pattern).
-        const payload = event.payload as {
-          readonly storageKey: string;
-          readonly fileName: string;
-          readonly mimeType: string;
-          readonly size: number;
-        };
+        // entity-event payloads are generic Record<string, unknown> — parse
+        // at the MSP boundary (same pattern as storage-tracking's readNumber).
+        const parsed = fileRefCreatedPayloadSchema.safeParse(event.payload);
+        // skip: malformed / incomplete payload — don't poison the consumer
+        if (!parsed.success) return;
+        const payload = parsed.data;
 
         // skip: over the fixed cap — no ingest requested, upload itself already succeeded
         if (payload.size > MAX_FILE_BYTES) return;

package/src/document-ingest-foundation/index.ts

@@ -9,3 +9,4 @@ export {
   documentIngestRequestedPayloadSchema,
 } from "./events";
 export { documentIngestFoundationFeature } from "./feature";
+export { readIngestPages, writeIngestPages } from "./pages";

package/src/document-ingest-foundation/pages.ts

@@ -0,0 +1,34 @@
+// Wire-format helpers for `documentExtract.pages` — the column is encrypted
+// longText (jsonb has no encryption support), so writers must persist
+// JSON.stringify(IngestPage[]) and readers must parse back. Centralising
+// the codec here keeps liteparse + recipe consumers lockstep with the
+// entity shape (kumiko-framework#1549/1).
+
+import type { IngestPage } from "./entity";
+
+export function writeIngestPages(pages: readonly IngestPage[]): string {
+  return JSON.stringify(pages);
+}
+
+export function readIngestPages(raw: unknown): IngestPage[] {
+  if (raw == null) return [];
+  // Executor decrypt returns the plaintext string; raw SQL / stale jsonb
+  // rows may still surface as an already-parsed array — accept both.
+  const parsed: unknown = typeof raw === "string" ? parseJson(raw) : raw;
+  if (!Array.isArray(parsed)) return [];
+  return parsed.filter(isIngestPage);
+}
+
+function parseJson(raw: string): unknown {
+  try {
+    return JSON.parse(raw);
+  } catch {
+    return null;
+  }
+}
+
+function isIngestPage(value: unknown): value is IngestPage {
+  if (value === null || typeof value !== "object") return false;
+  const page = value as { pageNumber?: unknown; text?: unknown };
+  return typeof page.pageNumber === "number" && typeof page.text === "string";
+}