@cosmicdrift/kumiko-samples 0.338.0

package/packages/bundled-features/src/derivatives-sharp/render.ts

@@ -0,0 +1,431 @@
+// sharp-backed renderer for the `derivativeRenderer` extension point —
+// resize/fit, format+quality re-encode, whole-image blur, and pre-resize
+// region blur (see BlurRegion). Server-only: sharp is a native binding.
+
+import type {
+  BlurRegion,
+  DerivativeRendererPlugin,
+  OverlayGravity,
+  ResolvedOverlayLayer,
+  VariantFit,
+  VariantSpec,
+} from "@cosmicdrift/kumiko-types/derivatives-types";
+import QRCode from "qrcode";
+import type { OverlayOptions, ResizeOptions, Sharp } from "sharp";
+import sharp from "sharp";
+
+const MIN_SHARP_SIGMA = 0.3;
+const MAX_SHARP_SIGMA = 1000;
+// Above this, a decoded pixel buffer runs into hundreds of MB before any
+// resize even happens — sharp's own default (~268MP) is still large enough
+// to be an OOM primitive for a single request.
+const MAX_INPUT_PIXELS = 100_000_000;
+// Longest output edge a caller may request via `size`/`maxEdge`. Without a
+// cap, `{ maxEdge: 50000 }` allocates a multi-gigabyte buffer per encoder
+// step before any client sees a byte back.
+const MAX_OUTPUT_EDGE = 8192;
+// applyBlurRegions decodes the full image fresh for every region — an
+// unbounded region list is a CPU/RAM-multiplying DoS knob independent of
+// output size.
+const MAX_BLUR_REGIONS = 64;
+// Every layer decodes and resizes separately before the composite — an
+// unbounded list is the same CPU/RAM multiplier MAX_BLUR_REGIONS caps.
+const MAX_OVERLAY_LAYERS = 8;
+// Layer bytes ride inside the spec, which is hashed on every variant() call.
+const MAX_OVERLAY_IMAGE_BYTES = 512 * 1024;
+const MAX_QR_DATA_LENGTH = 1024;
+// Below roughly this, a QR's modules land on too few pixels to survive
+// camera capture — the AC claims phone-scannable, so an under-sized layer
+// throws instead of caching a silently unusable image.
+const MIN_QR_PIXEL_WIDTH = 160;
+// sequentialRead trades random pixel access for lower peak memory on
+// streamed formats — applied at every decode entry point below since none
+// of them need non-sequential access to the *original* bytes (extract runs
+// against a single already-buffered region per call).
+const SHARP_INPUT_OPTIONS = { limitInputPixels: MAX_INPUT_PIXELS, sequentialRead: true } as const;
+
+function normalizeMimeType(mimeType: string): string {
+  return mimeType.toLowerCase().split(";")[0]?.trim() ?? "";
+}
+
+function clampSigma(sigma: number): number {
+  return Math.min(MAX_SHARP_SIGMA, Math.max(MIN_SHARP_SIGMA, sigma));
+}
+
+// Shared trust boundary for every image byte source this renderer decodes —
+// the original upload (renderImage) and an overlay layer's bytes (applyOverlays)
+// alike: a base64 string in a field declaration isn't more trustworthy than
+// an upload just because it came from app code.
+function assertNotSvg(format: string | undefined): void {
+  if (format === "svg") {
+    throw new Error(
+      "derivatives-sharp: SVG is not supported — rendering untrusted SVG through libvips is a trust boundary this renderer doesn't need to cross.",
+    );
+  }
+}
+
+function assertValidRegion(region: BlurRegion): void {
+  const values = [region.x, region.y, region.width, region.height];
+  if (values.some((v) => !Number.isFinite(v)) || region.width <= 0 || region.height <= 0) {
+    throw new Error(
+      `derivatives-sharp: invalid blurRegion ${JSON.stringify(region)} — x/y/width/height must be finite and width/height must be > 0.`,
+    );
+  }
+}
+
+// Burns each region into the auto-oriented base image, one extract+blur+
+// composite pass per region. Runs BEFORE resize: the regions are relative to
+// the original pixels, and a `fit: "cover"` crop shifts pixel coordinates in
+// a way a post-resize mapping can't reconstruct.
+async function applyBlurRegions(
+  data: Buffer,
+  width: number,
+  height: number,
+  regions: readonly BlurRegion[],
+): Promise<Buffer> {
+  const overlays: OverlayOptions[] = [];
+
+  for (const region of regions) {
+    assertValidRegion(region);
+
+    // AI-detected/user-corrected coordinates can run slightly outside the
+    // frame — clamp instead of rejecting.
+    const left = Math.min(Math.max(Math.round(region.x * width), 0), width);
+    const top = Math.min(Math.max(Math.round(region.y * height), 0), height);
+    const rectWidth = Math.min(Math.round(region.width * width), width - left);
+    const rectHeight = Math.min(Math.round(region.height * height), height - top);
+    if (rectWidth < 1 || rectHeight < 1) continue;
+
+    // ponytail: fixed size-proportional sigma heuristic (no per-region sigma
+    // input) — good enough until a caller needs to tune blur strength itself.
+    const sigma = clampSigma(Math.max(4, Math.min(rectWidth, rectHeight) / 6));
+
+    const patch = await sharp(data, SHARP_INPUT_OPTIONS)
+      .extract({ left, top, width: rectWidth, height: rectHeight })
+      .blur(sigma)
+      .toBuffer();
+    overlays.push({ input: patch, left, top });
+  }
+
+  if (overlays.length === 0) return data;
+  return sharp(data, SHARP_INPUT_OPTIONS).composite(overlays).toBuffer();
+}
+
+// Placement is anchored to the OUTPUT box (overlays run after resize, see
+// applyOverlays), then clamped the same way applyBlurRegions clamps its
+// region coordinates — a caller-declared marginPct can't push a layer
+// off-canvas.
+function overlayPosition(
+  gravity: OverlayGravity,
+  marginPx: number,
+  outputWidth: number,
+  outputHeight: number,
+  layerWidth: number,
+  layerHeight: number,
+): { left: number; top: number } {
+  let left: number;
+  let top: number;
+  switch (gravity) {
+    case "north-west":
+      left = marginPx;
+      top = marginPx;
+      break;
+    case "north-east":
+      left = outputWidth - layerWidth - marginPx;
+      top = marginPx;
+      break;
+    case "south-west":
+      left = marginPx;
+      top = outputHeight - layerHeight - marginPx;
+      break;
+    case "south-east":
+      left = outputWidth - layerWidth - marginPx;
+      top = outputHeight - layerHeight - marginPx;
+      break;
+    case "center":
+      left = (outputWidth - layerWidth) / 2;
+      top = (outputHeight - layerHeight) / 2;
+      break;
+    default:
+      throw new Error(`derivatives-sharp: unknown overlay gravity "${gravity satisfies never}".`);
+  }
+  const maxLeft = Math.max(0, outputWidth - layerWidth);
+  const maxTop = Math.max(0, outputHeight - layerHeight);
+  return {
+    left: Math.min(Math.max(Math.round(left), 0), maxLeft),
+    top: Math.min(Math.max(Math.round(top), 0), maxTop),
+  };
+}
+
+// Runs AFTER resize (see the AC in renderImage) so a layer sized as a
+// fraction of the output is a fraction of what the caller actually gets,
+// regardless of the source's own dimensions or `fit` crop.
+async function applyOverlays(
+  pipeline: Sharp,
+  layers: readonly ResolvedOverlayLayer[],
+): Promise<Sharp> {
+  // PNG intermediate: a bare toBuffer() re-encodes in the source format, so a
+  // JPEG source would gain an extra lossy generation before applyEncoder runs.
+  const { data, info } = await pipeline.png().toBuffer({ resolveWithObject: true });
+
+  const composites: OverlayOptions[] = [];
+  for (const layer of layers) {
+    const targetWidth = Math.max(1, Math.round(info.width * layer.widthPct));
+    // The QR is square, so the output height bounds its side just like the width does.
+    const qrSide = Math.min(targetWidth, info.height);
+    if (layer.kind === "qr" && qrSide < MIN_QR_PIXEL_WIDTH) {
+      throw new Error(
+        `derivatives-sharp: qr overlay would render at ${qrSide}px wide, below the ${MIN_QR_PIXEL_WIDTH}px minimum a camera can reliably scan.`,
+      );
+    }
+
+    // Rendered at its final size so the modules stay hard-edged; resampling a
+    // small default-size QR to a non-integer module size blurs them.
+    const layerBytes =
+      layer.kind === "qr"
+        ? await QRCode.toBuffer(layer.data, {
+            width: qrSide,
+            margin: 1,
+            errorCorrectionLevel: "M",
+          })
+        : Buffer.from(layer.imageBase64, "base64");
+
+    const layerMeta = await sharp(layerBytes, SHARP_INPUT_OPTIONS).metadata();
+    assertNotSvg(layerMeta.format);
+
+    // PNG (not the layer's own format) to keep any alpha through the resize.
+    // height caps the other axis too — sharp's composite() rejects an input
+    // larger than the base image, which a tall/square layer would otherwise
+    // hit once widthPct alone drives its width past the output's height.
+    const resized = await sharp(layerBytes, SHARP_INPUT_OPTIONS)
+      .resize({
+        width: targetWidth,
+        height: info.height,
+        fit: "inside",
+        // Only ever a sub-pixel correction for a QR; keeps module edges crisp.
+        kernel: layer.kind === "qr" ? "nearest" : "lanczos3",
+      })
+      .png()
+      .toBuffer({ resolveWithObject: true });
+
+    const marginPx = Math.round(info.width * (layer.marginPct ?? 0));
+    const { left, top } = overlayPosition(
+      layer.gravity,
+      marginPx,
+      info.width,
+      info.height,
+      resized.info.width,
+      resized.info.height,
+    );
+    composites.push({ input: resized.data, left, top });
+  }
+
+  return sharp(data, SHARP_INPUT_OPTIONS).composite(composites);
+}
+
+function buildResizeOptions(width: number, height: number, fit: VariantFit): ResizeOptions {
+  const options: ResizeOptions = { width, height, fit };
+  if (fit === "inside") {
+    // Max-box: never enlarge past the source.
+    options.withoutEnlargement = true;
+  }
+  if (fit === "contain") {
+    // sharp pads "contain" with opaque black by default — transparent instead
+    // so only formats that keep alpha show a border at all (jpeg flattens it).
+    options.background = { r: 0, g: 0, b: 0, alpha: 0 };
+  }
+  return options;
+}
+
+// HARD CONTRACT (see outputMimeType in derivatives-context.ts): with
+// spec.format set, the output MUST be that format; with it unset, the
+// output MUST stay the source format. The caller derives the mimeType from
+// the spec alone and never inspects the bytes, so drifting from this silently
+// mislabels the stored variant. sourceMimeType is a client-declared upload
+// label, not sniffed from bytes, so we can't lean on sharp's "no encoder
+// call" passthrough to honor it — the encoder is always called explicitly
+// from the map below, and an unmapped source mimeType throws instead of
+// silently keeping whatever format sharp actually decoded.
+const SOURCE_FORMAT_ENCODERS: Record<string, (pipeline: Sharp, quality?: number) => Sharp> = {
+  "image/jpeg": (pipeline, quality) =>
+    pipeline.jpeg(quality !== undefined ? { quality } : undefined),
+  // The upload whitelist admits the sloppy-but-common image/jpg alias
+  "image/jpg": (pipeline, quality) =>
+    pipeline.jpeg(quality !== undefined ? { quality } : undefined),
+  // sharp's PngOptions.quality doesn't mean JPEG/WebP-style compression —
+  // it switches PNG into palette quantization (lossless -> color-reduced,
+  // banding instead of a softer/smaller image). Ignore it, same as gif.
+  "image/png": (pipeline) => pipeline.png(),
+  "image/webp": (pipeline, quality) =>
+    pipeline.webp(quality !== undefined ? { quality } : undefined),
+  "image/avif": (pipeline, quality) =>
+    pipeline.avif(quality !== undefined ? { quality } : undefined),
+  // sharp's GifOptions has no quality knob (palette/dither instead) — ignore
+  // the value here rather than pass it through.
+  "image/gif": (pipeline) => pipeline.gif(),
+  "image/tiff": (pipeline, quality) =>
+    pipeline.tiff(quality !== undefined ? { quality } : undefined),
+};
+
+function applyEncoder(pipeline: Sharp, spec: VariantSpec, sourceMimeType: string): Sharp {
+  const quality = spec.quality;
+
+  if (spec.format) {
+    switch (spec.format) {
+      case "webp":
+        return pipeline.webp(quality !== undefined ? { quality } : undefined);
+      case "avif":
+        return pipeline.avif(quality !== undefined ? { quality } : undefined);
+      case "jpeg":
+        return pipeline.jpeg(quality !== undefined ? { quality } : undefined);
+    }
+  }
+
+  const encode = SOURCE_FORMAT_ENCODERS[sourceMimeType];
+  if (!Object.hasOwn(SOURCE_FORMAT_ENCODERS, sourceMimeType) || encode === undefined) {
+    throw new Error(
+      `derivatives-sharp: no output encoder for source mimeType "${sourceMimeType}" — set spec.format to pick an explicit output format.`,
+    );
+  }
+  return encode(pipeline, quality);
+}
+
+// Split out of assertRenderSpecBounds (not inlined there) purely to stay
+// under the AST-guard complexity budget — see assertRenderSpecBounds's own
+// comment. Still the single call site for every overlay bound.
+function assertOverlayLayerBounds(layer: ResolvedOverlayLayer): void {
+  if (!Number.isFinite(layer.widthPct) || layer.widthPct <= 0 || layer.widthPct > 1) {
+    throw new Error(
+      `derivatives-sharp: overlay widthPct ${layer.widthPct} is out of range — must be a finite number in (0, 1].`,
+    );
+  }
+  if (
+    layer.marginPct !== undefined &&
+    (!Number.isFinite(layer.marginPct) || layer.marginPct < 0 || layer.marginPct >= 0.5)
+  ) {
+    throw new Error(
+      `derivatives-sharp: overlay marginPct ${layer.marginPct} is out of range — must be within [0, 0.5).`,
+    );
+  }
+  if (layer.kind === "qr") {
+    if (layer.data.length === 0 || layer.data.length > MAX_QR_DATA_LENGTH) {
+      throw new Error(
+        `derivatives-sharp: qr overlay data length ${layer.data.length} is out of range — must be 1–${MAX_QR_DATA_LENGTH} characters.`,
+      );
+    }
+    // skip: qr layers have no imageBase64 field to validate; this return also
+    // narrows the union below to the image variant without a cast.
+    return;
+  }
+  const byteLength = Buffer.byteLength(layer.imageBase64, "base64");
+  if (byteLength > MAX_OVERLAY_IMAGE_BYTES) {
+    throw new Error(
+      `derivatives-sharp: overlay image is ${byteLength} bytes, exceeding the limit of ${MAX_OVERLAY_IMAGE_BYTES}.`,
+    );
+  }
+}
+
+function assertOverlayBounds(layers: readonly ResolvedOverlayLayer[] | undefined): void {
+  // skip: no overlays configured for this variant, nothing to validate
+  if (!layers) return;
+  if (layers.length > MAX_OVERLAY_LAYERS) {
+    throw new Error(
+      `derivatives-sharp: overlays has ${layers.length} entries, exceeding the limit of ${MAX_OVERLAY_LAYERS} — each layer decodes and resizes separately before the composite.`,
+    );
+  }
+  for (const layer of layers) {
+    assertOverlayLayerBounds(layer);
+  }
+}
+
+// Input-validation for the DoS caps above — kept out of renderImage so the
+// complexity hotspot stays under the AST-guard budget.
+function assertRenderSpecBounds(spec: VariantSpec): void {
+  if (spec.quality !== undefined && (spec.quality < 1 || spec.quality > 100)) {
+    throw new Error(`derivatives-sharp: quality ${spec.quality} is out of range (1–100).`);
+  }
+  if (spec.blur !== undefined && (spec.blur < MIN_SHARP_SIGMA || spec.blur > MAX_SHARP_SIGMA)) {
+    throw new Error(
+      `derivatives-sharp: blur sigma ${spec.blur} is out of sharp's supported range (${MIN_SHARP_SIGMA}–${MAX_SHARP_SIGMA}).`,
+    );
+  }
+  if (spec.blurRegions && spec.blurRegions.length > MAX_BLUR_REGIONS) {
+    throw new Error(
+      `derivatives-sharp: blurRegions has ${spec.blurRegions.length} entries, exceeding the limit of ${MAX_BLUR_REGIONS} — each region re-decodes the full image.`,
+    );
+  }
+  for (const [dimension, value] of [
+    ["size.width", spec.size?.width],
+    ["size.height", spec.size?.height],
+    ["maxEdge", spec.maxEdge],
+  ] as const) {
+    if (value === undefined) continue;
+    if (!Number.isInteger(value) || value < 1 || value > MAX_OUTPUT_EDGE) {
+      throw new Error(
+        `derivatives-sharp: ${dimension} ${value} is out of range — must be an integer between 1 and ${MAX_OUTPUT_EDGE}.`,
+      );
+    }
+  }
+  assertOverlayBounds(spec.resolvedOverlays);
+}
+
+export const renderImage: DerivativeRendererPlugin["render"] = async (
+  input,
+  spec,
+  sourceMimeType,
+) => {
+  const normalizedSource = normalizeMimeType(sourceMimeType);
+
+  // sourceMimeType is a client-declared upload label, not sniffed — decide
+  // the SVG rejection from the actual bytes so a mislabeled upload can't
+  // reach libvips as SVG.
+  const sniffed = await sharp(input, SHARP_INPUT_OPTIONS).metadata();
+  assertNotSvg(sniffed.format);
+
+  assertRenderSpecBounds(spec);
+
+  // rotate() with no argument applies the EXIF orientation and normalizes it
+  // away. sharp drops all other metadata by default — we never call
+  // .withMetadata()/.keepExif(), which is exactly what strips GPS/EXIF from
+  // the output.
+  let pipeline = sharp(input, SHARP_INPUT_OPTIONS).rotate();
+
+  if (spec.blurRegions && spec.blurRegions.length > 0) {
+    // PNG intermediate for the same reason as in applyOverlays (no extra lossy generation).
+    const { data, info } = await pipeline.png().toBuffer({ resolveWithObject: true });
+    const blurred = await applyBlurRegions(data, info.width, info.height, spec.blurRegions);
+    pipeline = sharp(blurred, SHARP_INPUT_OPTIONS);
+  }
+
+  const fit = spec.fit ?? "inside";
+  if (spec.size) {
+    pipeline = pipeline.resize(buildResizeOptions(spec.size.width, spec.size.height, fit));
+  } else if (spec.maxEdge !== undefined) {
+    pipeline = pipeline.resize(buildResizeOptions(spec.maxEdge, spec.maxEdge, fit));
+  }
+
+  if (spec.blur !== undefined) {
+    pipeline = pipeline.blur(spec.blur);
+  }
+
+  if (spec.resolvedOverlays && spec.resolvedOverlays.length > 0) {
+    pipeline = await applyOverlays(pipeline, spec.resolvedOverlays);
+  }
+
+  pipeline = applyEncoder(pipeline, spec, normalizedSource);
+
+  const buffer = await pipeline.toBuffer();
+  return new Uint8Array(buffer);
+};
+
+export async function imageMetadata(
+  input: Uint8Array,
+): Promise<{ width: number; height: number; format: string }> {
+  const metadata = await sharp(input, SHARP_INPUT_OPTIONS).metadata();
+  const width = metadata.autoOrient.width;
+  const height = metadata.autoOrient.height;
+  if (!Number.isFinite(width) || !Number.isFinite(height) || !metadata.format) {
+    throw new Error("derivatives-sharp: imageMetadata got a response without width/height/format.");
+  }
+  return { width, height, format: metadata.format };
+}

package/packages/bundled-features/src/document-ingest-foundation/boot-checks.ts

@@ -0,0 +1,58 @@
+// Every job triggered on documentIngest.requested must filter to exactly one
+// registered provider, and every registered provider must have a job wired
+// to it — otherwise requested ingests for that provider never get picked up.
+
+import type { FeatureDefinition, JobDefinition } from "@cosmicdrift/kumiko-framework/engine";
+import { DOCUMENT_INGEST_REQUESTED_EVENT_QN } from "./events.js";
+import { resolveDocumentIngestProviders } from "./providers.js";
+
+// undefined = job isn't triggered on documentIngest.requested at all (not our
+// concern here). "" = triggered without a usable where.provider filter.
+function providerFilterOf(trigger: JobDefinition["trigger"]): string | undefined {
+  if (!("on" in trigger)) return undefined;
+  const on = Array.isArray(trigger.on) ? trigger.on : [trigger.on];
+  if (!on.includes(DOCUMENT_INGEST_REQUESTED_EVENT_QN)) return undefined;
+  const provider = trigger.where?.["provider"];
+  return typeof provider === "string" && provider.length > 0 ? provider : "";
+}
+
+export function validateDocumentIngestProviderWiring(features: readonly FeatureDefinition[]): void {
+  const allUsages = features.flatMap((f) => f.extensionUsages);
+  // Throws on invalid options / mimeType collisions.
+  const byMime = resolveDocumentIngestProviders(allUsages);
+  const registeredProviderNames = new Set([...byMime.values()].map((p) => p.name));
+
+  const wiredProviderNames = new Set<string>();
+  for (const feature of features) {
+    for (const jobDef of Object.values(feature.jobs)) {
+      const provider = providerFilterOf(jobDef.trigger);
+      if (provider === undefined) continue;
+      if (provider === "") {
+        throw new Error(
+          `[kumiko:boot] Job "${jobDef.name}" (feature "${feature.name}") triggers on ` +
+            `document-ingest-foundation's documentIngest.requested without a provider filter — every job on ` +
+            `this event must route through documentIngestProviderTrigger("<name>"), or it fires for every ` +
+            `provider's uploads. Use: trigger: documentIngestProviderTrigger("<name>").`,
+        );
+      }
+      if (!registeredProviderNames.has(provider)) {
+        throw new Error(
+          `[kumiko:boot] Job "${jobDef.name}" (feature "${feature.name}") filters documentIngest.requested to ` +
+            `provider "${provider}", but no feature registers that provider via ` +
+            `r.useExtension(EXT_DOCUMENT_INGEST_PROVIDER, "${provider}", ...). Known: ` +
+            `${[...registeredProviderNames].join(", ") || "<none>"}.`,
+        );
+      }
+      wiredProviderNames.add(provider);
+    }
+  }
+
+  for (const name of registeredProviderNames) {
+    if (wiredProviderNames.has(name)) continue;
+    throw new Error(
+      `[kumiko:boot] Provider "${name}" is registered via EXT_DOCUMENT_INGEST_PROVIDER but no job triggers on ` +
+        `documentIngestProviderTrigger("${name}") — files matching its mimeTypes would be requested for ingest ` +
+        `and never processed. Add an r.job with that trigger, or unmount the provider.`,
+    );
+  }
+}

package/packages/bundled-features/src/document-ingest-foundation/changes.json

@@ -0,0 +1,44 @@
+[
+  {
+    "version": "0.335.0",
+    "type": "fix",
+    "title": "Writing a document extract is idempotent per fileRef",
+    "detail": "`writeDocumentExtractForLiveFileRef` now returns `{ kind: \"skipped\", reason: \"already_extracted\" }` when the fileRef already has a documentExtract, so a redelivered `documentIngest.requested` or a delete-restore race no longer creates a second extract. Providers need no existence check of their own; code that switches exhaustively on `reason` must handle the new value."
+  },
+  {
+    "version": "0.335.0",
+    "type": "breaking",
+    "title": "Provider name \"unknown\" is reserved for upcast legacy events",
+    "migration": "Registering a document-ingest provider under the name `unknown` now throws; rename any provider that uses it."
+  },
+  {
+    "version": "0.312.0",
+    "type": "breaking",
+    "title": "document-ingest-foundation is provider-driven; ALLOWED_MIME_TYPES/MAX_FILE_BYTES are gone",
+    "migration": "The hardcoded MIME allowlist and fixed size cap are removed. A provider feature (e.g. kumiko-enterprise's LiteParse) now registers via r.useExtension(EXT_DOCUMENT_INGEST_PROVIDER, \"<name>\", { mimeTypes, maxFileBytes }) and routes its r.job's trigger through documentIngestProviderTrigger(\"<name>\") instead of a bare { on: DOCUMENT_INGEST_REQUESTED_EVENT_QN }. A provider feature must also declare r.requires(\"document-ingest-foundation\") — validateBoot rejects r.useExtension(EXT_DOCUMENT_INGEST_PROVIDER, ...) without it as a missing feature dependency. Boot now throws if: two providers claim the same mimeType, a job triggers on documentIngest.requested without a provider filter, a job filters to an unregistered provider name, or a registered provider has no job wired to it. documentIngest.requested's payload gained a required provider field (event schema version 2; a migration upcasts existing v1 events to provider: \"unknown\", a sentinel that cannot match any real provider filter). A file whose mimeType has no registered provider now appends documentIngest.skipped with reason unsupported-mime-type instead of file-too-large. A fileRef.deleted or fileRef.forgotten event now forgets that fileRef's documentExtract row (new forget-extract-with-file-ref table-less MSP) — previously the extract silently outlived a deleted/forgotten source file. Mounting documentIngestFoundationFeature with zero providers is valid: every upload is simply skipped as unsupported-mime-type. Boot only fails for a misconfigured wiring (unfiltered job on documentIngest.requested, a job filtered to an unregistered provider name, two providers claiming the same mimeType, or a registered provider with no job wired to it)."
+  },
+  {
+    "version": "0.312.0",
+    "type": "fix",
+    "title": "document-ingest-foundation: no documentExtract survives a deleted/forgotten fileRef, even when a provider is still writing one concurrently",
+    "migration": "New export writeDocumentExtractForLiveFileRef(input) — providers that write documentExtract rows themselves (own executor call on ctx.db, outside the request/response flow) should write through this helper instead of calling the executor directly, and only emit their ready/failed follow-up event when the result's `kind` is \"written\" (a \"skipped\" result means the fileRef was already deleted/forgotten and nothing was written). The forget-extract-with-file-ref MSP now also reacts to documentExtract's own created event: if the extract's fileRef is not live at processing time, the consumer forgets that extract immediately. This closes the race where a provider's write and the fileRef's delete/forget interleave in either order — regardless of whether the provider uses the new helper or writes directly, no extract can now outlive its fileRef. The NotFound-tolerance check in the forget path is now a plain `error.code === \"not_found\"` comparison (WriteResult.error is always a plain WriteErrorInfo object, never a KumikoError instance, so `instanceof` never matched here). Test seeds of documentExtract rows with made-up fileRefIds, in any test stack that mounts document-ingest-foundation with a running dispatcher (calls runOnce), are now actively forgotten by this guard once the dispatcher processes the seed's created event — existing such seed patterns need a live (unsoftdeleted) fileRef backing the seeded fileRefId, or must avoid running the dispatcher over that event."
+  },
+  {
+    "version": "0.312.0",
+    "type": "fix",
+    "title": "document-ingest-foundation: a restored fileRef is re-ingested, and fileRef.deleted no longer forgets an extract that is live again",
+    "migration": "The request-ingest MSP now also reacts to fileRef.restored, parsing the entity fields from the event's `previous` snapshot and routing through the same provider-resolution/skip logic as fileRef.created — a delete→restore round-trip re-requests documentIngest.requested instead of leaving the file without an extract. forget-extract-with-file-ref's fileRef.deleted handler now checks the fileRef's current liveness (new isFileRefLive helper) before forgetting: since request-ingest and forget-extract-with-file-ref poll via separate consumer cursors, a fileRef.deleted event processed after a later restore already made the fileRef live again would otherwise forget the extract belonging to the now-live file. fileRef.forgotten still forgets unconditionally — forget stays a final Art. 17 erasure, restore on a forgotten fileRef fails NotFound as before, and no re-ingest happens for it. No consumer/action changes required."
+  },
+  {
+    "version": "0.306.0",
+    "type": "fix",
+    "title": "Tenant destroy no longer fails on documentExtract rows"
+  },
+  {
+    "version": "0.166.0",
+    "type": "breaking",
+    "title": "documentExtract.pages: tenantOwned instead of encrypted, and the feature now requires tenant-lifecycle",
+    "detail": "`pages` holds the full extracted text of ingested documents (invoices, IDs, contracts) and was `encrypted: true` — app-master-key ciphertext with no erasure subject, so no Art. 17 path could ever make it unreadable. It is now `tenantOwned: true`, which binds it to the tenant subject key that tenant-destroy's eraseSubjectKeys shreds. The feature registers an EXT_TENANT_DATA destroy hook for it, and since tenant-lifecycle hosts that extension point, `document-ingest-foundation` now declares it as a hard requirement — mounting the feature without tenant-lifecycle (plus its own tenant + compliance-profiles requires) makes createRegistry throw (#1621).",
+    "migration": "Mount createTenantFeature(), createComplianceProfilesFeature() and createTenantLifecycleFeature() alongside documentIngestFoundationFeature. Rows written before this version carry master-key envelope ciphertext that the subject-decrypt path does not recognise — readIngestPages returns [] for them. There is no reencrypt job; if you have existing extracts you care about, decrypt and rewrite them before upgrading."
+  }
+]

package/packages/bundled-features/src/document-ingest-foundation/entity.ts

@@ -0,0 +1,59 @@
+import { buildEntityTable } from "@cosmicdrift/kumiko-framework/db";
+import {
+  createEntity,
+  createJsonbField,
+  createLongTextField,
+  createTextField,
+} from "@cosmicdrift/kumiko-framework/engine";
+
+// Phase-1 (MVP) page shape — LiteParse fills this in (kumiko-enterprise#273).
+// Kept minimal on purpose: downstream themes (provider, recipe) extend it.
+// Stored as JSON.stringify(IngestPage[]) in the encrypted `pages` column —
+// jsonb has no encryption support in the engine, and this column holds the
+// full extracted text of ingested documents (invoices, IDs, contracts).
+// Writers/readers MUST use writeIngestPages / readIngestPages — do not pass
+// a raw IngestPage[] into executor.create (the encryption hook requires a
+// string).
+export type IngestPage = {
+  readonly pageNumber: number;
+  readonly text: string;
+};
+
+export type DocumentExtractMeta = {
+  readonly provider: string;
+  readonly ms: number;
+  readonly needsOcr: boolean;
+  readonly pagesParsed: number;
+  readonly totalPages: number;
+};
+
+// documentExtract — implicit entity-projection (r.entity, NOT r.projection):
+// an explicit r.projection would make `forget` non-rebuild-safe and require
+// archiveStream instead of the regular forget path (kumiko-framework#1495).
+// fileRefId points at the source file_refs row; storageKey duplicates it for
+// direct provider access without a join.
+export const documentExtractEntity = createEntity({
+  table: "read_document_extracts",
+  fields: {
+    fileRefId: createTextField({ required: true, personal: false, reason: "pseudonymous_fk" }),
+    storageKey: createTextField({
+      required: true,
+      personal: false,
+      reason: "technical_reference",
+    }),
+    // Holds the full extracted document text (PII). `personal: "tenant"`,
+    // NOT `encrypted: true` (#1621): the master-key path has no erasure subject,
+    // so nothing here would ever be shreddable. Tenant-subject ciphertext
+    // dies with eraseSubjectKeys on tenant-destroy (#800 pattern). Whose
+    // subject a third party named inside a document is stays open, same as
+    // inbound-mail-foundation (#957). meta is provider telemetry only and
+    // stays plaintext jsonb.
+    pages: createLongTextField({
+      personal: "tenant",
+      find: "none",
+    }),
+    meta: createJsonbField(),
+  },
+});
+
+export const documentExtractsTable = buildEntityTable("documentExtract", documentExtractEntity);

package/packages/bundled-features/src/document-ingest-foundation/events.ts

@@ -0,0 +1,43 @@
+import * as z from "zod";
+
+// documentIngest — own aggregate-type for the ingest-request stream, one
+// stream per fileRef (aggregateId = fileRefId). Not backed by an r.entity;
+// this is a pure saga-anchor event (kumiko-framework#1497) that the Phase-2
+// worker job (kumiko-enterprise LiteParse provider) subscribes to.
+export const DOCUMENT_INGEST_AGGREGATE_TYPE = "document-ingest";
+
+export const DOCUMENT_INGEST_REQUESTED_EVENT_SHORT = "documentIngest.requested" as const;
+
+// Qualified name — what ctx.unsafeAppendEvent's `type` actually needs
+// (r.defineEvent registers under qn(toKebab(feature), "event",
+// toKebab(shortName)); appendEvent looks up the registry by QN, not the
+// short name — see MSP-apply "event not registered" error otherwise).
+export const DOCUMENT_INGEST_REQUESTED_EVENT_QN =
+  "document-ingest-foundation:event:document-ingest-requested" as const;
+
+export const documentIngestRequestedPayloadSchema = z.object({
+  fileRefId: z.string().min(1),
+  storageKey: z.string().min(1),
+  fileName: z.string().min(1),
+  mimeType: z.string().min(1),
+  size: z.number().int().min(0),
+  // Which registered provider claimed this mimeType — the job-trigger
+  // where-filter every provider's job routes through (documentIngestProviderTrigger).
+  provider: z.string().min(1),
+});
+export type DocumentIngestRequestedPayload = z.infer<typeof documentIngestRequestedPayloadSchema>;
+
+export const DOCUMENT_INGEST_SKIPPED_EVENT_SHORT = "documentIngest.skipped" as const;
+
+export const DOCUMENT_INGEST_SKIPPED_EVENT_QN =
+  "document-ingest-foundation:event:document-ingest-skipped" as const;
+
+export const documentIngestSkippedPayloadSchema = z.object({
+  fileRefId: z.string().min(1),
+  storageKey: z.string().min(1),
+  fileName: z.string().min(1),
+  mimeType: z.string().min(1),
+  size: z.number().int().min(0),
+  reason: z.enum(["file-too-large", "unsupported-mime-type"]),
+});
+export type DocumentIngestSkippedPayload = z.infer<typeof documentIngestSkippedPayloadSchema>;

package/packages/bundled-features/src/document-ingest-foundation/executor.ts

@@ -0,0 +1,12 @@
+// Single shared instance — one createEventStoreExecutor per entity per
+// process is the framework convention, reused by every write path outside
+// the entity's own inline projection (tenant-destroy-hook, forget-extract-with-file-ref).
+
+import { createEventStoreExecutor } from "@cosmicdrift/kumiko-framework/db";
+import { documentExtractEntity, documentExtractsTable } from "./entity.js";
+
+export const documentExtractExecutor = createEventStoreExecutor(
+  documentExtractsTable,
+  documentExtractEntity,
+  { entityName: "document-extract" },
+);