@cosmicdrift/kumiko-bundled-features 0.311.0 → 0.312.0

package/src/document-ingest-foundation/forget-extract-with-file-ref.ts

@@ -0,0 +1,84 @@
+// documentExtract is derived data — once its source fileRef is gone (soft
+// delete or forget), the extract has no reason to survive it. A restored
+// fileRef is re-ingested by feature.ts's request-ingest MSP
+// (fileRef.restored), so a fresh extract replaces the forgotten one.
+//
+// forget(), not deleteMany: documentExtract is an ES-managed implicit
+// projection, so forget() replays correctly on rebuild. This MSP itself has
+// no `table`, so rebuildMultiStreamProjection never targets it directly.
+//
+// Also reacts to documentExtract.created: providers write extracts
+// themselves (own executor call), racing this consumer's fileRef.deleted/
+// forgotten handling. Each handler re-checks current fileRef liveness at
+// processing time, so no extract survives regardless of write/delete order —
+// this covers providers that bypass write-document-extract.ts's helper too.
+
+import { createTenantDb, type TenantDb } from "@cosmicdrift/kumiko-framework/db";
+import type { MultiStreamApplyFn } from "@cosmicdrift/kumiko-framework/engine";
+import { createSystemUser } from "@cosmicdrift/kumiko-framework/engine";
+import type { WriteErrorInfo } from "@cosmicdrift/kumiko-framework/errors";
+import * as z from "zod";
+import { documentExtractsTable } from "./entity";
+import { documentExtractExecutor } from "./executor";
+import { isFileRefLive } from "./file-ref-liveness";
+
+// WriteResult.error is always a plain WriteErrorInfo object, never a
+// KumikoError instance (JSON-serializable for the dispatcher's idempotency-key
+// storage) — compare on .code, not instanceof.
+function isNotFoundWriteError(error: WriteErrorInfo): boolean {
+  return error.code === "not_found";
+}
+
+async function forgetExtractsForFileRef(
+  event: Parameters<MultiStreamApplyFn>[0],
+  tenantDb: TenantDb,
+): Promise<void> {
+  const rows = await tenantDb.selectMany<{ id: string }>(documentExtractsTable, {
+    fileRefId: event.aggregateId,
+  });
+  const user = createSystemUser(event.tenantId);
+  for (const row of rows) {
+    const result = await documentExtractExecutor.forget({ id: row.id }, user, tenantDb);
+    if (result.isSuccess) continue;
+    // Idempotent: a concurrent/redelivered apply that already forgot this
+    // row is not an error — anything else (version conflict, ownership
+    // denial) must surface so the dispatcher retries/dead-letters instead of
+    // silently leaving the extract behind.
+    if (isNotFoundWriteError(result.error)) continue;
+    throw new Error(
+      `document-ingest-foundation: failed to forget documentExtract ${row.id} for fileRef ${event.aggregateId}: ${result.error.message}`,
+    );
+  }
+}
+
+const documentExtractCreatedPayloadSchema = z.object({ fileRefId: z.string().min(1) });
+
+export const forgetOrphanedDocumentExtractHook: MultiStreamApplyFn = async (event, tx) => {
+  const parsed = documentExtractCreatedPayloadSchema.safeParse(event.payload);
+  // skip: malformed payload — don't poison the consumer
+  if (!parsed.success) return;
+  const tenantDb = createTenantDb(tx, event.tenantId);
+  // skip: fileRef still live — the extract is legitimate
+  if (await isFileRefLive(tenantDb, parsed.data.fileRefId)) return;
+  const user = createSystemUser(event.tenantId);
+  const result = await documentExtractExecutor.forget({ id: event.aggregateId }, user, tenantDb);
+  // skip: forgotten now, or already gone via a concurrent/redelivered apply
+  if (result.isSuccess || isNotFoundWriteError(result.error)) return;
+  throw new Error(
+    `document-ingest-foundation: failed to forget orphaned documentExtract ${event.aggregateId}: ${result.error.message}`,
+  );
+};
+
+export const forgetExtractOnFileRefDeletedHook: MultiStreamApplyFn = async (event, tx) => {
+  const tenantDb = createTenantDb(tx, event.tenantId);
+  // skip: fileRef is live again — request-ingest and this consumer poll via
+  // separate cursors, so a delete→restore round-trip can finish before this
+  // consumer sees fileRef.deleted; forgetting would drop the live file's extract.
+  if (await isFileRefLive(tenantDb, event.aggregateId)) return;
+  await forgetExtractsForFileRef(event, tenantDb);
+};
+
+export const forgetExtractOnFileRefForgottenHook: MultiStreamApplyFn = async (event, tx) => {
+  const tenantDb = createTenantDb(tx, event.tenantId);
+  await forgetExtractsForFileRef(event, tenantDb);
+};

package/src/document-ingest-foundation/index.ts

@@ -10,8 +10,25 @@ export {
   DOCUMENT_INGEST_AGGREGATE_TYPE,
   DOCUMENT_INGEST_REQUESTED_EVENT_QN,
   DOCUMENT_INGEST_REQUESTED_EVENT_SHORT,
+  DOCUMENT_INGEST_SKIPPED_EVENT_QN,
+  DOCUMENT_INGEST_SKIPPED_EVENT_SHORT,
   type DocumentIngestRequestedPayload,
+  type DocumentIngestSkippedPayload,
   documentIngestRequestedPayloadSchema,
+  documentIngestSkippedPayloadSchema,
 } from "./events";
 export { documentIngestFoundationFeature } from "./feature";
 export { readIngestPages, writeIngestPages } from "./pages";
+export {
+  type DocumentIngestProviderOptions,
+  documentIngestProviderOptionsSchema,
+  documentIngestProviderTrigger,
+  EXT_DOCUMENT_INGEST_PROVIDER,
+  listIngestibleMimeTypes,
+  type ResolvedDocumentIngestProvider,
+  resolveDocumentIngestProviders,
+} from "./providers";
+export {
+  type DocumentExtractWriteResult,
+  writeDocumentExtractForLiveFileRef,
+} from "./write-document-extract";

package/src/document-ingest-foundation/providers.ts

@@ -0,0 +1,78 @@
+// document-ingest-foundation — provider extension-point. One shared
+// documentIngest.requested QN fans out to every mounted provider's job,
+// partitioned by documentIngestProviderTrigger's where-filter.
+
+import type { FeatureDefinition, Registry } from "@cosmicdrift/kumiko-framework/engine";
+import { normalizeMimeType } from "@cosmicdrift/kumiko-framework/files";
+import * as z from "zod";
+import { DOCUMENT_INGEST_REQUESTED_EVENT_QN } from "./events";
+
+export const EXT_DOCUMENT_INGEST_PROVIDER = "documentIngestProvider" as const;
+
+export const documentIngestProviderOptionsSchema = z.object({
+  mimeTypes: z.array(z.string().min(1)).min(1),
+  maxFileBytes: z.number().int().positive(),
+});
+export type DocumentIngestProviderOptions = z.infer<typeof documentIngestProviderOptionsSchema>;
+
+// r.useExtension options-shape, co-located since the framework never imports upward.
+declare module "@cosmicdrift/kumiko-framework/engine" {
+  interface KumikoExtensionOptionsMap {
+    [EXT_DOCUMENT_INGEST_PROVIDER]: DocumentIngestProviderOptions;
+  }
+}
+
+export type ResolvedDocumentIngestProvider = {
+  readonly name: string;
+  readonly maxFileBytes: number;
+};
+
+type ExtensionUsages = FeatureDefinition["extensionUsages"];
+
+// Throws on an invalid options shape or two providers claiming the same
+// normalized mimeType — callers decide whether that's boot-fatal.
+export function resolveDocumentIngestProviders(
+  usages: ExtensionUsages,
+): ReadonlyMap<string, ResolvedDocumentIngestProvider> {
+  const byMime = new Map<string, ResolvedDocumentIngestProvider>();
+  for (const usage of usages) {
+    if (usage.extensionName !== EXT_DOCUMENT_INGEST_PROVIDER) continue;
+    const parsed = documentIngestProviderOptionsSchema.safeParse(usage.options);
+    if (!parsed.success) {
+      throw new Error(
+        `document-ingest-foundation: provider "${usage.entityName}" registered invalid options via ` +
+          `r.useExtension(EXT_DOCUMENT_INGEST_PROVIDER, "${usage.entityName}", ...) — ${parsed.error.message}`,
+      );
+    }
+    for (const mimeType of parsed.data.mimeTypes) {
+      const normalized = normalizeMimeType(mimeType);
+      const existing = byMime.get(normalized);
+      if (existing && existing.name !== usage.entityName) {
+        throw new Error(
+          `document-ingest-foundation: providers "${existing.name}" and "${usage.entityName}" both claim mimeType ` +
+            `"${normalized}" — each mimeType may have exactly one registered provider.`,
+        );
+      }
+      byMime.set(normalized, { name: usage.entityName, maxFileBytes: parsed.data.maxFileBytes });
+    }
+  }
+  return byMime;
+}
+
+export function listIngestibleMimeTypes(registry: Registry): readonly string[] {
+  const usages = registry.getExtensionUsages(EXT_DOCUMENT_INGEST_PROVIDER);
+  return [...resolveDocumentIngestProviders(usages).keys()].sort();
+}
+
+export type DocumentIngestProviderJobTrigger = {
+  readonly on: typeof DOCUMENT_INGEST_REQUESTED_EVENT_QN;
+  readonly where: { readonly provider: string };
+};
+
+// The trigger every provider's job must use — filters the shared
+// documentIngest.requested QN down to just this provider's requests.
+export function documentIngestProviderTrigger(
+  providerName: string,
+): DocumentIngestProviderJobTrigger {
+  return { on: DOCUMENT_INGEST_REQUESTED_EVENT_QN, where: { provider: providerName } };
+}

package/src/document-ingest-foundation/tenant-destroy-hook.ts

@@ -10,13 +10,9 @@
 // (kumiko-framework#1495). The `pages` ciphertext dies separately when the
 // pipeline's later `subject-keys` stage erases the tenant subject key.
 
-import { createEventStoreExecutor } from "@cosmicdrift/kumiko-framework/db";
 import { createSystemUser, type TenantDataDestroyHook } from "@cosmicdrift/kumiko-framework/engine";
-import { documentExtractEntity, documentExtractsTable } from "./entity";
-
-const executor = createEventStoreExecutor(documentExtractsTable, documentExtractEntity, {
-  entityName: "document-extract",
-});
+import { documentExtractsTable } from "./entity";
+import { documentExtractExecutor } from "./executor";
 
 export const documentExtractTenantDestroyHook: TenantDataDestroyHook = async (ctx) => {
   const rows = await ctx.db.selectMany<{ id: string }>(documentExtractsTable, {
@@ -24,7 +20,7 @@ export const documentExtractTenantDestroyHook: TenantDataDestroyHook = async (ct
   });
   const user = createSystemUser(ctx.tenantId);
   for (const row of rows) {
-    const result = await executor.forget({ id: row.id }, user, ctx.db);
+    const result = await documentExtractExecutor.forget({ id: row.id }, user, ctx.db);
     // Executor writes return {isSuccess:false} instead of throwing — a
     // discarded result would report this destroy stage "succeeded" while the
     // extracted document text survives. Throw so the pipeline's retry/abandon

package/src/document-ingest-foundation/write-document-extract.ts

@@ -0,0 +1,48 @@
+// Providers write documentExtract rows themselves (their own executor
+// call on ctx.db) rather than through a request/response API, so this is
+// the one place that write must go through to stay race-safe against a
+// concurrent fileRef delete/forget — see forget-extract-with-file-ref.ts.
+
+import type { TenantDb } from "@cosmicdrift/kumiko-framework/db";
+import type { SessionUser } from "@cosmicdrift/kumiko-framework/engine";
+import { InternalError } from "@cosmicdrift/kumiko-framework/errors";
+import type { DocumentExtractMeta, IngestPage } from "./entity";
+import { documentExtractExecutor } from "./executor";
+import { isFileRefLive } from "./file-ref-liveness";
+import { writeIngestPages } from "./pages";
+
+export type DocumentExtractWriteResult =
+  | { readonly kind: "written"; readonly documentExtractId: string }
+  | { readonly kind: "skipped"; readonly reason: "file_ref_deleted" };
+
+export async function writeDocumentExtractForLiveFileRef(input: {
+  readonly tenantDb: TenantDb;
+  readonly actor: SessionUser;
+  readonly fileRefId: string;
+  readonly storageKey: string;
+  readonly pages: readonly IngestPage[];
+  readonly meta: DocumentExtractMeta & Readonly<Record<string, unknown>>;
+}): Promise<DocumentExtractWriteResult> {
+  // Early-skip optimization only — a fileRef deleted/forgotten right after
+  // this check still gets caught by forget-extract-with-file-ref.ts's
+  // documentExtract.created handler, which is the actual race-safety guarantee.
+  if (!(await isFileRefLive(input.tenantDb, input.fileRefId))) {
+    return { kind: "skipped", reason: "file_ref_deleted" };
+  }
+  const result = await documentExtractExecutor.create(
+    {
+      fileRefId: input.fileRefId,
+      storageKey: input.storageKey,
+      pages: writeIngestPages(input.pages),
+      meta: input.meta,
+    },
+    input.actor,
+    input.tenantDb,
+  );
+  if (!result.isSuccess) {
+    throw new InternalError({
+      message: `document-ingest-foundation: failed to write documentExtract for fileRef ${input.fileRefId}: ${result.error.message}`,
+    });
+  }
+  return { kind: "written", documentExtractId: String(result.data.id) };
+}