@cosmicdrift/kumiko-bundled-features 0.310.0 → 0.312.0

package/src/document-ingest-foundation/forget-extract-with-file-ref.ts

@@ -0,0 +1,84 @@
+// documentExtract is derived data — once its source fileRef is gone (soft
+// delete or forget), the extract has no reason to survive it. A restored
+// fileRef is re-ingested by feature.ts's request-ingest MSP
+// (fileRef.restored), so a fresh extract replaces the forgotten one.
+//
+// forget(), not deleteMany: documentExtract is an ES-managed implicit
+// projection, so forget() replays correctly on rebuild. This MSP itself has
+// no `table`, so rebuildMultiStreamProjection never targets it directly.
+//
+// Also reacts to documentExtract.created: providers write extracts
+// themselves (own executor call), racing this consumer's fileRef.deleted/
+// forgotten handling. Each handler re-checks current fileRef liveness at
+// processing time, so no extract survives regardless of write/delete order —
+// this covers providers that bypass write-document-extract.ts's helper too.
+
+import { createTenantDb, type TenantDb } from "@cosmicdrift/kumiko-framework/db";
+import type { MultiStreamApplyFn } from "@cosmicdrift/kumiko-framework/engine";
+import { createSystemUser } from "@cosmicdrift/kumiko-framework/engine";
+import type { WriteErrorInfo } from "@cosmicdrift/kumiko-framework/errors";
+import * as z from "zod";
+import { documentExtractsTable } from "./entity";
+import { documentExtractExecutor } from "./executor";
+import { isFileRefLive } from "./file-ref-liveness";
+
+// WriteResult.error is always a plain WriteErrorInfo object, never a
+// KumikoError instance (JSON-serializable for the dispatcher's idempotency-key
+// storage) — compare on .code, not instanceof.
+function isNotFoundWriteError(error: WriteErrorInfo): boolean {
+  return error.code === "not_found";
+}
+
+async function forgetExtractsForFileRef(
+  event: Parameters<MultiStreamApplyFn>[0],
+  tenantDb: TenantDb,
+): Promise<void> {
+  const rows = await tenantDb.selectMany<{ id: string }>(documentExtractsTable, {
+    fileRefId: event.aggregateId,
+  });
+  const user = createSystemUser(event.tenantId);
+  for (const row of rows) {
+    const result = await documentExtractExecutor.forget({ id: row.id }, user, tenantDb);
+    if (result.isSuccess) continue;
+    // Idempotent: a concurrent/redelivered apply that already forgot this
+    // row is not an error — anything else (version conflict, ownership
+    // denial) must surface so the dispatcher retries/dead-letters instead of
+    // silently leaving the extract behind.
+    if (isNotFoundWriteError(result.error)) continue;
+    throw new Error(
+      `document-ingest-foundation: failed to forget documentExtract ${row.id} for fileRef ${event.aggregateId}: ${result.error.message}`,
+    );
+  }
+}
+
+const documentExtractCreatedPayloadSchema = z.object({ fileRefId: z.string().min(1) });
+
+export const forgetOrphanedDocumentExtractHook: MultiStreamApplyFn = async (event, tx) => {
+  const parsed = documentExtractCreatedPayloadSchema.safeParse(event.payload);
+  // skip: malformed payload — don't poison the consumer
+  if (!parsed.success) return;
+  const tenantDb = createTenantDb(tx, event.tenantId);
+  // skip: fileRef still live — the extract is legitimate
+  if (await isFileRefLive(tenantDb, parsed.data.fileRefId)) return;
+  const user = createSystemUser(event.tenantId);
+  const result = await documentExtractExecutor.forget({ id: event.aggregateId }, user, tenantDb);
+  // skip: forgotten now, or already gone via a concurrent/redelivered apply
+  if (result.isSuccess || isNotFoundWriteError(result.error)) return;
+  throw new Error(
+    `document-ingest-foundation: failed to forget orphaned documentExtract ${event.aggregateId}: ${result.error.message}`,
+  );
+};
+
+export const forgetExtractOnFileRefDeletedHook: MultiStreamApplyFn = async (event, tx) => {
+  const tenantDb = createTenantDb(tx, event.tenantId);
+  // skip: fileRef is live again — request-ingest and this consumer poll via
+  // separate cursors, so a delete→restore round-trip can finish before this
+  // consumer sees fileRef.deleted; forgetting would drop the live file's extract.
+  if (await isFileRefLive(tenantDb, event.aggregateId)) return;
+  await forgetExtractsForFileRef(event, tenantDb);
+};
+
+export const forgetExtractOnFileRefForgottenHook: MultiStreamApplyFn = async (event, tx) => {
+  const tenantDb = createTenantDb(tx, event.tenantId);
+  await forgetExtractsForFileRef(event, tenantDb);
+};

package/src/document-ingest-foundation/index.ts

@@ -10,8 +10,25 @@ export {
   DOCUMENT_INGEST_AGGREGATE_TYPE,
   DOCUMENT_INGEST_REQUESTED_EVENT_QN,
   DOCUMENT_INGEST_REQUESTED_EVENT_SHORT,
+  DOCUMENT_INGEST_SKIPPED_EVENT_QN,
+  DOCUMENT_INGEST_SKIPPED_EVENT_SHORT,
   type DocumentIngestRequestedPayload,
+  type DocumentIngestSkippedPayload,
   documentIngestRequestedPayloadSchema,
+  documentIngestSkippedPayloadSchema,
 } from "./events";
 export { documentIngestFoundationFeature } from "./feature";
 export { readIngestPages, writeIngestPages } from "./pages";
+export {
+  type DocumentIngestProviderOptions,
+  documentIngestProviderOptionsSchema,
+  documentIngestProviderTrigger,
+  EXT_DOCUMENT_INGEST_PROVIDER,
+  listIngestibleMimeTypes,
+  type ResolvedDocumentIngestProvider,
+  resolveDocumentIngestProviders,
+} from "./providers";
+export {
+  type DocumentExtractWriteResult,
+  writeDocumentExtractForLiveFileRef,
+} from "./write-document-extract";

package/src/document-ingest-foundation/providers.ts

@@ -0,0 +1,78 @@
+// document-ingest-foundation — provider extension-point. One shared
+// documentIngest.requested QN fans out to every mounted provider's job,
+// partitioned by documentIngestProviderTrigger's where-filter.
+
+import type { FeatureDefinition, Registry } from "@cosmicdrift/kumiko-framework/engine";
+import { normalizeMimeType } from "@cosmicdrift/kumiko-framework/files";
+import * as z from "zod";
+import { DOCUMENT_INGEST_REQUESTED_EVENT_QN } from "./events";
+
+export const EXT_DOCUMENT_INGEST_PROVIDER = "documentIngestProvider" as const;
+
+export const documentIngestProviderOptionsSchema = z.object({
+  mimeTypes: z.array(z.string().min(1)).min(1),
+  maxFileBytes: z.number().int().positive(),
+});
+export type DocumentIngestProviderOptions = z.infer<typeof documentIngestProviderOptionsSchema>;
+
+// r.useExtension options-shape, co-located since the framework never imports upward.
+declare module "@cosmicdrift/kumiko-framework/engine" {
+  interface KumikoExtensionOptionsMap {
+    [EXT_DOCUMENT_INGEST_PROVIDER]: DocumentIngestProviderOptions;
+  }
+}
+
+export type ResolvedDocumentIngestProvider = {
+  readonly name: string;
+  readonly maxFileBytes: number;
+};
+
+type ExtensionUsages = FeatureDefinition["extensionUsages"];
+
+// Throws on an invalid options shape or two providers claiming the same
+// normalized mimeType — callers decide whether that's boot-fatal.
+export function resolveDocumentIngestProviders(
+  usages: ExtensionUsages,
+): ReadonlyMap<string, ResolvedDocumentIngestProvider> {
+  const byMime = new Map<string, ResolvedDocumentIngestProvider>();
+  for (const usage of usages) {
+    if (usage.extensionName !== EXT_DOCUMENT_INGEST_PROVIDER) continue;
+    const parsed = documentIngestProviderOptionsSchema.safeParse(usage.options);
+    if (!parsed.success) {
+      throw new Error(
+        `document-ingest-foundation: provider "${usage.entityName}" registered invalid options via ` +
+          `r.useExtension(EXT_DOCUMENT_INGEST_PROVIDER, "${usage.entityName}", ...) — ${parsed.error.message}`,
+      );
+    }
+    for (const mimeType of parsed.data.mimeTypes) {
+      const normalized = normalizeMimeType(mimeType);
+      const existing = byMime.get(normalized);
+      if (existing && existing.name !== usage.entityName) {
+        throw new Error(
+          `document-ingest-foundation: providers "${existing.name}" and "${usage.entityName}" both claim mimeType ` +
+            `"${normalized}" — each mimeType may have exactly one registered provider.`,
+        );
+      }
+      byMime.set(normalized, { name: usage.entityName, maxFileBytes: parsed.data.maxFileBytes });
+    }
+  }
+  return byMime;
+}
+
+export function listIngestibleMimeTypes(registry: Registry): readonly string[] {
+  const usages = registry.getExtensionUsages(EXT_DOCUMENT_INGEST_PROVIDER);
+  return [...resolveDocumentIngestProviders(usages).keys()].sort();
+}
+
+export type DocumentIngestProviderJobTrigger = {
+  readonly on: typeof DOCUMENT_INGEST_REQUESTED_EVENT_QN;
+  readonly where: { readonly provider: string };
+};
+
+// The trigger every provider's job must use — filters the shared
+// documentIngest.requested QN down to just this provider's requests.
+export function documentIngestProviderTrigger(
+  providerName: string,
+): DocumentIngestProviderJobTrigger {
+  return { on: DOCUMENT_INGEST_REQUESTED_EVENT_QN, where: { provider: providerName } };
+}

package/src/document-ingest-foundation/tenant-destroy-hook.ts

@@ -10,13 +10,9 @@
 // (kumiko-framework#1495). The `pages` ciphertext dies separately when the
 // pipeline's later `subject-keys` stage erases the tenant subject key.
 
-import { createEventStoreExecutor } from "@cosmicdrift/kumiko-framework/db";
 import { createSystemUser, type TenantDataDestroyHook } from "@cosmicdrift/kumiko-framework/engine";
-import { documentExtractEntity, documentExtractsTable } from "./entity";
-
-const executor = createEventStoreExecutor(documentExtractsTable, documentExtractEntity, {
-  entityName: "document-extract",
-});
+import { documentExtractsTable } from "./entity";
+import { documentExtractExecutor } from "./executor";
 
 export const documentExtractTenantDestroyHook: TenantDataDestroyHook = async (ctx) => {
   const rows = await ctx.db.selectMany<{ id: string }>(documentExtractsTable, {
@@ -24,7 +20,7 @@ export const documentExtractTenantDestroyHook: TenantDataDestroyHook = async (ct
   });
   const user = createSystemUser(ctx.tenantId);
   for (const row of rows) {
-    const result = await executor.forget({ id: row.id }, user, ctx.db);
+    const result = await documentExtractExecutor.forget({ id: row.id }, user, ctx.db);
     // Executor writes return {isSuccess:false} instead of throwing — a
     // discarded result would report this destroy stage "succeeded" while the
     // extracted document text survives. Throw so the pipeline's retry/abandon

package/src/document-ingest-foundation/write-document-extract.ts

@@ -0,0 +1,48 @@
+// Providers write documentExtract rows themselves (their own executor
+// call on ctx.db) rather than through a request/response API, so this is
+// the one place that write must go through to stay race-safe against a
+// concurrent fileRef delete/forget — see forget-extract-with-file-ref.ts.
+
+import type { TenantDb } from "@cosmicdrift/kumiko-framework/db";
+import type { SessionUser } from "@cosmicdrift/kumiko-framework/engine";
+import { InternalError } from "@cosmicdrift/kumiko-framework/errors";
+import type { DocumentExtractMeta, IngestPage } from "./entity";
+import { documentExtractExecutor } from "./executor";
+import { isFileRefLive } from "./file-ref-liveness";
+import { writeIngestPages } from "./pages";
+
+export type DocumentExtractWriteResult =
+  | { readonly kind: "written"; readonly documentExtractId: string }
+  | { readonly kind: "skipped"; readonly reason: "file_ref_deleted" };
+
+export async function writeDocumentExtractForLiveFileRef(input: {
+  readonly tenantDb: TenantDb;
+  readonly actor: SessionUser;
+  readonly fileRefId: string;
+  readonly storageKey: string;
+  readonly pages: readonly IngestPage[];
+  readonly meta: DocumentExtractMeta & Readonly<Record<string, unknown>>;
+}): Promise<DocumentExtractWriteResult> {
+  // Early-skip optimization only — a fileRef deleted/forgotten right after
+  // this check still gets caught by forget-extract-with-file-ref.ts's
+  // documentExtract.created handler, which is the actual race-safety guarantee.
+  if (!(await isFileRefLive(input.tenantDb, input.fileRefId))) {
+    return { kind: "skipped", reason: "file_ref_deleted" };
+  }
+  const result = await documentExtractExecutor.create(
+    {
+      fileRefId: input.fileRefId,
+      storageKey: input.storageKey,
+      pages: writeIngestPages(input.pages),
+      meta: input.meta,
+    },
+    input.actor,
+    input.tenantDb,
+  );
+  if (!result.isSuccess) {
+    throw new InternalError({
+      message: `document-ingest-foundation: failed to write documentExtract for fileRef ${input.fileRefId}: ${result.error.message}`,
+    });
+  }
+  return { kind: "written", documentExtractId: String(result.data.id) };
+}

package/src/inbound-provider-imap/__tests__/plugin-mocked.test.ts

@@ -5,6 +5,7 @@
 import { afterAll, beforeEach, describe, expect, mock, test } from "bun:test";
 import { EventEmitter } from "node:events";
 import { createSecret } from "@cosmicdrift/kumiko-framework/secrets";
+import { sleep, waitFor } from "@cosmicdrift/kumiko-framework/testing";
 import {
   type InboundMailContext,
   isInboundAuthError,
@@ -152,13 +153,6 @@ const account: MailAccountRecord = {
   watchState: "idle",
 };
 
-async function waitFor(predicate: () => boolean, timeoutMs = 2000): Promise<void> {
-  const t0 = Date.now();
-  while (!predicate() && Date.now() - t0 < timeoutMs) {
-    await Bun.sleep(5);
-  }
-}
-
 function ctxWithDoc(doc: string | null): InboundMailContext {
   return {
     _userId: "imap-plugin-mocked",
@@ -332,7 +326,9 @@ describe("imapInboundMailPlugin — mocked imapflow", () => {
 
     // drainNew() does MIME-parsing before onMessages fires — a fixed sleep
     // flakes on a loaded CI runner; poll instead.
-    await waitFor(() => received.some((batch) => batch.some((m) => m.subject === "pushed")));
+    await waitFor(() => received.some((batch) => batch.some((m) => m.subject === "pushed")), {
+      delays: Array(40).fill(5),
+    });
     expect(received.some((batch) => batch.some((m) => m.subject === "pushed"))).toBe(true);
 
     await stop();
@@ -347,11 +343,13 @@ describe("imapInboundMailPlugin — mocked imapflow", () => {
       },
     });
     lastIdleClient?.emit("error", new Error("socket hang up"));
-    await waitFor(() => errors >= 1);
+    await waitFor(() => errors >= 1, { delays: Array(40).fill(5) });
     lastIdleClient?.emit("error", new Error("second"));
-    // Confirms onError stays unsubscribed after the first error — polls the
-    // same bounded window rather than betting on a fixed sleep outlasting it.
-    await waitFor(() => errors >= 2, 100);
+    // Confirms onError stays unsubscribed after the first error. This is a
+    // grace period for a negative outcome, not a wait-for-true condition, so
+    // it's a single sleep (not a poll loop) rather than waitFor, which would
+    // throw once errors never reaches 2.
+    await sleep(100);
     expect(errors).toBe(1);
     await stop().catch(() => {});
   });

package/src/jobs/__tests__/projection-rebuild-job.integration.test.ts

@@ -40,7 +40,7 @@ import {
   unsafeCreateEntityTable,
   unsafePushTables,
 } from "@cosmicdrift/kumiko-framework/stack";
-import { sleep } from "@cosmicdrift/kumiko-framework/testing";
+import { waitFor } from "@cosmicdrift/kumiko-framework/testing";
 import { createJobsFeature } from "../feature";
 import { createJobRunLogger } from "../job-run-logger";
 import { jobRunLogsTable, jobRunsTable } from "../job-run-table";
@@ -148,7 +148,7 @@ describe("projection-rebuild job (jobs feature composed)", () => {
     }
 
     // Poll until the worker drained the queue and the rebuild refilled.
-    for (let i = 0; i < 40 && (await getCount()) !== 2; i++) await sleep(200);
+    await waitFor(async () => (await getCount()) === 2, { delays: Array(40).fill(200) });
     expect(await getCount()).toBe(2);
 
     // getCount()==2 only proves rebuildProjection's own writes landed — the
@@ -156,13 +156,15 @@ describe("projection-rebuild job (jobs feature composed)", () => {
     // async append that starts only after the handler returns, so it can
     // still be in flight here. Poll status too instead of racing it.
     let runs: readonly { jobName: string; status: string }[] = [];
-    for (let i = 0; i < 40; i++) {
-      runs = await selectMany<{ jobName: string; status: string }>(db, jobRunsTable, {
-        jobName: PROJECTION_REBUILD_JOB,
-      });
-      if (runs.some((r) => r.status === "completed")) break;
-      await sleep(200);
-    }
+    await waitFor(
+      async () => {
+        runs = await selectMany<{ jobName: string; status: string }>(db, jobRunsTable, {
+          jobName: PROJECTION_REBUILD_JOB,
+        });
+        return runs.some((r) => r.status === "completed");
+      },
+      { delays: Array(40).fill(200) },
+    );
     expect(runs.length).toBeGreaterThanOrEqual(1);
     expect(runs.some((r) => r.status === "completed")).toBe(true);
   }, 30000);