@cosmicdrift/kumiko-framework 0.306.0 → 0.308.0

package/src/search/__tests__/search-consumer-batch.integration.test.ts

@@ -0,0 +1,306 @@
+// #3227 — the search consumer's batchHandler against a real Meilisearch:
+// N created events collapse into one indexBatch task, and same-turn
+// collisions on one aggregate resolve to the LAST op (update-then-delete
+// removes, delete-then-restore indexes the restored state).
+import { afterAll, beforeAll, describe, expect, test } from "bun:test";
+import { generateId as uuid } from "@cosmicdrift/kumiko-framework/utils";
+import { Meilisearch } from "meilisearch";
+import * as z from "zod";
+import { createEntity, createTextField, defineFeature } from "../../engine";
+import type { ConsumerStateRow, StoredEventRow } from "../../pipeline/event-dispatcher-delivery";
+import { deliverEvents } from "../../pipeline/event-dispatcher-delivery";
+import { createSearchEventConsumer } from "../../pipeline/system-hooks";
+import { setupTestStack, type TestStack } from "../../stack";
+import { createMeilisearchAdapter, meilisearchTenantIndex } from "../meilisearch-adapter";
+import type { SearchAdapter } from "../types";
+
+const MEILI_URL = process.env["MEILI_URL"] ?? "http://localhost:17700";
+const MEILI_KEY = process.env["MEILI_MASTER_KEY"] ?? "kumiko-dev-key";
+
+async function meiliAvailable(): Promise<boolean> {
+  try {
+    const health = await fetch(`${MEILI_URL}/health`, { signal: AbortSignal.timeout(2000) });
+    return health.ok;
+  } catch {
+    return false;
+  }
+}
+
+const MEILI_UP = await meiliAvailable();
+if (!MEILI_UP) {
+  if (process.env["REQUIRE_MEILI"] === "1") {
+    throw new Error(
+      `REQUIRE_MEILI=1 but Meilisearch not reachable at ${MEILI_URL} — integration job misconfigured`,
+    );
+  }
+  console.warn(
+    `Meilisearch not reachable at ${MEILI_URL} — skipping live batch-consumer tests ` +
+      `(docker compose up -d meilisearch, default port 17700)`,
+  );
+}
+
+const docEntity = createEntity({
+  table: "read_batch_search_docs",
+  fields: {
+    label: createTextField({
+      personal: false,
+      reason: "test_fixture",
+      required: true,
+      maxLength: 100,
+      searchable: true,
+    }),
+  },
+});
+
+const batchSearchFeature = defineFeature("batch-search", (r) => {
+  r.entity("doc", docEntity);
+  r.defineEvent("noop", z.object({}), { piiFields: "none" });
+});
+
+const TENANT = uuid();
+
+let client: Meilisearch;
+let indexPrefix: string;
+let meiliAdapter: SearchAdapter;
+let stack: TestStack;
+let consumer: ReturnType<typeof createSearchEventConsumer>;
+let nextId = 1n;
+
+function stubState(): ConsumerStateRow {
+  return {
+    name: "search",
+    instanceId: "shared",
+    lastProcessedEventId: 0n,
+    status: "idle",
+    attempts: 0,
+    rearmCount: 0,
+    pendingGaps: [],
+    lastError: null,
+    updatedAt: Temporal.Now.instant(),
+  };
+}
+
+function eventRow(
+  aggregateId: string,
+  type: "created" | "updated" | "deleted" | "restored",
+  payload: Record<string, unknown>,
+): StoredEventRow {
+  return {
+    id: nextId++,
+    aggregateId,
+    aggregateType: "doc",
+    tenantId: TENANT,
+    version: 1,
+    type: `doc.${type}`,
+    eventVersion: 1,
+    payload,
+    metadata: { userId: "system" },
+    createdAt: Temporal.Now.instant(),
+    createdBy: "system",
+  };
+}
+
+async function deliverBatch(rows: readonly StoredEventRow[]): Promise<void> {
+  const outcome = await deliverEvents(
+    consumer,
+    rows,
+    { db: stack.db, redis: stack.redis.redis, registry: stack.registry },
+    10,
+    stubState(),
+  );
+  if (outcome.failed > 0) throw new Error(`test setup: delivery failed (${outcome.lastError})`);
+}
+
+async function taskCountSince(baselineUid: number, types: readonly string[]): Promise<number> {
+  const indexUid = meilisearchTenantIndex(indexPrefix, TENANT);
+  const page = await client.tasks.getTasks({
+    indexUids: [indexUid],
+    types: types as ("documentAdditionOrUpdate" | "documentDeletion")[],
+    limit: 100,
+  });
+  return page.results.filter((task) => task.uid > baselineUid).length;
+}
+
+async function highestTaskUid(): Promise<number> {
+  const indexUid = meilisearchTenantIndex(indexPrefix, TENANT);
+  const page = await client.tasks.getTasks({ indexUids: [indexUid], limit: 1 });
+  return page.results[0]?.uid ?? -1;
+}
+
+describe.skipIf(!MEILI_UP)("search consumer batchHandler (live Meilisearch)", () => {
+  beforeAll(async () => {
+    client = new Meilisearch({ host: MEILI_URL, apiKey: MEILI_KEY });
+    indexPrefix = `test_batch_${uuid()}_`;
+    meiliAdapter = createMeilisearchAdapter({ url: MEILI_URL, apiKey: MEILI_KEY, indexPrefix });
+
+    stack = await setupTestStack({ features: [batchSearchFeature], systemHooks: [] });
+    consumer = createSearchEventConsumer(meiliAdapter, stack.registry);
+
+    await meiliAdapter.configure(TENANT, { searchableFields: ["label"] });
+  });
+
+  afterAll(async () => {
+    await stack.cleanup();
+    const indices = await client.getIndexes();
+    for (const idx of indices.results) {
+      if (idx.uid.startsWith(indexPrefix)) {
+        try {
+          await client.index(idx.uid).delete().waitTask();
+        } catch {
+          /* ok */
+        }
+      }
+    }
+  });
+
+  test("5 created events for one tenant produce exactly one indexing task and 5 searchable docs", async () => {
+    const baseline = await highestTaskUid();
+    const rows = Array.from({ length: 5 }, (_, i) =>
+      eventRow(uuid(), "created", { label: `batchdoc-${i}` }),
+    );
+
+    await deliverBatch(rows);
+
+    const indexTasks = await taskCountSince(baseline, ["documentAdditionOrUpdate"]);
+    expect(indexTasks).toBe(1);
+
+    const hits = await meiliAdapter.search(TENANT, "batchdoc", { limit: 20, filterType: "doc" });
+    expect(hits.length).toBe(5);
+  });
+
+  test("update-then-delete of the same aggregate in one batch leaves the doc absent", async () => {
+    const aggregateId = uuid();
+    await deliverBatch([eventRow(aggregateId, "created", { label: "collapse-update-delete" })]);
+    expect(
+      (await meiliAdapter.search(TENANT, "collapse-update-delete", { filterType: "doc" })).some(
+        (h) => h.entityId === aggregateId,
+      ),
+    ).toBe(true);
+
+    await deliverBatch([
+      eventRow(aggregateId, "updated", {
+        previous: { label: "collapse-update-delete" },
+        changes: { label: "collapse-update-delete-updated" },
+      }),
+      eventRow(aggregateId, "deleted", {}),
+    ]);
+
+    const hits = await meiliAdapter.search(TENANT, "collapse-update-delete", {
+      filterType: "doc",
+    });
+    expect(hits.some((h) => h.entityId === aggregateId)).toBe(false);
+  });
+
+  test("delete-then-restore of the same aggregate in one batch indexes the restored state", async () => {
+    const aggregateId = uuid();
+    await deliverBatch([eventRow(aggregateId, "created", { label: "collapse-delete-restore" })]);
+
+    await deliverBatch([
+      eventRow(aggregateId, "deleted", {}),
+      eventRow(aggregateId, "restored", { previous: { label: "collapse-delete-restore-back" } }),
+    ]);
+
+    const hits = await meiliAdapter.search(TENANT, "collapse-delete-restore-back", {
+      filterType: "doc",
+    });
+    expect(hits.some((h) => h.entityId === aggregateId)).toBe(true);
+  });
+
+  test("a batch mixing creates and deletes produces at most one index task and one deletion task", async () => {
+    const toDelete = uuid();
+    await deliverBatch([eventRow(toDelete, "created", { label: "mixed-batch-to-delete" })]);
+
+    const baseline = await highestTaskUid();
+    await deliverBatch([
+      eventRow(uuid(), "created", { label: "mixed-batch-created-1" }),
+      eventRow(uuid(), "created", { label: "mixed-batch-created-2" }),
+      eventRow(toDelete, "deleted", {}),
+    ]);
+
+    const indexTasks = await taskCountSince(baseline, ["documentAdditionOrUpdate"]);
+    const deletionTasks = await taskCountSince(baseline, ["documentDeletion"]);
+    expect(indexTasks).toBe(1);
+    expect(deletionTasks).toBe(1);
+  });
+
+  test("remove and removeBatch on a tenant whose index was never created resolve", async () => {
+    const tenantWithoutIndex = uuid();
+    await expect(meiliAdapter.remove(tenantWithoutIndex, "doc", uuid())).resolves.toBeUndefined();
+    await expect(
+      meiliAdapter.removeBatch?.(tenantWithoutIndex, [{ entityType: "doc", entityId: uuid() }]),
+    ).resolves.toBeUndefined();
+  });
+
+  test("indexBatch rejects when one doc in the batch fails on the Meilisearch side", async () => {
+    // Meilisearch caps document identifiers at 511 bytes — meilisearchDocId
+    // prefixes with `${entityType}_`, so an id this long overflows that
+    // limit and the server fails the whole task with invalid_document_id.
+    const oversizedId = `${uuid()}${"a".repeat(520)}`;
+    await expect(
+      meiliAdapter.indexBatch?.(TENANT, [
+        { entityType: "doc", entityId: uuid(), weight: 1, fields: { label: "valid-doc" } },
+        { entityType: "doc", entityId: oversizedId, weight: 1, fields: { label: "poison-doc" } },
+      ]),
+    ).rejects.toThrow(/invalid_document_id/);
+  });
+
+  test("a poison doc in the batch halts the turn at the last good event; a second pass dead-letters it", async () => {
+    // Real UUID aggregateIds always produce a valid Meili doc id via
+    // meilisearchDocId (all illegal characters are sanitized away) — the
+    // only realistic way to fail Meili's own id validation is length, so
+    // event #3's aggregateId is an oversized synthetic id, not a shape
+    // production ever emits.
+    const poisonConsumer = {
+      ...createSearchEventConsumer(meiliAdapter, stack.registry),
+      errorPolicy: { maxAttempts: 2 },
+    };
+
+    const good1 = eventRow(uuid(), "created", { label: "poisondoc-1" });
+    const good2 = eventRow(uuid(), "created", { label: "poisondoc-2" });
+    const poisonAggregateId = `${uuid()}${"a".repeat(520)}`;
+    const poison = eventRow(poisonAggregateId, "created", { label: "poisondoc-3" });
+    const good4 = eventRow(uuid(), "created", { label: "poisondoc-4" });
+    const good5 = eventRow(uuid(), "created", { label: "poisondoc-5" });
+    const rows = [good1, good2, poison, good4, good5];
+
+    const first = await deliverEvents(
+      poisonConsumer,
+      rows,
+      { db: stack.db, redis: stack.redis.redis, registry: stack.registry },
+      10,
+      stubState(),
+    );
+
+    expect(first.cursor).toBe(good2.id);
+    expect(first.attempts).toBe(1);
+    expect(first.deadLettered).toBe(false);
+
+    const hitsAfterFirst = await meiliAdapter.search(TENANT, "poisondoc", {
+      limit: 20,
+      filterType: "doc",
+    });
+    const foundLabels = new Set(hitsAfterFirst.map((h) => h.entityId));
+    expect(foundLabels.has(good1.aggregateId)).toBe(true);
+    expect(foundLabels.has(good2.aggregateId)).toBe(true);
+    expect(foundLabels.has(poison.aggregateId)).toBe(false);
+    expect(foundLabels.has(good4.aggregateId)).toBe(false);
+    expect(foundLabels.has(good5.aggregateId)).toBe(false);
+
+    const second = await deliverEvents(
+      poisonConsumer,
+      rows.slice(2),
+      { db: stack.db, redis: stack.redis.redis, registry: stack.registry },
+      10,
+      {
+        ...stubState(),
+        lastProcessedEventId: first.cursor,
+        attempts: first.attempts,
+        lastError: first.lastError,
+      },
+    );
+
+    expect(second.deadLettered).toBe(true);
+    expect(second.attempts).toBe(2);
+    expect(second.cursor).toBe(good2.id);
+  });
+});

package/src/search/derive-search-adapter-config.ts

@@ -0,0 +1,27 @@
+import type { Registry } from "../engine/types";
+import type { SearchAdapterConfig } from "./types";
+
+// Union of getSearchableFields() across every registered entity, in stable
+// first-occurrence order and deduplicated. Feeds SearchAdapter.setDefaultConfig
+// so the framework can hand the adapter a config derived purely from the
+// registry, without any app-side wiring.
+//
+// searchPayloadExtension keys (e.g. customFields) are dynamic per-tenant and
+// therefore not part of any entity's static searchableFields — they stay out
+// of searchableAttributes here too. Not solved by this change, not worsened.
+export function deriveSearchAdapterConfig(registry: Registry): SearchAdapterConfig | undefined {
+  const searchableFields: string[] = [];
+  const seen = new Set<string>();
+
+  for (const entityName of registry.getAllEntities().keys()) {
+    for (const field of registry.getSearchableFields(entityName)) {
+      if (seen.has(field)) continue;
+      seen.add(field);
+      searchableFields.push(field);
+    }
+  }
+
+  if (searchableFields.length === 0) return undefined;
+
+  return { searchableFields, rankingFields: searchableFields };
+}

package/src/search/index.ts

@@ -2,6 +2,7 @@
 // Damit lädt der Main-Barrel keinen Meilisearch-Client beim bloßen Anfassen
 // von SearchAdapter-Types. Apps die Meilisearch nicht nutzen, ziehen den
 // Client-Code nicht mit rein.
+export { deriveSearchAdapterConfig } from "./derive-search-adapter-config";
 export { createInMemorySearchAdapter } from "./in-memory-adapter";
 export { purgeSearchDocumentsForSubject } from "./purge-subject";
 export type {

package/src/search/meilisearch-adapter.ts

@@ -1,6 +1,33 @@
-import { Meilisearch } from "meilisearch";
+import { type EnqueuedTaskPromise, ErrorStatusCode, Meilisearch, type Task } from "meilisearch";
 import type { EntityId, TenantId } from "../engine/types/identifiers";
-import type { SearchAdapter, SearchResult } from "./types";
+import type { SearchAdapter, SearchAdapterConfig, SearchResult } from "./types";
+
+// meilisearch's waitTask() RESOLVES (never rejects) once the server task
+// reaches a terminal state, even "failed" — a rejected doc (e.g. an id over
+// Meili's 511-byte limit) otherwise fails silently and the caller thinks
+// the write succeeded. Route every waitTask() through here so a non-
+// "succeeded" status always throws, except for the error codes a caller
+// declares as an already-reached end state.
+async function awaitSucceededTask(
+  enqueued: EnqueuedTaskPromise,
+  toleratedErrorCodes: readonly string[] = [],
+): Promise<Task> {
+  const task = await enqueued.waitTask();
+  const tolerated = task.error ? toleratedErrorCodes.includes(task.error.code) : false;
+  if (task.status !== "succeeded" && !tolerated) {
+    const detail = task.error ? `${task.error.code}: ${task.error.message}` : "no error detail";
+    throw new Error(
+      `Meilisearch task ${task.uid} (${task.type}) ended with status "${task.status}": ${detail}`,
+    );
+  }
+  return task;
+}
+
+// A tenant whose index was never created (no default config, nothing ever
+// indexed) has nothing to delete: the remove's end state already holds.
+// Without this, a delete event or a DSGVO subject purge for such a tenant
+// would fail forever.
+const REMOVE_TOLERATED_ERROR_CODES: readonly string[] = [ErrorStatusCode.INDEX_NOT_FOUND];
 
 export type MeilisearchAdapterOptions = {
   url: string;
@@ -23,19 +50,67 @@ export function createMeilisearchAdapter(options: MeilisearchAdapterOptions): Se
   const client = new Meilisearch({ host: options.url, apiKey: options.apiKey });
   const prefix = options.indexPrefix ?? "kumiko_";
 
+  // Set via setDefaultConfig once the registry is known (server.ts boot).
+  // undefined until then — tenants stay unconfigured, matching prior behavior.
+  let defaultConfig: SearchAdapterConfig | undefined;
+  // Per-tenant configure state. An explicit `configure(tenantId, ...)` call
+  // always wins and is never overwritten by the lazy default — its promise
+  // sits in the map exactly like a lazily-triggered one.
+  const configuredTenants = new Map<TenantId, Promise<void>>();
+
+  async function applyConfig(tenantId: TenantId, config: SearchAdapterConfig): Promise<void> {
+    const index = client.index(meilisearchTenantIndex(prefix, tenantId));
+    const fields = config.rankingFields ?? config.searchableFields;
+    await awaitSucceededTask(index.updateSearchableAttributes([...fields]));
+    await awaitSucceededTask(index.updateFilterableAttributes(["_type", "_weight"]));
+    await awaitSucceededTask(index.updateSortableAttributes(["_weight"]));
+  }
+
+  // Lazily configures a tenant's index off the default config on its first
+  // access, so callers that never call `configure` still get a filterable/
+  // sortable index instead of failing with "not filterable"/"index not
+  // found". A failed configure is never memoized — remove the map entry (iff
+  // it's still the same in-flight promise) and rethrow, so a transient
+  // Meili outage doesn't permanently poison the tenant for the process.
+  async function ensureConfigured(tenantId: TenantId): Promise<void> {
+    // skip: no registry-derived default — callers configure tenants explicitly.
+    if (!defaultConfig) return;
+    const existing = configuredTenants.get(tenantId);
+    if (existing) {
+      await existing;
+      // skip: configured (or in flight) already for this tenant.
+      return;
+    }
+    await trackConfigure(tenantId, defaultConfig);
+  }
+
+  async function trackConfigure(tenantId: TenantId, config: SearchAdapterConfig): Promise<void> {
+    const pending = applyConfig(tenantId, config);
+    configuredTenants.set(tenantId, pending);
+    try {
+      await pending;
+    } catch (err) {
+      if (configuredTenants.get(tenantId) === pending) {
+        configuredTenants.delete(tenantId);
+      }
+      throw err;
+    }
+  }
+
   return {
+    setDefaultConfig(config) {
+      defaultConfig = config;
+    },
+
     async configure(tenantId, config) {
-      const index = client.index(meilisearchTenantIndex(prefix, tenantId));
-      const fields = config.rankingFields ?? config.searchableFields;
-      await index.updateSearchableAttributes([...fields]).waitTask();
-      await index.updateFilterableAttributes(["_type", "_weight"]).waitTask();
-      await index.updateSortableAttributes(["_weight"]).waitTask();
+      await trackConfigure(tenantId, config);
     },
 
     async index(tenantId, doc) {
+      await ensureConfigured(tenantId);
       const index = client.index(meilisearchTenantIndex(prefix, tenantId));
-      await index
-        .addDocuments(
+      await awaitSucceededTask(
+        index.addDocuments(
           [
             {
               _id: meilisearchDocId(doc.entityType, doc.entityId),
@@ -46,13 +121,14 @@ export function createMeilisearchAdapter(options: MeilisearchAdapterOptions): Se
             },
           ],
           { primaryKey: "_id" },
-        )
-        .waitTask();
+        ),
+      );
     },
 
     async indexBatch(tenantId, docs) {
       // skip: empty batch — avoid an unnecessary Meilisearch round-trip
       if (docs.length === 0) return;
+      await ensureConfigured(tenantId);
       const index = client.index(meilisearchTenantIndex(prefix, tenantId));
       const payload = docs.map((doc) => ({
         _id: meilisearchDocId(doc.entityType, doc.entityId),
@@ -64,18 +140,20 @@ export function createMeilisearchAdapter(options: MeilisearchAdapterOptions): Se
       // Single Meilisearch task covering all N docs. Meilisearch processes
       // the payload server-side as one indexing job — waitTask blocks until
       // that job is done, but it's one round-trip instead of N.
-      await index.addDocuments(payload, { primaryKey: "_id" }).waitTask();
+      await awaitSucceededTask(index.addDocuments(payload, { primaryKey: "_id" }));
     },
 
     async removeBatch(tenantId, items) {
       // skip: empty batch — avoid an unnecessary Meilisearch round-trip
       if (items.length === 0) return;
+      await ensureConfigured(tenantId);
       const index = client.index(meilisearchTenantIndex(prefix, tenantId));
       const ids = items.map((i) => meilisearchDocId(i.entityType, i.entityId));
-      await index.deleteDocuments(ids).waitTask();
+      await awaitSucceededTask(index.deleteDocuments(ids), REMOVE_TOLERATED_ERROR_CODES);
     },
 
     async search(tenantId, query, options) {
+      await ensureConfigured(tenantId);
       const index = client.index(meilisearchTenantIndex(prefix, tenantId));
 
       const filter: string[] = [];
@@ -100,8 +178,12 @@ export function createMeilisearchAdapter(options: MeilisearchAdapterOptions): Se
     },
 
     async remove(tenantId, entityType, entityId) {
+      await ensureConfigured(tenantId);
       const index = client.index(meilisearchTenantIndex(prefix, tenantId));
-      await index.deleteDocument(meilisearchDocId(entityType, entityId)).waitTask();
+      await awaitSucceededTask(
+        index.deleteDocument(meilisearchDocId(entityType, entityId)),
+        REMOVE_TOLERATED_ERROR_CODES,
+      );
     },
   };
 }

package/src/secrets/__tests__/contains-secret.test.ts

@@ -1,5 +1,5 @@
 import { describe, expect, test } from "bun:test";
-import { z } from "zod";
+import * as z from "zod";
 import type { TenantId } from "../../engine";
 import { defineQueryHandler } from "../../engine/define-handler";
 import { type ContainsSecret, createSecret, type Secret } from "../index";

package/src/stack/__tests__/request-helper.integration.test.ts

@@ -4,7 +4,7 @@
 // stack.http.
 
 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
-import { z } from "zod";
+import * as z from "zod";
 import { defineFeature } from "../../engine";
 import { NotFoundError, UnprocessableError, writeFailure } from "../../errors";
 import { setupTestStack, type TestStack } from "../test-stack";

package/src/stack/__tests__/setup-test-stack-jobs.integration.test.ts

@@ -4,7 +4,7 @@
 // context slot and isn't guaranteed by this test passing.
 
 import { afterEach, describe, expect, test } from "bun:test";
-import { z } from "zod";
+import * as z from "zod";
 import { defineFeature } from "../../engine";
 import { InternalError, writeFailure } from "../../errors";
 import { waitFor } from "../../testing";

package/src/stack/__tests__/setup-test-stack-metrics.integration.test.ts

@@ -1,5 +1,5 @@
 import { afterEach, describe, expect, test } from "bun:test";
-import { z } from "zod";
+import * as z from "zod";
 import { defineFeature } from "../../engine";
 import { resolveObservabilityWiring } from "../../observability/metrics-wiring";
 import { setupTestStack, type TestStack } from "../test-stack";

package/src/stack/test-stack.ts

@@ -470,7 +470,7 @@ export async function setupTestStack(options: TestStackOptions): Promise<TestSta
 
     // Pre-register consumer state rows so tests can call runOnce() directly
     // without a preceding explicit start(). Timer fires at pollIntervalMs=50
-    // but passInFlight serialises concurrent passes — tests that drain via
+    // but each consumer runs at most one turn at a time — tests that drain via
     // runOnce() remain deterministic. Tests that specifically exercise the
     // timer loop call start() again (idempotent) after setup.
     if (eventDispatcher) await eventDispatcher.ensureRegistered();

package/src/testing/__tests__/e2e-generator.test.ts

@@ -1,5 +1,5 @@
 import { describe, expect, test } from "bun:test";
-import { z } from "zod";
+import * as z from "zod";
 import {
   createBooleanField,
   createDateField,

package/src/testing/e2e-generator.ts

@@ -24,7 +24,7 @@
 // "not supported yet" — wir füllen nach, wenn ein echter Caller es braucht.
 // Fake-komplexe-Werte würden nur False-Green-Tests produzieren.
 
-import type { z } from "zod";
+import type * as z from "zod";
 import {
   type EntityDefinition,
   type EntityEditScreenDefinition,