@cosmicdrift/kumiko-framework 0.326.1 → 0.328.0

package/src/migrations/pii-event-backfill.ts

@@ -0,0 +1,422 @@
+// Boot-time PII backfill for kumiko_events: when a field gains a PII annotation
+// after events were already stored, the rebuild copies payloads unchanged, so
+// old events stay plaintext until they are re-encrypted in place. This runs the
+// batch primitive from backfill-pii.ts once per annotation state (fingerprint),
+// then cheap catch-up passes, and rebuilds the projections whose source events
+// changed (including multi-stream projections with their own table) so
+// ciphertext and blind-index columns get materialized.
+
+import { createHash } from "node:crypto";
+import { configuredEventPiiCatalog } from "../crypto/event-pii";
+import { isLocalKeyKmsAdapter } from "../crypto/kms-adapter";
+import { configuredPiiSubjectKms } from "../crypto/pii-field-encryption";
+import { collectPiiSubjectFields } from "../crypto/subject-resolver";
+import { extractTableName } from "../db";
+import type { DbConnection, DbRunner } from "../db/connection";
+import { bigint, instant, table as pgTable, sql, text } from "../db/dialect";
+import { acquireNamespacedAdvisoryLock } from "../db/queries/advisory-lock";
+import {
+  backfillEventPiiEncryptionBatch,
+  type PiiBackfillBatchResult,
+  type PiiBackfillFailure,
+  type PiiBackfillScanCache,
+} from "../db/queries/backfill-pii";
+import {
+  type PiiBackfillStateRow,
+  persistPiiBackfillState,
+  readPiiBackfillState,
+} from "../db/queries/pii-backfill-state";
+import { upsertOnConflict } from "../db/query";
+import { tableExists } from "../db/schema-inspection";
+import type { Registry } from "../engine/types";
+import type { Logger } from "../logging/types";
+import { createFallbackLogger } from "../logging/utils";
+import { unsafePushTables } from "../stack";
+import {
+  clearPendingRebuilds,
+  createPendingRebuildsTable,
+  listPendingRebuildRows,
+  pendingRebuildsTable,
+  rebuildProjectionOrMultiStream,
+} from "./pending-rebuilds";
+import { buildProjectionTableIndex } from "./projection-table-index";
+
+const PII_EVENT_BACKFILL_LOCK_NAMESPACE = 0x70_69_69_62; // 'piib'
+const PII_EVENT_BACKFILL_LOCK_KEY = "pii-event-backfill";
+const PII_BACKFILL_MIGRATION_PREFIX = "pii-backfill:";
+const DEFAULT_BATCH_SIZE = 500;
+const MAX_LOGGED_FAILURE_IDS = 50;
+
+// Events can commit with an id lower than an already-scanned one (late commits),
+// and old pods in a rolling deploy keep writing the newly annotated field as
+// plaintext. The transformation is idempotent, so overlap only costs reads.
+const CATCH_UP_OVERLAP_EVENT_IDS = 1000n;
+
+export const piiBackfillStateTable = pgTable("kumiko_pii_backfill_state", {
+  fingerprint: text("fingerprint").primaryKey(),
+  cursorEventId: bigint("cursor_event_id", { mode: "bigint" }).notNull().default(0),
+  completedAt: instant("completed_at", { precision: 3 }),
+  updatedAt: instant("updated_at", { precision: 3 }).notNull().default(sql`now()`),
+});
+
+export async function createPiiBackfillStateTable(db: DbConnection): Promise<void> {
+  // skip: table already exists — boot runs on every replica
+  if (await tableExists(db, "public.kumiko_pii_backfill_state")) return;
+  await unsafePushTables(db, { kumikoPiiBackfillState: piiBackfillStateTable });
+}
+
+// Replicas boot concurrently: two of them can both pass the tableExists gate and
+// one CREATE then fails. The table existing afterwards means the other won.
+async function ensureInfraTable(
+  db: DbConnection,
+  tableName: string,
+  create: (db: DbConnection) => Promise<void>,
+): Promise<void> {
+  try {
+    await create(db);
+  } catch (e) {
+    if (!(await tableExists(db, `public.${tableName}`))) throw e;
+  }
+}
+
+export function piiAnnotationEntries(registry: Registry): readonly string[] {
+  const entries: string[] = [];
+  for (const [name, entity] of registry.getAllEntities()) {
+    for (const field of collectPiiSubjectFields(entity)) entries.push(`entity:${name}.${field}`);
+  }
+  for (const [eventType, fields] of configuredEventPiiCatalog()) {
+    for (const field of Object.keys(fields)) entries.push(`event:${eventType}.${field}`);
+  }
+  return entries.sort();
+}
+
+export function piiAnnotationFingerprint(registry: Registry): string {
+  return createHash("sha256").update(piiAnnotationEntries(registry).join("\n")).digest("hex");
+}
+
+export type PiiEventBackfillOptions = {
+  readonly signal?: AbortSignal;
+  readonly batchSize?: number;
+  readonly logger?: Pick<Logger, "error"> & Partial<Pick<Logger, "warn" | "debug">>;
+};
+
+export type PiiRebuildRun = {
+  readonly rebuilt: readonly { readonly projection: string; readonly eventsProcessed: number }[];
+  // Stay queued and are retried on the next boot.
+  readonly failed: readonly { readonly projection: string; readonly error: string }[];
+  // Queued by this backfill but without a projection in this process; left queued.
+  readonly skippedTables: readonly string[];
+};
+
+export type PiiEventBackfillResult =
+  | { readonly status: "skipped"; readonly reason: "no_kms" | "no_pii" }
+  | {
+      readonly status: "ran";
+      readonly mode: "full" | "catch-up";
+      readonly aborted: boolean;
+      readonly scannedEvents: number;
+      readonly updatedEvents: number;
+      readonly encryptedFields: number;
+      readonly erasedFields: number;
+      readonly failures: readonly PiiBackfillFailure[];
+      readonly queuedTables: readonly string[];
+      readonly rebuild: PiiRebuildRun | null;
+    };
+
+async function queueRebuildTables(
+  db: DbRunner,
+  tableNames: readonly string[],
+  migrationId: string,
+): Promise<void> {
+  for (const tableName of tableNames) {
+    await upsertOnConflict(
+      db,
+      pendingRebuildsTable,
+      { tableName, migrationId },
+      { conflictKeys: ["tableName"], update: { migrationId } },
+    );
+  }
+}
+
+function affectedProjectionTables(
+  registry: Registry,
+  touchedAggregateTypes: ReadonlySet<string>,
+  touchedEventTypes: ReadonlySet<string>,
+): readonly string[] {
+  const tables = new Set<string>();
+  for (const [name, def] of registry.getAllProjections()) {
+    const sources = [
+      ...(Array.isArray(def.source) ? def.source : [def.source]),
+      ...(def.extraSources ?? []),
+    ];
+    const isAffected =
+      sources.some((source) => touchedAggregateTypes.has(source)) ||
+      Object.keys(def.apply).some((eventType) => touchedEventTypes.has(eventType));
+    if (isAffected) tables.add(extractTableName(def.table, `pii-event-backfill(${name})`));
+  }
+  // MSP apply gets the raw event payloads, so their tables hold plaintext until rebuilt.
+  for (const [name, def] of registry.getAllMultiStreamProjections()) {
+    if (def.table === undefined) continue;
+    if (Object.keys(def.apply).some((eventType) => touchedEventTypes.has(eventType))) {
+      tables.add(extractTableName(def.table, `pii-event-backfill(${name})`));
+    }
+  }
+  return [...tables];
+}
+
+type BatchStep = {
+  readonly batch: PiiBackfillBatchResult;
+  readonly tables: readonly string[];
+};
+
+type RunProgress = {
+  scanCursor: bigint;
+  failureSeen: boolean;
+  readonly scanCache: PiiBackfillScanCache;
+  readonly totals: {
+    scannedEvents: number;
+    updatedEvents: number;
+    encryptedFields: number;
+    erasedFields: number;
+  };
+  readonly failures: PiiBackfillFailure[];
+  readonly queuedTables: Set<string>;
+};
+
+type RunContext = {
+  readonly db: DbConnection;
+  readonly registry: Registry;
+  readonly fingerprint: string;
+  readonly migrationId: string;
+  readonly mode: "full" | "catch-up";
+  readonly batchSize: number;
+};
+
+function initialScanCursor(state: PiiBackfillStateRow | null): {
+  mode: "full" | "catch-up";
+  scanCursor: bigint;
+} {
+  const cursor = BigInt(state?.cursor ?? "0");
+  if (state?.completed !== true) return { mode: "full", scanCursor: cursor };
+  return {
+    mode: "catch-up",
+    scanCursor: cursor > CATCH_UP_OVERLAP_EVENT_IDS ? cursor - CATCH_UP_OVERLAP_EVENT_IDS : 0n,
+  };
+}
+
+// One transaction per batch: the advisory lock serializes pods, and the payload
+// is re-read inside it so a value is never encrypted twice and two backfills
+// never mint the same DEK. Returns null when another pod already finished a full run.
+async function runOneBatch(ctx: RunContext, progress: RunProgress): Promise<BatchStep | null> {
+  return ctx.db.begin(async (tx: DbRunner) => {
+    await acquireNamespacedAdvisoryLock(
+      tx,
+      PII_EVENT_BACKFILL_LOCK_NAMESPACE,
+      PII_EVENT_BACKFILL_LOCK_KEY,
+    );
+    const state = await readPiiBackfillState(tx, ctx.fingerprint);
+    if (ctx.mode === "full" && state?.completed === true) return null;
+    const startCursor =
+      ctx.mode === "full" && state
+        ? maxBigInt(progress.scanCursor, BigInt(state.cursor))
+        : progress.scanCursor;
+
+    const batch = await backfillEventPiiEncryptionBatch(tx, ctx.registry, {
+      afterEventId: startCursor,
+      batchSize: ctx.batchSize,
+      resolveOwnerFromProjection: true,
+      scanCache: progress.scanCache,
+    });
+
+    const batchTouchedEventTypes = new Set(batch.touchedEventTypes);
+    const tables =
+      batch.updatedEvents > 0
+        ? affectedProjectionTables(
+            ctx.registry,
+            new Set(batch.touchedAggregateTypes),
+            batchTouchedEventTypes,
+          )
+        : [];
+    if (tables.length > 0) await queueRebuildTables(tx, tables, ctx.migrationId);
+
+    if (!progress.failureSeen) {
+      const completesRun = batch.scannedAll && batch.failures.length === 0;
+      await persistPiiBackfillState(
+        tx,
+        ctx.fingerprint,
+        persistableCursor(batch, startCursor),
+        completesRun,
+      );
+    }
+    return { batch, tables };
+  });
+}
+
+function recordBatch(progress: RunProgress, step: BatchStep): void {
+  const { batch } = step;
+  progress.totals.scannedEvents += batch.scannedEvents;
+  progress.totals.updatedEvents += batch.updatedEvents;
+  progress.totals.encryptedFields += batch.encryptedFields;
+  progress.totals.erasedFields += batch.erasedFields;
+  progress.failures.push(...batch.failures);
+  for (const table of step.tables) progress.queuedTables.add(table);
+  if (batch.failures.length > 0) progress.failureSeen = true;
+}
+
+// Returns true when the run was aborted between batches.
+async function scanAllBatches(
+  ctx: RunContext,
+  progress: RunProgress,
+  signal: AbortSignal | undefined,
+): Promise<boolean> {
+  for (;;) {
+    if (signal?.aborted) return true;
+    const step = await runOneBatch(ctx, progress);
+    if (step === null) return false;
+    recordBatch(progress, step);
+    if (step.batch.lastEventId === null || step.batch.scannedAll) return false;
+    progress.scanCursor = BigInt(step.batch.lastEventId);
+  }
+}
+
+function reportRunOutcome(
+  log: ReturnType<typeof createFallbackLogger>,
+  progress: RunProgress,
+): void {
+  if (progress.failures.length > 0) {
+    log.error(
+      `${progress.failures.length} event(s) could not be PII-encrypted and will be retried on the next boot`,
+      {
+        failureCount: progress.failures.length,
+        eventIds: progress.failures.slice(0, MAX_LOGGED_FAILURE_IDS).map((f) => f.eventId),
+      },
+    );
+  }
+}
+
+export async function runPiiEventBackfill(
+  db: DbConnection,
+  registry: Registry,
+  options: PiiEventBackfillOptions = {},
+): Promise<PiiEventBackfillResult> {
+  const log = createFallbackLogger("pii-event-backfill", options.logger);
+  const kms = configuredPiiSubjectKms();
+  if (!kms || !isLocalKeyKmsAdapter(kms)) return { status: "skipped", reason: "no_kms" };
+  if (piiAnnotationEntries(registry).length === 0) {
+    return { status: "skipped", reason: "no_pii" };
+  }
+
+  const fingerprint = piiAnnotationFingerprint(registry);
+  await ensureInfraTable(db, "kumiko_pii_backfill_state", createPiiBackfillStateTable);
+  await ensureInfraTable(db, "kumiko_pending_rebuilds", createPendingRebuildsTable);
+
+  const { mode, scanCursor } = initialScanCursor(await readPiiBackfillState(db, fingerprint));
+  const ctx: RunContext = {
+    db,
+    registry,
+    fingerprint,
+    migrationId: `${PII_BACKFILL_MIGRATION_PREFIX}${fingerprint.slice(0, 12)}`,
+    mode,
+    batchSize: options.batchSize ?? DEFAULT_BATCH_SIZE,
+  };
+  const progress: RunProgress = {
+    scanCursor,
+    failureSeen: false,
+    scanCache: {},
+    totals: { scannedEvents: 0, updatedEvents: 0, encryptedFields: 0, erasedFields: 0 },
+    failures: [],
+    queuedTables: new Set(),
+  };
+
+  const aborted = await scanAllBatches(ctx, progress, options.signal);
+  reportRunOutcome(log, progress);
+
+  const rebuild = aborted ? null : await rebuildQueuedTables(db, registry, options.signal, log);
+  const queuedTables = [...progress.queuedTables];
+  log.debug("PII event backfill finished", {
+    mode,
+    aborted,
+    ...progress.totals,
+    failureCount: progress.failures.length,
+    queuedTables,
+  });
+  return {
+    status: "ran",
+    mode,
+    aborted,
+    ...progress.totals,
+    failures: progress.failures,
+    queuedTables,
+    rebuild,
+  };
+}
+
+// Never advance past a failed event so the next boot retries it.
+function persistableCursor(batch: PiiBackfillBatchResult, startCursor: bigint): bigint {
+  if (batch.firstFailedEventId !== null) {
+    const beforeFailure = BigInt(batch.firstFailedEventId) - 1n;
+    return beforeFailure > startCursor ? beforeFailure : startCursor;
+  }
+  return batch.lastEventId === null ? startCursor : BigInt(batch.lastEventId);
+}
+
+function maxBigInt(a: bigint, b: bigint): bigint {
+  return a > b ? a : b;
+}
+
+// Only our own queue rows: runPendingRebuilds would drain the whole queue and
+// silently clear entries this process cannot map (a worker with a narrower
+// feature set would swallow rebuilds queued by the API pod or by schema apply).
+async function rebuildQueuedTables(
+  db: DbConnection,
+  registry: Registry,
+  signal: AbortSignal | undefined,
+  log: ReturnType<typeof createFallbackLogger>,
+): Promise<PiiRebuildRun | null> {
+  const ownRows = (await listPendingRebuildRows(db)).filter((row) =>
+    row.migrationId.startsWith(PII_BACKFILL_MIGRATION_PREFIX),
+  );
+  if (ownRows.length === 0) return null;
+
+  const tableToProjection = buildProjectionTableIndex(registry);
+  const rowsByProjection = new Map<string, (typeof ownRows)[number][]>();
+  const skippedTables: string[] = [];
+  for (const row of ownRows) {
+    const projection = tableToProjection.get(row.tableName);
+    if (projection === undefined) {
+      skippedTables.push(row.tableName);
+      continue;
+    }
+    rowsByProjection.set(projection, [...(rowsByProjection.get(projection) ?? []), row]);
+  }
+  if (skippedTables.length > 0) {
+    log.warn(
+      "Queued tables have no projection in this process and stay queued for a process that mounts them",
+      { tables: skippedTables },
+    );
+  }
+
+  const rebuilt: { projection: string; eventsProcessed: number }[] = [];
+  const failed: { projection: string; error: string }[] = [];
+  for (const [projection, rows] of rowsByProjection) {
+    if (signal?.aborted) break;
+    try {
+      const result = await rebuildProjectionOrMultiStream(projection, {
+        db,
+        registry,
+        ...(signal && { signal }),
+      });
+      await clearPendingRebuilds(db, rows);
+      rebuilt.push({ projection, eventsProcessed: result.eventsProcessed });
+    } catch (e) {
+      failed.push({ projection, error: e instanceof Error ? e.message : String(e) });
+    }
+  }
+  if (failed.length > 0) {
+    log.error(
+      "Projection rebuild after PII backfill failed; tables stay queued and are retried on the next boot",
+      { failed },
+    );
+  }
+  return { rebuilt, failed, skippedTables };
+}

package/src/pipeline/append-event-core.ts

@@ -1,4 +1,5 @@
 import { requestContext } from "../api/request-context";
+import { SYSTEM_EVENT_PREFIX } from "../crypto/system-event-pii";
 import type { DbRunner } from "../db/connection";
 import { qnScope, toKebab } from "../engine/qualified-name";
 import type { AppendEventArgs, Registry, TenantId } from "../engine/types";
@@ -32,14 +33,6 @@ export type AppendDomainEventCoreDeps = {
   readonly callerFeature?: string;
 };
 
-// System-event prefix: events under this namespace bypass the registry +
-// ownership checks. Reserved for framework-internal coordination (step-
-// engine deferred dispatch, lifecycle signals). The matching MSP filters
-// on the literal type-string. Apps cannot write into "kumiko:system:*"
-// directly — only framework step implementations call unsafeAppendEvent
-// with these types.
-export const SYSTEM_EVENT_PREFIX = "kumiko:system:";
-
 export async function appendDomainEventCore(
   deps: AppendDomainEventCoreDeps,
   args: AppendEventArgs,

package/src/pipeline/irreversible-operation-gate.ts

@@ -15,10 +15,11 @@ export function isIrreversibleEntityVerb(
 
 // The gate reads the directly dispatched entry handler; a write dispatched inside a deferred
 // context (afterCommit hook, job, consumer) gets its own entryHandler and is gated too. Direct
-// executor/KMS calls without one stay ungated in exactly two cases: deleting derived data
+// executor/KMS calls without one stay ungated in exactly three cases: deleting derived data
 // rebuildable from a reversible source (document-ingest-foundation's forget-extract-with-file-ref
-// consumer, re-created on fileRef.restored), and pipelines started by a risk "high" handler or a
-// cron scan (forget cleanup, retention, tenant destroy). A deferred context can't return the
+// consumer, re-created on fileRef.restored), pipelines started by a risk "high" handler or a
+// cron scan (forget cleanup, retention, tenant destroy), and the step-dispatcher erasing the
+// per-dispatch record key once the outcome is recorded. A deferred context can't return the
 // reason to the agent synchronously, so the gate sits at the primitives.
 function assertEntryHandlerHighRisk(args: {
   reason: FrameworkReason;

package/src/pipeline/msp-rebuild.ts

@@ -101,6 +101,10 @@ export type MspRebuildDeps = {
   readonly meter?: Meter;
   // Test-hook — independent of `meter`, fires on success only.
   readonly onMetrics?: (result: RebuildResult) => void;
+  // Default true. The rebuild TX rolls back completely (cursor reset included),
+  // so the live consumer is intact; an automatic background rebuild passes false
+  // to not park it as dead and logs the failure itself.
+  readonly markDeadOnFailure?: boolean;
 };
 
 function createRebuildCtx(
@@ -270,7 +274,9 @@ export async function rebuildMultiStreamProjection(
     });
   } catch (e) {
     const message = e instanceof Error ? e.message : String(e);
-    await markConsumerRebuildFailed(db, mspName, SHARED_INSTANCE_SENTINEL, message);
+    if (deps.markDeadOnFailure !== false) {
+      await markConsumerRebuildFailed(db, mspName, SHARED_INSTANCE_SENTINEL, message);
+    }
     if (deps.meter) {
       emitProjectionRebuild(
         deps.meter,

package/src/search/__tests__/meilisearch-adapter.integration.test.ts

@@ -346,3 +346,68 @@ describe.skipIf(!MEILI_UP)("meilisearch adapter — lazy default config", () =>
     expect(await index.getSearchableAttributes()).toEqual(["a"]);
   });
 });
+
+describe.skipIf(!MEILI_UP)("meilisearch adapter — dropAllIndexes", () => {
+  const dropClient = new Meilisearch({ host: MEILI_URL, apiKey: MEILI_KEY });
+  const createdPrefixes: string[] = [];
+
+  function adapterWithFreshPrefix(): { adapter: SearchAdapter; prefix: string } {
+    const prefix = `test_drop_${uuid()}_`;
+    createdPrefixes.push(prefix);
+    const created = createMeilisearchAdapter({
+      url: MEILI_URL,
+      apiKey: MEILI_KEY,
+      indexPrefix: prefix,
+    });
+    created.setDefaultConfig?.({ searchableFields: ["firstName"] });
+    return { adapter: created, prefix };
+  }
+
+  async function indexUids(prefix: string): Promise<string[]> {
+    const all = await dropClient.getIndexes({ limit: 1000 });
+    return all.results.map((i) => i.uid).filter((uid) => uid.startsWith(prefix));
+  }
+
+  afterAll(async () => {
+    for (const prefix of createdPrefixes) {
+      for (const uid of await indexUids(prefix)) {
+        await dropClient.deleteIndex(uid).waitTask();
+      }
+    }
+  });
+
+  const doc = { entityType: "user", entityId: 1, weight: 1, fields: { firstName: "Dropme" } };
+
+  test("drops only this adapter's prefix and returns the count", async () => {
+    const { adapter: dropAdapter, prefix } = adapterWithFreshPrefix();
+    const { adapter: foreignAdapter, prefix: foreignPrefix } = adapterWithFreshPrefix();
+    await dropAdapter.index(uuid(), doc);
+    await dropAdapter.index(uuid(), doc);
+    await foreignAdapter.index(uuid(), doc);
+
+    expect(await dropAdapter.dropAllIndexes?.()).toBe(2);
+
+    expect(await indexUids(prefix)).toEqual([]);
+    expect(await indexUids(foreignPrefix)).toHaveLength(1);
+  });
+
+  test("index and filtered search work again after a drop (configured state is reset)", async () => {
+    const { adapter: dropAdapter } = adapterWithFreshPrefix();
+    const tenant = uuid();
+    await dropAdapter.index(tenant, doc);
+    await dropAdapter.dropAllIndexes?.();
+
+    await dropAdapter.index(tenant, doc);
+    const results = await dropAdapter.search(tenant, "dropme", { filterType: "user" });
+    expect(results.map((r) => r.entityId)).toEqual([1]);
+  });
+
+  test("refuses an empty prefix", async () => {
+    const noPrefix = createMeilisearchAdapter({
+      url: MEILI_URL,
+      apiKey: MEILI_KEY,
+      indexPrefix: "",
+    });
+    await expect(noPrefix.dropAllIndexes?.()).rejects.toThrow("empty prefix");
+  });
+});

package/src/search/__tests__/search-adapter.test.ts

@@ -254,3 +254,18 @@ describe("limit", () => {
     expect(results).toHaveLength(3);
   });
 });
+
+describe("dropAllIndexes", () => {
+  test("empties every tenant and returns how many were dropped", async () => {
+    const otherTenant = "00000000-0000-4000-8000-000000000002";
+    const doc = { entityType: "user", entityId: 1, weight: 1, fields: { firstName: "Marc" } };
+    await adapter.index(TENANT, doc);
+    await adapter.index(otherTenant, doc);
+    expect(await adapter.search(TENANT, "marc")).toHaveLength(1);
+
+    expect(await adapter.dropAllIndexes?.()).toBe(2);
+
+    expect(await adapter.search(TENANT, "marc")).toEqual([]);
+    expect(await adapter.search(otherTenant, "marc")).toEqual([]);
+  });
+});

package/src/search/in-memory-adapter.ts

@@ -111,6 +111,12 @@ export function createInMemorySearchAdapter(): SearchAdapter {
       }
     },
 
+    async dropAllIndexes() {
+      const dropped = tenants.size;
+      tenants.clear();
+      return dropped;
+    },
+
     async removeBatch(tenantId, items) {
       const tenant = tenants.get(tenantId);
       // skip: tenant has no in-memory index (never configured) — nothing to remove

package/src/search/meilisearch-adapter.ts

@@ -29,6 +29,8 @@ async function awaitSucceededTask(
 // would fail forever.
 const REMOVE_TOLERATED_ERROR_CODES: readonly string[] = [ErrorStatusCode.INDEX_NOT_FOUND];
 
+const INDEX_LIST_PAGE_SIZE = 100;
+
 export type MeilisearchAdapterOptions = {
   url: string;
   apiKey: string;
@@ -143,6 +145,28 @@ export function createMeilisearchAdapter(options: MeilisearchAdapterOptions): Se
       await awaitSucceededTask(index.addDocuments(payload, { primaryKey: "_id" }));
     },
 
+    async dropAllIndexes() {
+      // An empty prefix matches every index, which would wipe the whole Meili instance.
+      if (prefix === "") {
+        throw new Error("refusing to drop Meilisearch indexes with an empty prefix");
+      }
+      const uids: string[] = [];
+      for (let offset = 0; ; offset += INDEX_LIST_PAGE_SIZE) {
+        const page = await client.getIndexes({ limit: INDEX_LIST_PAGE_SIZE, offset });
+        for (const index of page.results) {
+          if (index.uid.startsWith(prefix)) uids.push(index.uid);
+        }
+        if (page.results.length === 0 || offset + page.results.length >= page.total) break;
+      }
+      for (const uid of uids) {
+        await awaitSucceededTask(client.deleteIndex(uid), REMOVE_TOLERATED_ERROR_CODES);
+      }
+      // Otherwise ensureConfigured treats the dropped tenants as configured and the
+      // next index() recreates an unconfigured index ("not filterable" on filtered search).
+      configuredTenants.clear();
+      return uids.length;
+    },
+
     async removeBatch(tenantId, items) {
       // skip: empty batch — avoid an unnecessary Meilisearch round-trip
       if (items.length === 0) return;

package/src/stack/test-stack.ts

@@ -189,6 +189,14 @@ export type TestStackOptions = {
      *  query, same as prod. Only needed for suites that fan a job over
      *  tenants without mounting the tenant feature. */
     getActiveTenantIds?: JobRunnerOptions["getActiveTenantIds"];
+    /** Run-log callbacks (job_runs / tenant job failures), built once registry
+     *  and db exist. The framework can't import the bundled jobs feature, so the
+     *  dev-server wrapper supplies prod's `jobRunLoggerCallbacks` here. They are
+     *  awaited before drainJobs() observes the job's outcome. */
+    runLogger?: (deps: {
+      registry: Registry;
+      db: import("../db").DbConnection;
+    }) => Pick<JobRunnerOptions, "onJobStart" | "onJobComplete" | "onJobFailed"> | undefined;
   };
   /** Override the event dispatcher's polling-timer interval. Default 50ms.
    *  Tests that assert LISTEN/NOTIFY wake-up latency need this pushed far
@@ -378,6 +386,7 @@ export async function setupTestStack(options: TestStackOptions): Promise<TestSta
   // existing, so the returned drainJobs() closure always has one to read.
   const jobFailureTracker = createJobFailureTracker();
   if (options.jobs && registry.getAllJobs().size > 0) {
+    const runLogger = options.jobs.runLogger?.({ registry, db: testDb.db });
     jobRunner = createJobRunner({
       registry,
       context: { ...appContext, tracer: observability.tracer, meter: observability.meter },
@@ -390,8 +399,23 @@ export async function setupTestStack(options: TestStackOptions): Promise<TestSta
       ...(options.jobs.getActiveTenantIds !== undefined && {
         getActiveTenantIds: options.jobs.getActiveTenantIds,
       }),
-      onJobComplete: (jobName, jobId) => jobFailureTracker.onJobComplete(jobName, jobId),
-      onJobFailed: (jobName, jobId, error) => jobFailureTracker.onJobFailed(jobName, jobId, error),
+      ...(runLogger?.onJobStart !== undefined && { onJobStart: runLogger.onJobStart }),
+      // Tracker fires in finally: drainJobs wakes on it, so the log row must
+      // be written first, and a throwing logger must not swallow the signal.
+      onJobComplete: async (jobName, jobId, ...rest) => {
+        try {
+          await runLogger?.onJobComplete?.(jobName, jobId, ...rest);
+        } finally {
+          jobFailureTracker.onJobComplete(jobName, jobId);
+        }
+      },
+      onJobFailed: async (jobName, jobId, error, ...rest) => {
+        try {
+          await runLogger?.onJobFailed?.(jobName, jobId, error, ...rest);
+        } finally {
+          jobFailureTracker.onJobFailed(jobName, jobId, error);
+        }
+      },
     });
   }
 

package/src/ui-types/index.ts

@@ -54,6 +54,7 @@ export type {
   ImagesFieldDef,
   NumberFieldDef,
   SelectFieldDef,
+  SelectOptionTone,
   TextFieldDef,
 } from "../engine/types/fields";
 export type {