@cosmicdrift/kumiko-framework 0.235.1 → 0.235.3

package/src/engine/pattern-library/mixed-schemas.ts

@@ -75,6 +75,20 @@ export const writeHandlerSchema: PatternFormSchema = {
       readOnly: true,
     },
     accessRuleField,
+    {
+      path: "description",
+      label: { en: "Description", de: "Beschreibung" },
+      hint: {
+        en: "Prose the AI agent sees as the tool description. Without it the handler stays invisible to the agent.",
+      },
+      input: "textarea",
+    },
+    {
+      path: "agent",
+      label: { en: "Agent hints", de: "Agent-Hinweise" },
+      hint: { en: "expose override + risk level for the AI-agent manifest." },
+      input: "json-readonly",
+    },
     {
       path: "rateLimit",
       label: { en: "Rate limit", de: "Rate-Limit" },
@@ -117,6 +131,20 @@ export const queryHandlerSchema: PatternFormSchema = {
       readOnly: true,
     },
     accessRuleField,
+    {
+      path: "description",
+      label: { en: "Description", de: "Beschreibung" },
+      hint: {
+        en: "Prose the AI agent sees as the tool description. Without it the handler stays invisible to the agent.",
+      },
+      input: "textarea",
+    },
+    {
+      path: "agent",
+      label: { en: "Agent hints", de: "Agent-Hinweise" },
+      hint: { en: "expose override + risk level for the AI-agent manifest." },
+      input: "json-readonly",
+    },
     {
       path: "rateLimit",
       label: { en: "Rate limit", de: "Rate-Limit" },

package/src/engine/registry-facade.ts

@@ -73,6 +73,10 @@ export function buildRegistryFacade(state: RegistryState): Registry {
       return state.queryHandlerMap;
     },
 
+    getAllWriteHandlers(): ReadonlyMap<string, WriteHandlerDef> {
+      return state.writeHandlerMap;
+    },
+
     getStreamHandler(name: string): StreamHandlerDef | undefined {
       return state.streamHandlerMap.get(name);
     },

package/src/engine/types/index.ts

@@ -1,6 +1,7 @@
 // Barrel: re-exports all types from @cosmicdrift/kumiko-types, plus the
 // runtime helpers below that stay framework-side.
 
+export { resolveAgentExposure } from "@cosmicdrift/kumiko-types/agent-exposure";
 export type {
   ConfigAccessor,
   ConfigAccessorFactory,
@@ -133,6 +134,9 @@ export type {
 } from "@cosmicdrift/kumiko-types/fields";
 export type {
   AccessRule,
+  AgentExposure,
+  AgentHandlerHints,
+  AgentRisk,
   AggregateStreamHandle,
   AppContext,
   AppendEventArgs,

package/src/errors/__tests__/error-helpers.test.ts

@@ -1,6 +1,7 @@
 import { describe, expect, test } from "bun:test";
 import { NotFoundError } from "../classes";
-import { FrameworkReasons } from "../reasons";
+import { AgentReasons as BarrelAgentReasons } from "../index";
+import { AgentReasons, FrameworkReasons } from "../reasons";
 import { buildInvalidTransitionDetails } from "../transition-details";
 import { reraiseAsKumikoError, toWriteErrorInfo, writeFailure } from "../write-error-info";
 
@@ -42,3 +43,16 @@ describe("FrameworkReasons", () => {
     expect(FrameworkReasons.staleState).toBe("stale_state");
   });
 });
+
+describe("AgentReasons", () => {
+  test("exposes stable agent-prefixed reason codes", () => {
+    expect(AgentReasons.toolNotAllowed).toBe("agent.tool_not_allowed");
+    expect(AgentReasons.iterationLimit).toBe("agent.iteration_limit");
+    expect(AgentReasons.permissionDenied).toBe("agent.permission_denied");
+    expect(AgentReasons.highRiskNoAlways).toBe("agent.high_risk_no_always");
+  });
+
+  test("is importable from the errors barrel", () => {
+    expect(BarrelAgentReasons).toBe(AgentReasons);
+  });
+});

package/src/errors/index.ts

@@ -25,8 +25,8 @@ export {
 } from "./classes";
 export type { ErrorCtorInput, ErrorOpts } from "./kumiko-error";
 export { isKumikoError, KumikoError } from "./kumiko-error";
-export type { FrameworkReason } from "./reasons";
-export { FrameworkReasons } from "./reasons";
+export type { AgentReason, FrameworkReason } from "./reasons";
+export { AgentReasons, FrameworkReasons } from "./reasons";
 export type { ErrorLogEntry, ErrorResponseBody } from "./serialize";
 export { buildErrorLog, serializeError } from "./serialize";
 export { toKumikoError } from "./to-kumiko-error";

package/src/errors/reasons.ts

@@ -8,6 +8,13 @@
 // framework deliberately does NOT enforce uniqueness across features — two
 // features may use the same reason string if the semantics match. The
 // convention: snake_case, no spaces, no feature prefix for framework reasons.
+//
+// AgentReasons below is a deliberate exception to "no feature prefix": those
+// reasons surface from the AI-agent runtime (tool dispatch, permission and
+// risk gates), not from a handler's own domain logic, and the `agent.`
+// prefix keeps them visibly distinct from FrameworkReasons in error payloads.
+// Both the docs site (`errors/` pages) and the enterprise repo read this same
+// source, so the prefix stays a naming convention here, not a new registry.
 
 export const FrameworkReasons = {
   // ConflictError: atomic UPDATE lost the race (another writer moved the row
@@ -34,3 +41,26 @@ export const FrameworkReasons = {
 } as const;
 
 export type FrameworkReason = (typeof FrameworkReasons)[keyof typeof FrameworkReasons];
+
+// Reason codes the AI-agent runtime surfaces via `details.reason`. Kept as a
+// sister catalog to FrameworkReasons instead of extending it — see the
+// file-header note above for why these carry the `agent.` prefix.
+export const AgentReasons = {
+  // AccessDeniedError: the agent called a tool name outside its allowed
+  // catalog for this run.
+  toolNotAllowed: "agent.tool_not_allowed",
+
+  // UnprocessableError: the agent loop hit its maximum round count before
+  // producing a final answer.
+  iterationLimit: "agent.iteration_limit",
+
+  // AccessDeniedError: the tool exists, but the caller's roles don't cover
+  // the handler behind it.
+  permissionDenied: "agent.permission_denied",
+
+  // AccessDeniedError: an `agent.risk: "high"` handler was invoked without
+  // the required explicit confirmation.
+  highRiskNoAlways: "agent.high_risk_no_always",
+} as const;
+
+export type AgentReason = (typeof AgentReasons)[keyof typeof AgentReasons];

package/src/migrations/__tests__/pending-rebuilds.integration.test.ts

@@ -55,6 +55,13 @@ const countsTable = pgTable("read_pending_counts", {
   itemCount: integer("item_count").notNull().default(0),
 });
 
+// Exists in the DB but has no registered projection — models an owning
+// feature missing from the composition (distinct from a deliberately DROPped
+// table, which is never pushed at all).
+const orphanProjectionTable = pgTable("read_orphan_projection", {
+  groupId: uuid("group_id").primaryKey(),
+});
+
 // Steuerbarer Fail: simuliert einen transienten Rebuild-Fehler.
 let failApply = false;
 
@@ -92,7 +99,10 @@ beforeAll(async () => {
   await unsafeCreateEntityTable(testDb.db, itemEntity, "pending-item");
   await createEventsTable(testDb.db);
   await createProjectionStateTable(testDb.db);
-  await unsafePushTables(testDb.db, { readPendingCounts: countsTable });
+  await unsafePushTables(testDb.db, {
+    readPendingCounts: countsTable,
+    readOrphanProjection: orphanProjectionTable,
+  });
   tdb = createTenantDb(testDb.db, admin.tenantId);
   markerDir = mkdtempSync(join(tmpdir(), "pending-rebuilds-"));
 });
@@ -105,7 +115,7 @@ afterAll(async () => {
 beforeEach(async () => {
   failApply = false;
   await asRawClient(testDb.db).unsafe(
-    `TRUNCATE kumiko_events, read_pending_items, read_pending_counts, kumiko_projections RESTART IDENTITY CASCADE`,
+    `TRUNCATE kumiko_events, read_pending_items, read_pending_counts, read_orphan_projection, kumiko_projections RESTART IDENTITY CASCADE`,
   );
   await asRawClient(testDb.db).unsafe(`DROP TABLE IF EXISTS kumiko_pending_rebuilds`);
 });
@@ -233,6 +243,30 @@ describe("pending-rebuilds queue", () => {
     expect(await listPendingRebuilds(testDb.db)).toEqual([]);
   });
 
+  // A later migration in the SAME run can deliberately DROP a table it just
+  // emptied (e.g. after copying its rows elsewhere, phronexsis 0013/0025) —
+  // that's not data loss, so it must drain as unmapped instead of firing
+  // unresolvedManaged.
+  test("table emptied THIS run then dropped by a later migration → unmapped, not flagged", async () => {
+    await asRawClient(testDb.db).unsafe(
+      `CREATE TABLE IF NOT EXISTS read_dropped_by_later_migration (group_id uuid PRIMARY KEY)`,
+    );
+    writeRebuildMarker(markerDir, "0007_dropped.sql", ["read_dropped_by_later_migration"]);
+    const queued = await queueRebuildsFromMarkers(testDb.db, {
+      migrationsDir: markerDir,
+      appliedIds: ["0007_dropped"],
+    });
+    expect(queued).toEqual(["read_dropped_by_later_migration"]);
+
+    // Migration 0008 in the same run migrates the data elsewhere and drops the table.
+    await asRawClient(testDb.db).unsafe(`DROP TABLE read_dropped_by_later_migration`);
+
+    const run = await runPendingRebuilds(testDb.db, registry, { thisRunTables: queued });
+    expect(run.unresolvedManaged).toEqual([]);
+    expect(run.unmapped).toEqual(["read_dropped_by_later_migration"]);
+    expect(await listPendingRebuilds(testDb.db)).toEqual([]);
+  });
+
   // #361: dieselbe unmapped-Tabelle, aber NICHT in thisRunTables (pre-existing
   // pending aus einem früheren Run / evtl. altem unmanaged-Marker) → benign,
   // still gedraint, NICHT als unresolvedManaged geflaggt (kein False-Positive).

package/src/migrations/pending-rebuilds.ts

@@ -94,9 +94,10 @@ export type PendingRebuildRun = {
   readonly rebuilt: readonly { readonly projection: string; readonly eventsProcessed: number }[];
   /** Fehlgeschlagene Projektionen — ihre Tabellen BLEIBEN pending. */
   readonly failed: readonly { readonly projection: string; readonly error: string }[];
-  /** Pending-Tabellen ohne registrierte Projektion, die NICHT in diesem Run
-   *  frisch via Marker geleert wurden (pre-existing / Legacy-unmanaged-Marker)
-   *  — geräumt, still (nicht von echten Legacy-Tabellen unterscheidbar). */
+  /** Pending tables without a registered projection that are either NOT
+   *  freshly emptied via marker in this run (pre-existing / legacy unmanaged
+   *  marker) OR were deliberately DROPped by a later migration in the same
+   *  run (no data loss, no projection left to resolve) — drained, silent. */
   readonly unmapped: readonly string[];
   /** In DIESEM Run via Marker geleerte managed-Tabellen ohne auflösbare
    *  Projektion = das owning-Feature fehlt in der Komposition. Geräumt (kein
@@ -148,13 +149,18 @@ export async function runPendingRebuilds(
   for (const tableName of pending) {
     const projection = tableToProjection.get(tableName);
     if (projection === undefined) {
-      // Marker tragen nur managed Tabellen (rebuild-marker.ts). Eine in DIESEM
-      // Run frisch geleerte Tabelle ohne auflösbare Projektion ist daher ein
-      // echter Defekt (owning-Feature fehlt in der Komposition) → laut. Pre-
-      // existing pending Tabellen sind nicht von alten unmanaged-Markern
-      // unterscheidbar → still drainen wie bisher (kein Hard-Throw, siehe #361).
-      if (thisRun.has(tableName)) unresolvedManaged.push(tableName);
-      else unmapped.push(tableName);
+      // Markers only carry managed tables (rebuild-marker.ts). A table emptied
+      // in THIS run with no resolving projection is a real defect (owning
+      // feature missing from the composition) → loud (#361) — unless the table
+      // no longer exists: then a later migration in the same run deliberately
+      // DROPped it after migrating its data elsewhere, no data loss. Pre-
+      // existing pending tables stay indistinguishable from old unmanaged
+      // markers → still drain silently as before.
+      if (thisRun.has(tableName) && (await tableExists(db, `public.${tableName}`))) {
+        unresolvedManaged.push(tableName);
+      } else {
+        unmapped.push(tableName);
+      }
       continue;
     }
     byProjection.set(projection, [...(byProjection.get(projection) ?? []), tableName]);

package/src/pipeline/__tests__/event-dispatcher-pass-failure.integration.test.ts

@@ -0,0 +1,266 @@
+// kumiko-framework#2616 — processConsumer's db.begin rolled back the WHOLE
+// pass (including markProcessing/persistConsumerOutcome) whenever anything
+// inside it threw AFTER delivery succeeded, silently reverting attempts/
+// last_error/cursor to their pre-pass values. Combined with `context.log?.`,
+// a process without a logger lost the error entirely — in prod this looked
+// like a healthy consumer (status=idle, attempts=0, last_error=NULL) that
+// was actually looping at 24 rollbacks/sec.
+//
+// The real prod trigger was never identified (see the issue), so this test
+// reproduces the SHAPE of the bug via a legitimate fault-injection point
+// instead of the unknown original cause: EventDispatcherOptions.meter is
+// dependency-injected, and emitLagFromTx (called INSIDE the same db.begin,
+// after delivery+persistConsumerOutcome already ran) is where a thrown
+// metric emission rolls back an otherwise-successful pass. No production
+// code is touched to make this reproducible.
+
+import { afterEach, beforeAll, describe, expect, test } from "bun:test";
+import { createEventStoreExecutor } from "../../db/event-store-executor";
+import { asRawClient } from "../../db/query";
+import { createTenantDb, type TenantDb } from "../../db/tenant-db";
+import { defineFeature } from "../../engine";
+import { type Gauge, getFallbackMeter, type Meter } from "../../observability";
+import {
+  createEventDispatcher,
+  type EventConsumer,
+  type EventDispatcher,
+  getConsumerState,
+} from "../../pipeline";
+import {
+  resetEventStore,
+  setupTestStack,
+  type TestStack,
+  TestUsers,
+  unsafeCreateEntityTable,
+} from "../../stack";
+import { sharedWidgetEntity, sharedWidgetTable } from "../../testing";
+
+const executor = createEventStoreExecutor(sharedWidgetTable, sharedWidgetEntity, {
+  entityName: "widget",
+});
+
+const feature = defineFeature("passfailure", (r) => {
+  r.entity("widget", sharedWidgetEntity);
+});
+
+const admin = TestUsers.admin;
+let stack: TestStack;
+let tdb: TenantDb;
+
+beforeAll(async () => {
+  stack = await setupTestStack({ features: [feature], systemHooks: [] });
+  await unsafeCreateEntityTable(stack.db, sharedWidgetEntity, "widget");
+  tdb = createTenantDb(stack.db, admin.tenantId);
+});
+
+afterEach(async () => {
+  await resetEventStore(stack, ["read_widgets"]);
+});
+
+async function appendWidget(name: string): Promise<void> {
+  await executor.create({ name }, admin, tdb);
+}
+
+// A meter that throws on the lag-gauge emission for one specific consumer —
+// the same injection point prod's unidentified exception would have hit,
+// since emitLagFromTx runs last inside processConsumer's db.begin.
+function makeLagFailingMeter(shouldFail: (consumer: string) => boolean): Meter {
+  const base = getFallbackMeter();
+  return {
+    registerMetric: (def) => base.registerMetric(def),
+    counter: (name) => base.counter(name),
+    histogram: (name) => base.histogram(name),
+    gauge: (name): Gauge => {
+      const real = base.gauge(name);
+      if (name !== "kumiko_event_consumer_lag_events") return real;
+      return {
+        set: (value, labels) => {
+          const consumer = String(labels?.["consumer"] ?? "");
+          if (shouldFail(consumer)) {
+            throw new Error(`injected-lag-failure-for-${consumer}`);
+          }
+          real.set(value, labels);
+        },
+        inc: (value, labels) => real.inc(value, labels),
+        dec: (value, labels) => real.dec(value, labels),
+      };
+    },
+    definitions: () => base.definitions(),
+  };
+}
+
+function buildDispatcher(consumers: readonly EventConsumer[], meter: Meter): EventDispatcher {
+  return createEventDispatcher({
+    db: stack.db,
+    consumers,
+    // Deliberately no `log` field — proves the console.error fallback, not
+    // the pre-existing context.log?.error(...) path.
+    context: { db: stack.db, redis: stack.redis.redis, registry: stack.registry },
+    meter,
+    batchSize: 200,
+    pollIntervalMs: 5000,
+  });
+}
+
+describe("event-dispatcher — pass-level throw survives the rolled-back tx (#2616)", () => {
+  test("logs via console.error when context.log is absent", async () => {
+    const flaky = "passfailure:flaky-log";
+    const consumer: EventConsumer = { name: flaky, handler: async () => {} };
+    const dispatcher = buildDispatcher(
+      [consumer],
+      makeLagFailingMeter((c) => c === flaky),
+    );
+    await dispatcher.ensureRegistered();
+    await appendWidget("one");
+
+    const originalError = console.error;
+    const logged: string[] = [];
+    console.error = (...args: unknown[]) => {
+      logged.push(args.map(String).join(" "));
+    };
+    try {
+      await dispatcher.runOnce();
+    } finally {
+      console.error = originalError;
+    }
+
+    expect(logged.some((l) => l.includes(flaky) && l.includes("injected-lag-failure"))).toBe(true);
+  });
+
+  test("last_error survives the rollback while attempts stays untouched", async () => {
+    const flaky = "passfailure:attempts";
+    const consumer: EventConsumer = { name: flaky, handler: async () => {} };
+    const dispatcher = buildDispatcher(
+      [consumer],
+      makeLagFailingMeter((c) => c === flaky),
+    );
+    await dispatcher.ensureRegistered();
+    await appendWidget("one");
+
+    const before = await getConsumerState(stack.db, flaky);
+    expect(before?.attempts).toBe(0);
+    expect(before?.lastError).toBeNull();
+
+    await dispatcher.runOnce();
+
+    const after = await getConsumerState(stack.db, flaky);
+    // Delivery itself succeeded (the handler never threw) — only the lag
+    // emission did, rolling back markProcessing/persistConsumerOutcome. The
+    // dedicated post-catch write in its own transaction lands last_error,
+    // but must NOT touch attempts: that counter is deliverEvents' dead-letter
+    // budget, and this is an infra-level pass failure, not a handler throw.
+    expect(after?.attempts).toBe(0);
+    expect(after?.lastError).toMatch(/injected-lag-failure/);
+    // The cursor update rolled back with the rest of the transaction — the
+    // consumer will redeliver "one" once its backoff clears.
+    expect(after?.lastProcessedEventId).toBe(0n);
+  });
+
+  test("backs off after a thrown pass instead of retrying every tick, other consumers unaffected", async () => {
+    const flaky = "passfailure:backoff";
+    const healthy = "passfailure:healthy";
+    const healthySeen: string[] = [];
+
+    const flakyConsumer: EventConsumer = { name: flaky, handler: async () => {} };
+    const healthyConsumer: EventConsumer = {
+      name: healthy,
+      handler: async (event) => {
+        healthySeen.push(String(event.payload["name"]));
+      },
+    };
+    const dispatcher = buildDispatcher(
+      [flakyConsumer, healthyConsumer],
+      makeLagFailingMeter((c) => c === flaky),
+    );
+    await dispatcher.ensureRegistered();
+    await appendWidget("first");
+
+    await dispatcher.runOnce();
+    const afterFirst = await getConsumerState(stack.db, flaky);
+    expect(afterFirst?.attempts).toBe(0);
+    expect(afterFirst?.lastProcessedEventId).toBe(0n);
+
+    // Retry on the very next tick: the backoff must skip the flaky consumer
+    // (no delivery attempt, so no change to attempts) while the healthy one
+    // keeps advancing.
+    await appendWidget("second-for-healthy");
+    const second = await dispatcher.runOnce();
+    const afterSecond = await getConsumerState(stack.db, flaky);
+    expect(afterSecond?.attempts).toBe(0);
+    expect(healthySeen).toEqual(["first", "second-for-healthy"]);
+    expect(second.byConsumer[healthy]?.processed).toBeGreaterThan(0);
+  });
+
+  test("a successful pass clears the backoff so a later failure isn't skipped", async () => {
+    const name = "passfailure:recovers";
+    let failOnce = true;
+    const consumer: EventConsumer = { name, handler: async () => {} };
+    const meter = makeLagFailingMeter((c) => {
+      if (c === name && failOnce) {
+        failOnce = false;
+        return true;
+      }
+      return false;
+    });
+
+    const dispatcher = buildDispatcher([consumer], meter);
+    await dispatcher.ensureRegistered();
+    await appendWidget("one");
+
+    await dispatcher.runOnce();
+    const afterFailure = await getConsumerState(stack.db, name);
+    expect(afterFailure?.attempts).toBe(0);
+    expect(afterFailure?.lastProcessedEventId).toBe(0n);
+
+    // Wait out the (short, base-case) backoff window rather than reaching
+    // into dispatcher internals.
+    await new Promise((r) => setTimeout(r, 1100));
+    await dispatcher.runOnce();
+
+    const afterSuccess = await getConsumerState(stack.db, name);
+    expect(afterSuccess?.status).toBe("idle");
+    expect(afterSuccess?.attempts).toBe(0);
+    expect(afterSuccess?.lastProcessedEventId).toBe(1n);
+
+    // A fresh failure right after the successful pass must NOT be skipped —
+    // proves the backoff entry was cleared, not left at its escalated delay.
+    // attempts stays at 0: this is the same infra-level pass failure as
+    // above, which never touches deliverEvents' dead-letter budget.
+    failOnce = true;
+    await appendWidget("two");
+    await dispatcher.runOnce();
+    const afterSecondFailure = await getConsumerState(stack.db, name);
+    expect(afterSecondFailure?.attempts).toBe(0);
+  });
+
+  test("repeated infra-level pass failures never spend deliverEvents' dead-letter budget", async () => {
+    const flaky = "passfailure:budget";
+    const consumer: EventConsumer = { name: flaky, handler: async () => {} };
+    const meter = makeLagFailingMeter((c) => c === flaky);
+    await buildDispatcher([consumer], meter).ensureRegistered();
+
+    // Seed as if 9 of the real deliverEvents budget (maxAttempts=10) were
+    // already spent on genuine handler throws.
+    await asRawClient(stack.db).unsafe(
+      `UPDATE "kumiko_event_consumers" SET "attempts" = 9 WHERE "name" = $1`,
+      [flaky],
+    );
+    await appendWidget("one");
+
+    // Three infra-level pass failures in a row. Each uses a fresh dispatcher
+    // instance so its in-memory backoff never delays the next runOnce() —
+    // the DB row (and its seeded attempts=9) is what's shared and under test.
+    for (let i = 0; i < 3; i++) {
+      const dispatcher = buildDispatcher([consumer], meter);
+      await dispatcher.ensureRegistered();
+      await dispatcher.runOnce();
+    }
+
+    const after = await getConsumerState(stack.db, flaky);
+    // A 10th genuine handler throw would now dead-letter the consumer if
+    // these infra failures had bumped attempts past the seeded 9 — they
+    // must not, since none of them ran a real delivery pass.
+    expect(after?.attempts).toBe(9);
+    expect(after?.status).not.toBe("dead");
+  });
+});

package/src/pipeline/event-dispatcher-delivery.ts

@@ -8,6 +8,7 @@ import {
   insertConsumerIfAbsent,
   markConsumerProcessing,
   rearmDeadConsumer,
+  recordConsumerPassFailure,
   selectConsumerForUpdateSkipLocked,
   updateConsumerDeliveryOutcome,
 } from "../db/queries/event-consumer";
@@ -289,6 +290,19 @@ export async function persistConsumerOutcome(
   await updateConsumerDeliveryOutcome(tx, name, instanceId, outcome);
 }
 
+// Best-effort record of a pass that threw before persistConsumerOutcome
+// could run (event-dispatcher.ts's processConsumer catch). The caller opens
+// its OWN transaction for this — the one that rolled back never committed
+// anything, including this write, if it happened inside the same tx.
+export async function persistConsumerPassFailure(
+  tx: DbTx,
+  name: string,
+  instanceId: string,
+  errorMessage: string,
+): Promise<void> {
+  await recordConsumerPassFailure(tx, name, instanceId, errorMessage);
+}
+
 // Emit the lag gauge inside the consumer pass's tx so ops sees a snapshot
 // consistent with the cursor we just advanced to. `MAX(id)` on the events
 // table is an O(1) reverse-index scan — cheap even under load.

package/src/pipeline/event-dispatcher.ts

@@ -20,6 +20,7 @@ import {
   fetchPendingEvents,
   markProcessing,
   persistConsumerOutcome,
+  persistConsumerPassFailure,
   preRegisterConsumers,
 } from "./event-dispatcher-delivery";
 
@@ -175,6 +176,13 @@ const DEFAULT_POLL_MS = 100;
 const DEFAULT_MAX_ATTEMPTS = 10;
 const DEFAULT_REARM_COOLDOWN_MS = 5 * 60_000;
 const DEFAULT_MAX_REARM_COUNT = 3;
+// Backoff for a consumer whose pass THREW (the processConsumer catch below)
+// — an infra-level failure (db.begin's callback rejected outside deliverEvents'
+// own per-event try/catch), not a poisoned event. Doubles per consecutive
+// failure, capped at PASS_FAILURE_MAX_BACKOFF_MS so a consumer stuck failing
+// still gets retried within a minute; a successful pass resets it to 0.
+const PASS_FAILURE_BASE_BACKOFF_MS = 1_000;
+const PASS_FAILURE_MAX_BACKOFF_MS = 60_000;
 
 export function createEventDispatcher(options: EventDispatcherOptions): EventDispatcher {
   const {
@@ -217,6 +225,14 @@ export function createEventDispatcher(options: EventDispatcherOptions): EventDis
   // exactly the "still needs a human" signal ops wants.
   const reportedDeadConsumers = new Set<string>();
 
+  // Per-(consumer, instanceId) backoff after a thrown pass (see the catch in
+  // processConsumer). Keyed the same way as reportedDeadConsumers.
+  // consecutiveFailures drives the exponential delay; retryAtMs is the
+  // Date.now() timestamp doPass compares against to decide whether this
+  // consumer's turn is skipped this tick. A successful pass deletes the
+  // entry — the next failure (if any) starts a fresh backoff from scratch.
+  const consumerBackoff = new Map<string, { consecutiveFailures: number; retryAtMs: number }>();
+
   let running = false;
   // Separate from `running` on purpose: pre-registration of consumer state
   // rows is a one-time boot action, while running/timer/LISTEN is a
@@ -282,6 +298,16 @@ export function createEventDispatcher(options: EventDispatcherOptions): EventDis
         byConsumer[consumer.name] = { processed: 0, failed: 0 };
         continue;
       }
+      // Backoff-gate: a consumer whose last pass threw waits out its
+      // exponential delay before being tried again — see the catch in
+      // processConsumer. Other consumers are unaffected; only this one's
+      // turn is skipped this tick.
+      const backoffKey = `${consumer.name}:${consumerInstanceId(consumer, options.instanceId)}`;
+      const backoff = consumerBackoff.get(backoffKey);
+      if (backoff && backoff.retryAtMs > Date.now()) {
+        byConsumer[consumer.name] = { processed: 0, failed: 0 };
+        continue;
+      }
       const perConsumer = await processConsumer(consumer);
       byConsumer[consumer.name] = perConsumer;
       totalProcessed += perConsumer.processed;
@@ -352,6 +378,11 @@ export function createEventDispatcher(options: EventDispatcherOptions): EventDis
         await emitLagFromTx(tx, consumer.name, instanceId, outcome.cursor, meter);
       });
 
+      // Pass completed without throwing — clear any backoff from a prior
+      // failure so the next unrelated outage gets its own fresh delay
+      // instead of inheriting a partially-escalated one.
+      consumerBackoff.delete(`${consumer.name}:${instanceId}`);
+
       emitEventConsumerPassOutcome(
         meter,
         { consumer: consumer.name, instanceId },
@@ -370,8 +401,44 @@ export function createEventDispatcher(options: EventDispatcherOptions): EventDis
       // callers and at-least-once-with-duplicate-delivery on the next pass;
       // neither is what we want, so ops needs to see it.
       const msg = e instanceof Error ? e.message : String(e);
-      context.log?.error(`[event-dispatcher] ${consumer.name} pass failed: ${msg}`);
+      if (context.log) {
+        context.log.error(`[event-dispatcher] ${consumer.name} pass failed: ${msg}`);
+      } else {
+        // No logger wired onto this context (e.g. createEventDispatcher used
+        // standalone) — console.error is the only way ops still sees this in
+        // pod logs instead of the failure vanishing with the rolled-back tx.
+        console.error(`[event-dispatcher] ${consumer.name} pass failed: ${msg}`);
+      }
       span.setStatus("error", msg);
+
+      // The whole pass rolled back, so markProcessing/persistConsumerOutcome
+      // never committed — attempts/last_error are stuck at their pre-pass
+      // values. Record the failure in its OWN transaction so it survives.
+      // Best-effort: this write must never itself take down the dispatcher,
+      // so any failure here is swallowed after logging.
+      try {
+        await db.begin(async (tx: DbTx) => {
+          await persistConsumerPassFailure(tx, consumer.name, instanceId, msg);
+        });
+      } catch (persistErr) {
+        const persistMsg = persistErr instanceof Error ? persistErr.message : String(persistErr);
+        const failureMsg = `[event-dispatcher] ${consumer.name} failed to persist pass-failure state: ${persistMsg}`;
+        if (context.log) {
+          context.log.error(failureMsg);
+        } else {
+          console.error(failureMsg);
+        }
+      }
+
+      // Escalate the backoff so this consumer isn't retried on the very
+      // next tick — see PASS_FAILURE_BASE_BACKOFF_MS/MAX_MS above.
+      const key = `${consumer.name}:${instanceId}`;
+      const consecutiveFailures = (consumerBackoff.get(key)?.consecutiveFailures ?? 0) + 1;
+      const delayMs = Math.min(
+        PASS_FAILURE_BASE_BACKOFF_MS * 2 ** (consecutiveFailures - 1),
+        PASS_FAILURE_MAX_BACKOFF_MS,
+      );
+      consumerBackoff.set(key, { consecutiveFailures, retryAtMs: Date.now() + delayMs });
     } finally {
       span.end();
     }