@cosmicdrift/kumiko-framework 0.105.2 → 0.109.0

package/src/event-store/rebuild-dead-letter.ts

@@ -0,0 +1,111 @@
+// Dead-letter storage for apply-handler failures during projection rebuild.
+//
+// Background: rebuildProjection / rebuildMultiStreamProjection replay the
+// event log through apply handlers inside ONE transaction. A single event
+// whose (possibly years-old) payload makes its apply throw rolls the whole
+// replay back — the projection stays permanently un-rebuildable until the
+// event is repaired by hand. The upcaster dead-letter only quarantines
+// upcast-TRANSFORM failures, not apply failures (#760).
+//
+// Quarantine mode (errorPolicy.skipApplyErrors on RebuildDeps, or
+// MspErrorMode.rebuild.skipApplyErrors on the MSP definition) confines each
+// apply to a savepoint: a throwing apply is rolled back, captured into
+// `kumiko_rebuild_dead_letters`, and the replay continues. Replay-after-fix
+// is a separate ops step — same stance as the upcaster dead-letter.
+
+import type { DbConnection, DbRunner } from "../db/connection";
+import { bigint, index, jsonb, table as pgTable, text, timestamp, uuid } from "../db/dialect";
+import { tableExists } from "../db/schema-inspection";
+import { unsafePushTables } from "../stack";
+import type { StoredEvent } from "./event-store";
+
+export const rebuildDeadLetterTable = pgTable(
+  "kumiko_rebuild_dead_letters",
+  {
+    // Surrogate PK — the same event can land here across multiple rebuild
+    // attempts before the fix ships, without unique-violation noise.
+    id: bigint("id", { mode: "bigint" }).primaryKey().generatedAlwaysAsIdentity(),
+    projectionName: text("projection_name").notNull(),
+    // StoredEvent.id is surfaced as `string` (bigint serialised for JSON
+    // safety) — stored as text to keep the round-trip identity.
+    eventId: text("event_id").notNull(),
+    tenantId: uuid("tenant_id").notNull(),
+    aggregateId: text("aggregate_id").notNull(),
+    aggregateType: text("aggregate_type").notNull(),
+    eventType: text("event_type").notNull(),
+    errorMessage: text("error_message").notNull(),
+    payload: jsonb("payload").notNull(),
+    createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(),
+  },
+  (t) => ({
+    projectionIdx: index("rebuild_dead_letters_projection_idx").on(t.projectionName),
+    createdAtIdx: index("rebuild_dead_letters_created_at_idx").on(t.createdAt),
+  }),
+);
+
+// Idempotent table-create. The rebuild runners call this before the rebuild
+// tx whenever quarantine mode is active, so the opt-in works on databases
+// provisioned before this table existed.
+export async function createRebuildDeadLetterTable(db: DbConnection): Promise<void> {
+  // skip: table already exists — bootstrap called from multiple paths
+  if (await tableExists(db, "public.kumiko_rebuild_dead_letters")) return;
+  await unsafePushTables(db, { kumikoRebuildDeadLetters: rebuildDeadLetterTable });
+}
+
+export type SkippedApply = {
+  readonly event: StoredEvent;
+  readonly error: unknown;
+};
+
+// Bulk-write, called ONCE at the end of a quarantining rebuild (inside its
+// tx). Unqualified insert — during the rebuild the search_path points at the
+// shadow schema, and this table exists only in public, so it falls through.
+export async function recordRebuildDeadLetters(
+  db: DbRunner,
+  projectionName: string,
+  skipped: readonly SkippedApply[],
+): Promise<void> {
+  const { insertMany } = await import("../bun-db/query");
+  await insertMany(
+    db,
+    rebuildDeadLetterTable,
+    skipped.map(({ event, error }) => ({
+      projectionName,
+      eventId: event.id,
+      tenantId: event.tenantId,
+      aggregateId: event.aggregateId,
+      aggregateType: event.aggregateType,
+      eventType: event.type,
+      errorMessage: error instanceof Error ? error.message : String(error),
+      payload: event.payload,
+    })),
+  );
+}
+
+export type RebuildDeadLetterRow = {
+  readonly id: bigint;
+  readonly projectionName: string;
+  readonly eventId: string;
+  readonly tenantId: string;
+  readonly aggregateId: string;
+  readonly aggregateType: string;
+  readonly eventType: string;
+  readonly errorMessage: string;
+  readonly payload: Record<string, unknown>;
+  readonly createdAt: Date;
+};
+
+// Ops-side triage query, optionally scoped to one projection.
+export async function listRebuildDeadLetters(
+  db: DbConnection,
+  options: { projectionName?: string; limit?: number } = {},
+): Promise<readonly RebuildDeadLetterRow[]> {
+  const { selectMany } = await import("../bun-db/query");
+  const limit = options.limit ?? 100;
+  const where =
+    options.projectionName !== undefined ? { projectionName: options.projectionName } : undefined;
+  return selectMany<RebuildDeadLetterRow>(db, rebuildDeadLetterTable, where, {
+    orderBy: { col: "createdAt", direction: "desc" },
+    limit,
+  });
+}

package/src/pipeline/__tests__/load-aggregate-query.integration.test.ts

@@ -9,10 +9,10 @@
 // Bun.SQL-only setup. KEIN postgres-js, KEIN setupTestStack.
 
 import { afterAll, afterEach, beforeAll, describe, expect, test } from "bun:test";
+import { seedRow } from "@cosmicdrift/kumiko-framework/testing";
 import { z } from "zod";
 import type { DbConnection, DbTx } from "../../db/connection";
 import { createEventStoreExecutor } from "../../db/event-store-executor";
-import { insertOne } from "../../db/query";
 import { buildEntityTable } from "../../db/table-builder";
 import { createEntity, createTextField, defineFeature } from "../../engine";
 import { append, loadAggregate as loadAggregateRaw } from "../../event-store";
@@ -219,7 +219,7 @@ describe("ctx.loadAggregate via queryHandler — Marten AggregateStreamAsync equ
     // produce garbage.
     const invoiceId = "00000000-0000-4000-8000-000000000042";
     await (stack.db as DbConnection).begin(async (tx: DbTx) => {
-      await insertOne(tx, invoiceTable, {
+      await seedRow(tx, invoiceTable, {
         id: invoiceId,
         tenantId: admin.tenantId,
         customer: "LegacyCo",

package/src/pipeline/__tests__/rebuild-poison-quarantine.integration.test.ts

@@ -0,0 +1,274 @@
+// #760 — poison-event quarantine during projection rebuild.
+//
+// A single historical event whose apply throws used to abort the WHOLE
+// rebuild (tx rollback, status failed) with no recovery path short of
+// hand-editing the event. With quarantine mode the rebuild confines each
+// apply to a savepoint: the poison event is skipped, recorded into
+// kumiko_rebuild_dead_letters, and the replay completes.
+//
+// Covers both rebuild flavors:
+//   - single-stream: RebuildDeps.errorPolicy.skipApplyErrors (per run)
+//   - MSP: MspErrorMode.rebuild.skipApplyErrors (per definition — declared
+//     API that was previously never honored by rebuildMultiStreamProjection)
+// and both poison flavors:
+//   - JS throw (no SQL executed)
+//   - SQL error (aborts the tx → proves the savepoint is load-bearing)
+
+import { afterAll, beforeAll, beforeEach, describe, expect, test } from "bun:test";
+import { z } from "zod";
+import { integer as pgInteger, table as pgTable, uuid as pgUuid } from "../../db/dialect";
+import { createEventStoreExecutor } from "../../db/event-store-executor";
+import { asRawClient } from "../../db/query";
+import { buildEntityTable } from "../../db/table-builder";
+import { createTenantDb, type TenantDb } from "../../db/tenant-db";
+import { createEntity, createTextField, defineApply, defineFeature } from "../../engine";
+import type { ProjectionDefinition } from "../../engine/types";
+import { append, createEventsTable } from "../../event-store";
+import { listRebuildDeadLetters } from "../../event-store/rebuild-dead-letter";
+import {
+  createProjectionStateTable,
+  getConsumerState,
+  getProjectionState,
+  rebuildMultiStreamProjection,
+  rebuildProjection,
+} from "../../pipeline";
+import {
+  setupTestStack,
+  type TestStack,
+  TestUsers,
+  unsafeCreateEntityTable,
+  unsafePushTables,
+} from "../../stack";
+
+// --- Fixtures ---
+
+const POISON_JS = "poison-js";
+const POISON_SQL = "poison-sql";
+
+const itemEntity = createEntity({
+  table: "read_poison_items",
+  fields: {
+    groupId: createTextField({ required: true }),
+    name: createTextField({ required: true }),
+  },
+});
+const itemTable = buildEntityTable("poison-item", itemEntity);
+
+const counterTable = pgTable("read_poison_counter", {
+  groupId: pgUuid("group_id").primaryKey(),
+  tenantId: pgUuid("tenant_id").notNull(),
+  itemCount: pgInteger("item_count").notNull().default(0),
+});
+
+const mspCounterTable = pgTable("read_poison_msp_counter", {
+  groupId: pgUuid("group_id").primaryKey(),
+  tenantId: pgUuid("tenant_id").notNull(),
+  itemCount: pgInteger("item_count").notNull().default(0),
+});
+
+async function bump(tx: unknown, tableName: string, groupId: string, tenantId: string) {
+  await asRawClient(tx).unsafe(
+    `INSERT INTO "${tableName}" (group_id, tenant_id, item_count) VALUES ($1::uuid, $2::uuid, 1) ON CONFLICT (group_id) DO UPDATE SET item_count = "${tableName}".item_count + 1`,
+    [groupId, tenantId],
+  );
+}
+
+// Poison by payload marker. POISON_SQL runs a genuinely failing statement so
+// the surrounding tx would be in 25P02 without the savepoint.
+async function applyOrPoison(
+  tx: unknown,
+  tableName: string,
+  payload: { groupId: string; name: string },
+  tenantId: string,
+): Promise<void> {
+  if (payload.name === POISON_JS) throw new Error("intentional js poison");
+  if (payload.name === POISON_SQL) {
+    await asRawClient(tx).unsafe(
+      `INSERT INTO "${tableName}" (group_id, tenant_id, item_count) VALUES ('not-a-uuid', $1::uuid, 1)`,
+      [tenantId],
+    );
+    return;
+  }
+  await bump(tx, tableName, payload.groupId, tenantId);
+}
+
+type ItemPayload = { groupId: string; name: string };
+
+const poisonProjection: ProjectionDefinition = {
+  name: "poison-counter",
+  source: "poison-item",
+  table: counterTable,
+  apply: {
+    "poison-item.created": defineApply<ItemPayload>(async (event, tx) => {
+      await applyOrPoison(tx, "read_poison_counter", event.payload, event.tenantId);
+    }),
+  },
+};
+
+const MSP_EVENT_SHORT = "poison-noted";
+
+const feature = defineFeature("poisontest", (r) => {
+  r.entity("poison-item", itemEntity);
+  r.projection(poisonProjection);
+  const noted = r.defineEvent(MSP_EVENT_SHORT, z.object({ groupId: z.uuid(), name: z.string() }));
+  r.multiStreamProjection({
+    name: "poison-msp-counter",
+    table: mspCounterTable,
+    errorMode: { rebuild: { skipApplyErrors: true } },
+    apply: {
+      [noted.name]: async (event, tx) => {
+        const p = event.payload as ItemPayload; // @cast-boundary engine-payload
+        await applyOrPoison(tx, "read_poison_msp_counter", p, event.tenantId);
+      },
+    },
+  });
+});
+
+const admin = TestUsers.admin;
+const PROJECTION = "poisontest:projection:poison-counter";
+const MSP = "poisontest:projection:poison-msp-counter";
+const MSP_EVENT = "poisontest:event:poison-noted";
+const GROUP = "00000000-0000-4000-8000-00000000cafe";
+
+let stack: TestStack;
+let tdb: TenantDb;
+
+const executor = createEventStoreExecutor(itemTable, itemEntity, { entityName: "poison-item" });
+
+beforeAll(async () => {
+  stack = await setupTestStack({ features: [feature] });
+  await unsafeCreateEntityTable(stack.db, itemEntity, "poison-item");
+  await createEventsTable(stack.db);
+  await createProjectionStateTable(stack.db);
+  await unsafePushTables(stack.db, {
+    poisonCounter: counterTable,
+    poisonMspCounter: mspCounterTable,
+  });
+  tdb = createTenantDb(stack.db, admin.tenantId);
+});
+
+afterAll(async () => {
+  await stack.cleanup();
+});
+
+beforeEach(async () => {
+  await asRawClient(stack.db).unsafe(
+    `TRUNCATE kumiko_events, read_poison_items, read_poison_counter, read_poison_msp_counter, kumiko_projections RESTART IDENTITY CASCADE`,
+  );
+  await asRawClient(stack.db).unsafe(`DROP TABLE IF EXISTS kumiko_rebuild_dead_letters`);
+  if (stack.eventDispatcher) await stack.eventDispatcher.ensureRegistered();
+});
+
+async function createItem(name: string): Promise<void> {
+  await executor.create({ groupId: GROUP, name }, admin, tdb);
+}
+
+async function getCount(table: string): Promise<number | undefined> {
+  const rows = (await asRawClient(stack.db).unsafe(
+    `SELECT item_count FROM "${table}" WHERE group_id = $1::uuid`,
+    [GROUP],
+  )) as ReadonlyArray<{ item_count: number }>;
+  return rows[0]?.item_count;
+}
+
+describe("rebuildProjection — poison event, strict default (#760)", () => {
+  test("apply throw aborts the rebuild: status failed, no dead letters", async () => {
+    await createItem("good-1");
+    await createItem(POISON_JS);
+    await createItem("good-2");
+
+    await expect(
+      rebuildProjection(PROJECTION, { db: stack.db, registry: stack.registry }),
+    ).rejects.toThrow("intentional js poison");
+
+    const state = await getProjectionState(stack.db, PROJECTION);
+    expect(state?.status).toBe("failed");
+    // Strict mode never provisions/records dead letters.
+    const rows = (await asRawClient(stack.db).unsafe(
+      `SELECT to_regclass('public.kumiko_rebuild_dead_letters') AS t`,
+    )) as ReadonlyArray<{ t: string | null }>;
+    expect(rows[0]?.t).toBeNull();
+  });
+});
+
+describe("rebuildProjection — quarantine mode (#760)", () => {
+  test("js- and sql-poison events are skipped, recorded, and the rebuild completes", async () => {
+    await createItem("good-1");
+    await createItem(POISON_JS);
+    // sql-poison proves the savepoint: without it the failed statement puts
+    // the whole rebuild tx into 25P02 and every later apply fails too.
+    await createItem(POISON_SQL);
+    await createItem("good-2");
+
+    const result = await rebuildProjection(PROJECTION, {
+      db: stack.db,
+      registry: stack.registry,
+      errorPolicy: { skipApplyErrors: true },
+    });
+
+    expect(result.eventsSkipped).toBe(2);
+    expect(result.eventsProcessed).toBe(4);
+    expect(await getCount("read_poison_counter")).toBe(2);
+
+    const state = await getProjectionState(stack.db, PROJECTION);
+    expect(state?.status).toBe("idle");
+
+    const deadLetters = await listRebuildDeadLetters(stack.db, { projectionName: PROJECTION });
+    expect(deadLetters).toHaveLength(2);
+    const messages = deadLetters.map((d) => d.errorMessage).sort();
+    expect(messages[0]).toContain("poison");
+    expect(deadLetters.every((d) => d.eventType === "poison-item.created")).toBe(true);
+    expect(deadLetters.every((d) => d.aggregateType === "poison-item")).toBe(true);
+  });
+
+  test("clean run in quarantine mode: zero skipped, no dead letters", async () => {
+    await createItem("good-1");
+    await createItem("good-2");
+
+    const result = await rebuildProjection(PROJECTION, {
+      db: stack.db,
+      registry: stack.registry,
+      errorPolicy: { skipApplyErrors: true },
+    });
+
+    expect(result.eventsSkipped).toBe(0);
+    expect(await getCount("read_poison_counter")).toBe(2);
+    expect(await listRebuildDeadLetters(stack.db, { projectionName: PROJECTION })).toHaveLength(0);
+  });
+});
+
+describe("rebuildMultiStreamProjection — errorMode.rebuild.skipApplyErrors (#760)", () => {
+  async function appendMspEvent(name: string, expectedVersion: number): Promise<void> {
+    await append(stack.db, {
+      aggregateId: GROUP,
+      aggregateType: "poison-note",
+      tenantId: admin.tenantId,
+      expectedVersion,
+      type: MSP_EVENT,
+      payload: { groupId: GROUP, name },
+      metadata: { userId: admin.id },
+    });
+  }
+
+  test("declared rebuild policy is honored: poison skipped + recorded, cursor advanced", async () => {
+    await appendMspEvent("good-1", 0);
+    await appendMspEvent(POISON_SQL, 1);
+    await appendMspEvent("good-2", 2);
+
+    const result = await rebuildMultiStreamProjection(MSP, {
+      db: stack.db,
+      registry: stack.registry,
+    });
+
+    expect(result.eventsSkipped).toBe(1);
+    expect(result.eventsProcessed).toBe(3);
+    expect(await getCount("read_poison_msp_counter")).toBe(2);
+
+    const deadLetters = await listRebuildDeadLetters(stack.db, { projectionName: MSP });
+    expect(deadLetters).toHaveLength(1);
+    expect(deadLetters[0]?.eventType).toBe(MSP_EVENT);
+
+    const consumer = await getConsumerState(stack.db, MSP);
+    expect(consumer?.lastProcessedEventId).toBe(result.lastProcessedEventId);
+  });
+});

package/src/pipeline/dispatcher.ts

@@ -1,7 +1,7 @@
 import { requestContext } from "../api/request-context";
 import type { DbConnection, DbRow, DbTx } from "../db/connection";
 import { selectRowForUpdateById } from "../db/queries/entity-read";
-import { selectMany, transaction } from "../db/query";
+import { asEntityTableMeta, selectMany, transaction } from "../db/query";
 import { buildEntityTable, toSnakeCase } from "../db/table-builder";
 import { createTenantDb } from "../db/tenant-db";
 import { hasAccess } from "../engine/access";
@@ -717,12 +717,8 @@ export function createDispatcher(
       user,
       ip: reqCtx?.ip,
     });
-    // skip: ip-bucketed handler called from a non-HTTP entry point (job, seed,
-    //       MSP-apply) — no client IP to bucket on, nothing to enforce. Pass
-    //       through BEFORE requiring a resolver, so system/seed writes through
-    //       such a handler don't need a RateLimitResolver wired (the es-ops
-    //       seed dispatcher has none). L1/L2 middleware handle the HTTP-side
-    //       ip caps.
+    // skip: ip-bucket + no IP (non-HTTP entry point) — pass through before
+    // requiring a resolver; HTTP path always has an IP + L1/L2 middleware.
     if (bucket.kind === "skip") return;
     if (!context.rateLimit) {
       throw new InternalError({
@@ -1162,9 +1158,7 @@ export function createDispatcher(
           // can false-pass; optimistic locking would catch it later, but with
           // a less specific error. Falls back to a plain SELECT if no tx is
           // active (tests without a DB connection).
-          const tableName = String(
-            (table as { [key: symbol]: unknown })[Symbol.for("kumiko:schema:Name")],
-          );
+          const tableName = asEntityTableMeta(table)?.tableName ?? "";
           const rows = tx
             ? await selectRowForUpdateById(handlerContext.db, tableName, id)
             : await selectMany(handlerContext.db, table, { id });

package/src/pipeline/msp-rebuild.ts

@@ -11,11 +11,16 @@ import {
   rebuildMetaOrThrow,
   swapShadowIntoLive,
 } from "../db/queries/shadow-swap";
-import { selectMany } from "../db/query";
+import { runInSavepoint, selectMany } from "../db/query";
 import type { Registry, TenantId } from "../engine/types";
 import { InternalError } from "../errors";
 import { eventsTable, type StoredEvent, upcastStoredEvent } from "../event-store";
 import { loadAggregate, loadAggregateAsOf } from "../event-store/event-store";
+import {
+  createRebuildDeadLetterTable,
+  recordRebuildDeadLetters,
+  type SkippedApply,
+} from "../event-store/rebuild-dead-letter";
 import { upcastStoredEvents } from "../event-store/upcaster";
 import { emitProjectionRebuild } from "../observability/standard-metrics";
 import type { Meter } from "../observability/types/metric";
@@ -123,12 +128,21 @@ export async function rebuildMultiStreamProjection(
 
   const meta = rebuildMetaOrThrow(msp.table, mspName);
 
+  // Declared per-projection (MspErrorMode): the rebuild policy falls back to
+  // continuous when omitted — previously declared but never honored here.
+  const skipApplyErrors =
+    (msp.errorMode?.rebuild ?? msp.errorMode?.continuous)?.skipApplyErrors === true;
+
   const startedAt = Date.now();
   let eventsProcessed = 0;
   let lastProcessedEventId = 0n;
+  const skipped: SkippedApply[] = [];
 
   try {
     await ensureRebuildSchema(db);
+    // Outside the rebuild tx, like the schema: idempotent DDL colliding
+    // inside the tx would roll the whole replay back.
+    if (skipApplyErrors) await createRebuildDeadLetterTable(db);
     await db.begin(async (tx: DbTx) => {
       await resetConsumerForMspRebuild(tx, mspName, SHARED_INSTANCE_SENTINEL);
       await selectConsumerForUpdate(tx, mspName, SHARED_INSTANCE_SENTINEL);
@@ -178,13 +192,31 @@ export async function rebuildMultiStreamProjection(
           });
           const applyFn = msp.apply[row.type];
           if (!applyFn) continue;
-          const rebuildCtx = createRebuildCtx(registry, tx, row.tenantId);
-          await applyFn(storedEvent, tx, rebuildCtx);
+          if (skipApplyErrors) {
+            // Driver-native savepoint: a throwing SQL statement would
+            // otherwise poison the rebuild tx (25P02) AND make the driver
+            // reject the whole begin() even after a caught error. Apply +
+            // ctx run on the savepoint-scoped handle.
+            try {
+              await runInSavepoint(tx, async (sp) => {
+                const rebuildCtx = createRebuildCtx(registry, sp as DbRunner, row.tenantId);
+                await applyFn(storedEvent, sp as DbRunner, rebuildCtx);
+              });
+            } catch (e) {
+              skipped.push({ event: storedEvent, error: e });
+            }
+          } else {
+            const rebuildCtx = createRebuildCtx(registry, tx, row.tenantId);
+            await applyFn(storedEvent, tx, rebuildCtx);
+          }
           eventsProcessed++;
           lastProcessedEventId = row.id;
         }
       }
 
+      if (skipped.length > 0) {
+        await recordRebuildDeadLetters(tx, mspName, skipped);
+      }
       await updateConsumerRebuildCursor(
         tx,
         mspName,
@@ -210,6 +242,7 @@ export async function rebuildMultiStreamProjection(
   const result: RebuildResult = {
     projection: mspName,
     eventsProcessed,
+    eventsSkipped: skipped.length,
     lastProcessedEventId,
     durationMs: Date.now() - startedAt,
   };

package/src/pipeline/projection-rebuild.ts

@@ -13,7 +13,7 @@ import {
   rebuildMetaOrThrow,
   swapShadowIntoLive,
 } from "../db/queries/shadow-swap";
-import { coerceRow, extractTableInfo, selectMany } from "../db/query";
+import { coerceRow, extractTableInfo, runInSavepoint, selectMany } from "../db/query";
 import type { Registry, TenantId } from "../engine/types";
 import {
   eventsTable,
@@ -22,6 +22,11 @@ import {
   upcastStoredEvent,
 } from "../event-store";
 import type { EventMetadata } from "../event-store/event-store";
+import {
+  createRebuildDeadLetterTable,
+  recordRebuildDeadLetters,
+  type SkippedApply,
+} from "../event-store/rebuild-dead-letter";
 import { emitProjectionRebuild } from "../observability/standard-metrics";
 import type { Meter } from "../observability/types/metric";
 import { projectionStateTable } from "./projection-state";
@@ -119,7 +124,12 @@ function rowToStoredEvent(row: StoredEventRow): StoredEvent {
 
 export type RebuildResult = {
   readonly projection: string;
+  // Events consumed from the log — includes quarantined ones (the cursor
+  // advanced past them).
   readonly eventsProcessed: number;
+  // Poison events quarantined into kumiko_rebuild_dead_letters this run.
+  // Always 0 unless quarantine mode was active (#760).
+  readonly eventsSkipped: number;
   readonly lastProcessedEventId: bigint;
   readonly durationMs: number;
 };
@@ -145,6 +155,15 @@ type RebuildDeps = {
   // on the live table; if a long-running writer holds it past this, the rebuild
   // fails loud instead of hanging. Defaults to DEFAULT_FENCE_LOCK_TIMEOUT_MS.
   readonly fenceLockTimeoutMs?: number;
+  // Apply-error policy for THIS run. Default strict: the first throwing
+  // apply aborts the rebuild (tx rollback, status "failed") — a poison
+  // event makes the projection permanently un-rebuildable (#760).
+  // skipApplyErrors: true confines every apply to a savepoint; a throwing
+  // apply is rolled back, recorded into kumiko_rebuild_dead_letters and
+  // skipped, and the rebuild completes without its effect. Opt-in for
+  // operators unblocking a pinned rebuild — the projection then knowingly
+  // misses the quarantined events until they are repaired and replayed.
+  readonly errorPolicy?: { readonly skipApplyErrors?: boolean };
   // Test-only seam: fires once after the unlocked bulk drain and before the
   // cutover fence. Lets a concurrency test inject a committed write into the
   // replay window deterministically. The `__test_` prefix marks it test-only:
@@ -174,7 +193,7 @@ export async function rebuildProjection(
   const meta = rebuildMetaOrThrow(projection.table, projectionName);
 
   const sources = Array.isArray(projection.source) ? projection.source : [projection.source];
-  const sourcesList = [...sources];
+  const sourcesList = [...sources, ...(projection.extraSources ?? [])];
   const subscribedList = Object.keys(projection.apply);
   // Upcasters run at read time: older stored payloads get walked through the
   // registered r.eventMigration chain until their shape matches the current
@@ -185,6 +204,11 @@ export async function rebuildProjection(
   const startedAt = Date.now();
   let eventsProcessed = 0;
   let lastProcessedEventId = 0n;
+  const skipApplyErrors = deps.errorPolicy?.skipApplyErrors === true;
+  // Quarantined events, collected in memory and written once after the
+  // replay settles — the #443 full-re-replay path would otherwise duplicate
+  // the dead-letter rows.
+  let skipped: SkippedApply[] = [];
 
   // One chronological batch of events after lastProcessedEventId, applied into
   // the shadow. Returns the batch size so the caller can detect the tail
@@ -208,7 +232,21 @@ export async function rebuildProjection(
       // skip: apply-key validation ensures every subscribed type has a handler;
       //       defensive check against runtime-mutated registry
       if (!applyFn) continue;
-      await applyFn(storedEvent, tx);
+      if (skipApplyErrors) {
+        // Driver-native savepoint: a throwing SQL statement would otherwise
+        // poison the rebuild tx (25P02) AND make the driver reject the whole
+        // begin() even after a caught error. The apply runs on the
+        // savepoint-scoped handle so its statements are confined.
+        try {
+          await runInSavepoint(tx, async (sp) => {
+            await applyFn(storedEvent, sp as DbTx, projection.table);
+          });
+        } catch (e) {
+          skipped.push({ event: storedEvent, error: e });
+        }
+      } else {
+        await applyFn(storedEvent, tx, projection.table);
+      }
       eventsProcessed++;
       lastProcessedEventId = row.id;
     }
@@ -217,6 +255,9 @@ export async function rebuildProjection(
 
   try {
     await ensureRebuildSchema(db);
+    // Outside the rebuild tx, like the schema: idempotent DDL colliding
+    // inside the tx would roll the whole replay back.
+    if (skipApplyErrors) await createRebuildDeadLetterTable(db);
     await db.begin(async (tx: DbTx) => {
       await markProjectionRebuilding(tx, projectionName);
       await buildShadowTable(tx, meta);
@@ -261,12 +302,16 @@ export async function rebuildProjection(
           deps.__test_onBuildShadowTable?.();
           lastProcessedEventId = 0n;
           eventsProcessed = 0;
+          skipped = [];
           while ((await drainBatch(tx)) === REBUILD_BATCH_SIZE) {
             // re-replay the full log into the fresh shadow
           }
         }
       }
 
+      if (skipped.length > 0) {
+        await recordRebuildDeadLetters(tx, projectionName, skipped);
+      }
       await finalizeProjectionRebuild(tx, projectionName, lastProcessedEventId);
       await swapShadowIntoLive(tx, meta.tableName);
     });
@@ -292,6 +337,7 @@ export async function rebuildProjection(
   const result: RebuildResult = {
     projection: projectionName,
     eventsProcessed,
+    eventsSkipped: skipped.length,
     lastProcessedEventId,
     durationMs: Date.now() - startedAt,
   };

package/src/pipeline/projections-runner.ts

@@ -51,6 +51,6 @@ export async function runProjectionsForEvent(
     const applyFn = proj.apply[event.type];
     // skip: this projection doesn't care about this event type
     if (!applyFn) continue;
-    await applyFn(event, tx);
+    await applyFn(event, tx, proj.table);
   }
 }