@cosmicdrift/kumiko-framework 0.327.0 → 0.328.1

package/dist/pipeline/projection-rebuild.js

@@ -0,0 +1,285 @@
+import { countSubscribedEvents, finalizeProjectionRebuild, markProjectionRebuildFailed, markProjectionRebuilding, selectEventsForProjectionRebuildBatch, } from "../db/queries/projection-rebuild.js";
+import { assertLiveColumnsMatchMeta, assertLiveTableHasNoRowLevelSecurity, assertNoBlindIndexLoss, assertNoUnreachableLiveRows, buildShadowTable, countColumnDrift, ensureRebuildSchema, fenceLiveTable, rebuildMetaOrThrow, swapShadowIntoLive, } from "../db/queries/shadow-swap.js";
+import { coerceRow, extractTableInfo, runInSavepoint, selectMany } from "../db/query.js";
+import { eventsTable, getEventsHighWaterMark, upcastStoredEvent, } from "../event-store/index.js";
+import { createRebuildDeadLetterTable, recordRebuildDeadLetters, } from "../event-store/rebuild-dead-letter.js";
+import { emitProjectionRebuild } from "../observability/standard-metrics.js";
+import { projectionStateTable } from "./projection-state.js";
+// Events replayed per catch-up batch. Each batch is a fresh READ COMMITTED
+// SELECT, so a batch shorter than this means the currently-committed tail is
+// drained.
+const REBUILD_BATCH_SIZE = 1000;
+// Cap on UNLOCKED catch-up batches before forcing the cutover fence. Bounds the
+// lock-free phase under sustained writes that never momentarily quiesce; the
+// fenced final drain then always terminates (no new event can commit once the
+// live table is held ACCESS EXCLUSIVE).
+const MAX_UNLOCKED_BATCHES = 10_000;
+const DEFAULT_FENCE_LOCK_TIMEOUT_MS = 5_000;
+function rowToStoredEvent(row) {
+    return {
+        id: String(row.id),
+        aggregateId: row.aggregateId,
+        aggregateType: row.aggregateType,
+        tenantId: row.tenantId,
+        version: row.version,
+        type: row.type,
+        eventVersion: row.eventVersion,
+        payload: row.payload,
+        metadata: row.metadata,
+        createdAt: row.createdAt,
+        createdBy: row.createdBy,
+    };
+}
+// Ops-facing log line for countColumnDrift's result. No-op when clean. #916.
+function warnOnColumnDrift(projectionName, tableName, drift) {
+    // skip: no drift found — nothing to warn about
+    if (drift.rowCount === 0)
+        return;
+    const sampledIds = new Set(drift.sample.map((s) => s.slice(0, s.lastIndexOf("."))));
+    const countLabel = sampledIds.size < drift.rowCount
+        ? `${drift.rowCount} (sample truncated)`
+        : String(drift.rowCount);
+    console.warn(`projection-rebuild "${projectionName}": ${countLabel} live row(s) in "${tableName}" have ` +
+        `column values that differ from the freshly-replayed shadow (sample: ${drift.sample.join(", ")}). ` +
+        "Not blocking — replay wins on swap. Likely cause: a legacy direct-write predating the row's " +
+        "event history (#494), or a sensitive custom field pending #972. Investigate if unexpected.");
+}
+export async function rebuildProjection(projectionName, deps) {
+    const { db, registry } = deps;
+    const projection = registry.getAllProjections().get(projectionName);
+    if (!projection) {
+        throw new Error(`Projection "${projectionName}" is not registered. Known: ${[...registry.getAllProjections().keys()].join(", ") || "(none)"}`);
+    }
+    const meta = rebuildMetaOrThrow(projection.table, projectionName);
+    const sources = Array.isArray(projection.source) ? projection.source : [projection.source];
+    const sourcesList = [...sources, ...(projection.extraSources ?? [])];
+    const subscribedList = Object.keys(projection.apply);
+    // Upcasters run at read time: older stored payloads get walked through the
+    // registered r.eventMigration chain until their shape matches the current
+    // event version.
+    const upcasters = registry.getEventUpcasters();
+    const eventsInfo = extractTableInfo(eventsTable);
+    const fenceLockTimeoutMs = deps.fenceLockTimeoutMs ?? DEFAULT_FENCE_LOCK_TIMEOUT_MS;
+    const startedAt = Date.now();
+    let eventsProcessed = 0;
+    let columnDriftCount = 0;
+    let lastProcessedEventId = 0n;
+    const skipApplyErrors = deps.errorPolicy?.skipApplyErrors === true;
+    // Quarantined events, collected in memory and written once after the
+    // replay settles — the #443 full-re-replay path would otherwise duplicate
+    // the dead-letter rows.
+    let skipped = [];
+    // One chronological batch of events after lastProcessedEventId, applied into
+    // the shadow. Returns the batch size so the caller can detect the tail
+    // (a short batch = no more currently-committed events).
+    const drainBatch = async (tx) => {
+        const rawEvents = await selectEventsForProjectionRebuildBatch(tx, sourcesList, subscribedList, lastProcessedEventId, REBUILD_BATCH_SIZE);
+        for (const r of rawEvents) {
+            deps.signal?.throwIfAborted();
+            const row = coerceRow(r, eventsInfo);
+            const storedEvent = await upcastStoredEvent(rowToStoredEvent(row), upcasters, {
+                db: tx,
+                tenantId: row.tenantId, // @cast-boundary db-row
+            });
+            const applyFn = projection.apply[row.type];
+            // skip: apply-key validation ensures every subscribed type has a handler;
+            //       defensive check against runtime-mutated registry
+            if (!applyFn)
+                continue;
+            if (skipApplyErrors) {
+                // Driver-native savepoint: a throwing SQL statement would otherwise
+                // poison the rebuild tx (25P02) AND make the driver reject the whole
+                // begin() even after a caught error. The apply runs on the
+                // savepoint-scoped handle so its statements are confined.
+                try {
+                    await runInSavepoint(tx, async (sp) => {
+                        await applyFn(storedEvent, sp, projection.table);
+                    });
+                }
+                catch (e) {
+                    skipped.push({ event: storedEvent, error: e });
+                }
+            }
+            else {
+                await applyFn(storedEvent, tx, projection.table);
+            }
+            eventsProcessed++;
+            lastProcessedEventId = row.id;
+        }
+        return rawEvents.length;
+    };
+    try {
+        await ensureRebuildSchema(db);
+        // Outside the rebuild tx, like the schema: idempotent DDL colliding
+        // inside the tx would roll the whole replay back.
+        if (skipApplyErrors)
+            await createRebuildDeadLetterTable(db);
+        await db.begin(async (tx) => {
+            await markProjectionRebuilding(tx, projectionName);
+            await assertLiveColumnsMatchMeta(tx, meta, projectionName);
+            await assertLiveTableHasNoRowLevelSecurity(tx, meta.tableName, projectionName);
+            await buildShadowTable(tx, meta);
+            deps.__test_onBuildShadowTable?.();
+            // A projection that subscribes to nothing has no events to replay and no
+            // live writer touching its table — skip straight to swapping the empty
+            // shadow (no fence needed).
+            if (subscribedList.length > 0) {
+                // Unlocked catch-up: drain batches until a short batch signals the
+                // currently-committed tail. Live synchronous applies keep writing to
+                // public.<table> meanwhile; READ COMMITTED makes each fresh batch see
+                // their newly-committed events. Capped so sustained writes can't keep
+                // the lock-free phase running forever.
+                for (let batches = 0; batches < MAX_UNLOCKED_BATCHES; batches++) {
+                    if ((await drainBatch(tx)) < REBUILD_BATCH_SIZE)
+                        break;
+                }
+                // Test seam: inject a mid-replay committed write here to prove the
+                // fenced final drain catches it instead of losing it at swap.
+                await deps.__test_onBeforeFence?.();
+                // Fence the live table, then drain the final delta. Once ACCESS
+                // EXCLUSIVE is held no concurrent apply can commit a new event, so this
+                // loop terminates and the shadow ends up reflecting every committed
+                // event — closing Phase 1's write-loss window for single-pod rebuilds.
+                await fenceLiveTable(tx, meta.tableName, fenceLockTimeoutMs);
+                while ((await drainBatch(tx)) === REBUILD_BATCH_SIZE) {
+                    // keep draining full batches; a short batch ends the loop
+                }
+                // Fenced → the subscribed-event set is final (every live apply blocks
+                // on the live-table lock). If fewer events were applied than exist, a
+                // lower-id event committed late during the unlocked phase and the
+                // id-cursor leapt past it (#443). Rebuild the shadow from scratch and
+                // replay the whole log under the fence: a fresh shadow means no
+                // double-apply (the rejected "re-drain into a populated shadow" hazard),
+                // and the full-replay cost is paid only on this rare detected path.
+                const expectedEvents = await countSubscribedEvents(tx, sourcesList, subscribedList);
+                if (expectedEvents > BigInt(eventsProcessed)) {
+                    await buildShadowTable(tx, meta);
+                    deps.__test_onBuildShadowTable?.();
+                    lastProcessedEventId = 0n;
+                    eventsProcessed = 0;
+                    skipped = [];
+                    while ((await drainBatch(tx)) === REBUILD_BATCH_SIZE) {
+                        // re-replay the full log into the fresh shadow
+                    }
+                    // drainBatch increments eventsProcessed for every drained row
+                    // regardless of quarantine (skipApplyErrors), so this must hold
+                    // exactly under the fence — a mismatch means drainBatch silently
+                    // dropped a row instead of counting it, which the rest of this
+                    // function has no other way to detect.
+                    if (eventsProcessed < Number(expectedEvents)) {
+                        throw new Error(`projection-rebuild "${projectionName}": fenced re-replay processed ` +
+                            `${eventsProcessed} events but ${expectedEvents} were expected — drainBatch ` +
+                            "silently dropped rows instead of counting them.");
+                    }
+                }
+            }
+            if (skipped.length > 0) {
+                await recordRebuildDeadLetters(tx, projectionName, skipped);
+            }
+            // Guard the swap: abort if the process has no blind-index key configured
+            // but the live table already has populated bidx columns — the rebuild
+            // would otherwise silently null them out (fw#3091). Applies to every
+            // projection, not just implicit ones: the shadow always gets a fresh
+            // computeBlindIndexValues() pass regardless of projection kind.
+            await assertNoBlindIndexLoss(tx, meta.tableName, meta, projectionName);
+            // Guard the swap: abort if the live table holds a row no event can
+            // reconstruct (#498 ghost — direct-inserted without a .created event),
+            // which the swap would silently drop. Implicit projections only.
+            if (projection.isImplicit === true) {
+                await assertNoUnreachableLiveRows(tx, projectionName, meta.tableName, sourcesList);
+                // Non-blocking counterpart: reports column-level drift (bidx excluded)
+                // without aborting — see countColumnDrift for why this isn't fail-hard.
+                const drift = await countColumnDrift(tx, meta.tableName, meta);
+                columnDriftCount = drift.rowCount;
+                warnOnColumnDrift(projectionName, meta.tableName, drift);
+            }
+            await finalizeProjectionRebuild(tx, projectionName, lastProcessedEventId);
+            await swapShadowIntoLive(tx, meta.tableName);
+        });
+    }
+    catch (e) {
+        // Outer catch: TX has been rolled back by Postgres already. Record the
+        // failure in a SEPARATE write so ops can see what happened.
+        const message = e instanceof Error ? e.message : String(e);
+        await markProjectionRebuildFailed(db, projectionName, message);
+        // Failure metric: duration until throw, 0 events "delivered" (the replayed
+        // rows were rolled back — counting them would overstate live delivery).
+        // success=false label distinguishes these in Prom dashboards.
+        if (deps.meter) {
+            emitProjectionRebuild(deps.meter, { projection: projectionName, success: false }, (Date.now() - startedAt) / 1000, 0);
+        }
+        throw e;
+    }
+    const result = {
+        projection: projectionName,
+        eventsProcessed,
+        eventsSkipped: skipped.length,
+        lastProcessedEventId,
+        durationMs: Date.now() - startedAt,
+        columnDriftCount,
+    };
+    if (deps.meter) {
+        emitProjectionRebuild(deps.meter, { projection: projectionName, success: true }, result.durationMs / 1000, eventsProcessed);
+    }
+    deps.onMetrics?.(result);
+    return result;
+}
+// Read-only status for one projection. Returns null if the projection was
+// registered but never rebuilt (no row yet).
+export async function getProjectionState(db, projectionName) {
+    const [row] = await selectMany(db, projectionStateTable, { name: projectionName });
+    if (!row)
+        return null;
+    return {
+        name: row.name,
+        status: row.status,
+        lastProcessedEventId: row.lastProcessedEventId,
+        lastRebuildAt: row.lastRebuildAt,
+        lastError: row.lastError,
+        updatedAt: row.updatedAt,
+    };
+}
+// List every registered projection with its current state (if any).
+// The registry is the source-of-truth for which projections exist; the
+// state table holds per-projection rebuild info and may be sparse.
+//
+// Implicit-Projections (auto-registered pro r.entity, eine pro entity)
+// werden default ausgefiltert — sie sind als rebuild-Ziele weiter mit
+// `<feature>:projection:<entity>-entity` adressierbar, aber in `kumiko
+// project list` würden sie das Bild dominieren ohne Mehrwert. Mit
+// `{ includeImplicit: true }` opt-in einschalten.
+export async function listProjectionsWithState(db, registry, options = {}) {
+    const projections = registry.getAllProjections();
+    const stateRows = await selectMany(db, projectionStateTable);
+    const stateByName = new Map(stateRows.map((r) => [r.name, r]));
+    return [...projections.values()]
+        .filter((proj) => options.includeImplicit === true || !proj.isImplicit)
+        .map((proj) => {
+        const state = stateByName.get(proj.name);
+        const sources = Array.isArray(proj.source) ? proj.source : [proj.source];
+        return {
+            name: proj.name,
+            sources,
+            status: state?.status ?? "never-rebuilt",
+            lastProcessedEventId: state?.lastProcessedEventId ?? 0n,
+            lastRebuildAt: state?.lastRebuildAt ?? null,
+            lastError: state?.lastError ?? null,
+        };
+    });
+}
+// Extended variant of listProjectionsWithState that also returns HWM and lag
+// per projection. One extra cheap MAX-aggregate query — no additional
+// roundtrip per projection. Programmatic callers (e.g. a Prometheus gauge
+// exporter) can map the result directly to a `kumiko_projection_lag{name}`
+// gauge.
+export async function getAllProjectionProgress(db, registry) {
+    const [projections, highWaterMark] = await Promise.all([
+        listProjectionsWithState(db, registry),
+        getEventsHighWaterMark(db),
+    ]);
+    return projections.map((p) => ({
+        ...p,
+        highWaterMark,
+        lag: highWaterMark - p.lastProcessedEventId,
+    }));
+}

package/dist/pipeline/projection-state.d.ts

@@ -0,0 +1,9 @@
+import type { DbConnection } from "../db/connection.js";
+export declare const projectionStateTable: import("@cosmicdrift/kumiko-types/schema-table-types").SchemaTable;
+export declare const ProjectionStatuses: {
+    readonly idle: "idle";
+    readonly rebuilding: "rebuilding";
+    readonly failed: "failed";
+};
+export type ProjectionStatus = (typeof ProjectionStatuses)[keyof typeof ProjectionStatuses];
+export declare function createProjectionStateTable(db: DbConnection): Promise<void>;

package/dist/pipeline/projection-state.js

@@ -0,0 +1,51 @@
+import { bigint, index, instant, table as pgTable, sql, text } from "../db/dialect.js";
+import { tableExists } from "../db/schema-inspection.js";
+import { unsafePushTables } from "../stack/index.js";
+// Framework-level state for every registered projection. One row per qualified
+// projection name. Written by the rebuild machinery; read by the CLI + any
+// status dashboard. Lives alongside the events table as framework infra —
+// user projection tables stay separate and user-owned.
+//
+// Columns:
+//   - name: projection's qualified name (feature:projection:shortname)
+//   - lastProcessedEventId: the bigserial `events.id` of the most recent
+//     event that was applied. Rebuild uses it as the cursor for what's
+//     done; live writes DON'T currently update it (synchronous apply means
+//     no meaningful lag, see projections-runner.ts). Once async apply lands
+//     in B.3+, this becomes the lag source.
+//   - status: "idle" (normal) | "rebuilding" (in-progress) | "failed"
+//   - lastRebuildAt: wall-clock time the last full rebuild finished
+//   - lastError: last error message when status = "failed" — rebuild sets
+//     this from the thrown message so ops can see it in `project status`
+// last_processed_event_id uses a raw DEFAULT 0 instead of .default(0n) because
+// drizzle-kit's JSON snapshot generator cannot serialise bigint literals —
+// `TypeError: Do not know how to serialize a BigInt` bubbles through
+// unsafePushTables → generateMigration. `sql\`0\`` yields the same server-side
+// default without ever putting a bigint in a generated-JSON path.
+export const projectionStateTable = pgTable("kumiko_projections", {
+    name: text("name").primaryKey(),
+    lastProcessedEventId: bigint("last_processed_event_id", { mode: "bigint" })
+        .notNull()
+        .default(sql `0`),
+    status: text("status").notNull().default("idle"),
+    lastRebuildAt: instant("last_rebuild_at", { precision: 3 }),
+    lastError: text("last_error"),
+    updatedAt: instant("updated_at", { precision: 3 }).notNull().default(sql `now()`),
+}, (t) => ({
+    statusIdx: index("kumiko_projections_status_idx").on(t.status),
+}));
+export const ProjectionStatuses = {
+    idle: "idle",
+    rebuilding: "rebuilding",
+    failed: "failed",
+};
+// Idempotent table bootstrap. Called by setupTestStack (and createApp once
+// that wires it up) — same pattern as createEventsTable. If the table is
+// already there (second stack in same test DB, production boot after
+// migration), skip cleanly.
+export async function createProjectionStateTable(db) {
+    // skip: table already exists — bootstrap is called from multiple paths
+    if (await tableExists(db, "public.kumiko_projections"))
+        return;
+    await unsafePushTables(db, { kumikoProjections: projectionStateTable });
+}

package/dist/pipeline/projections-runner.d.ts

@@ -0,0 +1,5 @@
+import type { DbRunner } from "../db/index.js";
+import type { LifecycleResult, Registry } from "../engine/types/index.js";
+import type { StoredEvent } from "../event-store/index.js";
+export declare function runProjections(result: LifecycleResult, registry: Registry, runner: DbRunner | undefined): Promise<void>;
+export declare function runProjectionsForEvent(event: StoredEvent, registry: Registry, tx: DbRunner): Promise<void>;

package/dist/pipeline/projections-runner.js

@@ -0,0 +1,57 @@
+import { InternalError } from "../errors/index.js";
+// Run custom projections for a save or delete result. Lives INSIDE the
+// transaction that appended the event — a throw from apply() rolls the event
+// back along with any auto-projection write.
+//
+// Why in the pipeline, not in the executor:
+//   Executors used to take an optional `registry` per call. Every caller
+//   (crud-builder, manual handlers, seed scripts, future replay tools) had to
+//   remember to pass it — forgetting meant projections silently didn't fire.
+//   Putting the trigger here, keyed off the StoredEvent the executor surfaces
+//   on SaveContext/DeleteContext, closes that hole: every write that went
+//   through the dispatcher gets its projections, no opt-in needed.
+//
+// Contracts:
+//   - Projections receive the exact StoredEvent from the executor. If you
+//     hand-craft a SaveContext (tests, non-executor writes), just don't set
+//     `event` and the runner no-ops.
+//   - `runner` is the caller's already-resolved DbRunner — projections-runner
+//     has no TenantDb access of its own to derive one from.
+//   - Apply-function throws bubble up unchanged. The dispatcher wraps the
+//     whole lifecycle in a try/catch that rolls the tx back; the event is
+//     gone from the events table just like a rolled-back state change.
+export async function runProjections(result, registry, runner) {
+    // skip: hand-crafted result with no event — nothing to project
+    if (!result.event)
+        return;
+    if (!runner) {
+        throw new InternalError({
+            message: `runProjections("${result.event.aggregateType}") requires a database connection — none is configured.`,
+        });
+    }
+    await runProjectionsForEvent(result.event, registry, runner);
+}
+// Fire every projection whose source matches the event's aggregate type AND
+// that declares an apply-handler for the event's type. Used by both the
+// CRUD path (via runProjections) and the ctx.appendEvent path (domain events
+// emitted inside a write handler). Keeping one function means an auto-event
+// and a r.defineEvent-event land in the same inline-projection pipeline.
+export async function runProjectionsForEvent(event, registry, tx) {
+    const projections = registry.getProjectionsForSource(event.aggregateType);
+    // skip: no projection feeds off this entity — fast path for the common case
+    if (projections.length === 0)
+        return;
+    for (const proj of projections) {
+        // ImplicitProjections existieren nur für rebuildProjection — der
+        // EventStoreExecutor schreibt im Live-Pfad bereits direkt in die
+        // Tabelle. Live-Apply der Implicit würde doppelt schreiben → unique
+        // key violation. Filter ist Pflicht.
+        if (proj.isImplicit)
+            continue;
+        const applyFn = proj.apply[event.type];
+        // skip: this projection doesn't care about this event type
+        if (!applyFn)
+            continue;
+        await applyFn(event, tx, proj.table);
+    }
+}

package/dist/pipeline/redis-keys.d.ts

@@ -0,0 +1,8 @@
+export declare const RedisKeys: {
+    readonly idempotency: "kumiko:idempotency:";
+    readonly eventDedup: "kumiko:event-dedup:";
+    readonly entityCache: "kumiko:cache:";
+    readonly lock: "kumiko:lock:";
+    readonly events: "kumiko:events";
+    readonly rateLimit: "kumiko:rl:";
+};

package/dist/pipeline/redis-keys.js

@@ -0,0 +1,10 @@
+// Central registry of all Redis key prefixes used by the framework.
+// Prevents prefix collisions and makes key usage discoverable.
+export const RedisKeys = {
+    idempotency: "kumiko:idempotency:",
+    eventDedup: "kumiko:event-dedup:",
+    entityCache: "kumiko:cache:",
+    lock: "kumiko:lock:",
+    events: "kumiko:events",
+    rateLimit: "kumiko:rl:",
+};

package/dist/pipeline/system-hooks.d.ts

@@ -0,0 +1,25 @@
+import type { SseBroker } from "../api/sse-broker.js";
+import type { EntityId, JobRunnerRef, Registry, TenantId } from "../engine/types/index.js";
+import type { SearchAdapter, SearchDocument } from "../search/types.js";
+import type { EventConsumer } from "./event-dispatcher.js";
+export declare const SEARCH_CONSUMER_NAME = "system:consumer:search";
+export type SearchOperation = {
+    readonly kind: "index";
+    readonly tenantId: TenantId;
+    readonly doc: SearchDocument;
+} | {
+    readonly kind: "remove";
+    readonly tenantId: TenantId;
+    readonly entityType: string;
+    readonly entityId: EntityId;
+};
+export declare function createSearchEventConsumer(searchAdapter: SearchAdapter, registry: Registry): EventConsumer;
+export declare function decryptSearchableSubjectFields(entityName: string, state: Record<string, unknown>, registry: Registry): Promise<Record<string, unknown>>;
+export declare function hasErasedSearchableSubjectField(entityName: string, state: Record<string, unknown>, registry: Registry): boolean;
+export declare function buildSearchDocument(entityName: string, entityId: EntityId, state: Record<string, unknown>, registry: Registry): Promise<SearchDocument | null>;
+export declare const SSE_BROADCAST_CONSUMER_NAME = "system:consumer:sse-broadcast";
+export declare function createSseBroadcastEventConsumer(sseBroker: SseBroker): EventConsumer;
+export declare const JOB_TRIGGER_CONSUMER_NAME = "system:consumer:job-trigger";
+export declare function createJobTriggerEventConsumer(jobRunner: JobRunnerRef, registry: Registry): EventConsumer;
+export declare const ACCESS_INVALIDATION_CONSUMER_NAME = "system:consumer:access-invalidation";
+export declare function createAccessInvalidationEventConsumer(sseBroker: SseBroker): EventConsumer;