@cosmicdrift/kumiko-framework 0.1.0

package/src/pipeline/multi-stream-apply-context.ts

@@ -0,0 +1,115 @@
+import type { DbRunner } from "../db/connection";
+import type {
+  AppendEventArgs,
+  AppendEventFn,
+  AppendEventUnsafeFn,
+  KumikoEventTypeMap,
+  Registry,
+  TenantId,
+} from "../engine/types";
+import { loadAggregate, loadAggregateAsOf, type StoredEvent } from "../event-store/event-store";
+import { upcastStoredEvents } from "../event-store/upcaster";
+import type { FileContext } from "../files/file-handle";
+import { appendDomainEventCore } from "./append-event-core";
+
+// Minimal, read+write surface handed to a MultiStreamProjection's apply()
+// when it needs to produce follow-up events (saga / process-manager
+// pattern). Keeps the MSP feature-decoupled: applies don't reach into
+// handler-bridge (no query/write/writeAs), they just read the aggregate
+// stream and append new events — Marten's session scope for projections.
+//
+// TMap propagates the strict event-type-map (see HandlerContext). Default
+// matches the global KumikoEventTypeMap; runtime-pluggable callers route
+// through appendEventUnsafe.
+export type MultiStreamApplyContext<TMap extends object = KumikoEventTypeMap> = {
+  // Append a domain event onto an aggregate stream in the CURRENT tx.
+  // Schema-validated, archive-guarded, stream-version derived. Metadata
+  // inherits from the triggering event (correlationId) + requestContext
+  // (causationId is already set to the triggering event.id by the
+  // dispatcher wrap). Strict against KumikoEventTypeMap — same contract
+  // as HandlerContext.appendEvent (compile-time-validated payload).
+  readonly appendEvent: AppendEventFn<TMap>;
+  // Escape hatch for runtime-pluggable events without compile-time
+  // augmentation. Same runtime semantics; type-surface is `payload: unknown`.
+  readonly appendEventUnsafe: AppendEventUnsafeFn;
+  // Read an aggregate stream — useful when a saga needs to inspect the
+  // current state of a different aggregate before deciding what to emit.
+  readonly loadAggregate: (
+    aggregateId: string,
+    options?: { readonly asOf?: Temporal.Instant },
+  ) => Promise<readonly StoredEvent[]>;
+  // Binary storage handle factory, mirrors AppContext.files. Present when
+  // the app booted with `files.storageProvider`; undefined otherwise.
+  // Post-processing MSPs (resize, EXIF-strip, virus-scan) read bytes via
+  // `ctx.files.ref(payload.storageKey).read()` and write derivates via
+  // `.derive("thumb").write(...)` — binaries never ride through events.
+  readonly files?: FileContext;
+};
+
+export type MultiStreamApplyContextDeps = {
+  readonly registry: Registry;
+  // TX-scoped DbRunner — the same `tx` the applyFn receives as the 2nd
+  // arg. ctx.appendEvent + inline-projections run inside this tx so a
+  // throw rolls the whole hop back (consumer retries the triggering
+  // event on the next pass).
+  readonly db: DbRunner;
+  // tenantId + userId of the TRIGGERING event. appendEvent stamps these
+  // onto the new event so the causal chain stays tenant-consistent and
+  // the downstream audit-trail can reconstruct the acting principal.
+  readonly tenantId: TenantId;
+  readonly userId: string;
+  // MSP's owning feature (prefix of its qualified name). Enforced at
+  // emit-site: the MSP cannot ctx.appendEvent a type owned by another
+  // feature. Cross-feature reactions are fine inbound (this MSP is
+  // subscribed to events from any feature), but outbound appends must
+  // stay within the MSP's own feature.
+  readonly callerFeature?: string;
+  // Same FileContext the outer AppContext carries, passed through so
+  // MSP applies can reach binaries without another wiring indirection.
+  readonly files?: FileContext;
+};
+
+export function createMultiStreamApplyContext(
+  deps: MultiStreamApplyContextDeps,
+): MultiStreamApplyContext {
+  return {
+    ...(deps.files ? { files: deps.files } : {}),
+    // @cast-boundary engine-bridge — concrete impl conforms to AppendEventFn overload
+    appendEvent: (async (args: AppendEventArgs) => {
+      await appendDomainEventCore(
+        {
+          registry: deps.registry,
+          db: deps.db,
+          tenantId: deps.tenantId,
+          userId: deps.userId,
+          callSiteLabel: "MSP-apply ctx.appendEvent",
+          ...(deps.callerFeature && { callerFeature: deps.callerFeature }),
+        },
+        args,
+      );
+    }) as AppendEventFn,
+    appendEventUnsafe: async (args) => {
+      await appendDomainEventCore(
+        {
+          registry: deps.registry,
+          db: deps.db,
+          tenantId: deps.tenantId,
+          userId: deps.userId,
+          callSiteLabel: "MSP-apply ctx.appendEventUnsafe",
+          ...(deps.callerFeature && { callerFeature: deps.callerFeature }),
+        },
+        args,
+      );
+    },
+
+    loadAggregate: async (aggregateId, options) => {
+      const events = options?.asOf
+        ? await loadAggregateAsOf(deps.db, aggregateId, deps.tenantId, options.asOf)
+        : await loadAggregate(deps.db, aggregateId, deps.tenantId);
+      return upcastStoredEvents(events, deps.registry.getEventUpcasters(), {
+        db: deps.db,
+        tenantId: deps.tenantId,
+      });
+    },
+  };
+}

package/src/pipeline/projection-rebuild.ts

@@ -0,0 +1,334 @@
+import { asc, eq, getTableName, inArray, sql } from "drizzle-orm";
+import type { DbConnection } from "../db/connection";
+import type { Registry, TenantId } from "../engine/types";
+import {
+  eventsTable,
+  getEventsHighWaterMark,
+  type StoredEvent,
+  upcastStoredEvent,
+} from "../event-store";
+import { emitProjectionRebuild } from "../observability/standard-metrics";
+import type { Meter } from "../observability/types/metric";
+import { projectionStateTable } from "./projection-state";
+
+// Rebuild a projection from the event log.
+//
+// Mechanics:
+//   1. Lock the projection's state row FOR UPDATE. Concurrent rebuild
+//      attempts of the same projection block here instead of racing.
+//   2. Mark status = "rebuilding".
+//   3. TRUNCATE the projection's backing table.
+//   4. Stream events in chronological order, for every apply-key match
+//      invoke apply(event, tx). Event-by-event, so two projections of the
+//      same source stay semantically identical to the live pipeline.
+//   5. Store the last processed event-id + mark status = "idle".
+//
+// All of that runs in ONE transaction. If apply throws partway through,
+// Postgres rolls back everything — the old projection is still there,
+// status goes back to "idle" via the outer catch, and lastError records
+// what went wrong. A partial/empty projection is never observable.
+//
+// This is an ops-time operation. While a rebuild is in progress, live
+// writes that touch the same projection will also try to insert into the
+// TRUNCATE'd table, triggering either a serialization conflict or (for a
+// new row after TRUNCATE) a noisy conflict. Intended behaviour: rebuild
+// on a quiet entity, or during a deliberate write-pause.
+//
+// Scale limit: single-TX TRUNCATE + replay works as long as your
+// maintenance window absorbs the replay. Effective ceiling depends on
+// payload size, apply() cost, and DB load — measure before trusting it.
+// Beyond that window, plan for a shadow-swap variant. For v1 that's
+// documented as a known boundary in docs/projections.md.
+
+export type RebuildResult = {
+  readonly projection: string;
+  readonly eventsProcessed: number;
+  readonly lastProcessedEventId: bigint;
+  readonly durationMs: number;
+};
+
+type RebuildDeps = {
+  readonly db: DbConnection;
+  readonly registry: Registry;
+  // Optional framework meter. When provided, the runner emits the two
+  // projection-rebuild metrics (duration histogram + events counter) on both
+  // success and failure paths — the Prometheus-facing surface. CLI callers
+  // can leave it undefined and rely on stdout feedback.
+  readonly meter?: Meter;
+  // Lightweight observation callback for tests that want to assert the
+  // RebuildResult without spinning up a full meter. Independent of `meter`.
+  readonly onMetrics?: (result: RebuildResult) => void;
+  // Cancellation. Checked before each event-apply. The transaction is
+  // rolled back on abort — a partial rebuild is never observable. Useful
+  // when an HTTP-triggered rebuild needs to honour client disconnect, or
+  // when a CLI/Job wraps the rebuild in its own AbortController for ops
+  // timeout enforcement.
+  readonly signal?: AbortSignal;
+};
+
+export async function rebuildProjection(
+  projectionName: string,
+  deps: RebuildDeps,
+): Promise<RebuildResult> {
+  const { db, registry } = deps;
+  const projection = registry.getAllProjections().get(projectionName);
+  if (!projection) {
+    throw new Error(
+      `Projection "${projectionName}" is not registered. Known: ${
+        [...registry.getAllProjections().keys()].join(", ") || "(none)"
+      }`,
+    );
+  }
+
+  const sources = Array.isArray(projection.source) ? projection.source : [projection.source];
+  const startedAt = Date.now();
+  let eventsProcessed = 0;
+  let lastProcessedEventId = 0n;
+
+  try {
+    await db.transaction(async (tx) => {
+      // Lock the state row. Use upsert so a never-rebuilt projection also
+      // gets a row. FOR UPDATE would need the row to exist — upsert-first
+      // keeps it idempotent.
+      await tx
+        .insert(projectionStateTable)
+        .values({ name: projectionName, status: "rebuilding" })
+        .onConflictDoUpdate({
+          target: projectionStateTable.name,
+          set: {
+            status: "rebuilding",
+            lastError: null,
+            updatedAt: sql`now()`,
+          },
+        });
+
+      // Wipe the projection table. drizzle-orm's public API doesn't expose
+      // TRUNCATE, so we issue raw SQL — but `getTableName()` is the public
+      // accessor for the table's registered name, avoiding Symbol.for()
+      // internal lookups. The identifier is still quoted defensively.
+      const tableName = getTableName(projection.table);
+      await tx.execute(sql.raw(`TRUNCATE TABLE ${quoteIdent(tableName)}`));
+
+      // Stream events in chronological order for every source. The event
+      // type filter (inArray(type, validTypes)) prunes events the projection
+      // doesn't care about early — important when a single source has more
+      // event types than the projection subscribes to.
+      const subscribed = Object.keys(projection.apply);
+      if (subscribed.length === 0) {
+        // nothing to replay, just mark idle — projection exists but doesn't
+        // subscribe to any event types on its sources yet.
+      } else {
+        const events = (await tx
+          .select()
+          .from(eventsTable)
+          .where(
+            sql`${inArray(eventsTable.aggregateType, sources)} AND ${inArray(
+              eventsTable.type,
+              subscribed,
+            )}`,
+          )
+          .orderBy(asc(eventsTable.id))) as ReadonlyArray<typeof eventsTable.$inferSelect>;
+
+        // Upcasters run at read time: older stored payloads get walked
+        // through the registered r.eventMigration chain until their shape
+        // matches the current event version. An apply() written against the
+        // v3 shape stays oblivious to v1 payloads still on disk.
+        const upcasters = registry.getEventUpcasters();
+        for (const row of events) {
+          deps.signal?.throwIfAborted();
+          const raw: StoredEvent = {
+            id: String(row.id),
+            aggregateId: row.aggregateId,
+            aggregateType: row.aggregateType,
+            tenantId: row.tenantId,
+            version: row.version,
+            type: row.type,
+            eventVersion: row.eventVersion,
+            payload: row.payload,
+            metadata: row.metadata,
+            createdAt: row.createdAt,
+            createdBy: row.createdBy,
+          };
+          const storedEvent = await upcastStoredEvent(raw, upcasters, {
+            db: tx,
+            tenantId: row.tenantId as TenantId,
+          });
+          const applyFn = projection.apply[row.type];
+          // skip: apply-key validation ensures every subscribed type has a
+          //       handler; defensive check against runtime-mutated registry
+          if (!applyFn) continue;
+          await applyFn(storedEvent, tx);
+          eventsProcessed++;
+          lastProcessedEventId = row.id;
+        }
+      }
+
+      // Finalize state row.
+      await tx
+        .update(projectionStateTable)
+        .set({
+          lastProcessedEventId,
+          status: "idle",
+          lastRebuildAt: sql`now()`,
+          lastError: null,
+          updatedAt: sql`now()`,
+        })
+        .where(eq(projectionStateTable.name, projectionName));
+    });
+  } catch (e) {
+    // Outer catch: TX has been rolled back by Postgres already. Record the
+    // failure in a SEPARATE write so ops can see what happened — the
+    // rolled-back status change is gone, so we write failed+error now.
+    const message = e instanceof Error ? e.message : String(e);
+    await db
+      .insert(projectionStateTable)
+      .values({ name: projectionName, status: "failed", lastError: message })
+      .onConflictDoUpdate({
+        target: projectionStateTable.name,
+        set: { status: "failed", lastError: message, updatedAt: sql`now()` },
+      });
+    // Failure metric: duration until throw, 0 events "delivered" (the replayed
+    // rows were rolled back — counting them would overstate live delivery).
+    // success=false label distinguishes these in Prom dashboards.
+    if (deps.meter) {
+      emitProjectionRebuild(
+        deps.meter,
+        { projection: projectionName, success: false },
+        (Date.now() - startedAt) / 1000,
+        0,
+      );
+    }
+    throw e;
+  }
+
+  const result: RebuildResult = {
+    projection: projectionName,
+    eventsProcessed,
+    lastProcessedEventId,
+    durationMs: Date.now() - startedAt,
+  };
+  if (deps.meter) {
+    emitProjectionRebuild(
+      deps.meter,
+      { projection: projectionName, success: true },
+      result.durationMs / 1000,
+      eventsProcessed,
+    );
+  }
+  deps.onMetrics?.(result);
+  return result;
+}
+
+// Identifier quoting for raw TRUNCATE. Drizzle doesn't expose a safe helper
+// for table-name interpolation in raw SQL; double-quote + escape double-quote
+// matches Postgres identifier rules.
+function quoteIdent(name: string): string {
+  return `"${name.replace(/"/g, '""')}"`;
+}
+
+// Read-only status for one projection. Returns null if the projection was
+// registered but never rebuilt (no row yet).
+export async function getProjectionState(
+  db: DbConnection,
+  projectionName: string,
+): Promise<{
+  readonly name: string;
+  readonly status: string;
+  readonly lastProcessedEventId: bigint;
+  readonly lastRebuildAt: Temporal.Instant | null;
+  readonly lastError: string | null;
+  readonly updatedAt: Temporal.Instant;
+} | null> {
+  const [row] = await db
+    .select()
+    .from(projectionStateTable)
+    .where(eq(projectionStateTable.name, projectionName));
+  if (!row) return null;
+  return {
+    name: row.name,
+    status: row.status,
+    lastProcessedEventId: row.lastProcessedEventId,
+    lastRebuildAt: row.lastRebuildAt,
+    lastError: row.lastError,
+    updatedAt: row.updatedAt,
+  };
+}
+
+// List every registered projection with its current state (if any).
+// The registry is the source-of-truth for which projections exist; the
+// state table holds per-projection rebuild info and may be sparse.
+//
+// Implicit-Projections (auto-registered pro r.entity, eine pro entity)
+// werden default ausgefiltert — sie sind als rebuild-Ziele weiter mit
+// `<feature>:projection:<entity>-entity` adressierbar, aber in `kumiko
+// project list` würden sie das Bild dominieren ohne Mehrwert. Mit
+// `{ includeImplicit: true }` opt-in einschalten.
+export async function listProjectionsWithState(
+  db: DbConnection,
+  registry: Registry,
+  options: { readonly includeImplicit?: boolean } = {},
+): Promise<
+  ReadonlyArray<{
+    readonly name: string;
+    readonly sources: readonly string[];
+    readonly status: string;
+    readonly lastProcessedEventId: bigint;
+    readonly lastRebuildAt: Temporal.Instant | null;
+    readonly lastError: string | null;
+  }>
+> {
+  const projections = registry.getAllProjections();
+  const stateRows = await db.select().from(projectionStateTable);
+  const stateByName = new Map(stateRows.map((r) => [r.name, r]));
+
+  return [...projections.values()]
+    .filter((proj) => options.includeImplicit === true || !proj.isImplicit)
+    .map((proj) => {
+      const state = stateByName.get(proj.name);
+      const sources = Array.isArray(proj.source) ? proj.source : [proj.source];
+      return {
+        name: proj.name,
+        sources,
+        status: state?.status ?? "never-rebuilt",
+        lastProcessedEventId: state?.lastProcessedEventId ?? 0n,
+        lastRebuildAt: state?.lastRebuildAt ?? null,
+        lastError: state?.lastError ?? null,
+      };
+    });
+}
+
+export type ProjectionProgress = {
+  readonly name: string;
+  readonly sources: readonly string[];
+  readonly status: string;
+  readonly lastProcessedEventId: bigint;
+  readonly lastRebuildAt: Temporal.Instant | null;
+  readonly lastError: string | null;
+  // Global MAX(events.id) at query time.
+  readonly highWaterMark: bigint;
+  // HWM - cursor. 0n when caught-up. Cannot be negative (that would mean
+  // the projection is ahead of HWM = bug). Used by ops dashboards to
+  // visualize projection lag.
+  readonly lag: bigint;
+};
+
+// Extended variant of listProjectionsWithState that also returns HWM and lag
+// per projection. One extra cheap MAX-aggregate query — no additional
+// roundtrip per projection. Programmatic callers (e.g. a Prometheus gauge
+// exporter) can map the result directly to a `kumiko_projection_lag{name}`
+// gauge.
+export async function getAllProjectionProgress(
+  db: DbConnection,
+  registry: Registry,
+): Promise<readonly ProjectionProgress[]> {
+  const [projections, highWaterMark] = await Promise.all([
+    listProjectionsWithState(db, registry),
+    getEventsHighWaterMark(db),
+  ]);
+
+  return projections.map((p) => ({
+    ...p,
+    highWaterMark,
+    lag: highWaterMark - p.lastProcessedEventId,
+  }));
+}

package/src/pipeline/projection-state.ts

@@ -0,0 +1,72 @@
+import { sql } from "drizzle-orm";
+import type { DbConnection } from "../db/connection";
+import { bigint, index, instant, table as pgTable, text } from "../db/dialect";
+import { tableExists } from "../db/schema-inspection";
+import { pushTables } from "../stack";
+
+// Framework-level state for every registered projection. One row per qualified
+// projection name. Written by the rebuild machinery; read by the CLI + any
+// status dashboard. Lives alongside the events table as framework infra —
+// user projection tables stay separate and user-owned.
+//
+// Columns:
+//   - name: projection's qualified name (feature:projection:shortname)
+//   - lastProcessedEventId: the bigserial `events.id` of the most recent
+//     event that was applied. Rebuild uses it as the cursor for what's
+//     done; live writes DON'T currently update it (synchronous apply means
+//     no meaningful lag, see projections-runner.ts). Once async apply lands
+//     in B.3+, this becomes the lag source.
+//   - status: "idle" (normal) | "rebuilding" (in-progress) | "failed"
+//   - lastRebuildAt: wall-clock time the last full rebuild finished
+//   - lastError: last error message when status = "failed" — rebuild sets
+//     this from the thrown message so ops can see it in `project status`
+// last_processed_event_id uses a raw DEFAULT 0 instead of .default(0n) because
+// drizzle-kit's JSON snapshot generator cannot serialise bigint literals —
+// `TypeError: Do not know how to serialize a BigInt` bubbles through
+// pushTables → generateMigration. `sql\`0\`` yields the same server-side
+// default without ever putting a bigint in a generated-JSON path.
+export const projectionStateTable = pgTable(
+  "kumiko_projections",
+  {
+    name: text("name").primaryKey(),
+    lastProcessedEventId: bigint("last_processed_event_id", { mode: "bigint" })
+      .notNull()
+      .default(sql`0`),
+    status: text("status").notNull().default("idle"),
+    lastRebuildAt: instant("last_rebuild_at", { precision: 3 }),
+    lastError: text("last_error"),
+    updatedAt: instant("updated_at", { precision: 3 }).notNull().default(sql`now()`),
+  },
+  (t) => ({
+    statusIdx: index("kumiko_projections_status_idx").on(t.status),
+  }),
+);
+
+export const ProjectionStatuses = {
+  idle: "idle",
+  rebuilding: "rebuilding",
+  failed: "failed",
+} as const;
+export type ProjectionStatus = (typeof ProjectionStatuses)[keyof typeof ProjectionStatuses];
+
+/**
+ * @deprecated Use `ProjectionStatuses` (object form) or the `ProjectionStatus`
+ * union type. Tuple alias kept for back-compat with callers that relied on
+ * the array form (`z.enum(...)`, runtime iteration) — scheduled for removal
+ * after downstream migration.
+ */
+export const PROJECTION_STATUSES = [
+  "idle",
+  "rebuilding",
+  "failed",
+] as const satisfies readonly ProjectionStatus[];
+
+// Idempotent table bootstrap. Called by setupTestStack (and createApp once
+// that wires it up) — same pattern as createEventsTable. If the table is
+// already there (second stack in same test DB, production boot after
+// migration), skip cleanly.
+export async function createProjectionStateTable(db: DbConnection): Promise<void> {
+  // skip: table already exists — bootstrap is called from multiple paths
+  if (await tableExists(db, "public.kumiko_projections")) return;
+  await pushTables(db, { kumikoProjections: projectionStateTable });
+}

package/src/pipeline/projections-runner.ts

@@ -0,0 +1,56 @@
+import type { DbRunner } from "../db";
+import type { HandlerContext, LifecycleResult, Registry } from "../engine/types";
+import type { StoredEvent } from "../event-store";
+
+// Run custom projections for a save or delete result. Lives INSIDE the
+// transaction that appended the event — a throw from apply() rolls the event
+// back along with any auto-projection write.
+//
+// Why in the pipeline, not in the executor:
+//   Executors used to take an optional `registry` per call. Every caller
+//   (crud-builder, manual handlers, seed scripts, future replay tools) had to
+//   remember to pass it — forgetting meant projections silently didn't fire.
+//   Putting the trigger here, keyed off the StoredEvent the executor surfaces
+//   on SaveContext/DeleteContext, closes that hole: every write that went
+//   through the dispatcher gets its projections, no opt-in needed.
+//
+// Contracts:
+//   - Projections receive the exact StoredEvent from the executor. If you
+//     hand-craft a SaveContext (tests, non-executor writes), just don't set
+//     `event` and the runner no-ops.
+//   - `tx`-scoped DbRunner is passed via the registered apply() — we reuse
+//     `ctx.db.raw`, which the dispatcher already scoped to the active tx.
+//   - Apply-function throws bubble up unchanged. The dispatcher wraps the
+//     whole lifecycle in a try/catch that rolls the tx back; the event is
+//     gone from the events table just like a rolled-back state change.
+export async function runProjections(result: LifecycleResult, ctx: HandlerContext): Promise<void> {
+  // skip: hand-crafted result with no event — nothing to project
+  if (!result.event) return;
+  await runProjectionsForEvent(result.event, ctx.registry, ctx.db.raw);
+}
+
+// Fire every projection whose source matches the event's aggregate type AND
+// that declares an apply-handler for the event's type. Used by both the
+// CRUD path (via runProjections) and the ctx.appendEvent path (domain events
+// emitted inside a write handler). Keeping one function means an auto-event
+// and a r.defineEvent-event land in the same inline-projection pipeline.
+export async function runProjectionsForEvent(
+  event: StoredEvent,
+  registry: Registry,
+  tx: DbRunner,
+): Promise<void> {
+  const projections = registry.getProjectionsForSource(event.aggregateType);
+  // skip: no projection feeds off this entity — fast path for the common case
+  if (projections.length === 0) return;
+  for (const proj of projections) {
+    // ImplicitProjections existieren nur für rebuildProjection — der
+    // EventStoreExecutor schreibt im Live-Pfad bereits direkt in die
+    // Tabelle. Live-Apply der Implicit würde doppelt schreiben → unique
+    // key violation. Filter ist Pflicht.
+    if (proj.isImplicit) continue;
+    const applyFn = proj.apply[event.type];
+    // skip: this projection doesn't care about this event type
+    if (!applyFn) continue;
+    await applyFn(event, tx);
+  }
+}

package/src/pipeline/redis-keys.ts

@@ -0,0 +1,11 @@
+// Central registry of all Redis key prefixes used by the framework.
+// Prevents prefix collisions and makes key usage discoverable.
+
+export const RedisKeys = {
+  idempotency: "kumiko:idempotency:",
+  eventDedup: "kumiko:event-dedup:",
+  entityCache: "kumiko:cache:",
+  lock: "kumiko:lock:",
+  events: "kumiko:events",
+  rateLimit: "kumiko:rl:",
+} as const;