@cosmicdrift/kumiko-framework 0.1.0

package/src/event-store/event-store.ts

@@ -0,0 +1,405 @@
+import { and, asc, eq, gt, lte, max, sql } from "drizzle-orm";
+import type { DbRunner } from "../db";
+import { isUniqueViolation } from "../db/pg-error";
+import type { TenantId } from "../engine/types";
+import { isStreamArchived } from "./archive";
+import { VersionConflictError } from "./errors";
+import { eventsTable } from "./events-schema";
+
+export type EventMetadata = {
+  readonly userId: string;
+  readonly requestId?: string;
+  // End-to-end business-operation id. Root HTTP requests get it from the
+  // x-correlation-id header (default: requestId). MSP-applies inherit it
+  // from the triggering event. Lets you trace "which user click caused
+  // this email 3 streams later?".
+  readonly correlationId?: string;
+  // Stored event id that triggered this write. Null for root commands;
+  // set to event.id when an MSP-apply runs ctx.appendEvent. Together with
+  // correlationId forms a causation DAG across aggregate streams.
+  readonly causationId?: string;
+  // Marten-conform free key/value space for app-specific metadata that
+  // doesn't deserve its own EventMetadata field. Examples: A/B-test bucket,
+  // feature-flag snapshot, geo-region, client SDK version. Persisted into
+  // events.metadata jsonb (no schema change — it's already a free-form
+  // jsonb column), survives upcasters untouched, available on every
+  // StoredEvent.metadata.headers. Framework does not interpret values; the
+  // app reads them when filtering/auditing. Keep values JSON-primitive
+  // (string|number|boolean) so JSON serialization stays bulletproof.
+  readonly headers?: Readonly<Record<string, string | number | boolean>>;
+};
+
+export type EventToAppend = {
+  readonly aggregateId: string;
+  readonly aggregateType: string;
+  readonly tenantId: TenantId;
+  // Predecessor version. 0 for a brand-new aggregate — framework writes version 1.
+  readonly expectedVersion: number;
+  readonly type: string;
+  readonly eventVersion?: number;
+  readonly payload: Record<string, unknown>;
+  readonly metadata: EventMetadata;
+};
+
+// Generic über payload-shape. Default = Record<string, unknown> macht
+// alle existierenden Konsumenten backwards-compatible. Konkrete Apply-
+// Handler / Tests können `StoredEvent<MyEventPayload>` annotieren um
+// payload typed zu lesen. Type-Propagation kommt durch r.defineEvent +
+// SingleStreamApplyFn<T> in apply-Maps.
+export type StoredEvent<TPayload = Record<string, unknown>> = {
+  readonly id: string;
+  readonly aggregateId: string;
+  readonly aggregateType: string;
+  readonly tenantId: TenantId;
+  readonly version: number;
+  readonly type: string;
+  readonly eventVersion: number;
+  readonly payload: TPayload;
+  readonly metadata: EventMetadata;
+  readonly createdAt: Temporal.Instant;
+  readonly createdBy: string;
+};
+
+type SelectedEvent = typeof eventsTable.$inferSelect;
+
+// Append one event atomically. Two guarantees combined:
+//
+//   1. UNIQUE (tenant_id, aggregate_id, version) serializes concurrent writers
+//      within a tenant — a second writer racing the same expectedVersion
+//      receives a PG unique violation (SQLSTATE 23505) → VersionConflictError.
+//      Cross-tenant aggregate_id collisions are not conflicts by definition:
+//      two tenants owning a row with the same UUID is just isolation, not a
+//      race.
+//
+//   2. For updates (expectedVersion > 0), INSERT … SELECT … WHERE EXISTS
+//      requires the predecessor event to exist within the same tenant — i.e.
+//      "you can't append v6 to a stream whose v5 was never written." The
+//      tenant filter inside the EXISTS is belt-and-suspenders now that the
+//      unique index carries tenant_id; we keep it so the predecessor check
+//      stays semantically obvious when read in isolation.
+//
+// Creates (expectedVersion === 0) skip the predecessor check — no predecessor
+// exists yet. Colliding creates fall out via UNIQUE (tenant_id, aggregate_id, version=1).
+// Channel name used by append() → NOTIFY and the event-dispatcher → LISTEN
+// (Sprint E.4). The event-dispatcher subscribes to this channel on start and
+// fires a runOnce immediately on each commit, so delivery latency is bounded
+// by TCP round-trip instead of pollIntervalMs.
+export const EVENTS_PUBSUB_CHANNEL = "kumiko_events_new";
+
+export async function append(db: DbRunner, event: EventToAppend): Promise<StoredEvent> {
+  const newVersion = event.expectedVersion + 1;
+  const eventVersion = event.eventVersion ?? 1;
+
+  try {
+    const row =
+      event.expectedVersion === 0
+        ? await insertFirstEvent(db, event, newVersion, eventVersion)
+        : await insertSubsequentEvent(db, event, newVersion, eventVersion);
+
+    // NOTIFY fires on commit (PG buffers NOTIFY per TX), so subscribers never
+    // see a wake-up for an event that later rolled back. Harmless no-op when
+    // no LISTENer is attached.
+    await db.execute(sql`SELECT pg_notify(${EVENTS_PUBSUB_CHANNEL}, '')`);
+
+    return buildStoredEvent(event, newVersion, eventVersion, row);
+  } catch (e) {
+    if (isUniqueViolation(e)) {
+      // Only constraint left on the events table: events_aggregate_version_uq
+      // on (tenant_id, aggregate_id, version). A unique violation here always
+      // means a concurrent writer in the same tenant won the race to the
+      // next version — retry-able conflict.
+      throw new VersionConflictError(event.aggregateId, event.expectedVersion);
+    }
+    throw e;
+  }
+}
+
+type InsertReturn = { id: bigint; createdAt: Temporal.Instant };
+
+async function insertFirstEvent(
+  db: DbRunner,
+  event: EventToAppend,
+  newVersion: number,
+  eventVersion: number,
+): Promise<InsertReturn> {
+  const [row] = await db
+    .insert(eventsTable)
+    .values({
+      aggregateId: event.aggregateId,
+      aggregateType: event.aggregateType,
+      tenantId: event.tenantId,
+      version: newVersion,
+      type: event.type,
+      eventVersion,
+      payload: event.payload,
+      metadata: event.metadata,
+      createdBy: event.metadata.userId,
+    })
+    .returning({ id: eventsTable.id, createdAt: eventsTable.createdAt });
+  if (!row) throw new Error("insertFirstEvent: INSERT RETURNING produced no row");
+  return row;
+}
+
+// Subsequent event — predecessor must exist AND belong to the same tenant.
+// INSERT … SELECT … WHERE EXISTS is awkward in the typed builder, so this
+// one stays raw. Uses ${eventsTable} for the table reference so renames
+// don't silently break the statement.
+async function insertSubsequentEvent(
+  db: DbRunner,
+  event: EventToAppend,
+  newVersion: number,
+  eventVersion: number,
+): Promise<InsertReturn> {
+  const payloadJson = JSON.stringify(event.payload);
+  const metadataJson = JSON.stringify(event.metadata);
+  const rows = await db.execute<{ id: string; created_at: Date | string }>(sql`
+    INSERT INTO ${eventsTable} (
+      aggregate_id, aggregate_type, tenant_id, version,
+      type, event_version, payload, metadata, created_by
+    )
+    SELECT ${event.aggregateId}::uuid, ${event.aggregateType}, ${event.tenantId}::uuid, ${newVersion},
+           ${event.type}, ${eventVersion}, ${payloadJson}::jsonb,
+           ${metadataJson}::jsonb, ${event.metadata.userId}
+    WHERE EXISTS (
+      SELECT 1 FROM ${eventsTable}
+      WHERE aggregate_id = ${event.aggregateId}::uuid
+        AND version = ${event.expectedVersion}
+        AND tenant_id = ${event.tenantId}::uuid
+    )
+    RETURNING id, created_at;
+  `);
+  const row = rows[0];
+  if (!row) throw new VersionConflictError(event.aggregateId, event.expectedVersion);
+  return {
+    id: BigInt(row.id),
+    // Raw SQL bypasses Drizzle's customType — postgres-js returns Date or
+    // string depending on driver-config. Normalize through Temporal.Instant
+    // so the InsertReturn shape matches the typed-builder path.
+    createdAt:
+      row.created_at instanceof Date
+        ? Temporal.Instant.fromEpochMilliseconds(row.created_at.getTime())
+        : Temporal.Instant.from(row.created_at),
+  };
+}
+
+function buildStoredEvent(
+  event: EventToAppend,
+  newVersion: number,
+  eventVersion: number,
+  row: InsertReturn,
+): StoredEvent {
+  return {
+    id: String(row.id),
+    aggregateId: event.aggregateId,
+    aggregateType: event.aggregateType,
+    tenantId: event.tenantId,
+    version: newVersion,
+    type: event.type,
+    eventVersion,
+    payload: event.payload,
+    metadata: event.metadata,
+    createdAt: row.createdAt,
+    createdBy: event.metadata.userId,
+  };
+}
+
+// Load all events for an aggregate, ordered by version. Tenant check is
+// belt + suspenders: even if a caller passes a correct aggregate_id by
+// mistake, the tenant filter prevents cross-tenant reads.
+//
+// Archived streams return an empty slice by default. Pass
+// { includeArchived: true } for ops tools / audit that must see the tail
+// of an archived aggregate. The archive check is a single indexed lookup —
+// negligible on the hot path.
+export async function loadAggregate(
+  db: DbRunner,
+  aggregateId: string,
+  tenantId: TenantId,
+  options?: { readonly includeArchived?: boolean },
+): Promise<readonly StoredEvent[]> {
+  if (!options?.includeArchived) {
+    const archived = await isStreamArchived(db, tenantId, aggregateId);
+    if (archived) return [];
+  }
+  const rows = await db
+    .select()
+    .from(eventsTable)
+    .where(and(eq(eventsTable.aggregateId, aggregateId), eq(eventsTable.tenantId, tenantId)))
+    .orderBy(asc(eventsTable.version));
+  return rows.map(toStoredEvent);
+}
+
+// Load events up to a point in time. Used for asOf queries that reconstruct
+// historical state. Includes events whose created_at is <= asOf. Same
+// archive semantics as loadAggregate.
+export async function loadAggregateAsOf(
+  db: DbRunner,
+  aggregateId: string,
+  tenantId: TenantId,
+  asOf: Temporal.Instant,
+  options?: { readonly includeArchived?: boolean },
+): Promise<readonly StoredEvent[]> {
+  if (!options?.includeArchived) {
+    const archived = await isStreamArchived(db, tenantId, aggregateId);
+    if (archived) return [];
+  }
+  const rows = await db
+    .select()
+    .from(eventsTable)
+    .where(
+      and(
+        eq(eventsTable.aggregateId, aggregateId),
+        eq(eventsTable.tenantId, tenantId),
+        lte(eventsTable.createdAt, asOf),
+      ),
+    )
+    .orderBy(asc(eventsTable.version));
+  return rows.map(toStoredEvent);
+}
+
+// Cheapest possible read of "what's the latest version on this stream?". The
+// CRUD executor uses this as expectedVersion for its append, so a domain
+// event appended via ctx.appendEvent between two CRUD writes doesn't cause
+// the next CRUD write to fail with version_conflict. Indexed lookup on the
+// existing (aggregate_id, version) unique index. Returns 0 for empty/unknown
+// streams (matches create()'s expectedVersion=0 convention).
+export async function getStreamVersion(
+  db: DbRunner,
+  aggregateId: string,
+  tenantId: TenantId,
+): Promise<number> {
+  const [row] = await db
+    .select({ v: max(eventsTable.version) })
+    .from(eventsTable)
+    .where(and(eq(eventsTable.aggregateId, aggregateId), eq(eventsTable.tenantId, tenantId)));
+  return row?.v ?? 0;
+}
+
+// Global high-water-mark = MAX(events.id). Marten/Wolverine standard for
+// projection/consumer lag math: lag = HWM - cursor. Single-row aggregate over
+// the bigserial PK index — sub-millisecond cost. Returns 0n on an empty log
+// (boot, fresh tenant, post-archive).
+export async function getEventsHighWaterMark(db: DbRunner): Promise<bigint> {
+  const [row] = await db.select({ max: max(eventsTable.id) }).from(eventsTable);
+  return row?.max ?? 0n;
+}
+
+// Load events strictly newer than a given version. Used by snapshot-aware
+// reads: snapshot provides state up to version N, apply events v > N to
+// catch up to current.
+export async function loadEventsAfterVersion(
+  db: DbRunner,
+  aggregateId: string,
+  tenantId: TenantId,
+  afterVersion: number,
+): Promise<readonly StoredEvent[]> {
+  const rows = await db
+    .select()
+    .from(eventsTable)
+    .where(
+      and(
+        eq(eventsTable.aggregateId, aggregateId),
+        eq(eventsTable.tenantId, tenantId),
+        gt(eventsTable.version, afterVersion),
+      ),
+    )
+    .orderBy(asc(eventsTable.version));
+  return rows.map(toStoredEvent);
+}
+
+// Load every event for an aggregate_type across all tenants. Ordered by
+// (created_at, id) — chronological replay order for projection rebuilds.
+//
+// CAUTION — buffers ALL matching events in memory. Safe for smaller
+// aggregate-types (≤ 100k events), a memory cliff for large stores.
+// For >100k events use `streamAllEventsByType` (yields batchwise).
+// Mostly called from tests today — production rebuild goes through
+// projection-rebuild's own streaming path.
+export async function loadAllEventsByType(
+  db: DbRunner,
+  aggregateType: string,
+): Promise<readonly StoredEvent[]> {
+  const rows = await db
+    .select()
+    .from(eventsTable)
+    .where(eq(eventsTable.aggregateType, aggregateType))
+    .orderBy(asc(eventsTable.createdAt), asc(eventsTable.id));
+  return rows.map(toStoredEvent);
+}
+
+// Stream every event for an aggregate_type across all tenants, batchwise
+// instead of buffered. Memory-bounded: never more than `batchSize` rows
+// resident. Cursor walks `events.id` (bigserial monotonic — concurrent
+// inserts get distinct ids in commit order, so no duplicates and no skips
+// past the cursor).
+//
+// Use case: projection-rebuild on a large event log (>100k events per
+// aggregate-type). loadAllEventsByType would OOM; this iterator yields
+// in batches and the caller accumulates only what it needs.
+//
+// Default batchSize 1000 — trade-off between DB round-trips (smaller =
+// more queries) and memory (larger = more resident).
+//
+// The caller may pause / do async work between yields; the next batch is
+// only fetched when consumed.
+//
+// Cancellation: pass `signal` (typically `ctx.signal` from a handler) to
+// abort iteration. Checked both at batch boundaries (before the next
+// fetch) AND between yields (so abort lands within one event regardless
+// of batch size). Throws AbortError on the first check after abort;
+// in-flight queries are not actively cancelled (postgres-js connection-
+// cancel is a separate, riskier concern handled per-query at the TenantDb
+// boundary).
+export async function* streamAllEventsByType(
+  db: DbRunner,
+  aggregateType: string,
+  batchSize = 1000,
+  signal?: AbortSignal,
+): AsyncIterable<StoredEvent> {
+  let cursorId = 0n;
+  while (true) {
+    signal?.throwIfAborted();
+    const rows = await db
+      .select()
+      .from(eventsTable)
+      .where(and(eq(eventsTable.aggregateType, aggregateType), gt(eventsTable.id, cursorId)))
+      .orderBy(asc(eventsTable.id))
+      .limit(batchSize);
+
+    if (rows.length === 0) {
+      // skip: end of stream — generator exit is the natural termination.
+      return;
+    }
+
+    // Track the highest id seen in this batch as we yield. Avoids both the
+    // non-null assertion and a redundant array index — the cursor falls out
+    // of the same loop that produces the events.
+    let nextCursor = cursorId;
+    for (const row of rows) {
+      // Per-yield abort check. Cheap (one boolean read), keeps cancel
+      // semantics independent of the batchSize knob — at batchSize=1000
+      // a batch-boundary-only check would still yield 1000 events after
+      // an abort which isn't what callers expect.
+      signal?.throwIfAborted();
+      yield toStoredEvent(row);
+      nextCursor = row.id;
+    }
+    cursorId = nextCursor;
+  }
+}
+
+function toStoredEvent(row: SelectedEvent): StoredEvent {
+  return {
+    id: String(row.id),
+    aggregateId: row.aggregateId,
+    aggregateType: row.aggregateType,
+    tenantId: row.tenantId,
+    version: row.version,
+    type: row.type,
+    eventVersion: row.eventVersion,
+    payload: row.payload,
+    metadata: row.metadata,
+    createdAt: row.createdAt,
+    createdBy: row.createdBy,
+  };
+}

package/src/event-store/events-schema.ts

@@ -0,0 +1,90 @@
+import { sql } from "drizzle-orm";
+import { type DbConnection, tableExists } from "../db";
+import {
+  bigserial,
+  index,
+  instant,
+  integer,
+  jsonb,
+  table as pgTable,
+  text,
+  uniqueIndex,
+  uuid,
+} from "../db/dialect";
+import { pushTables } from "../stack";
+import { createArchivedStreamsTable } from "./archive";
+import { createSnapshotsTable } from "./snapshot";
+
+// Event-store schema as a Drizzle table. The typed select/insert path handles
+// most operations; append() for subsequent versions uses raw SQL because
+// INSERT ... SELECT ... WHERE EXISTS isn't ergonomic in the typed builder.
+//
+// HTTP-level retry idempotency is handled by pipeline/idempotency.ts
+// (Redis-backed check + cached-response replay). The event-store itself
+// imposes no idempotency index — a single HTTP request may write N events
+// freely, metadata.requestId is purely a trace marker.
+export type EventMetadata = {
+  readonly userId: string;
+  readonly requestId?: string;
+  readonly correlationId?: string;
+  readonly causationId?: string;
+  // App-specific free key/value (Marten "headers"). Mirror of the canonical
+  // type in event-store.ts — kept duplicate because events-schema must stay
+  // import-cycle-free vs the event-store module.
+  readonly headers?: Readonly<Record<string, string | number | boolean>>;
+};
+
+export const eventsTable = pgTable(
+  "kumiko_events",
+  {
+    // bigserial PK: global chronological ordering cheap to index, safe past
+    // 2^53 as long as we stay < ~9e15 events. Returned to JS as BigInt.
+    id: bigserial("id", { mode: "bigint" }).primaryKey(),
+    aggregateId: uuid("aggregate_id").notNull(),
+    aggregateType: text("aggregate_type").notNull(),
+    tenantId: uuid("tenant_id").notNull(),
+    version: integer("version").notNull(),
+    type: text("type").notNull(),
+    eventVersion: integer("event_version").notNull().default(1),
+    payload: jsonb("payload").$type<Record<string, unknown>>().notNull(),
+    metadata: jsonb("metadata").$type<EventMetadata>().notNull(),
+    // Millisecond precision: matches what asOf-queries can compare reliably.
+    // Sprint F: instant() = Temporal.Instant round-trip via dialect.ts customType.
+    createdAt: instant("created_at", { precision: 3 }).notNull().default(sql`now()`),
+    // Text rather than uuid: the framework's SessionUser.id is a number
+    // (serial) by default. Stringified here so both integer- and UUID-shaped
+    // user ids round-trip cleanly. Aggregate-IDs stay uuid because events are
+    // aggregated by UUID end-to-end.
+    createdBy: text("created_by").notNull(),
+  },
+  (t) => ({
+    // Tenant-scoped unique: two tenants that happen to pick the same
+    // aggregate_id (deterministic IDs, replay, restores) don't collide, and
+    // insertFirstEvent needs no extra tenant check for cross-tenant safety —
+    // the constraint itself guarantees it. For expectedVersion > 0 the raw
+    // INSERT … SELECT … WHERE EXISTS still pairs the predecessor with the
+    // same tenant, which is now just predecessor-existence and no longer
+    // doing double duty as an anti-hijack check.
+    aggregateVersionUq: uniqueIndex("events_aggregate_version_uq").on(
+      t.tenantId,
+      t.aggregateId,
+      t.version,
+    ),
+    loadIdx: index("events_load_idx").on(t.aggregateId, t.version),
+    tenantTypeIdx: index("events_tenant_type_idx").on(t.tenantId, t.aggregateType, t.createdAt),
+  }),
+);
+
+// Convenience used by framework integration tests. Creates the table via
+// drizzle-kit diffing. Also materializes kumiko_archived_streams and
+// kumiko_snapshots — loadAggregate / appendEvent / loadAggregateWithSnapshot
+// consult them on the hot path, so the three tables must come up together.
+export async function createEventsTable(db: DbConnection): Promise<void> {
+  // skip: events table already exists — createEventsTable is called from both
+  // setupTestStack and explicit test-setups, the guard keeps it idempotent.
+  if (!(await tableExists(db, "public.kumiko_events"))) {
+    await pushTables(db, { kumikoEvents: eventsTable });
+  }
+  await createArchivedStreamsTable(db);
+  await createSnapshotsTable(db);
+}

package/src/event-store/index.ts

@@ -0,0 +1,50 @@
+export {
+  type ArchiveStreamArgs,
+  archivedStreamsTable,
+  archiveStream,
+  createArchivedStreamsTable,
+  isStreamArchived,
+  restoreStream,
+} from "./archive";
+export { ArchivedStreamError, VersionConflictError } from "./errors";
+export {
+  append,
+  EVENTS_PUBSUB_CHANNEL,
+  type EventMetadata,
+  type EventToAppend,
+  getEventsHighWaterMark,
+  getStreamVersion,
+  loadAggregate,
+  loadAggregateAsOf,
+  loadAllEventsByType,
+  loadEventsAfterVersion,
+  type StoredEvent,
+  streamAllEventsByType,
+} from "./event-store";
+export { createEventsTable, eventsTable } from "./events-schema";
+export {
+  createSnapshotsTable,
+  type LoadAggregateWithSnapshotResult,
+  loadAggregateWithSnapshot,
+  loadLatestSnapshot,
+  type SaveSnapshotArgs,
+  type Snapshot,
+  type SnapshotReducer,
+  saveSnapshot,
+  snapshotsTable,
+} from "./snapshot";
+export {
+  type EventUpcasters,
+  makeUpcastCtx,
+  type UpcasterErrorPolicy,
+  type UpcastOptions,
+  upcastStoredEvent,
+  upcastStoredEvents,
+} from "./upcaster";
+export {
+  createUpcasterDeadLetterTable,
+  type DeadLetterRow,
+  listDeadLetters,
+  recordUpcasterDeadLetter,
+  upcasterDeadLetterTable,
+} from "./upcaster-dead-letter";