@cosmicdrift/kumiko-framework 0.327.0 → 0.328.1

package/src/pipeline/event-consumer-state.ts

@@ -1,190 +0,0 @@
-// sql now comes from native dialect
-import type { DbConnection } from "../db/connection";
-import {
-  bigint,
-  index,
-  instant,
-  integer,
-  jsonb,
-  table as pgTable,
-  primaryKey,
-  sql,
-  text,
-} from "../db/dialect";
-import { alterTableAddColumn } from "../db/queries/ddl";
-import { deleteOrphanedPerInstanceConsumerRows } from "../db/queries/event-consumer";
-import { tableExists } from "../db/schema-inspection";
-import { unsafePushTables } from "../stack";
-
-// Reserved sentinel used in the instance_id column for consumers whose
-// delivery is "shared" — i.e. one cursor across all dispatcher instances
-// (the default, pre-Welle-2.7 behaviour). Per-instance consumers store
-// the concrete instanceId. Postgres PK columns can't be NULL and
-// `NULL = NULL` is UNKNOWN in SQL — a nullable instance_id would break
-// both uniqueness and PK constraints. The boot-validator refuses to
-// start with KUMIKO_INSTANCE_ID === SHARED_INSTANCE_SENTINEL so the
-// sentinel can never collide with a real instance identifier.
-export const SHARED_INSTANCE_SENTINEL = "__shared__";
-
-// Framework-level state per event-consumer-shard. A "consumer" is anything
-// that walks the events-table via a persistent cursor: system consumers
-// (SSE, Search) and feature multiStreamProjections (async, cross-aggregate).
-//
-// One row per (consumer name, instance_id) shard. Shared-delivery consumers
-// have exactly one row with instance_id = SHARED_INSTANCE_SENTINEL — this
-// preserves the pre-Welle-2.7 single-cursor semantic unchanged. Per-instance
-// consumers get N rows (one per dispatcher instance), each with its own
-// cursor — used by SSE so every API process pushes the same events to its
-// own clients without a pub/sub transport. Read by the event-dispatcher
-// (cursor + locking), surfaced by the CLI for ops inspection.
-//
-// Columns:
-//   - name: consumer's qualified identifier, e.g. "system:consumer:sse" or
-//     "my-feature:consumer:analytics". Matches the qualified-name convention
-//     so two features can't accidentally collide on the same consumer name.
-//   - instanceId: SHARED_INSTANCE_SENTINEL for shared-delivery consumers;
-//     concrete process-local identifier (ServerOptions.instanceId, defaults
-//     to KUMIKO_INSTANCE_ID or a boot-time UUID) for per-instance consumers.
-//   - lastProcessedEventId: bigserial `events.id` of the most recent event
-//     this shard finished handling. Dispatcher reads events WHERE id > this.
-//   - status / attempts / lastError / updatedAt — per shard.
-//
-// Composite PK (name, instance_id): Postgres requires NOT NULL on all PK
-// columns, and `NULL = NULL` is UNKNOWN in SQL — nullable instance_id would
-// break uniqueness for shared rows. Sentinel avoids both hazards with one
-// uniform column shape.
-//
-// CAUTION (retention-guard + scale-down):
-//   pruneEvents refuses to delete past MIN(lastProcessedEventId) across ALL
-//   shards. A decommissioned instance leaves its row behind at its last
-//   cursor — prune is then pinned indefinitely. Before scaling down,
-//   delete stale per-instance shards:
-//     DELETE FROM kumiko_event_consumers WHERE instance_id = '<decommissioned>'
-//   TODO: auto-cleanup via heartbeat-liveness — follow-up, not in v1.
-//
-// The default(sql`0`) on lastProcessedEventId mirrors projection-state.ts:
-// drizzle-kit's JSON snapshot generator can't serialise a bigint literal, so
-// the server-side default is specified as raw SQL instead of .default(0n).
-
-// A contiguous run of ids below the cursor invisible on some earlier turn,
-// not yet resolved (delivered) or proven burnt (see event-dispatcher.ts's
-// processConsumer). Ranges, not individual ids — a large historical gap
-// (retention prune, or a new consumer starting "beginning" over pruned
-// history) then costs O(1) entries instead of one per missing id. xmax is
-// the pg_snapshot_xmax() captured when the range was recorded: once a later
-// turn's xmin passes it, every xact that could still produce a row in this
-// range has finished, so it's provably rolled back, not just slow.
-// bigint/xid8 travel as strings — JS bigint doesn't round-trip through jsonb.
-export type PendingGapEntry = {
-  readonly from: string;
-  readonly to: string;
-  readonly xmax: string;
-};
-export const eventConsumerStateTable = pgTable(
-  "kumiko_event_consumers",
-  {
-    name: text("name").notNull(),
-    instanceId: text("instance_id").notNull().default(SHARED_INSTANCE_SENTINEL),
-    lastProcessedEventId: bigint("last_processed_event_id", { mode: "bigint" })
-      .notNull()
-      .default(sql`0`),
-    status: text("status").notNull().default("idle"),
-    attempts: integer("attempts").notNull().default(0),
-    // Counts automatic dead→idle revivals since the last delivery that
-    // advanced the cursor (event-dispatcher.ts's cooldown re-arm, see
-    // acquireConsumerState). A poison event never advances the cursor, so
-    // this climbs to maxRearmCount and then stays dead permanently. A
-    // successful delivery resets it to 0 (proof the consumer isn't
-    // poisoned), as does a manual restartConsumer()/enableConsumer()/
-    // skipPoisonEvent() — an operator vouching the consumer is healthy.
-    rearmCount: integer("rearm_count").notNull().default(0),
-    pendingGaps: jsonb("pending_gaps").$type<PendingGapEntry[]>().default([]).notNull(),
-    lastError: text("last_error"),
-    updatedAt: instant("updated_at", { precision: 3 }).notNull().default(sql`now()`),
-  },
-  (t) => ({
-    pk: primaryKey({ columns: [t.name, t.instanceId] }),
-    statusIdx: index("kumiko_event_consumers_status_idx").on(t.status),
-  }),
-);
-
-// fw#2625: these three consumers moved from delivery: "per-instance" to
-// "shared" (see system-hooks.ts and bundled-features/feature-toggles/
-// feature.ts). Their old per-instance rows are now orphaned — no dispatcher
-// will ever advance them again — and pruneEvents stays pinned to whichever
-// one has the lowest cursor forever. Named here (not imported from
-// system-hooks.ts / feature.ts) to avoid a pipeline → system-hooks /
-// bundled-features dependency for three string literals; those modules'
-// own exported constants (or, for feature-toggles, the qualified consumer
-// name qualify() produces) are the source of truth these must match.
-const ORPHANED_PER_INSTANCE_CONSUMER_NAMES = [
-  "system:consumer:sse-broadcast",
-  "system:consumer:access-invalidation",
-  "feature-toggles:projection:toggle-cache-sync",
-] as const;
-
-// Object-const form lets call sites write `ConsumerStatuses.disabled` instead
-// of the raw string, which keeps status checks refactor-safe. The runtime
-// value on each field is the same string the DB stores — no mapping needed.
-export const ConsumerStatuses = {
-  idle: "idle",
-  processing: "processing",
-  dead: "dead",
-  disabled: "disabled",
-} as const;
-export type ConsumerStatus = (typeof ConsumerStatuses)[keyof typeof ConsumerStatuses];
-
-// Idempotent bootstrap. Called by setupTestStack + production boot path —
-// same pattern as createProjectionStateTable / createEventsTable. If the
-// table is already present (second stack in the same test DB, prod boot
-// after migration), skip cleanly.
-//
-// guard:dup-ok — intentionale Parallele zu createProjectionStateTable; symmetrische State-Tabellen by design
-export async function createEventConsumerStateTable(db: DbConnection): Promise<void> {
-  // skip: table already exists — bootstrap is called from multiple paths.
-  // Still check for columns added after the table's first deploy (e.g.
-  // rearm_count) so an older DB catches up without a dedicated migration.
-  if (await tableExists(db, "public.kumiko_event_consumers")) {
-    // ADD COLUMN IF NOT EXISTS is idempotent and race-safe on its own —
-    // no need to pre-check columnNamesOf first. Two instances booting
-    // concurrently against the same DB (this table is explicitly
-    // multi-instance) both ALTER, neither crashes on "column already
-    // exists" (#1362).
-    await alterTableAddColumn(
-      db,
-      "kumiko_event_consumers",
-      "rearm_count",
-      "integer",
-      " DEFAULT 0",
-      " NOT NULL",
-      /* ifNotExists */ true,
-    );
-    await alterTableAddColumn(
-      db,
-      "kumiko_event_consumers",
-      "pending_gaps",
-      "jsonb",
-      " DEFAULT '[]'::jsonb",
-      " NOT NULL",
-      /* ifNotExists */ true,
-    );
-    // Runs on every boot, including mid-rolling-deploy: a new pod can delete
-    // an old pod's still-in-use per-instance row out from under it. That old
-    // pod's next acquireConsumer then finds no row, delivers no more SSE/
-    // access-invalidation events for the rest of its lifetime — a one-time,
-    // few-second gap that resolves itself as clients reconnect onto new pods
-    // during the same rollout. Not a replay risk: preRegisterConsumers only
-    // (re-)inserts a row at dispatcher startup, so the deleted pod won't
-    // silently get a fresh cursor-0 row and replay history mid-flight.
-    await deleteOrphanedPerInstanceConsumerRows(
-      db,
-      ORPHANED_PER_INSTANCE_CONSUMER_NAMES,
-      SHARED_INSTANCE_SENTINEL,
-    );
-    // skip: table (+ any missing column) is already up to date
-    return;
-  }
-  // A freshly created table has no rows to clean up — the DELETE only
-  // matters for a DB that already ran the old per-instance consumers.
-  await unsafePushTables(db, { kumikoEventConsumers: eventConsumerStateTable });
-}

package/src/pipeline/event-dedup.ts

@@ -1,23 +0,0 @@
-import type Redis from "ioredis";
-import { RedisKeys } from "./redis-keys";
-
-export type EventDedup = {
-  /**
-   * Atomically try to acquire processing rights for an eventId.
-   * Returns true if this is the first call (proceed), false if already processed (skip).
-   */
-  tryAcquire(eventId: string): Promise<boolean>;
-};
-
-export function createEventDedup(redis: Redis, options: { ttlSeconds?: number } = {}): EventDedup {
-  const ttl = options.ttlSeconds ?? 300;
-  const prefix = RedisKeys.eventDedup;
-
-  return {
-    async tryAcquire(eventId) {
-      // SET NX = atomic check-and-set: only succeeds if key does not exist
-      const result = await redis.set(`${prefix}${eventId}`, "1", "EX", ttl, "NX");
-      return result === "OK";
-    },
-  };
-}

package/src/pipeline/event-dispatcher-admin.ts

@@ -1,320 +0,0 @@
-import type { DbConnection, DbTx } from "../db/connection";
-import {
-  advanceConsumerPastEventReturning,
-  removePendingGapReturning,
-  updateConsumerStatusReturning,
-} from "../db/queries/event-consumer";
-import { selectNextEventIdAfter, selectSmallestVisibleIdInRanges } from "../db/queries/event-store";
-import { coerceRow, extractTableInfo, selectMany } from "../db/query";
-import { getEventsHighWaterMark } from "../event-store";
-import { eventConsumerStateTable, SHARED_INSTANCE_SENTINEL } from "./event-consumer-state";
-import type { ConsumerStateRow, ConsumerStateRowShape } from "./event-dispatcher-delivery";
-import { rangeContainsId, splitRangeExcludingIds, toIdRanges } from "./pending-gap-ranges";
-
-// --- Ops recovery surface ---
-//
-// These are intentionally verb-distinct; each maps to a CLI sub-command.
-// They all target a single consumer row by name. Every call returns the
-// state after the write so the CLI can echo what actually changed.
-//
-// Semantics:
-//   restartConsumer   status="dead" → "idle", attempts=0, lastError=null.
-//                     Cursor unchanged → next pass retries the SAME event
-//                     that poisoned the consumer. For transient failures.
-//   disableConsumer   status=* → "disabled". Dispatcher skips this consumer
-//                     until enableConsumer() flips it back.
-//   enableConsumer    status="disabled" → "idle". No-op on any other state.
-//   skipPoisonEvent   cursor advances past the first event after the
-//                     current cursor (the one that's failing). attempts=0,
-//                     lastError=null, status="idle". For events that will
-//                     never succeed (broken payload, removed feature code).
-
-function normalizeConsumerState(row: ConsumerStateRowShape): ConsumerRecoveryState {
-  return {
-    name: row.name,
-    instanceId: row.instanceId,
-    status: row.status,
-    lastProcessedEventId: row.lastProcessedEventId,
-    attempts: row.attempts,
-    rearmCount: row.rearmCount,
-    lastError: row.lastError,
-    updatedAt: row.updatedAt,
-  };
-}
-
-export type ConsumerRecoveryState = {
-  readonly name: string;
-  readonly instanceId: string;
-  readonly status: string;
-  readonly lastProcessedEventId: bigint;
-  readonly attempts: number;
-  readonly rearmCount: number;
-  readonly lastError: string | null;
-  readonly updatedAt: Temporal.Instant;
-};
-
-// Ops calls default to the SHARED_INSTANCE_SENTINEL row — that's the only
-// row shared-delivery consumers have, so legacy CLI invocations without
-// --instance-id keep working. Per-instance consumers require an explicit
-// instanceId: picking one of N shards arbitrarily ("first row wins") or
-// mutating all shards simultaneously ("bounce every instance") are both
-// worse than a loud missing-arg error on the CLI.
-async function requireConsumerRow(
-  db: DbConnection,
-  name: string,
-  instanceId: string,
-): Promise<ConsumerStateRowShape> {
-  const [row] = await selectMany<ConsumerStateRow>(db, eventConsumerStateTable, {
-    name,
-    instanceId,
-  });
-  if (!row) {
-    throw new Error(
-      `Consumer "${name}" (instance_id="${instanceId}") has no state row — it hasn't run yet, the name is misspelled, or the instance is misspelled. ` +
-        `For per-instance consumers pass the instance_id explicitly; shared consumers use the default.`,
-    );
-  }
-  return row;
-}
-
-async function applyConsumerStatusTransition(
-  db: DbConnection,
-  name: string,
-  instanceId: string,
-  targetStatus: "idle" | "disabled",
-): Promise<ConsumerRecoveryState> {
-  const raw = await updateConsumerStatusReturning(db, name, instanceId, targetStatus);
-  const updated =
-    raw && (coerceRow(raw, extractTableInfo(eventConsumerStateTable)) as ConsumerStateRow);
-  if (!updated) {
-    throw new Error(
-      `Consumer "${name}" (instance_id="${instanceId}") vanished between read and write — retry.`,
-    );
-  }
-  return normalizeConsumerState(updated);
-}
-
-export async function restartConsumer(
-  db: DbConnection,
-  name: string,
-  instanceId: string = SHARED_INSTANCE_SENTINEL,
-): Promise<ConsumerRecoveryState> {
-  const before = await requireConsumerRow(db, name, instanceId);
-  if (before.status !== "dead") {
-    throw new Error(
-      `Consumer "${name}" (instance_id="${instanceId}") is not dead (status="${before.status}"). Restart only applies to dead consumers; use "enable" for a disabled one.`,
-    );
-  }
-  return applyConsumerStatusTransition(db, name, instanceId, "idle");
-}
-
-export async function disableConsumer(
-  db: DbConnection,
-  name: string,
-  instanceId: string = SHARED_INSTANCE_SENTINEL,
-): Promise<ConsumerRecoveryState> {
-  await requireConsumerRow(db, name, instanceId);
-  return applyConsumerStatusTransition(db, name, instanceId, "disabled");
-}
-
-export async function enableConsumer(
-  db: DbConnection,
-  name: string,
-  instanceId: string = SHARED_INSTANCE_SENTINEL,
-): Promise<ConsumerRecoveryState> {
-  const before = await requireConsumerRow(db, name, instanceId);
-  if (before.status !== "disabled") {
-    throw new Error(
-      `Consumer "${name}" (instance_id="${instanceId}") is not disabled (status="${before.status}"). Enable only flips disabled → idle; use "restart" for a dead consumer.`,
-    );
-  }
-  return applyConsumerStatusTransition(db, name, instanceId, "idle");
-}
-
-// skipPoisonEvent advances past whatever is currently blocking the consumer.
-// If the smallest visible id inside pending_gaps exists, THAT'S the poison —
-// halt-on-poison walks gap ids before any id past the cursor — so it's split
-// out of its range directly; the cursor stays put (it's already above this
-// id). Otherwise the poison is the usual next event after the cursor. Single
-// TX so concurrent dispatcher passes can't double-advance. Neither exists →
-// idempotent no-op.
-export async function skipPoisonEvent(
-  db: DbConnection,
-  name: string,
-  instanceId: string = SHARED_INSTANCE_SENTINEL,
-): Promise<ConsumerRecoveryState & { readonly skippedEventId: bigint | null }> {
-  const before = await requireConsumerRow(db, name, instanceId);
-  return db.begin(async (tx: DbTx) => {
-    const pendingGaps = before.pendingGaps;
-    const smallestVisiblePending = await selectSmallestVisibleIdInRanges(
-      tx,
-      toIdRanges(pendingGaps),
-    );
-
-    if (smallestVisiblePending !== null) {
-      const newPendingGaps = pendingGaps.flatMap((gap) =>
-        rangeContainsId(gap, smallestVisiblePending)
-          ? splitRangeExcludingIds(gap, [smallestVisiblePending])
-          : [gap],
-      );
-      const raw = await removePendingGapReturning(tx, name, instanceId, newPendingGaps);
-      const updated =
-        raw && (coerceRow(raw, extractTableInfo(eventConsumerStateTable)) as ConsumerStateRow);
-      if (!updated)
-        throw new Error(
-          `Consumer "${name}" (instance_id="${instanceId}") vanished mid-skip — retry.`,
-        );
-      return { ...normalizeConsumerState(updated), skippedEventId: smallestVisiblePending };
-    }
-
-    const poisonId = await selectNextEventIdAfter(tx, before.lastProcessedEventId);
-    if (poisonId === null) {
-      const [unchanged] = await selectMany<ConsumerStateRow>(tx, eventConsumerStateTable, {
-        name,
-        instanceId,
-      });
-      if (!unchanged)
-        throw new Error(`Consumer "${name}" (instance_id="${instanceId}") vanished — retry.`);
-      return { ...normalizeConsumerState(unchanged), skippedEventId: null };
-    }
-    const raw = await advanceConsumerPastEventReturning(tx, name, instanceId, poisonId);
-    const updated =
-      raw && (coerceRow(raw, extractTableInfo(eventConsumerStateTable)) as ConsumerStateRow);
-    if (!updated)
-      throw new Error(
-        `Consumer "${name}" (instance_id="${instanceId}") vanished mid-skip — retry.`,
-      );
-    return { ...normalizeConsumerState(updated), skippedEventId: poisonId };
-  });
-}
-
-// Read-only status for one consumer shard — CLI surface.
-export async function getConsumerState(
-  db: DbConnection,
-  name: string,
-  instanceId: string = SHARED_INSTANCE_SENTINEL,
-): Promise<{
-  readonly name: string;
-  readonly instanceId: string;
-  readonly status: string;
-  readonly lastProcessedEventId: bigint;
-  readonly attempts: number;
-  readonly rearmCount: number;
-  readonly lastError: string | null;
-  readonly updatedAt: Temporal.Instant;
-} | null> {
-  const [row] = await selectMany<ConsumerStateRow>(db, eventConsumerStateTable, {
-    name,
-    instanceId,
-  });
-  if (!row) return null;
-  return {
-    name: row.name,
-    instanceId: row.instanceId,
-    status: row.status,
-    lastProcessedEventId: row.lastProcessedEventId,
-    attempts: row.attempts,
-    rearmCount: row.rearmCount,
-    lastError: row.lastError,
-    updatedAt: row.updatedAt,
-  };
-}
-
-// List every consumer the registry knows about, joined with all shard rows
-// from the state table. One entry per (name, instance_id) shard. Consumers
-// that have never run appear with status="never-run" and instance_id =
-// SHARED_INSTANCE_SENTINEL — a placeholder, because without a running
-// dispatcher we can't know the instance-ids of per-instance consumers yet.
-// Mirrors listProjectionsWithState — the registry (not the DB) is the
-// source-of-truth for which consumer-names exist; the DB is the source-
-// of-truth for which instance-shards have been seen.
-export async function listConsumersWithState(
-  db: DbConnection,
-  registeredNames: readonly string[],
-): Promise<
-  ReadonlyArray<{
-    readonly name: string;
-    readonly instanceId: string;
-    readonly status: string;
-    readonly lastProcessedEventId: bigint;
-    readonly attempts: number;
-    readonly lastError: string | null;
-  }>
-> {
-  const stateRows = await selectMany<ConsumerStateRow>(db, eventConsumerStateTable);
-  const registered = new Set(registeredNames);
-
-  // Materialize one output row per (name, instance_id). Registered names
-  // without any shard (never-run) get a placeholder row so ops can still
-  // see the name exists.
-  const out: Array<{
-    name: string;
-    instanceId: string;
-    status: string;
-    lastProcessedEventId: bigint;
-    attempts: number;
-    lastError: string | null;
-  }> = [];
-
-  const seenNames = new Set<string>();
-  for (const r of stateRows) {
-    if (!registered.has(r.name)) continue; // stale row from an older deploy
-    seenNames.add(r.name);
-    out.push({
-      name: r.name,
-      instanceId: r.instanceId,
-      status: r.status,
-      lastProcessedEventId: r.lastProcessedEventId,
-      attempts: r.attempts,
-      lastError: r.lastError,
-    });
-  }
-  for (const name of registeredNames) {
-    if (seenNames.has(name)) continue;
-    out.push({
-      name,
-      instanceId: SHARED_INSTANCE_SENTINEL,
-      status: "never-run",
-      lastProcessedEventId: 0n,
-      attempts: 0,
-      lastError: null,
-    });
-  }
-  return out;
-}
-
-export type ConsumerProgress = {
-  readonly name: string;
-  readonly instanceId: string;
-  readonly status: string;
-  readonly lastProcessedEventId: bigint;
-  readonly attempts: number;
-  readonly lastError: string | null;
-  // Global MAX(events.id) at query time.
-  readonly highWaterMark: bigint;
-  // HWM - cursor. 0n when caught-up. Disabled consumers often show high
-  // lag intentionally (ops parks them before pruning).
-  readonly lag: bigint;
-};
-
-// Like listConsumersWithState, but also returns HWM + lag per consumer.
-// Async consumers (MSPs) lag behind inline projections because they run
-// post-commit — lag is the primary signal for backpressure, dead consumers,
-// or dispatcher stalls. Programmatic callers can map the result to a
-// `kumiko_consumer_lag{name}` Prometheus gauge.
-// guard:dup-ok — intentionale Parallele zu getAllProjectionProgress; Consumer ≠ Projection (verschiedene Subsysteme)
-export async function getAllConsumerProgress(
-  db: DbConnection,
-  registeredNames: readonly string[],
-): Promise<readonly ConsumerProgress[]> {
-  const [consumers, highWaterMark] = await Promise.all([
-    listConsumersWithState(db, registeredNames),
-    getEventsHighWaterMark(db),
-  ]);
-
-  return consumers.map((c) => ({
-    ...c,
-    highWaterMark,
-    lag: highWaterMark - c.lastProcessedEventId,
-  }));
-}