@cosmicdrift/kumiko-framework 0.1.0

package/src/pipeline/event-retention.ts

@@ -0,0 +1,154 @@
+import { and, getTableName, inArray, lt, sql } from "drizzle-orm";
+import type { DbConnection } from "../db/connection";
+import { eventsTable } from "../event-store";
+import { eventConsumerStateTable } from "./event-consumer-state";
+
+// Retention for the events-table. Aggregate events are source of truth —
+// they power loadAggregate, projection rebuilds, asOf queries, audit.
+// Pruning them is destructive and cannot be reversed, so the caller MUST
+// name the aggregateTypes explicitly; there is no default.
+//
+// Typical use: ops prunes archived aggregate streams (see archive.ts) after
+// legal/compliance retention has elapsed, or drops a specific aggregate
+// type that's been deprecated and replaced.
+//
+// Safety guard: before deleting, we check every row in
+// `kumiko_event_consumers`. If the minimum `lastProcessedEventId` across
+// non-disabled consumers is below the largest event id we'd delete, we
+// refuse with ConsumerLagError. A lagging consumer must either catch up,
+// be disabled, or the retention call must opt around it; pruning past its
+// cursor would silently drop deliveries.
+//
+// No background scheduler: the framework exposes the function. Ops wires
+// it into a cron, BullMQ repeating job, or pg_cron entry — whatever the
+// deployment already runs. That keeps the framework dependency-free for
+// retention and lets ops reason about timing alongside existing jobs.
+
+export type PruneEventsOptions = {
+  // Delete events whose createdAt is strictly older than this.
+  // Pass EITHER olderThan (explicit Temporal.Instant) OR olderThanDays.
+  readonly olderThan?: Temporal.Instant;
+  readonly olderThanDays?: number;
+  // Which aggregateTypes to prune. REQUIRED and non-empty. There is no
+  // default — pruning the event log is destructive, so the caller has to
+  // name what they're destroying.
+  readonly aggregateTypes: readonly string[];
+  // Dry-run: compute what would be deleted, return count, delete nothing.
+  readonly dryRun?: boolean;
+};
+
+export type PruneEventsResult = {
+  readonly deletedCount: number;
+  readonly cutoff: Temporal.Instant;
+  readonly aggregateTypes: readonly string[];
+  readonly dryRun: boolean;
+};
+
+export class ConsumerLagError extends Error {
+  constructor(
+    readonly laggingConsumer: string,
+    readonly consumerCursor: bigint,
+    readonly maxCandidateId: bigint,
+  ) {
+    super(
+      `Consumer "${laggingConsumer}" is behind the prune candidates ` +
+        `(cursor=${consumerCursor}, max candidate event id=${maxCandidateId}). ` +
+        `Pruning would silently drop deliveries. Catch up, disable, or skip the consumer first.`,
+    );
+    this.name = "ConsumerLagError";
+  }
+}
+
+function resolveCutoff(opts: PruneEventsOptions): Temporal.Instant {
+  if (opts.olderThan) return opts.olderThan;
+  const days = opts.olderThanDays;
+  if (days === undefined || days <= 0) {
+    throw new Error(
+      "pruneEvents: pass olderThan (Temporal.Instant) or olderThanDays (positive number).",
+    );
+  }
+  return Temporal.Now.instant().subtract({ hours: days * 24 });
+}
+
+export async function pruneEvents(
+  db: DbConnection,
+  options: PruneEventsOptions,
+): Promise<PruneEventsResult> {
+  const cutoff = resolveCutoff(options);
+  if (!options.aggregateTypes || options.aggregateTypes.length === 0) {
+    throw new Error(
+      "pruneEvents: aggregateTypes is required and must be non-empty. Pruning the event log is destructive — name the aggregate types to delete explicitly.",
+    );
+  }
+  const aggregateTypes = options.aggregateTypes;
+  const dryRun = options.dryRun === true;
+
+  return db.transaction(async (tx) => {
+    // Serialise against consumer-bootstrap INSERTs. Without this, the race
+    // is: prune reads consumers (snapshot misses a consumer bootstrapping
+    // in a parallel tx) → consumer commits its row with
+    // lastProcessedEventId=0 → prune deletes events below its new cursor
+    // → first pass of the new consumer silently skips the deleted events.
+    //
+    // SHARE is the lightest table-mode that conflicts with ROW EXCLUSIVE
+    // (the mode INSERT/UPDATE/DELETE take). Concurrent INSERTs on the
+    // consumer table queue until this TX commits; concurrent UPDATEs
+    // (cursor advances) do too, but prune is measured in milliseconds and
+    // pausing cursor advances for that window is cheap insurance against
+    // a silent data-loss bug.
+    //
+    // Drizzle can't express LOCK TABLE — drop to raw SQL with the table
+    // name identifier so a future table-rename is caught at compile time.
+    await tx.execute(sql.raw(`LOCK TABLE ${getTableName(eventConsumerStateTable)} IN SHARE MODE`));
+
+    // Step 1 — collect candidate event ids.
+    const candidates = await tx
+      .select({ id: eventsTable.id })
+      .from(eventsTable)
+      .where(
+        and(
+          inArray(eventsTable.aggregateType, [...aggregateTypes]),
+          lt(eventsTable.createdAt, cutoff),
+        ),
+      );
+
+    if (candidates.length === 0) {
+      return { deletedCount: 0, cutoff, aggregateTypes, dryRun };
+    }
+
+    const maxCandidateId = candidates.reduce(
+      (acc, row) => (row.id > acc ? row.id : acc),
+      candidates[0]?.id ?? 0n,
+    );
+
+    // Step 2 — safety guard: check every ACTIVE consumer has moved past
+    // the candidate set. Disabled consumers are intentionally excluded —
+    // ops disables them precisely to park them while pruning happens.
+    // The SHARE lock above guarantees this SELECT sees a complete view:
+    // no new consumer can INSERT a fresh-cursor row between here and the
+    // DELETE below.
+    const activeConsumers = await tx
+      .select()
+      .from(eventConsumerStateTable)
+      .where(sql`${eventConsumerStateTable.status} <> 'disabled'`);
+
+    for (const consumer of activeConsumers) {
+      if (consumer.lastProcessedEventId < maxCandidateId) {
+        throw new ConsumerLagError(consumer.name, consumer.lastProcessedEventId, maxCandidateId);
+      }
+    }
+
+    if (dryRun) {
+      return { deletedCount: candidates.length, cutoff, aggregateTypes, dryRun: true };
+    }
+
+    // Step 3 — actual delete, bounded to the candidate set.
+    const candidateIds = candidates.map((c) => c.id);
+    const deleted = await tx
+      .delete(eventsTable)
+      .where(inArray(eventsTable.id, candidateIds))
+      .returning({ id: eventsTable.id });
+
+    return { deletedCount: deleted.length, cutoff, aggregateTypes, dryRun: false };
+  });
+}

package/src/pipeline/idempotency.ts

@@ -0,0 +1,76 @@
+import type Redis from "ioredis";
+import { RedisKeys } from "./redis-keys";
+
+export type IdempotencyGuard = {
+  check(requestId: string): Promise<string | null>;
+  store(requestId: string, result: unknown): Promise<void>;
+};
+
+// Sentinel stored under the key while the handler is running. A second
+// request that sees this waits for the real result instead of racing.
+const PENDING_MARKER = "__pending__";
+
+export function createIdempotencyGuard(
+  redis: Redis,
+  options: {
+    ttlSeconds?: number;
+    pendingTtlSeconds?: number;
+    waitTimeoutMs?: number;
+    pollIntervalMs?: number;
+  } = {},
+): IdempotencyGuard {
+  const ttl = options.ttlSeconds ?? 300;
+  // Max time a single handler is allowed to hold the in-progress lock before
+  // a parallel retry is allowed to try again. Short enough that a crashed
+  // handler doesn't permanently block retries, long enough to cover normal
+  // batch durations.
+  const pendingTtl = options.pendingTtlSeconds ?? 30;
+  const waitTimeoutMs = options.waitTimeoutMs ?? 25_000;
+  const pollIntervalMs = options.pollIntervalMs ?? 50;
+  const prefix = RedisKeys.idempotency;
+
+  return {
+    // Returns:
+    //   null   — caller owns the in-progress lock, proceed to run the handler
+    //            and then call store() with the real result.
+    //   string — serialized cached result from a concurrent or prior request.
+    //
+    // The old behaviour (pure GET + SET-NX-store) let two parallel requests
+    // both see a cache miss, both execute side-effects, and only one persist
+    // the result. This version uses a pending-marker lock so the second caller
+    // waits for the first to finish and reuses its result.
+    async check(requestId) {
+      const key = `${prefix}${requestId}`;
+
+      // Try to acquire the in-progress lock.
+      const acquired = await redis.set(key, PENDING_MARKER, "EX", pendingTtl, "NX");
+      if (acquired === "OK") return null;
+
+      // Lost the race. Poll until the lock holder stores a result, or the
+      // lock expires (handler crashed) and we can try again.
+      const deadline = Date.now() + waitTimeoutMs;
+      while (Date.now() < deadline) {
+        const value = await redis.get(key);
+        if (value === null) {
+          // Lock expired before a result was stored — try to claim it
+          // ourselves and proceed as the new owner.
+          const reclaimed = await redis.set(key, PENDING_MARKER, "EX", pendingTtl, "NX");
+          if (reclaimed === "OK") return null;
+          continue;
+        }
+        if (value !== PENDING_MARKER) return value;
+        await new Promise((resolve) => setTimeout(resolve, pollIntervalMs));
+      }
+
+      // Gave up waiting. Treat as a fresh request — forces the caller to
+      // re-run the handler rather than hang indefinitely.
+      return null;
+    },
+
+    async store(requestId, result) {
+      // Overwrite the pending marker with the real result. Plain SET (no NX)
+      // on purpose: we own the lock; writing the result is the final step.
+      await redis.set(`${prefix}${requestId}`, JSON.stringify(result), "EX", ttl);
+    },
+  };
+}

package/src/pipeline/index.ts

@@ -0,0 +1,66 @@
+export type { ResolveAuthClaimsArgs } from "./auth-claims-resolver";
+export { resolveAuthClaims } from "./auth-claims-resolver";
+export { createCascadeDeleteHook } from "./cascade-handler";
+export type { Dispatcher } from "./dispatcher";
+export { createDispatcher } from "./dispatcher";
+export type { DistributedLock } from "./distributed-lock";
+export { createDistributedLock } from "./distributed-lock";
+export type { EntityCache, EntityCacheOptions } from "./entity-cache";
+export { createEntityCache } from "./entity-cache";
+export type { ConsumerStatus } from "./event-consumer-state";
+export {
+  CONSUMER_STATUSES,
+  ConsumerStatuses,
+  createEventConsumerStateTable,
+  eventConsumerStateTable,
+} from "./event-consumer-state";
+export type { EventDedup } from "./event-dedup";
+export { createEventDedup } from "./event-dedup";
+export type {
+  ConsumerProgress,
+  ConsumerRecoveryState,
+  DispatcherPassResult,
+  EventConsumer,
+  EventConsumerHandler,
+  EventDispatcher,
+  EventDispatcherOptions,
+} from "./event-dispatcher";
+export {
+  createEventDispatcher,
+  disableConsumer,
+  enableConsumer,
+  getAllConsumerProgress,
+  getConsumerState,
+  listConsumersWithState,
+  restartConsumer,
+  skipPoisonEvent,
+} from "./event-dispatcher";
+export type { PruneEventsOptions, PruneEventsResult } from "./event-retention";
+export { ConsumerLagError, pruneEvents } from "./event-retention";
+export type { IdempotencyGuard } from "./idempotency";
+export { createIdempotencyGuard } from "./idempotency";
+export type { LifecycleHooks, SystemHookDef, SystemHooks } from "./lifecycle-pipeline";
+export { createLifecycleHooks } from "./lifecycle-pipeline";
+export type { MspRebuildDeps } from "./msp-rebuild";
+export { rebuildMultiStreamProjection } from "./msp-rebuild";
+export type { ProjectionProgress, RebuildResult } from "./projection-rebuild";
+export {
+  getAllProjectionProgress,
+  getProjectionState,
+  listProjectionsWithState,
+  rebuildProjection,
+} from "./projection-rebuild";
+export type { ProjectionStatus } from "./projection-state";
+export {
+  createProjectionStateTable,
+  PROJECTION_STATUSES,
+  ProjectionStatuses,
+  projectionStateTable,
+} from "./projection-state";
+export { runProjectionsForEvent } from "./projections-runner";
+export {
+  createSearchEventConsumer,
+  createSseBroadcastEventConsumer,
+  SEARCH_CONSUMER_NAME,
+  SSE_BROADCAST_CONSUMER_NAME,
+} from "./system-hooks";