@cosmicdrift/kumiko-framework 0.1.0

package/src/pipeline/distributed-lock.ts

@@ -0,0 +1,37 @@
+import type Redis from "ioredis";
+import { generateId } from "../utils";
+import { RedisKeys } from "./redis-keys";
+
+export type DistributedLock = {
+  acquire(key: string, options?: { ttlSeconds?: number }): Promise<string | null>;
+  release(key: string, token: string): Promise<boolean>;
+};
+
+export function createDistributedLock(
+  redis: Redis,
+  prefix: string = RedisKeys.lock,
+): DistributedLock {
+  // Lua script for atomic check-and-delete (safe Redis server-side eval, not JS eval)
+  const releaseScript = `
+    if redis.call("get", KEYS[1]) == ARGV[1] then
+      return redis.call("del", KEYS[1])
+    else
+      return 0
+    end
+  `;
+
+  return {
+    async acquire(key, options = {}) {
+      const ttl = options.ttlSeconds ?? 30;
+      const token = generateId();
+      const result = await redis.set(`${prefix}${key}`, token, "EX", ttl, "NX");
+      return result === "OK" ? token : null;
+    },
+
+    async release(key, token) {
+      // Atomic: only release if we own the lock (compare token via Lua)
+      const result = (await redis.eval(releaseScript, 1, `${prefix}${key}`, token)) as number;
+      return result === 1;
+    },
+  };
+}

package/src/pipeline/entity-cache.ts

@@ -0,0 +1,113 @@
+import type { EntityId, TenantId } from "@cosmicdrift/kumiko-framework/engine";
+import type Redis from "ioredis";
+import { RedisKeys } from "./redis-keys";
+
+// JSON.stringify turns Date into an ISO string, but DB reads return Date
+// objects. Without a reviver the cache path would yield strings where the
+// DB path yields Dates — consumers would silently see a type mismatch.
+const ISO_DATE = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})$/;
+
+function dateReviver(_key: string, value: unknown): unknown {
+  if (typeof value === "string" && ISO_DATE.test(value)) {
+    const d = new Date(value);
+    if (!Number.isNaN(d.getTime())) return d;
+  }
+  return value;
+}
+
+function parseCached(raw: string): Record<string, unknown> | null {
+  try {
+    return JSON.parse(raw, dateReviver) as Record<string, unknown> | null; // @cast-boundary engine-payload
+  } catch {
+    return null;
+  }
+}
+
+export type EntityCache = {
+  /** Get a single cached entity. Returns null on miss. */
+  get(
+    tenantId: TenantId,
+    entityName: string,
+    id: EntityId,
+  ): Promise<Record<string, unknown> | null>;
+
+  /** Get multiple cached entities. Returns a Map of id → data (misses are absent). */
+  mget(
+    tenantId: TenantId,
+    entityName: string,
+    ids: readonly EntityId[],
+  ): Promise<Map<EntityId, Record<string, unknown>>>;
+
+  /** Cache a single entity. */
+  set(
+    tenantId: TenantId,
+    entityName: string,
+    id: EntityId,
+    data: Record<string, unknown>,
+  ): Promise<void>;
+
+  /** Cache multiple entities at once. */
+  mset(
+    tenantId: TenantId,
+    entityName: string,
+    entries: ReadonlyArray<{ id: EntityId; data: Record<string, unknown> }>,
+  ): Promise<void>;
+
+  /** Invalidate a single cached entity. */
+  del(tenantId: TenantId, entityName: string, id: EntityId): Promise<void>;
+};
+
+export type EntityCacheOptions = {
+  ttlSeconds?: number;
+};
+
+export function createEntityCache(redis: Redis, options: EntityCacheOptions = {}): EntityCache {
+  const ttl = options.ttlSeconds ?? 300;
+  const prefix = RedisKeys.entityCache;
+
+  function cacheKey(tenantId: TenantId, entityName: string, id: EntityId): string {
+    return `${prefix}${tenantId}:${entityName}:${id}`;
+  }
+
+  return {
+    async get(tenantId, entityName, id) {
+      const raw = await redis.get(cacheKey(tenantId, entityName, id));
+      if (!raw) return null;
+      return parseCached(raw);
+    },
+
+    async mget(tenantId, entityName, ids) {
+      if (ids.length === 0) return new Map();
+      const keys = ids.map((id) => cacheKey(tenantId, entityName, id));
+      const values = await redis.mget(...keys);
+
+      const result = new Map<EntityId, Record<string, unknown>>();
+      for (let i = 0; i < ids.length; i++) {
+        const raw = values[i];
+        if (raw) {
+          const parsed = parseCached(raw);
+          if (parsed) result.set(ids[i] as EntityId, parsed);
+        }
+      }
+      return result;
+    },
+
+    async set(tenantId, entityName, id, data) {
+      await redis.set(cacheKey(tenantId, entityName, id), JSON.stringify(data), "EX", ttl);
+    },
+
+    async mset(tenantId, entityName, entries) {
+      // skip: empty batch, nothing to cache
+      if (entries.length === 0) return;
+      const pipe = redis.pipeline();
+      for (const entry of entries) {
+        pipe.set(cacheKey(tenantId, entityName, entry.id), JSON.stringify(entry.data), "EX", ttl);
+      }
+      await pipe.exec();
+    },
+
+    async del(tenantId, entityName, id) {
+      await redis.del(cacheKey(tenantId, entityName, id));
+    },
+  };
+}

package/src/pipeline/event-consumer-state.ts

@@ -0,0 +1,108 @@
+import { sql } from "drizzle-orm";
+import type { DbConnection } from "../db/connection";
+import { bigint, index, instant, integer, table as pgTable, primaryKey, text } from "../db/dialect";
+import { tableExists } from "../db/schema-inspection";
+import { pushTables } from "../stack";
+
+// Reserved sentinel used in the instance_id column for consumers whose
+// delivery is "shared" — i.e. one cursor across all dispatcher instances
+// (the default, pre-Welle-2.7 behaviour). Per-instance consumers store
+// the concrete instanceId. Postgres PK columns can't be NULL and
+// `NULL = NULL` is UNKNOWN in SQL — a nullable instance_id would break
+// both uniqueness and PK constraints. The boot-validator refuses to
+// start with KUMIKO_INSTANCE_ID === SHARED_INSTANCE_SENTINEL so the
+// sentinel can never collide with a real instance identifier.
+export const SHARED_INSTANCE_SENTINEL = "__shared__";
+
+// Framework-level state per event-consumer-shard. A "consumer" is anything
+// that walks the events-table via a persistent cursor: system consumers
+// (SSE, Search) and feature multiStreamProjections (async, cross-aggregate).
+//
+// One row per (consumer name, instance_id) shard. Shared-delivery consumers
+// have exactly one row with instance_id = SHARED_INSTANCE_SENTINEL — this
+// preserves the pre-Welle-2.7 single-cursor semantic unchanged. Per-instance
+// consumers get N rows (one per dispatcher instance), each with its own
+// cursor — used by SSE so every API process pushes the same events to its
+// own clients without a pub/sub transport. Read by the event-dispatcher
+// (cursor + locking), surfaced by the CLI for ops inspection.
+//
+// Columns:
+//   - name: consumer's qualified identifier, e.g. "system:consumer:sse" or
+//     "my-feature:consumer:analytics". Matches the qualified-name convention
+//     so two features can't accidentally collide on the same consumer name.
+//   - instanceId: SHARED_INSTANCE_SENTINEL for shared-delivery consumers;
+//     concrete process-local identifier (ServerOptions.instanceId, defaults
+//     to KUMIKO_INSTANCE_ID or a boot-time UUID) for per-instance consumers.
+//   - lastProcessedEventId: bigserial `events.id` of the most recent event
+//     this shard finished handling. Dispatcher reads events WHERE id > this.
+//   - status / attempts / lastError / updatedAt — per shard.
+//
+// Composite PK (name, instance_id): Postgres requires NOT NULL on all PK
+// columns, and `NULL = NULL` is UNKNOWN in SQL — nullable instance_id would
+// break uniqueness for shared rows. Sentinel avoids both hazards with one
+// uniform column shape.
+//
+// CAUTION (retention-guard + scale-down):
+//   pruneEvents refuses to delete past MIN(lastProcessedEventId) across ALL
+//   shards. A decommissioned instance leaves its row behind at its last
+//   cursor — prune is then pinned indefinitely. Before scaling down,
+//   delete stale per-instance shards:
+//     DELETE FROM kumiko_event_consumers WHERE instance_id = '<decommissioned>'
+//   TODO: auto-cleanup via heartbeat-liveness — follow-up, not in v1.
+//
+// The default(sql`0`) on lastProcessedEventId mirrors projection-state.ts:
+// drizzle-kit's JSON snapshot generator can't serialise a bigint literal, so
+// the server-side default is specified as raw SQL instead of .default(0n).
+export const eventConsumerStateTable = pgTable(
+  "kumiko_event_consumers",
+  {
+    name: text("name").notNull(),
+    instanceId: text("instance_id").notNull().default(SHARED_INSTANCE_SENTINEL),
+    lastProcessedEventId: bigint("last_processed_event_id", { mode: "bigint" })
+      .notNull()
+      .default(sql`0`),
+    status: text("status").notNull().default("idle"),
+    attempts: integer("attempts").notNull().default(0),
+    lastError: text("last_error"),
+    updatedAt: instant("updated_at", { precision: 3 }).notNull().default(sql`now()`),
+  },
+  (t) => ({
+    pk: primaryKey({ columns: [t.name, t.instanceId] }),
+    statusIdx: index("kumiko_event_consumers_status_idx").on(t.status),
+  }),
+);
+
+// Object-const form lets call sites write `ConsumerStatuses.disabled` instead
+// of the raw string, which keeps status checks refactor-safe. The runtime
+// value on each field is the same string the DB stores — no mapping needed.
+export const ConsumerStatuses = {
+  idle: "idle",
+  processing: "processing",
+  dead: "dead",
+  disabled: "disabled",
+} as const;
+export type ConsumerStatus = (typeof ConsumerStatuses)[keyof typeof ConsumerStatuses];
+
+/**
+ * @deprecated Use `ConsumerStatuses` (object form) or the `ConsumerStatus`
+ * union type. The tuple form is kept as a back-compat alias for callers
+ * that were using it with `z.enum(...)` or runtime iteration — scheduled
+ * for removal once downstream consumers have migrated.
+ */
+export const CONSUMER_STATUSES = [
+  "idle",
+  "processing",
+  "dead",
+  "disabled",
+] as const satisfies readonly ConsumerStatus[];
+
+// Idempotent bootstrap. Called by setupTestStack + production boot path —
+// same pattern as createProjectionStateTable / createEventsTable. If the
+// table is already present (second stack in the same test DB, prod boot
+// after migration), skip cleanly.
+//
+export async function createEventConsumerStateTable(db: DbConnection): Promise<void> {
+  // skip: table already exists — bootstrap is called from multiple paths
+  if (await tableExists(db, "public.kumiko_event_consumers")) return;
+  await pushTables(db, { kumikoEventConsumers: eventConsumerStateTable });
+}

package/src/pipeline/event-dedup.ts

@@ -0,0 +1,23 @@
+import type Redis from "ioredis";
+import { RedisKeys } from "./redis-keys";
+
+export type EventDedup = {
+  /**
+   * Atomically try to acquire processing rights for an eventId.
+   * Returns true if this is the first call (proceed), false if already processed (skip).
+   */
+  tryAcquire(eventId: string): Promise<boolean>;
+};
+
+export function createEventDedup(redis: Redis, options: { ttlSeconds?: number } = {}): EventDedup {
+  const ttl = options.ttlSeconds ?? 300;
+  const prefix = RedisKeys.eventDedup;
+
+  return {
+    async tryAcquire(eventId) {
+      // SET NX = atomic check-and-set: only succeeds if key does not exist
+      const result = await redis.set(`${prefix}${eventId}`, "1", "EX", ttl, "NX");
+      return result === "OK";
+    },
+  };
+}