@cosmicdrift/kumiko-framework 0.131.0 → 0.133.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cosmicdrift/kumiko-framework",
-  "version": "0.131.0",
+  "version": "0.133.0",
   "description": "Framework core — engine, pipeline, API, DB, and every other bit that makes Kumiko go.",
   "license": "BUSL-1.1",
   "author": "Marc Frost <marc@cosmicdriftgamestudio.com>",
@@ -189,7 +189,7 @@
     "zod": "^4.4.3"
   },
   "devDependencies": {
-    "@cosmicdrift/kumiko-dispatcher-live": "0.131.0",
+    "@cosmicdrift/kumiko-dispatcher-live": "0.133.0",
     "bun-types": "^1.3.13",
     "pino-pretty": "^13.1.3"
   },

package/src/db/__tests__/assert-no-unreachable-live-rows.integration.test.ts

@@ -0,0 +1,139 @@
+// Runtime guard #722: before the rebuild swaps the shadow over the live table,
+// abort if a live row has NO event in the projection's source streams — a row
+// no replay can reconstruct (#498 ghost, direct-inserted without a .created
+// event), which the swap would silently drop.
+//
+// The guard is deliberately narrow (event EXISTENCE only, not a column diff):
+// the framework legitimately makes live diverge from a fresh replay in shipped
+// ways (blind-index erase→NULL, sensitive-strip, archived-stream wipe, the #494
+// backfill flow). Those rows all HAVE an event, so they are not ghosts and the
+// guard leaves them alone — proven by the "direct column-write" test below.
+
+import { afterAll, beforeAll, beforeEach, describe, expect, test } from "bun:test";
+import { type BunTestDb, createTestDb } from "../../bun-db/__tests__/bun-test-db";
+import { asRawClient, insertOne, selectMany } from "../../db/query";
+import { createBooleanField, createEntity, createTextField, defineFeature } from "../../engine";
+import { createRegistry } from "../../engine/registry";
+import { createEventsTable } from "../../event-store";
+import { rebuildProjection } from "../../pipeline";
+import { createProjectionStateTable } from "../../pipeline/projection-state";
+import { TestUsers, unsafeCreateEntityTable } from "../../stack";
+import { ensureTemporalPolyfill } from "../../time/polyfill";
+import { createEventStoreExecutor } from "../event-store-executor";
+import { buildEntityTable } from "../table-builder";
+import { createTenantDb, type TenantDb } from "../tenant-db";
+
+const userEntity = createEntity({
+  table: "read_unreachable_users",
+  fields: {
+    email: createTextField({ required: true }),
+    firstName: createTextField(),
+    isEnabled: createBooleanField({ default: true }),
+  },
+  softDelete: true,
+});
+
+const userFeature = defineFeature("unreachabletest", (r) => {
+  r.entity("user", userEntity);
+});
+
+const userTable = buildEntityTable("user", userEntity);
+const projectionName = "unreachabletest:projection:user-entity";
+const registry = createRegistry([userFeature]);
+
+let testDb: BunTestDb;
+let tdb: TenantDb;
+const adminUser = TestUsers.admin;
+
+beforeAll(async () => {
+  await ensureTemporalPolyfill();
+  testDb = await createTestDb();
+  await unsafeCreateEntityTable(testDb.db, userEntity, "user");
+  await createEventsTable(testDb.db);
+  await createProjectionStateTable(testDb.db);
+  tdb = createTenantDb(testDb.db, adminUser.tenantId);
+});
+
+afterAll(async () => {
+  await testDb.cleanup();
+});
+
+beforeEach(async () => {
+  await asRawClient(testDb.db).unsafe(
+    `TRUNCATE kumiko_events, read_unreachable_users, kumiko_projections RESTART IDENTITY CASCADE`,
+  );
+});
+
+async function snapshotTable(): Promise<readonly Record<string, unknown>[]> {
+  const rows = await selectMany(
+    testDb.db,
+    userTable,
+    {},
+    { orderBy: { col: "id", direction: "asc" } },
+  );
+  return rows as readonly Record<string, unknown>[];
+}
+
+describe("assert-no-unreachable-live-rows / #722 ghost-row guard", () => {
+  test("clean executor-only projection rebuilds without firing the guard", async () => {
+    const crud = createEventStoreExecutor(userTable, userEntity, { entityName: "user" });
+    await crud.create({ email: "a@test.de", firstName: "Alice" }, adminUser, tdb);
+    await crud.create({ email: "b@test.de", firstName: "Bob" }, adminUser, tdb);
+    const before = await snapshotTable();
+
+    const result = await rebuildProjection(projectionName, { db: testDb.db, registry });
+
+    expect(result.eventsProcessed).toBe(2);
+    expect(await snapshotTable()).toEqual(before);
+  });
+
+  test("ghost row (no backing event) → swap aborts, live untouched", async () => {
+    const crud = createEventStoreExecutor(userTable, userEntity, { entityName: "user" });
+    const a = await crud.create({ email: "a@test.de", firstName: "Alice" }, adminUser, tdb);
+    if (!a.isSuccess) throw new Error("setup failed");
+
+    // Drift: clone the event-backed row into a second id WITHOUT appending an
+    // event — the #498 ghost. No replay can ever reconstruct it.
+    const [liveRow] = await selectMany(testDb.db, userTable, { id: a.data.id as string });
+    if (!liveRow) throw new Error("live row missing");
+    const ghostId = crypto.randomUUID();
+    // Deliberately bypass the EXECUTOR_ONLY brand: the whole point is to write
+    // the entity table WITHOUT going through the executor — the production bug
+    // the guard defends against.
+    const writable = userTable as unknown as Parameters<typeof insertOne>[1];
+    await insertOne(tdb, writable, { ...liveRow, id: ghostId, email: "ghost@test.de" });
+    expect(await snapshotTable()).toHaveLength(2);
+
+    await expect(rebuildProjection(projectionName, { db: testDb.db, registry })).rejects.toThrow(
+      /have no\s+event in the projection's source streams/,
+    );
+
+    // Swap never ran: both rows — including the ghost — survive.
+    const after = await snapshotTable();
+    expect(after).toHaveLength(2);
+    expect(after.map((r) => r["id"])).toContain(ghostId);
+  });
+
+  test("direct column-write on an event-backed row does NOT trip the guard", async () => {
+    // The row has a .created event, so it is not a ghost. Its column state was
+    // direct-written without an event (the #494 / blind-index-erase class) — the
+    // guard is event-existence-only and leaves it alone; the replay overwrites
+    // the column from the event, which is the intended, shipped behavior.
+    const crud = createEventStoreExecutor(userTable, userEntity, { entityName: "user" });
+    const a = await crud.create({ email: "a@test.de", firstName: "Alice" }, adminUser, tdb);
+    if (!a.isSuccess) throw new Error("setup failed");
+
+    await asRawClient(testDb.db).unsafe(
+      `UPDATE read_unreachable_users SET first_name = 'DirectWrite' WHERE id = $1`,
+      [a.data.id],
+    );
+
+    // No throw — the row is event-backed. Rebuild replays the create event.
+    await rebuildProjection(projectionName, { db: testDb.db, registry });
+
+    const [rebuilt] = await selectMany(testDb.db, userTable, { id: a.data.id as string });
+    expect(rebuilt?.["email"]).toBe("a@test.de");
+    // The direct write is overwritten by the replay — deliberately allowed.
+    expect(rebuilt?.["firstName"]).toBe("Alice");
+  });
+});

package/src/db/queries/shadow-swap.ts

@@ -149,6 +149,68 @@ export async function fenceLiveTable(
   await raw.unsafe(`LOCK TABLE public.${quoteTableIdent(tableName)} IN ACCESS EXCLUSIVE MODE`);
 }
 
+// Ids reported when the swap is aborted — enough to locate the ghost rows
+// without dumping an unbounded set into the log.
+const UNREACHABLE_SAMPLE_LIMIT = 20;
+
+// Runs INSIDE the rebuild tx, under the fence, before swapShadowIntoLive. A live
+// row whose aggregate id has NO event in the projection's source streams is
+// UNREACHABLE: no replay can ever reconstruct it, so the swap would silently
+// drop it. That is the #498 ghost — a row direct-inserted without ever emitting
+// a .created event. The static CI guard cannot see it in data that already
+// exists in production, or on table identifiers it couldn't resolve; this
+// catches it at cutover and aborts (tx rolls back, live untouched).
+//
+// Deliberately NARROW — event EXISTENCE only, not a column or row-vs-shadow
+// diff. The framework legitimately makes live diverge from a fresh replay in
+// several SHIPPED ways, none of which is drift:
+//   - a blind-index column recomputed to NULL after the subject's key is
+//     shredded (GDPR erase) — the NULL is the intended end state;
+//   - a `sensitive` column stripped from the event log by design;
+//   - an archived stream that stops replaying (fw#832) — the row's wipe is the
+//     intended tombstone behavior, reported via backfill's `failed` list;
+//   - a legacy column direct-written before its handler emitted events, healed
+//     by the #494 backfill-then-rebuild flow.
+// Checking event existence INCLUDING archived streams leaves every one of them
+// alone: those rows all have a real event, so they are not ghosts. Column-level
+// drift detection (fail-hard vs. graceful repair) is the open question deferred
+// from #722.
+//
+// Implicit projections only (caller-gated). aggregate_id and the entity id are
+// both uuid, so the anti-join probes the events index without a cast.
+export async function assertNoUnreachableLiveRows(
+  tx: AnyDb,
+  projectionName: string,
+  tableName: string,
+  aggregateTypes: readonly string[],
+): Promise<void> {
+  // skip: no source streams → no events could back any row anyway; a rebuild
+  // of a subscription-less projection swaps an empty shadow (handled upstream).
+  if (aggregateTypes.length === 0) return;
+  const raw = asRawClient(tx);
+  const t = quoteTableIdent(tableName);
+  const ghosts = await raw.unsafe<{ id: unknown }>(
+    `SELECT l."id" FROM public.${t} l
+     WHERE NOT EXISTS (
+       SELECT 1 FROM "kumiko_events" e
+        WHERE e."aggregate_id" = l."id" AND e."aggregate_type" = ANY($1::text[])
+     )
+     LIMIT ${UNREACHABLE_SAMPLE_LIMIT}`,
+    [aggregateTypes],
+  );
+  // skip: every live row has a backing event — nothing unreachable, swap is safe
+  if (ghosts.length === 0) return;
+  const ids = ghosts.map((r) => String(r.id));
+  throw new Error(
+    `projection-rebuild "${projectionName}": ${ids.length}+ live rows in "${tableName}" have no ` +
+      `event in the projection's source streams and cannot be reconstructed by replay — the swap ` +
+      `would silently drop them (ids: ${ids.join(", ")}). A handler direct-inserted these rows ` +
+      `without emitting a .created event. Fix: register the table with r.unmanagedTable(meta, ` +
+      `{ reason }) to opt out of rebuild, or emit the missing events. See ` +
+      `docs/reference/entity-write-patterns.md. Rebuild aborted; live table untouched.`,
+  );
+}
+
 // Atomic swap, INSIDE the rebuild tx, AFTER replay. Schema-qualified so the
 // active shadow search_path can't redirect them. DROP without CASCADE: if any
 // object depends on the live table the swap fails loud and the whole rebuild

package/src/engine/__tests__/deep-link.test.ts

@@ -0,0 +1,35 @@
+import { describe, expect, test } from "bun:test";
+import { buildDeepLinkUrl } from "../deep-link";
+
+describe("buildDeepLinkUrl()", () => {
+  test("screenId only", () => {
+    expect(buildDeepLinkUrl("https://app.example.com", { screenId: "jobs" })).toBe(
+      "https://app.example.com/jobs",
+    );
+  });
+
+  test("screenId + entityId", () => {
+    expect(
+      buildDeepLinkUrl("https://app.example.com", { screenId: "jobs", entityId: "job-1" }),
+    ).toBe("https://app.example.com/jobs/job-1");
+  });
+
+  test("workspaceId + screenId + entityId, in path order", () => {
+    expect(
+      buildDeepLinkUrl("https://app.example.com", {
+        workspaceId: "admin",
+        screenId: "jobs",
+        entityId: "job-1",
+      }),
+    ).toBe("https://app.example.com/admin/jobs/job-1");
+  });
+
+  test("strips trailing slash(es) from baseUrl", () => {
+    expect(buildDeepLinkUrl("https://app.example.com/", { screenId: "jobs" })).toBe(
+      "https://app.example.com/jobs",
+    );
+    expect(buildDeepLinkUrl("https://app.example.com//", { screenId: "jobs" })).toBe(
+      "https://app.example.com/jobs",
+    );
+  });
+});

package/src/engine/deep-link.ts

@@ -0,0 +1,23 @@
+// Deep-Link-URL-Builder für Notification-Templates (#449, Lazy-Scope: nur
+// Notification→Screen, kein Permalink-Sharing-Layer). Server-seitig nutzbar
+// (kein React) — der Renderer hat mit `formatPath`
+// (packages/renderer/src/app/nav.tsx) dasselbe Pfad-Format fürs Client-Routing,
+// hier bewusst dupliziert statt importiert: `renderer` zieht React als
+// Dependency, Notification-Data-Fns laufen server-seitig im Write-Handler.
+//
+// baseUrl kommt vom App-Autor (analog `AuthMailOptions.baseUrl` /
+// magic-link-mail.ts appendToken) — kein Auto-Detect, kein Env-Var-Read hier.
+
+export type DeepLinkTarget = {
+  readonly screenId: string;
+  readonly entityId?: string;
+  readonly workspaceId?: string;
+};
+
+export function buildDeepLinkUrl(baseUrl: string, target: DeepLinkTarget): string {
+  const segments: string[] = [];
+  if (target.workspaceId !== undefined) segments.push(target.workspaceId);
+  segments.push(target.screenId);
+  if (target.entityId !== undefined) segments.push(target.entityId);
+  return `${baseUrl.replace(/\/+$/, "")}/${segments.join("/")}`;
+}

package/src/engine/index.ts

@@ -38,6 +38,8 @@ export {
 export type { App, AppConfig } from "./create-app";
 export { createApp } from "./create-app";
 export { crossTenantOverrideDenied } from "./cross-tenant";
+export type { DeepLinkTarget } from "./deep-link";
+export { buildDeepLinkUrl } from "./deep-link";
 export { defineFeature } from "./define-feature";
 export type {
   QueryHandlerDefinition,

package/src/jobs/__tests__/job-queue-depth.integration.test.ts

@@ -0,0 +1,82 @@
+import { afterAll, beforeAll, describe, expect, test } from "bun:test";
+import { createRegistry, defineFeature } from "../../engine";
+import type { AppContext, Registry } from "../../engine/types";
+import { createPrometheusMeter, registerStandardMetrics } from "../../observability";
+import { createTestRedis, type TestRedis } from "../../stack";
+import { sleep } from "../../testing";
+import { createJobRunner } from "../job-runner";
+
+let testRedis: TestRedis;
+let redisUrl: string;
+
+const testFeature = defineFeature("test-queue-depth", (r) => {
+  r.job("noop", { trigger: { manual: true } }, async () => {});
+});
+
+beforeAll(async () => {
+  testRedis = await createTestRedis();
+  redisUrl = `redis://${testRedis.redis.options.host}:${testRedis.redis.options.port}/${testRedis.redis.options.db}`;
+});
+
+afterAll(async () => {
+  await testRedis.cleanup();
+});
+
+describe("job-runner — kumiko_job_queue_depth", () => {
+  test("start() polls BullMQ counts into the gauge, stop() tears the poller down", async () => {
+    const registry: Registry = createRegistry([testFeature]);
+    const meter = createPrometheusMeter();
+    registerStandardMetrics(meter);
+    const context: AppContext = { meter };
+    const queueNamePrefix = `kumiko-test-qd-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
+
+    const runner = createJobRunner({
+      registry,
+      context,
+      redisUrl,
+      consumerLane: "worker",
+      queueNamePrefix,
+    });
+
+    try {
+      await runner.start();
+
+      const snapshot = meter.snapshot().get("kumiko_job_queue_depth");
+      expect(snapshot).toBeDefined();
+      const waitingSlot = snapshot?.slots.find(
+        (s) => s.labels?.["lane"] === "worker" && s.labels?.["state"] === "waiting",
+      );
+      expect(waitingSlot).toBeDefined();
+      expect((waitingSlot as { value: number }).value).toBe(0);
+      // BullMQ's fresh Worker connection is still settling right after
+      // start() returns (no boot/cron job here to have already warmed it
+      // up, unlike every other scenario in this suite) — stop() immediately
+      // after start() races the Worker's own connection teardown and throws
+      // an unrelated "Connection is closed" from ioredis. Production runners
+      // live far longer than this; the race is a test-only artifact of
+      // start()+stop() with zero work in between.
+      await sleep(50);
+    } finally {
+      await runner.stop();
+      const keys = await testRedis.redis.keys(`bull:${queueNamePrefix}-worker:*`);
+      if (keys.length > 0) await testRedis.redis.del(...keys);
+    }
+  });
+
+  test("enqueuer-only runner (no consumerLane) never polls — no gauge slots", async () => {
+    const registry: Registry = createRegistry([testFeature]);
+    const meter = createPrometheusMeter();
+    registerStandardMetrics(meter);
+    const context: AppContext = { meter };
+    const queueNamePrefix = `kumiko-test-qd-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
+
+    const runner = createJobRunner({ registry, context, redisUrl, queueNamePrefix });
+    try {
+      await runner.start();
+      const snapshot = meter.snapshot().get("kumiko_job_queue_depth");
+      expect(snapshot?.slots.length ?? 0).toBe(0);
+    } finally {
+      await runner.stop();
+    }
+  });
+});

package/src/jobs/job-runner.ts

@@ -11,7 +11,13 @@ import {
   SYSTEM_TENANT_ID,
 } from "../engine/types";
 import type { Logger } from "../logging/types";
-import { getFallbackTracer, type SerializedTraceContext, type Tracer } from "../observability";
+import {
+  emitJobQueueDepth,
+  getFallbackTracer,
+  type Meter,
+  type SerializedTraceContext,
+  type Tracer,
+} from "../observability";
 import { createDistributedLock, type DistributedLock } from "../pipeline/distributed-lock";
 import { RedisKeys } from "../pipeline/redis-keys";
 
@@ -119,6 +125,29 @@ function captureTraceContext(tracer: Tracer): SerializedTraceContext | undefined
   return { traceId: span.traceId, spanId: span.spanId };
 }
 
+const QUEUE_DEPTH_POLL_INTERVAL_MS = 15_000;
+
+// kumiko_job_queue_depth: only the lane's own consumer polls — the count is
+// a global Redis-backed value (BullMQ, not per-process), so one reporter per
+// lane is enough. Fires once immediately (metrics exist before the first
+// poll tick) then on an interval; the caller stores + clears the handle.
+async function startQueueDepthPolling(
+  queue: Queue,
+  lane: JobRunIn,
+  meter: Meter,
+): Promise<ReturnType<typeof setInterval>> {
+  const pollQueueDepth = async (): Promise<void> => {
+    try {
+      const counts = await queue.getJobCounts();
+      emitJobQueueDepth(meter, lane, counts);
+    } catch {
+      // skip: transient Redis hiccup — next poll retries
+    }
+  };
+  await pollQueueDepth();
+  return setInterval(() => void pollQueueDepth(), QUEUE_DEPTH_POLL_INTERVAL_MS);
+}
+
 function parseRedisOpts(url: string): { host: string; port: number; db?: number | undefined } {
   const parsed = new URL(url);
   const result: { host: string; port: number; db?: number | undefined } = {
@@ -181,6 +210,7 @@ export function createJobRunner(options: JobRunnerOptions): JobRunner {
     worker: new Queue(queueNameFor(queueNamePrefix, "worker"), { connection: redisOpts }),
   };
   let worker: Worker | null = null;
+  let queueDepthTimer: ReturnType<typeof setInterval> | null = null;
   // Forward reference to the runner's own API, exposed on the job-handler ctx
   // so a handler can dispatch a follow-up job (job→job chaining, e.g.
   // delivery.render → delivery.send). Assigned just before return; reads happen
@@ -431,9 +461,19 @@ export function createJobRunner(options: JobRunnerOptions): JobRunner {
           await consumerQueue.add(bootName, {}, { jobId: `boot-${name.replace(/\./g, "-")}` });
         }
       }
+
+      // Skipped when no meter is wired (context.meter is optional, e.g. in
+      // tests without an observability provider).
+      if (context.meter) {
+        queueDepthTimer = await startQueueDepthPolling(consumerQueue, consumerLane, context.meter);
+      }
     },
 
     async stop(): Promise<void> {
+      if (queueDepthTimer) {
+        clearInterval(queueDepthTimer);
+        queueDepthTimer = null;
+      }
       if (worker) {
         await worker.close();
         worker = null;

package/src/observability/index.ts

@@ -52,6 +52,7 @@ export {
   emitEventConsumerPassOutcome,
   emitEventDispatcherListenConnected,
   emitHttpRequest,
+  emitJobQueueDepth,
   registerStandardMetrics,
   STANDARD_METRIC_DEFS,
 } from "./standard-metrics";

package/src/observability/standard-metrics.ts

@@ -94,6 +94,21 @@ export const STANDARD_METRIC_DEFS: readonly MetricDefinition[] = [
       "1 if the event-dispatcher holds an active PG LISTEN subscription on the events channel, 0 otherwise.",
     labels: [],
   },
+  // BullMQ backlog per lane + state (waiting/active/delayed/failed/…). Only
+  // the process that owns a lane's consumerLane polls it (job-runner.ts
+  // start()) — the count is a global Redis-backed value, no per-instance
+  // label needed. A growing "waiting" count means the lane's worker can't
+  // keep up with dispatch; "failed" trending up means jobs are erroring
+  // silently. Motivated by apps that run every job on the "api" lane
+  // (no dedicated worker consumer) — fanout/reconcile jobs then share
+  // Redis-queue capacity with request-handling, and contention is
+  // otherwise invisible until requests slow down.
+  {
+    name: "kumiko_job_queue_depth",
+    type: "gauge",
+    description: "BullMQ job counts per lane and state.",
+    labels: ["lane", "state"],
+  },
 ] as const;
 
 export function registerStandardMetrics(meter: Meter): void {
@@ -211,3 +226,16 @@ export function emitEventConsumerPassOutcome(
     });
   }
 }
+
+// counts: BullMQ's Queue.getJobCounts() result — a state → count record
+// (waiting/active/delayed/failed/completed/…), passed through unmodified so
+// this stays agnostic of exactly which states BullMQ reports.
+export function emitJobQueueDepth(
+  meter: Meter,
+  lane: string,
+  counts: Readonly<Record<string, number>>,
+): void {
+  for (const [state, count] of Object.entries(counts)) {
+    meter.gauge("kumiko_job_queue_depth").set(count, { lane, state });
+  }
+}

package/src/pipeline/projection-rebuild.ts

@@ -8,6 +8,7 @@ import {
 } from "../db/queries/projection-rebuild";
 import {
   assertLiveColumnsMatchMeta,
+  assertNoUnreachableLiveRows,
   buildShadowTable,
   ensureRebuildSchema,
   fenceLiveTable,
@@ -326,6 +327,12 @@ export async function rebuildProjection(
       if (skipped.length > 0) {
         await recordRebuildDeadLetters(tx, projectionName, skipped);
       }
+      // Guard the swap: abort if the live table holds a row no event can
+      // reconstruct (#498 ghost — direct-inserted without a .created event),
+      // which the swap would silently drop. Implicit projections only.
+      if (projection.isImplicit === true) {
+        await assertNoUnreachableLiveRows(tx, projectionName, meta.tableName, sourcesList);
+      }
       await finalizeProjectionRebuild(tx, projectionName, lastProcessedEventId);
       await swapShadowIntoLive(tx, meta.tableName);
     });