@cosmicdrift/kumiko-framework 0.290.0 → 0.291.0

package/src/jobs/__tests__/job-last-success.integration.test.ts

@@ -0,0 +1,135 @@
+import { afterAll, beforeAll, describe, expect, test } from "bun:test";
+import { buildServer } from "../../api/server";
+import { createRegistry, defineFeature } from "../../engine";
+import type { AppContext, Registry } from "../../engine/types";
+import {
+  createNoopProvider,
+  createPrometheusMeter,
+  registerStandardMetrics,
+} from "../../observability";
+import { createTestRedis, type TestRedis } from "../../stack";
+import { waitFor } from "../../testing";
+import { createJobRunner } from "../job-runner";
+
+const JWT = "job-last-success-test-secret-minimum-32-chars!!";
+const SUCCEEDS = "liveness:job:succeeds";
+const FAILS = "liveness:job:fails-always";
+
+let testRedis: TestRedis;
+let redisUrl: string;
+
+const livenessFeature = defineFeature("liveness", (r) => {
+  r.job("succeeds", { trigger: { manual: true } }, async () => {});
+  r.job("failsAlways", { trigger: { manual: true } }, async () => {
+    throw new Error("intentional failure");
+  });
+});
+
+beforeAll(async () => {
+  testRedis = await createTestRedis();
+  redisUrl = `redis://${testRedis.redis.options.host}:${testRedis.redis.options.port}/${testRedis.redis.options.db}`;
+});
+
+afterAll(async () => {
+  await testRedis.cleanup();
+});
+
+function slotFor(meter: ReturnType<typeof createPrometheusMeter>, job: string) {
+  return meter
+    .snapshot()
+    .get("kumiko_job_last_success_timestamp_seconds")
+    ?.slots.find((s) => s.labels?.["job"] === job);
+}
+
+async function withRunner(
+  meter: ReturnType<typeof createPrometheusMeter>,
+  fn: (runner: ReturnType<typeof createJobRunner>, failures: string[]) => Promise<void>,
+): Promise<void> {
+  const registry: Registry = createRegistry([livenessFeature]);
+  const context: AppContext = { meter };
+  const failures: string[] = [];
+  const queueNamePrefix = `kumiko-test-ls-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
+  const runner = createJobRunner({
+    registry,
+    context,
+    redisUrl,
+    consumerLane: "worker",
+    queueNamePrefix,
+    onJobFailed: (jobName) => {
+      failures.push(jobName);
+    },
+  });
+  await runner.start();
+  try {
+    await fn(runner, failures);
+  } finally {
+    await runner.stop();
+    const keys = await testRedis.redis.keys(`bull:${queueNamePrefix}-worker:*`);
+    if (keys.length > 0) await testRedis.redis.del(...keys);
+  }
+}
+
+describe("job-runner — kumiko_job_last_success_timestamp_seconds", () => {
+  test("a successful run stamps the gauge, a failing run leaves no series", async () => {
+    const meter = createPrometheusMeter();
+    registerStandardMetrics(meter);
+
+    await withRunner(meter, async (runner, failures) => {
+      const before = Date.now() / 1000;
+      await runner.dispatch(SUCCEEDS, {});
+      await waitFor(() => slotFor(meter, SUCCEEDS) !== undefined);
+
+      const stamped = slotFor(meter, SUCCEEDS) as { value: number };
+      expect(stamped.value).toBeGreaterThanOrEqual(before);
+      expect(stamped.value).toBeLessThanOrEqual(Date.now() / 1000 + 1);
+
+      // Separate job name, so the "failure must not stamp" assertion cannot
+      // pass just because a prior success left a value inside the same second.
+      await runner.dispatch(FAILS, {});
+      await waitFor(() => failures.includes(FAILS));
+      expect(slotFor(meter, FAILS)).toBeUndefined();
+    });
+  });
+
+  test("a later success advances the stamp", async () => {
+    const meter = createPrometheusMeter();
+    registerStandardMetrics(meter);
+
+    await withRunner(meter, async (runner) => {
+      await runner.dispatch(SUCCEEDS, {});
+      await waitFor(() => slotFor(meter, SUCCEEDS) !== undefined);
+      const first = (slotFor(meter, SUCCEEDS) as { value: number }).value;
+
+      await runner.dispatch(SUCCEEDS, {});
+      await waitFor(() => (slotFor(meter, SUCCEEDS) as { value: number }).value > first, {
+        delays: [250, 1000, 3000],
+      });
+      expect((slotFor(meter, SUCCEEDS) as { value: number }).value).toBeGreaterThan(first);
+    });
+  });
+
+  test("the stamp reaches the real /metrics scrape output", async () => {
+    const meter = createPrometheusMeter();
+    registerStandardMetrics(meter);
+
+    await withRunner(meter, async (runner) => {
+      await runner.dispatch(SUCCEEDS, {});
+      await waitFor(() => slotFor(meter, SUCCEEDS) !== undefined);
+
+      // Same meter instance the runner wrote into — that sharing is what
+      // buildServer + job-runner do in a real process (fw#1046).
+      const { app } = buildServer({
+        registry: createRegistry([livenessFeature]),
+        context: {},
+        jwtSecret: JWT,
+        observability: { ...createNoopProvider(), meter },
+        metrics: {},
+      });
+      const res = await app.request("/metrics");
+      expect(res.status).toBe(200);
+      const body = await res.text();
+      expect(body).toContain("# TYPE kumiko_job_last_success_timestamp_seconds gauge");
+      expect(body).toContain(`kumiko_job_last_success_timestamp_seconds{job="${SUCCEEDS}"} `);
+    });
+  });
+});

package/src/jobs/index.ts

@@ -1,2 +1,8 @@
-export type { JobLogEntry, JobMeta, JobRunner, JobRunnerOptions } from "./job-runner";
+export type {
+  JobLogEntry,
+  JobMeta,
+  JobOutcomeMeta,
+  JobRunner,
+  JobRunnerOptions,
+} from "./job-runner";
 export { createJobRunner } from "./job-runner";

package/src/jobs/job-runner.ts

@@ -18,10 +18,12 @@ import {
   SYSTEM_TENANT_ID,
   type TenantId,
 } from "../engine/types";
+import { isKumikoError } from "../errors/kumiko-error";
 import { createFileContext } from "../files/file-handle";
 import { createFallbackLogger } from "../logging";
 import type { Logger } from "../logging/types";
 import {
+  emitJobLastSuccess,
   emitJobQueueDepth,
   getFallbackTracer,
   type Meter,
@@ -139,6 +141,55 @@ export type JobMeta = {
   priority?: number | undefined;
 };
 
+// What a finished run tells the run-logger about the tenant-visible failure
+// record (`JobDefinition.tenantVisibleFailure`). `tenantVisible` is set only
+// when the job opted in; `messageKey` is null on the success path, where the
+// record is cleared rather than written.
+export type JobOutcomeMeta = {
+  readonly tenantId: string;
+  readonly finalAttempt: boolean;
+  readonly tenantVisible?:
+    | { readonly subject: string | null; readonly messageKey: string | null }
+    | undefined;
+};
+
+// Stable identity for the declared payload fields: sorted keys, so two runs
+// with the same subject values produce the same string and the later one
+// replaces the earlier record. Non-primitives would serialize into something
+// no caller can reconstruct for a lookup, so they fail the run loudly.
+function jobSubjectKey(
+  jobName: string,
+  payload: Record<string, unknown>,
+  fields: readonly string[] | undefined,
+): string | null {
+  if (fields === undefined || fields.length === 0) return null;
+  const entries: [string, string | number | boolean | null][] = [];
+  for (const field of [...fields].sort()) {
+    const value = payload[field] ?? null;
+    if (value !== null && typeof value === "object") {
+      throw new Error(
+        `Job "${jobName}": tenantVisibleFailure.subjectFields["${field}"] must be a primitive, got ${typeof value}`,
+      );
+    }
+    entries.push([field, value as string | number | boolean | null]);
+  }
+  return JSON.stringify(entries);
+}
+
+// Only a translation key ever travels to the tenant: the thrown error's own
+// i18nKey when it carries one, otherwise the key declared at the job. The
+// error message itself can echo provider or payload content and stays on the
+// run row and in the run log.
+function tenantFailureMessageKey(
+  err: unknown,
+  declaration: JobDefinition["tenantVisibleFailure"],
+): string | null {
+  // skip: job did not opt in — nothing tenant-visible to record
+  if (!declaration) return null;
+  if (isKumikoError(err) && err.i18nKey.length > 0) return err.i18nKey;
+  return declaration.messageKey;
+}
+
 export type JobRunner = {
   start(): Promise<void>;
   stop(): Promise<void>;
@@ -176,8 +227,20 @@ export type JobRunnerOptions = {
   bootRedisTimeoutMs?: number | undefined;
   getActiveTenantIds?: () => Promise<TenantId[]>;
   onJobStart?: (jobName: string, jobId: string, meta: JobMeta) => void;
-  onJobComplete?: (jobName: string, jobId: string, duration: number, logs: JobLogEntry[]) => void;
-  onJobFailed?: (jobName: string, jobId: string, error: string, logs: JobLogEntry[]) => void;
+  onJobComplete?: (
+    jobName: string,
+    jobId: string,
+    duration: number,
+    logs: JobLogEntry[],
+    outcome?: JobOutcomeMeta,
+  ) => void;
+  onJobFailed?: (
+    jobName: string,
+    jobId: string,
+    error: string,
+    logs: JobLogEntry[],
+    outcome?: JobOutcomeMeta,
+  ) => void;
 };
 
 // Serialized trace context lives under this key in the BullMQ job data.
@@ -527,6 +590,24 @@ export function createJobRunner(options: JobRunnerOptions): JobRunner {
       SYSTEM_TENANT_ID;
     const triggeredById = (rawData["_triggeredById"] as string | undefined) ?? null; // @cast-boundary dynamic-key
 
+    // Tenant-visible failure record (JobDefinition.tenantVisibleFailure). The
+    // subject is read from the handler payload, so it is resolved here where
+    // the payload is built, not in the callbacks.
+    const tenantVisibleDecl = jobDef.tenantVisibleFailure;
+    const tenantVisibleSubject = tenantVisibleDecl
+      ? jobSubjectKey(jobName, payload, tenantVisibleDecl.subjectFields)
+      : null;
+    // BullMQ stops retrying once attemptsMade reaches the configured attempts
+    // (`retries + 1`), so this is the attempt whose failure is final.
+    const finalAttempt = bullJob.attemptsMade + 1 >= (jobDef.retries ?? 0) + 1;
+    const outcomeMeta = (messageKey: string | null): JobOutcomeMeta => ({
+      tenantId,
+      finalAttempt,
+      ...(tenantVisibleDecl && {
+        tenantVisible: { subject: tenantVisibleSubject, messageKey },
+      }),
+    });
+
     // Carry `_triggerName` from rawData when set — handleEvent injects it on
     // multi-trigger dispatch; exposed as jobContext.triggerName so handlers
     // don't dig through the raw payload themselves.
@@ -676,12 +757,21 @@ export function createJobRunner(options: JobRunnerOptions): JobRunner {
           },
           () => jobDef.handler(payload, jobContext),
         );
+        // Stamped before the observer hook: a throwing onJobComplete must not
+        // make a run that actually succeeded look dead to the liveness alert.
+        if (context.meter) emitJobLastSuccess(context.meter, jobName);
         const duration = Date.now() - startTime;
-        await options.onJobComplete?.(jobName, jobId, duration, logs);
+        await options.onJobComplete?.(jobName, jobId, duration, logs, outcomeMeta(null));
       } catch (err) {
         const errorMsg = err instanceof Error ? err.message : String(err);
         logs.push({ level: "error", message: errorMsg, timestamp: Temporal.Now.instant() });
-        await options.onJobFailed?.(jobName, jobId, errorMsg, logs);
+        await options.onJobFailed?.(
+          jobName,
+          jobId,
+          errorMsg,
+          logs,
+          outcomeMeta(tenantFailureMessageKey(err, tenantVisibleDecl)),
+        );
         throw err;
       }
     };

package/src/logging/utils.ts

@@ -2,18 +2,27 @@ import type { Logger } from "./types";
 
 type FallbackLogger = {
   error(msg: string, data?: Record<string, unknown>): void;
+  warn(msg: string, data?: Record<string, unknown>): void;
   debug(msg: string, data?: Record<string, unknown>): void;
 };
 
 export function createFallbackLogger(
   namespace: string,
-  logger?: (Pick<Logger, "error"> & Partial<Pick<Logger, "debug">>) | undefined,
+  logger?: (Pick<Logger, "error"> & Partial<Pick<Logger, "warn" | "debug">>) | undefined,
 ): FallbackLogger {
   if (logger) {
     return {
       error(msg, data) {
         logger.error(`[${namespace}] ${msg}`, data);
       },
+      warn(msg, data) {
+        if (logger.warn) {
+          logger.warn(`[${namespace}] ${msg}`, data);
+        } else {
+          // biome-ignore lint/suspicious/noConsole: ops-visible fallback when the wrapped logger has no warn method
+          console.warn(`[${namespace}] ${msg}`, data);
+        }
+      },
       debug(msg, data) {
         if (logger.debug) {
           logger.debug(`[${namespace}] ${msg}`, data);
@@ -29,6 +38,10 @@ export function createFallbackLogger(
       // biome-ignore lint/suspicious/noConsole: ops-visible fallback when no logger is wired
       console.error(`[${namespace}] ${msg}`, data);
     },
+    warn(msg, data) {
+      // biome-ignore lint/suspicious/noConsole: ops-visible fallback when no logger is wired
+      console.warn(`[${namespace}] ${msg}`, data);
+    },
     debug(msg, data) {
       // biome-ignore lint/suspicious/noConsole: ops-visible fallback when no logger is wired
       console.debug(`[${namespace}] ${msg}`, data);

package/src/observability/index.ts

@@ -54,6 +54,7 @@ export {
   emitEventConsumerRearmExhausted,
   emitEventDispatcherListenConnected,
   emitHttpRequest,
+  emitJobLastSuccess,
   emitJobQueueDepth,
   registerStandardMetrics,
   STANDARD_METRIC_DEFS,

package/src/observability/standard-metrics.ts

@@ -123,6 +123,19 @@ export const STANDARD_METRIC_DEFS: readonly MetricDefinition[] = [
     description: "BullMQ job counts per lane and state.",
     labels: ["lane", "state"],
   },
+  // Dead-man for in-process jobs: the k8s CronJob alerts never fire for them
+  // (no CronJob object exists), and a gauge the job itself sets freezes on its
+  // last value when the job dies, so neither `absent()` nor a threshold
+  // catches a silently stopped cron. This one is stamped by the runner, so
+  // `time() - kumiko_job_last_success_timestamp_seconds{job="…"} > interval`
+  // is a true liveness check. Absent until the first success after a restart —
+  // see docs/reference/job-liveness-metric.md for the `for:` that implies.
+  {
+    name: "kumiko_job_last_success_timestamp_seconds",
+    type: "gauge",
+    description: "Unix timestamp of the last successful run, per registered job.",
+    labels: ["job"],
+  },
 ] as const;
 
 export function registerStandardMetrics(meter: Meter): void {
@@ -273,3 +286,10 @@ export function emitJobQueueDepth(
     meter.gauge("kumiko_job_queue_depth").set(count, { lane, state });
   }
 }
+
+// `job` is the registry-declared job name — handleJob rejects an unknown name
+// before it can reach here, so the label set is bounded by the app's r.job
+// registrations and carries no tenant or user input.
+export function emitJobLastSuccess(meter: Meter, job: string): void {
+  meter.gauge("kumiko_job_last_success_timestamp_seconds").set(Date.now() / 1000, { job });
+}

package/src/pipeline/__tests__/blind-index-rebuild-guard.integration.test.ts

@@ -0,0 +1,96 @@
+// projection-rebuild aborts instead of silently NULLing an already-populated
+// blind-index column when this process has no blind-index key configured
+// (fw#3091) — see assertNoBlindIndexLoss in db/queries/shadow-swap.ts.
+
+import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test";
+import { resetBlindIndexKeyForTests } from "@cosmicdrift/kumiko-framework/testing";
+import { computeBlindIndex, configureBlindIndexKey, decodeBlindIndexKey } from "../../crypto";
+import { createEventStoreExecutor } from "../../db/event-store-executor";
+import { asRawClient } from "../../db/query";
+import { buildEntityTable } from "../../db/table-builder";
+import { createTenantDb, type TenantDb } from "../../db/tenant-db";
+import { createEntity, createRegistry, createTextField, defineFeature } from "../../engine";
+import { createEventsTable } from "../../event-store";
+import { createProjectionStateTable, rebuildProjection } from "../../pipeline";
+import { createTestDb, type TestDb, TestUsers, unsafeCreateEntityTable } from "../../stack";
+
+const TEST_KEY_B64 = Buffer.alloc(32, 9).toString("base64");
+const TEST_KEY = decodeBlindIndexKey(TEST_KEY_B64);
+
+const personEntity = createEntity({
+  table: "read_bidx_guard_persons",
+  fields: {
+    email: createTextField({ required: true, personal: "self", find: "exact" }),
+  },
+});
+const personFeature = defineFeature("bidxguardtest", (r) => {
+  r.entity("person", personEntity);
+});
+const personTable = buildEntityTable("person", personEntity);
+const implicitName = "bidxguardtest:projection:person-entity";
+
+const admin = TestUsers.admin;
+let testDb: TestDb;
+let tdb: TenantDb;
+const registry = createRegistry([personFeature]);
+const crud = createEventStoreExecutor(personTable, personEntity, { entityName: "person" });
+
+beforeAll(async () => {
+  testDb = await createTestDb();
+  await unsafeCreateEntityTable(testDb.db, personEntity, "person");
+  await createEventsTable(testDb.db);
+  await createProjectionStateTable(testDb.db);
+  tdb = createTenantDb(testDb.db, admin.tenantId);
+});
+
+afterAll(async () => {
+  await testDb.cleanup();
+});
+
+beforeEach(async () => {
+  await asRawClient(testDb.db).unsafe(
+    `TRUNCATE kumiko_events, read_bidx_guard_persons, kumiko_projections RESTART IDENTITY CASCADE`,
+  );
+  configureBlindIndexKey(TEST_KEY_B64);
+});
+
+afterEach(() => {
+  resetBlindIndexKeyForTests();
+});
+
+async function rawRow(id: string): Promise<Record<string, unknown>> {
+  const rows = await asRawClient(testDb.db).unsafe<Record<string, unknown>>(
+    `SELECT * FROM read_bidx_guard_persons WHERE id = $1`,
+    [id],
+  );
+  const row = rows[0];
+  if (!row) throw new Error(`no row for ${id}`);
+  return row;
+}
+
+describe("projection-rebuild — blind-index loss guard (fw#3091)", () => {
+  test("populated bidx column + no key in this process → rebuild throws, live table untouched", async () => {
+    const created = await crud.create({ email: "marc@example.com" }, admin, tdb);
+    if (!created.isSuccess) throw new Error("create failed");
+    const before = await rawRow(String(created.data.id));
+    expect(before["email_bidx"]).toBe(computeBlindIndex(TEST_KEY, "marc@example.com"));
+
+    resetBlindIndexKeyForTests();
+    await expect(rebuildProjection(implicitName, { db: testDb.db, registry })).rejects.toThrow(
+      /KUMIKO_BLIND_INDEX_KEY/,
+    );
+
+    const after = await rawRow(String(created.data.id));
+    expect(after).toEqual(before);
+  });
+
+  test("populated bidx column + key configured → rebuild succeeds, bidx recomputed", async () => {
+    const created = await crud.create({ email: "marc@example.com" }, admin, tdb);
+    if (!created.isSuccess) throw new Error("create failed");
+
+    await rebuildProjection(implicitName, { db: testDb.db, registry });
+
+    const after = await rawRow(String(created.data.id));
+    expect(after["email_bidx"]).toBe(computeBlindIndex(TEST_KEY, "marc@example.com"));
+  });
+});

package/src/pipeline/projection-rebuild.ts

@@ -9,6 +9,7 @@ import {
 import {
   assertLiveColumnsMatchMeta,
   assertLiveTableHasNoRowLevelSecurity,
+  assertNoBlindIndexLoss,
   assertNoUnreachableLiveRows,
   buildShadowTable,
   type ColumnDriftResult,
@@ -358,6 +359,12 @@ export async function rebuildProjection(
       if (skipped.length > 0) {
         await recordRebuildDeadLetters(tx, projectionName, skipped);
       }
+      // Guard the swap: abort if the process has no blind-index key configured
+      // but the live table already has populated bidx columns — the rebuild
+      // would otherwise silently null them out (fw#3091). Applies to every
+      // projection, not just implicit ones: the shadow always gets a fresh
+      // computeBlindIndexValues() pass regardless of projection kind.
+      await assertNoBlindIndexLoss(tx, meta.tableName, meta, projectionName);
       // Guard the swap: abort if the live table holds a row no event can
       // reconstruct (#498 ghost — direct-inserted without a .created event),
       // which the swap would silently drop. Implicit projections only.

package/src/schema-cli.ts

@@ -10,6 +10,12 @@
 
 import { existsSync, mkdirSync, readdirSync, writeFileSync } from "node:fs";
 import { join, resolve as resolvePath } from "node:path";
+import {
+  configureBlindIndexKey,
+  configurePiiSubjectKms,
+  type KmsWiring,
+  resolveKmsWiringAsync,
+} from "./crypto";
 import {
   assertValidMigrationName,
   baselineMigrations,
@@ -316,6 +322,7 @@ export async function runSchemaCli(
         return 1;
       }
       const { db, close } = createDbConnection(dbUrl);
+      let wiring: KmsWiring | undefined;
       try {
         const result = await runMigrationsFromDir(db, migrationsDir);
         // Framework-Infra-Tabellen (event-store + pipeline-state) — die erfasst
@@ -346,6 +353,19 @@ export async function runSchemaCli(
         // retry it — otherwise a failed rebuild is silently never retried,
         // since the migration itself is already tracked applied (#2464).
         if (options.features) {
+          // A rebuild replays applyEntityEvent, which computes bidx columns from
+          // configureBlindIndexKey/configurePiiSubjectKms — both boot-injected by
+          // run{Prod,Dev}App but never by this standalone CLI (the migrate-db
+          // initContainer). Without wiring them here, a rebuild would silently
+          // null out bidx columns and, for ciphertext, throw only once it hits
+          // the row (fw#3091).
+          wiring = await resolveKmsWiringAsync(process.env, {
+            logPrefix: "[kumiko schema apply]",
+          });
+          if ("kms" in wiring) {
+            configurePiiSubjectKms(wiring.kms);
+            configureBlindIndexKey(wiring.blindIndexKey);
+          }
           const thisRunTables = await queueRebuildsFromMarkers(db, {
             migrationsDir,
             appliedIds: result.applied,
@@ -376,6 +396,7 @@ export async function runSchemaCli(
         out.err("");
         return 1;
       } finally {
+        await wiring?.close();
         await close();
       }
     }