@cosmicdrift/kumiko-framework 0.289.0 → 0.291.0

package/src/event-store/__tests__/event-attribution.integration.test.ts

@@ -0,0 +1,186 @@
+// #3043 — event attribution. append() stamps metadata.feature +
+// metadata.handler from the ambient execution scope, below every envelope
+// builder, so no writer can forget it.
+//
+// Claims pinned here:
+//   1. Dispatch path: an event a write-handler appends carries the handler's
+//      qualified name and its owning feature.
+//   2. In-process path: the entity-executor's CRUD event carries the SAME
+//      attribution although nothing was passed through its signature.
+//   3. MSP-apply: an event an apply writes is attributed to the consumer, not
+//      to the handler that started the chain.
+//   4. No scope: a bare append() stamps the sentinel, never a guess.
+//   5. appendRaw stays unstamped — historical rows keep their metadata verbatim
+//      and stay readable without the new fields.
+
+import { afterAll, afterEach, beforeAll, describe, expect, test } from "bun:test";
+import { UNATTRIBUTED_ORIGIN } from "@cosmicdrift/kumiko-types/event-store-types";
+import { z } from "zod";
+import { createEventStoreExecutor } from "../../db/event-store-executor";
+import { selectMany } from "../../db/query";
+import { buildEntityTable } from "../../db/table-builder";
+import { createEntity, createTextField, defineFeature } from "../../engine";
+import {
+  resetEventStore,
+  setupTestStack,
+  type TestStack,
+  TestUsers,
+  unsafeCreateEntityTable,
+} from "../../stack";
+import { generateId as uuid } from "../../utils";
+import { appendRaw } from "../admin-api";
+import { append } from "../event-store";
+import { eventsTable } from "../events-schema";
+
+const orderEntity = createEntity({
+  table: "read_attribution_orders",
+  fields: {
+    item: createTextField({ personal: false, reason: "test_fixture", required: true }),
+  },
+});
+
+const orderTable = buildEntityTable("attr-order", orderEntity);
+
+const PLACED = "attribution:event:placed";
+const CONFIRMED = "attribution:event:confirmed";
+const PLACE_HANDLER = "attribution:write:order:place";
+const CONFIRMER_MSP = "attribution:projection:confirmer";
+
+const attributionFeature = defineFeature("attribution", (r) => {
+  r.entity("attr-order", orderEntity);
+
+  const placed = r.defineEvent("placed", z.object({ orderId: z.uuid() }), { piiFields: "none" });
+  const confirmed = r.defineEvent("confirmed", z.object({ orderId: z.uuid() }), {
+    piiFields: "none",
+  });
+
+  const orderExecutor = createEventStoreExecutor(orderTable, orderEntity, {
+    entityName: "attr-order",
+  });
+
+  r.writeHandler(
+    "order:place",
+    z.object({ item: z.string() }),
+    async (event, ctx) => {
+      const created = await orderExecutor.create({ item: event.payload.item }, event.user, ctx.db);
+      if (!created.isSuccess) return created;
+      await ctx.unsafeAppendEvent({
+        aggregateId: String(created.data.id),
+        aggregateType: "attr-order",
+        type: placed.name,
+        payload: { orderId: String(created.data.id) },
+      });
+      return created;
+    },
+    { access: { roles: ["Admin"] } },
+  );
+
+  r.multiStreamProjection({
+    name: "confirmer",
+    apply: {
+      [placed.name]: async (event, _tx, ctx) => {
+        if (!ctx) throw new Error("MSP-apply ctx missing — regression of C.2b wiring");
+        await ctx.unsafeAppendEvent({
+          aggregateId: event.aggregateId,
+          aggregateType: "attr-order",
+          type: confirmed.name,
+          payload: { orderId: event.aggregateId },
+        });
+      },
+    },
+  });
+});
+
+let stack: TestStack;
+const admin = TestUsers.admin;
+
+beforeAll(async () => {
+  stack = await setupTestStack({ features: [attributionFeature], systemHooks: [] });
+  await unsafeCreateEntityTable(stack.db, orderEntity, "attr-order");
+});
+
+afterAll(async () => {
+  await stack.cleanup();
+});
+
+afterEach(async () => {
+  await resetEventStore(stack, ["read_attribution_orders"]);
+});
+
+type Origin = { feature?: string; handler?: string };
+
+async function originOf(type: string): Promise<Origin> {
+  const rows = await selectMany(stack.db, eventsTable);
+  const row = rows.find((r: Record<string, unknown>) => r["type"] === type);
+  expect(row).toBeDefined();
+  return (row?.["metadata"] ?? {}) as Origin;
+}
+
+describe("#3043 — event attribution from the execution scope", () => {
+  test("dispatch path: ctx.appendEvent stamps the handler and its feature", async () => {
+    await stack.http.writeOk(PLACE_HANDLER, { item: "widget" }, admin);
+
+    expect(await originOf(PLACED)).toMatchObject({
+      feature: "attribution",
+      handler: PLACE_HANDLER,
+    });
+  });
+
+  test("in-process path: the entity-executor CRUD event carries the same attribution", async () => {
+    await stack.http.writeOk(PLACE_HANDLER, { item: "sprocket" }, admin);
+
+    // Nothing is threaded through createEntityExecutor → EventStoreExecutor.write —
+    // the write runs inside the handler's scope, so the stamp finds it anyway.
+    expect(await originOf("attr-order.created")).toMatchObject({
+      feature: "attribution",
+      handler: PLACE_HANDLER,
+    });
+  });
+
+  test("MSP-apply: the follow-up event is attributed to the consumer, not the trigger", async () => {
+    await stack.http.writeOk(PLACE_HANDLER, { item: "gasket" }, admin);
+    await stack.eventDispatcher?.runOnce();
+
+    expect(await originOf(CONFIRMED)).toMatchObject({
+      feature: "attribution",
+      handler: CONFIRMER_MSP,
+    });
+  });
+
+  test("no scope: a bare append() stamps the sentinel instead of guessing", async () => {
+    const aggregateId = uuid();
+    await append(stack.db, {
+      aggregateId,
+      aggregateType: "attr-order",
+      tenantId: admin.tenantId,
+      expectedVersion: 0,
+      type: PLACED,
+      payload: { orderId: aggregateId },
+      metadata: { userId: admin.id },
+    });
+
+    expect(await originOf(PLACED)).toMatchObject({
+      feature: UNATTRIBUTED_ORIGIN,
+      handler: UNATTRIBUTED_ORIGIN,
+    });
+  });
+
+  test("appendRaw keeps historical metadata verbatim and stays readable", async () => {
+    const aggregateId = uuid();
+    await appendRaw(stack.db, {
+      aggregateId,
+      aggregateType: "attr-order",
+      tenantId: admin.tenantId,
+      expectedVersion: 0,
+      type: CONFIRMED,
+      payload: { orderId: aggregateId },
+      metadata: { userId: admin.id },
+      createdAt: Temporal.Instant.from("2023-01-15T10:00:00Z"),
+      createdBy: admin.id,
+    });
+
+    const origin = await originOf(CONFIRMED);
+    expect(origin.feature).toBeUndefined();
+    expect(origin.handler).toBeUndefined();
+  });
+});

package/src/event-store/event-store.ts

@@ -1,4 +1,8 @@
-import type { EventMetadata, StoredEvent } from "@cosmicdrift/kumiko-types/event-store-types";
+import {
+  type EventMetadata,
+  type StoredEvent,
+  UNATTRIBUTED_ORIGIN,
+} from "@cosmicdrift/kumiko-types/event-store-types";
 // Value-only import, aliased to avoid shadowing the ambient global
 // `Temporal` TYPE this file's other Temporal.Instant annotations resolve
 // against (StoredEvent.createdAt et al. — importing the bare name here
@@ -6,6 +10,7 @@ import type { EventMetadata, StoredEvent } from "@cosmicdrift/kumiko-types/event
 // as a runtime value on globalThis, so the un-aliased call below crashed
 // with "Temporal is not defined" outside boot paths that install it (#1480).
 import { Temporal as TemporalPolyfill } from "temporal-polyfill";
+import { requestContext } from "../api/request-context";
 import { encryptEventPayloadPii } from "../crypto/event-pii";
 import type { DbRunner } from "../db";
 import { constraintOf, isUniqueViolation } from "../db/pg-error";
@@ -83,7 +88,7 @@ export async function append(db: DbRunner, event: EventToAppend): Promise<Stored
     aggregateType: event.aggregateType,
     aggregateId: event.aggregateId,
   });
-  const toStore = payload === event.payload ? event : { ...event, payload };
+  const toStore = stampOrigin(payload === event.payload ? event : { ...event, payload });
   const newVersion = toStore.expectedVersion + 1;
   const eventVersion = toStore.eventVersion ?? 1;
 
@@ -116,6 +121,22 @@ export async function append(db: DbRunner, event: EventToAppend): Promise<Stored
   }
 }
 
+// #3043 — attribution is derived from the execution scope, never taken from
+// the caller: a passed-in value can lie about who wrote the row, a derived
+// one cannot. Hence overwrite rather than merge. appendRaw/appendRawBatch
+// bypass this deliberately — they replay historical rows verbatim.
+function stampOrigin(event: EventToAppend): EventToAppend {
+  const origin = requestContext.get();
+  return {
+    ...event,
+    metadata: {
+      ...event.metadata,
+      feature: origin?.feature ?? UNATTRIBUTED_ORIGIN,
+      handler: origin?.handler ?? UNATTRIBUTED_ORIGIN,
+    },
+  };
+}
+
 type InsertReturn = { id: bigint; createdAt: Temporal.Instant };
 
 async function insertFirstEvent(

package/src/i18n/required-surface-keys.ts

@@ -328,6 +328,7 @@ export function requiredKeysFromScreen(
         if (isWriteFormEditSection(section)) {
           pushKey(out, section.title);
           pushKey(out, section.submitLabel);
+          // kumiko-lint-ignore section-fields-raw writeForm sections carry no groups (EditWriteFormSection)
           for (const f of section.fields) {
             const fieldName = editFieldName(f);
             out.add(fieldLabelKey(featureName, WRITE_FORM_SECTION_ENTITY, fieldName));

package/src/jobs/__tests__/job-last-success.integration.test.ts

@@ -0,0 +1,135 @@
+import { afterAll, beforeAll, describe, expect, test } from "bun:test";
+import { buildServer } from "../../api/server";
+import { createRegistry, defineFeature } from "../../engine";
+import type { AppContext, Registry } from "../../engine/types";
+import {
+  createNoopProvider,
+  createPrometheusMeter,
+  registerStandardMetrics,
+} from "../../observability";
+import { createTestRedis, type TestRedis } from "../../stack";
+import { waitFor } from "../../testing";
+import { createJobRunner } from "../job-runner";
+
+const JWT = "job-last-success-test-secret-minimum-32-chars!!";
+const SUCCEEDS = "liveness:job:succeeds";
+const FAILS = "liveness:job:fails-always";
+
+let testRedis: TestRedis;
+let redisUrl: string;
+
+const livenessFeature = defineFeature("liveness", (r) => {
+  r.job("succeeds", { trigger: { manual: true } }, async () => {});
+  r.job("failsAlways", { trigger: { manual: true } }, async () => {
+    throw new Error("intentional failure");
+  });
+});
+
+beforeAll(async () => {
+  testRedis = await createTestRedis();
+  redisUrl = `redis://${testRedis.redis.options.host}:${testRedis.redis.options.port}/${testRedis.redis.options.db}`;
+});
+
+afterAll(async () => {
+  await testRedis.cleanup();
+});
+
+function slotFor(meter: ReturnType<typeof createPrometheusMeter>, job: string) {
+  return meter
+    .snapshot()
+    .get("kumiko_job_last_success_timestamp_seconds")
+    ?.slots.find((s) => s.labels?.["job"] === job);
+}
+
+async function withRunner(
+  meter: ReturnType<typeof createPrometheusMeter>,
+  fn: (runner: ReturnType<typeof createJobRunner>, failures: string[]) => Promise<void>,
+): Promise<void> {
+  const registry: Registry = createRegistry([livenessFeature]);
+  const context: AppContext = { meter };
+  const failures: string[] = [];
+  const queueNamePrefix = `kumiko-test-ls-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
+  const runner = createJobRunner({
+    registry,
+    context,
+    redisUrl,
+    consumerLane: "worker",
+    queueNamePrefix,
+    onJobFailed: (jobName) => {
+      failures.push(jobName);
+    },
+  });
+  await runner.start();
+  try {
+    await fn(runner, failures);
+  } finally {
+    await runner.stop();
+    const keys = await testRedis.redis.keys(`bull:${queueNamePrefix}-worker:*`);
+    if (keys.length > 0) await testRedis.redis.del(...keys);
+  }
+}
+
+describe("job-runner — kumiko_job_last_success_timestamp_seconds", () => {
+  test("a successful run stamps the gauge, a failing run leaves no series", async () => {
+    const meter = createPrometheusMeter();
+    registerStandardMetrics(meter);
+
+    await withRunner(meter, async (runner, failures) => {
+      const before = Date.now() / 1000;
+      await runner.dispatch(SUCCEEDS, {});
+      await waitFor(() => slotFor(meter, SUCCEEDS) !== undefined);
+
+      const stamped = slotFor(meter, SUCCEEDS) as { value: number };
+      expect(stamped.value).toBeGreaterThanOrEqual(before);
+      expect(stamped.value).toBeLessThanOrEqual(Date.now() / 1000 + 1);
+
+      // Separate job name, so the "failure must not stamp" assertion cannot
+      // pass just because a prior success left a value inside the same second.
+      await runner.dispatch(FAILS, {});
+      await waitFor(() => failures.includes(FAILS));
+      expect(slotFor(meter, FAILS)).toBeUndefined();
+    });
+  });
+
+  test("a later success advances the stamp", async () => {
+    const meter = createPrometheusMeter();
+    registerStandardMetrics(meter);
+
+    await withRunner(meter, async (runner) => {
+      await runner.dispatch(SUCCEEDS, {});
+      await waitFor(() => slotFor(meter, SUCCEEDS) !== undefined);
+      const first = (slotFor(meter, SUCCEEDS) as { value: number }).value;
+
+      await runner.dispatch(SUCCEEDS, {});
+      await waitFor(() => (slotFor(meter, SUCCEEDS) as { value: number }).value > first, {
+        delays: [250, 1000, 3000],
+      });
+      expect((slotFor(meter, SUCCEEDS) as { value: number }).value).toBeGreaterThan(first);
+    });
+  });
+
+  test("the stamp reaches the real /metrics scrape output", async () => {
+    const meter = createPrometheusMeter();
+    registerStandardMetrics(meter);
+
+    await withRunner(meter, async (runner) => {
+      await runner.dispatch(SUCCEEDS, {});
+      await waitFor(() => slotFor(meter, SUCCEEDS) !== undefined);
+
+      // Same meter instance the runner wrote into — that sharing is what
+      // buildServer + job-runner do in a real process (fw#1046).
+      const { app } = buildServer({
+        registry: createRegistry([livenessFeature]),
+        context: {},
+        jwtSecret: JWT,
+        observability: { ...createNoopProvider(), meter },
+        metrics: {},
+      });
+      const res = await app.request("/metrics");
+      expect(res.status).toBe(200);
+      const body = await res.text();
+      expect(body).toContain("# TYPE kumiko_job_last_success_timestamp_seconds gauge");
+      expect(body).toContain(`kumiko_job_last_success_timestamp_seconds{job="${SUCCEEDS}"} `);
+    });
+  });
+});

package/src/jobs/index.ts

@@ -1,2 +1,8 @@
-export type { JobLogEntry, JobMeta, JobRunner, JobRunnerOptions } from "./job-runner";
+export type {
+  JobLogEntry,
+  JobMeta,
+  JobOutcomeMeta,
+  JobRunner,
+  JobRunnerOptions,
+} from "./job-runner";
 export { createJobRunner } from "./job-runner";

package/src/jobs/job-runner.ts

@@ -4,6 +4,7 @@ import { requestContext } from "../api/request-context";
 import type { DbConnection, DbRow } from "../db/connection";
 import { createTenantDb, createUncheckedSystemDb, type TenantDb } from "../db/tenant-db";
 import { createDerivativesContext } from "../derivatives/derivatives-context";
+import { qnScope } from "../engine/qualified-name";
 import { createSystemUser } from "../engine/system-user";
 import {
   type AppContext,
@@ -17,10 +18,12 @@ import {
   SYSTEM_TENANT_ID,
   type TenantId,
 } from "../engine/types";
+import { isKumikoError } from "../errors/kumiko-error";
 import { createFileContext } from "../files/file-handle";
 import { createFallbackLogger } from "../logging";
 import type { Logger } from "../logging/types";
 import {
+  emitJobLastSuccess,
   emitJobQueueDepth,
   getFallbackTracer,
   type Meter,
@@ -138,6 +141,55 @@ export type JobMeta = {
   priority?: number | undefined;
 };
 
+// What a finished run tells the run-logger about the tenant-visible failure
+// record (`JobDefinition.tenantVisibleFailure`). `tenantVisible` is set only
+// when the job opted in; `messageKey` is null on the success path, where the
+// record is cleared rather than written.
+export type JobOutcomeMeta = {
+  readonly tenantId: string;
+  readonly finalAttempt: boolean;
+  readonly tenantVisible?:
+    | { readonly subject: string | null; readonly messageKey: string | null }
+    | undefined;
+};
+
+// Stable identity for the declared payload fields: sorted keys, so two runs
+// with the same subject values produce the same string and the later one
+// replaces the earlier record. Non-primitives would serialize into something
+// no caller can reconstruct for a lookup, so they fail the run loudly.
+function jobSubjectKey(
+  jobName: string,
+  payload: Record<string, unknown>,
+  fields: readonly string[] | undefined,
+): string | null {
+  if (fields === undefined || fields.length === 0) return null;
+  const entries: [string, string | number | boolean | null][] = [];
+  for (const field of [...fields].sort()) {
+    const value = payload[field] ?? null;
+    if (value !== null && typeof value === "object") {
+      throw new Error(
+        `Job "${jobName}": tenantVisibleFailure.subjectFields["${field}"] must be a primitive, got ${typeof value}`,
+      );
+    }
+    entries.push([field, value as string | number | boolean | null]);
+  }
+  return JSON.stringify(entries);
+}
+
+// Only a translation key ever travels to the tenant: the thrown error's own
+// i18nKey when it carries one, otherwise the key declared at the job. The
+// error message itself can echo provider or payload content and stays on the
+// run row and in the run log.
+function tenantFailureMessageKey(
+  err: unknown,
+  declaration: JobDefinition["tenantVisibleFailure"],
+): string | null {
+  // skip: job did not opt in — nothing tenant-visible to record
+  if (!declaration) return null;
+  if (isKumikoError(err) && err.i18nKey.length > 0) return err.i18nKey;
+  return declaration.messageKey;
+}
+
 export type JobRunner = {
   start(): Promise<void>;
   stop(): Promise<void>;
@@ -175,8 +227,20 @@ export type JobRunnerOptions = {
   bootRedisTimeoutMs?: number | undefined;
   getActiveTenantIds?: () => Promise<TenantId[]>;
   onJobStart?: (jobName: string, jobId: string, meta: JobMeta) => void;
-  onJobComplete?: (jobName: string, jobId: string, duration: number, logs: JobLogEntry[]) => void;
-  onJobFailed?: (jobName: string, jobId: string, error: string, logs: JobLogEntry[]) => void;
+  onJobComplete?: (
+    jobName: string,
+    jobId: string,
+    duration: number,
+    logs: JobLogEntry[],
+    outcome?: JobOutcomeMeta,
+  ) => void;
+  onJobFailed?: (
+    jobName: string,
+    jobId: string,
+    error: string,
+    logs: JobLogEntry[],
+    outcome?: JobOutcomeMeta,
+  ) => void;
 };
 
 // Serialized trace context lives under this key in the BullMQ job data.
@@ -526,6 +590,24 @@ export function createJobRunner(options: JobRunnerOptions): JobRunner {
       SYSTEM_TENANT_ID;
     const triggeredById = (rawData["_triggeredById"] as string | undefined) ?? null; // @cast-boundary dynamic-key
 
+    // Tenant-visible failure record (JobDefinition.tenantVisibleFailure). The
+    // subject is read from the handler payload, so it is resolved here where
+    // the payload is built, not in the callbacks.
+    const tenantVisibleDecl = jobDef.tenantVisibleFailure;
+    const tenantVisibleSubject = tenantVisibleDecl
+      ? jobSubjectKey(jobName, payload, tenantVisibleDecl.subjectFields)
+      : null;
+    // BullMQ stops retrying once attemptsMade reaches the configured attempts
+    // (`retries + 1`), so this is the attempt whose failure is final.
+    const finalAttempt = bullJob.attemptsMade + 1 >= (jobDef.retries ?? 0) + 1;
+    const outcomeMeta = (messageKey: string | null): JobOutcomeMeta => ({
+      tenantId,
+      finalAttempt,
+      ...(tenantVisibleDecl && {
+        tenantVisible: { subject: tenantVisibleSubject, messageKey },
+      }),
+    });
+
     // Carry `_triggerName` from rawData when set — handleEvent injects it on
     // multi-trigger dispatch; exposed as jobContext.triggerName so handlers
     // don't dig through the raw payload themselves.
@@ -665,15 +747,31 @@ export function createJobRunner(options: JobRunnerOptions): JobRunner {
 
     const runInSpan = async (): Promise<void> => {
       try {
-        await requestContext.run({ requestId: jobRequestId, correlationId: jobCorrelationId }, () =>
-          jobDef.handler(payload, jobContext),
+        await requestContext.run(
+          {
+            requestId: jobRequestId,
+            correlationId: jobCorrelationId,
+            // #3043 — events a job writes carry the job as their origin.
+            handler: jobName,
+            feature: qnScope(jobName),
+          },
+          () => jobDef.handler(payload, jobContext),
         );
+        // Stamped before the observer hook: a throwing onJobComplete must not
+        // make a run that actually succeeded look dead to the liveness alert.
+        if (context.meter) emitJobLastSuccess(context.meter, jobName);
         const duration = Date.now() - startTime;
-        await options.onJobComplete?.(jobName, jobId, duration, logs);
+        await options.onJobComplete?.(jobName, jobId, duration, logs, outcomeMeta(null));
       } catch (err) {
         const errorMsg = err instanceof Error ? err.message : String(err);
         logs.push({ level: "error", message: errorMsg, timestamp: Temporal.Now.instant() });
-        await options.onJobFailed?.(jobName, jobId, errorMsg, logs);
+        await options.onJobFailed?.(
+          jobName,
+          jobId,
+          errorMsg,
+          logs,
+          outcomeMeta(tenantFailureMessageKey(err, tenantVisibleDecl)),
+        );
         throw err;
       }
     };

package/src/logging/utils.ts

@@ -2,18 +2,27 @@ import type { Logger } from "./types";
 
 type FallbackLogger = {
   error(msg: string, data?: Record<string, unknown>): void;
+  warn(msg: string, data?: Record<string, unknown>): void;
   debug(msg: string, data?: Record<string, unknown>): void;
 };
 
 export function createFallbackLogger(
   namespace: string,
-  logger?: (Pick<Logger, "error"> & Partial<Pick<Logger, "debug">>) | undefined,
+  logger?: (Pick<Logger, "error"> & Partial<Pick<Logger, "warn" | "debug">>) | undefined,
 ): FallbackLogger {
   if (logger) {
     return {
       error(msg, data) {
         logger.error(`[${namespace}] ${msg}`, data);
       },
+      warn(msg, data) {
+        if (logger.warn) {
+          logger.warn(`[${namespace}] ${msg}`, data);
+        } else {
+          // biome-ignore lint/suspicious/noConsole: ops-visible fallback when the wrapped logger has no warn method
+          console.warn(`[${namespace}] ${msg}`, data);
+        }
+      },
       debug(msg, data) {
         if (logger.debug) {
           logger.debug(`[${namespace}] ${msg}`, data);
@@ -29,6 +38,10 @@ export function createFallbackLogger(
       // biome-ignore lint/suspicious/noConsole: ops-visible fallback when no logger is wired
       console.error(`[${namespace}] ${msg}`, data);
     },
+    warn(msg, data) {
+      // biome-ignore lint/suspicious/noConsole: ops-visible fallback when no logger is wired
+      console.warn(`[${namespace}] ${msg}`, data);
+    },
     debug(msg, data) {
       // biome-ignore lint/suspicious/noConsole: ops-visible fallback when no logger is wired
       console.debug(`[${namespace}] ${msg}`, data);

package/src/observability/index.ts

@@ -54,6 +54,7 @@ export {
   emitEventConsumerRearmExhausted,
   emitEventDispatcherListenConnected,
   emitHttpRequest,
+  emitJobLastSuccess,
   emitJobQueueDepth,
   registerStandardMetrics,
   STANDARD_METRIC_DEFS,

package/src/observability/standard-metrics.ts

@@ -123,6 +123,19 @@ export const STANDARD_METRIC_DEFS: readonly MetricDefinition[] = [
     description: "BullMQ job counts per lane and state.",
     labels: ["lane", "state"],
   },
+  // Dead-man for in-process jobs: the k8s CronJob alerts never fire for them
+  // (no CronJob object exists), and a gauge the job itself sets freezes on its
+  // last value when the job dies, so neither `absent()` nor a threshold
+  // catches a silently stopped cron. This one is stamped by the runner, so
+  // `time() - kumiko_job_last_success_timestamp_seconds{job="…"} > interval`
+  // is a true liveness check. Absent until the first success after a restart —
+  // see docs/reference/job-liveness-metric.md for the `for:` that implies.
+  {
+    name: "kumiko_job_last_success_timestamp_seconds",
+    type: "gauge",
+    description: "Unix timestamp of the last successful run, per registered job.",
+    labels: ["job"],
+  },
 ] as const;
 
 export function registerStandardMetrics(meter: Meter): void {
@@ -273,3 +286,10 @@ export function emitJobQueueDepth(
     meter.gauge("kumiko_job_queue_depth").set(count, { lane, state });
   }
 }
+
+// `job` is the registry-declared job name — handleJob rejects an unknown name
+// before it can reach here, so the label set is bounded by the app's r.job
+// registrations and carries no tenant or user input.
+export function emitJobLastSuccess(meter: Meter, job: string): void {
+  meter.gauge("kumiko_job_last_success_timestamp_seconds").set(Date.now() / 1000, { job });
+}