@cosmicdrift/kumiko-bundled-features 0.289.0 → 0.291.0

package/src/jobs/__tests__/tenant-job-failures.integration.test.ts

@@ -0,0 +1,275 @@
+// Tenant-visible job failures (fw#3079) end to end: a write triggers a job,
+// the job fails, and the triggering tenant reads the failure back through
+// `jobs:query:failures` — over real HTTP, a real BullMQ worker and a real
+// Postgres, with the real createJobRunLogger callbacks wired in.
+//
+// Not setupTestStack: that helper builds a JobRunner but wires none of the
+// bundled run-logger callbacks (test-stack.ts), which are the write path
+// under test here. Same buildServer + createJobRunner + createJobRunLogger
+// harness the neighbouring jobs integration tests use.
+
+import { afterAll, beforeAll, describe, expect, test } from "bun:test";
+import { buildServer, type JwtHelper } from "@cosmicdrift/kumiko-framework/api";
+import { insertOne, selectMany } from "@cosmicdrift/kumiko-framework/bun-db";
+import type { DbConnection } from "@cosmicdrift/kumiko-framework/db";
+import {
+  createRegistry,
+  defineFeature,
+  defineWriteHandler,
+  type SessionUser,
+} from "@cosmicdrift/kumiko-framework/engine";
+import { UnprocessableError } from "@cosmicdrift/kumiko-framework/errors";
+import { createEventsTable } from "@cosmicdrift/kumiko-framework/event-store";
+import { createJobRunner, type JobRunner } from "@cosmicdrift/kumiko-framework/jobs";
+import {
+  createTestDb,
+  createTestRedis,
+  createTestUser,
+  type TestDb,
+  type TestRedis,
+  testTenantId,
+  unsafePushTables,
+} from "@cosmicdrift/kumiko-framework/stack";
+import { sleep } from "@cosmicdrift/kumiko-framework/testing";
+import type { Hono } from "hono";
+import { z } from "zod";
+import { JobQueries } from "../constants";
+import { createJobsFeature } from "../feature";
+import { createJobRunLogger } from "../job-run-logger";
+import { jobRunLogsTable, jobRunsTable } from "../job-run-table";
+import { tenantJobFailuresTable } from "../tenant-job-failure-table";
+
+const JWT_SECRET = "tenant-job-failures-integration-secret-key-0123456789";
+const DECLARED_KEY = "app:errors.generationFailed";
+const BUDGET_KEY = "app:errors.budgetExceeded";
+// A provider message that must never reach the tenant.
+const PROVIDER_MESSAGE = "openai 429: prompt 'Herr Schmidt, Kennzeichen B-XY-123' rejected";
+
+const tenantA = testTenantId(1);
+const tenantB = testTenantId(2);
+const userA = createTestUser({ id: 1, tenantId: tenantA, roles: ["Admin"] });
+const userB = createTestUser({ id: 2, tenantId: tenantB, roles: ["Admin"] });
+const systemAdmin = createTestUser({ id: 3, tenantId: tenantA, roles: ["SystemAdmin"] });
+
+const generateWrite = defineWriteHandler({
+  name: "generate",
+  description: "Test-only: starts the text generation for one campaign.",
+  schema: z.object({ campaignId: z.string(), mode: z.enum(["fail", "budget", "succeed"]) }),
+  access: { roles: ["Admin"] },
+  handler: async (event) => ({ isSuccess: true as const, data: { ...event.payload } }),
+});
+
+const retryWrite = defineWriteHandler({
+  name: "startFlaky",
+  description: "Test-only: starts a job that fails on every attempt.",
+  schema: z.object({}),
+  access: { roles: ["Admin"] },
+  handler: async () => ({ isSuccess: true as const, data: {} }),
+});
+
+const appFeature = defineFeature("app", (r) => {
+  r.writeHandler(generateWrite);
+  r.writeHandler(retryWrite);
+
+  r.job(
+    "generateTexts",
+    {
+      trigger: { on: "app:write:generate" },
+      tenantVisibleFailure: { messageKey: DECLARED_KEY, subjectFields: ["campaignId"] },
+    },
+    async (payload) => {
+      if (payload["mode"] === "budget") {
+        throw new UnprocessableError("budget_exceeded", { i18nKey: BUDGET_KEY });
+      }
+      if (payload["mode"] === "fail") throw new Error(PROVIDER_MESSAGE);
+    },
+  );
+
+  // retries: 1 — two attempts, both failing. Only the last one may record.
+  r.job(
+    "flaky",
+    {
+      trigger: { on: "app:write:start-flaky" },
+      retries: 1,
+      tenantVisibleFailure: { messageKey: DECLARED_KEY },
+    },
+    async () => {
+      throw new Error(PROVIDER_MESSAGE);
+    },
+  );
+});
+
+let testDb: TestDb;
+let testRedis: TestRedis;
+let db: DbConnection;
+let app: Hono;
+let jwt: JwtHelper;
+let jobRunner: JobRunner;
+
+beforeAll(async () => {
+  testDb = await createTestDb();
+  testRedis = await createTestRedis();
+  db = testDb.db;
+
+  const registry = createRegistry([appFeature, createJobsFeature()]);
+  await unsafePushTables(db, { jobRunsTable, jobRunLogsTable, tenantJobFailuresTable });
+  await createEventsTable(db);
+
+  const redisUrl = `redis://${testRedis.redis.options.host}:${testRedis.redis.options.port}/${testRedis.redis.options.db}`;
+  jobRunner = createJobRunner({
+    registry,
+    context: { db },
+    redisUrl,
+    consumerLane: "worker",
+    queueNamePrefix: `kumiko-tenant-job-failures-${Date.now()}`,
+    ...createJobRunLogger({ db, registry }),
+  });
+
+  const server = buildServer({
+    registry,
+    context: { db, registry, jobRunner },
+    jwtSecret: JWT_SECRET,
+    // Event-triggered jobs enqueue from dispatch-write's afterCommit hooks,
+    // which read the runner off the dispatcher — not off the AppContext.
+    dispatcherOptions: { jobRunner },
+  });
+  app = server.app;
+  jwt = server.jwt;
+
+  await jobRunner.start();
+});
+
+afterAll(async () => {
+  await jobRunner.stop();
+  await testDb.cleanup();
+  await testRedis.cleanup();
+});
+
+async function post(path: string, user: SessionUser, body: unknown): Promise<Response> {
+  const token = await jwt.sign(user);
+  return app.request(path, {
+    method: "POST",
+    headers: { "Content-Type": "application/json", Authorization: `Bearer ${token}` },
+    body: JSON.stringify(body),
+  });
+}
+
+type FailureRow = {
+  readonly jobName: string;
+  readonly subject: Record<string, unknown> | null;
+  readonly messageKey: string;
+};
+
+async function generate(
+  user: SessionUser,
+  campaignId: string,
+  mode: "fail" | "budget" | "succeed",
+): Promise<void> {
+  const res = await post("/api/write", user, {
+    type: "app:write:generate",
+    payload: { campaignId, mode },
+  });
+  expect((await res.json()).isSuccess).toBe(true);
+  await sleep(1500);
+}
+
+async function failures(user: SessionUser, payload: unknown = {}): Promise<FailureRow[]> {
+  const res = await post("/api/query", user, { type: JobQueries.failures, payload });
+  const body = await res.json();
+  expect(res.status, JSON.stringify(body)).toBe(200);
+  return body.data.rows;
+}
+
+describe("jobs:query:failures (fw#3079)", () => {
+  test("the triggering tenant reads its own failed job, scoped by subject", async () => {
+    await generate(userA, "campaign-1", "fail");
+    await generate(userA, "campaign-2", "budget");
+
+    const rows = await failures(userA);
+    expect(rows).toHaveLength(2);
+    const byCampaign = new Map(rows.map((row) => [row.subject?.["campaignId"], row]));
+    expect(byCampaign.get("campaign-1")?.jobName).toBe("app:job:generate-texts");
+    // Plain Error → the key declared at the job.
+    expect(byCampaign.get("campaign-1")?.messageKey).toBe(DECLARED_KEY);
+    // KumikoError → its own i18nKey wins over the declared fallback.
+    expect(byCampaign.get("campaign-2")?.messageKey).toBe(BUDGET_KEY);
+  });
+
+  test("another tenant sees none of them, and only its own", async () => {
+    expect(await failures(userB)).toHaveLength(0);
+
+    await generate(userB, "campaign-1", "fail");
+
+    const rowsB = await failures(userB);
+    expect(rowsB).toHaveLength(1);
+    expect(rowsB[0]?.subject?.["campaignId"]).toBe("campaign-1");
+    // Tenant A's two records are untouched by B's own run of the same job
+    // and the same campaign id.
+    expect(await failures(userA)).toHaveLength(2);
+  });
+
+  test("no provider message reaches the tenant", async () => {
+    const res = await post("/api/query", userA, { type: JobQueries.failures, payload: {} });
+    const raw = await res.text();
+    expect(raw).toContain(DECLARED_KEY);
+    expect(raw).not.toContain(PROVIDER_MESSAGE);
+    expect(raw).not.toContain("openai");
+  });
+
+  test("a later successful run of the same subject clears the record", async () => {
+    await generate(userA, "campaign-1", "succeed");
+
+    const rows = await failures(userA);
+    expect(rows.map((row) => row.subject?.["campaignId"])).toEqual(["campaign-2"]);
+  });
+
+  test("only the final attempt records — a retried job leaves one record", async () => {
+    const res = await post("/api/write", userB, { type: "app:write:start-flaky", payload: {} });
+    expect((await res.json()).isSuccess).toBe(true);
+    await sleep(2500);
+
+    const rows = await failures(userB, { jobName: "app:job:flaky" });
+    expect(rows).toHaveLength(1);
+    expect(rows[0]?.subject).toBeNull();
+
+    // Both attempts did land as their own failed run — the single record
+    // above is the final-attempt gate, not a missing second attempt.
+    const runs = await selectMany(db, jobRunsTable, { jobName: "app:job:flaky" });
+    expect(runs).toHaveLength(2);
+  });
+
+  test("the subject filter selects one record", async () => {
+    const rows = await failures(userA, { subject: { campaignId: "campaign-2" } });
+    expect(rows).toHaveLength(1);
+    expect(rows[0]?.messageKey).toBe(BUDGET_KEY);
+
+    expect(await failures(userA, { subject: { campaignId: "campaign-unknown" } })).toHaveLength(0);
+  });
+
+  test("SystemAdmin still sees every tenant's run with its provider message", async () => {
+    const res = await post("/api/query", systemAdmin, {
+      type: JobQueries.list,
+      payload: { jobName: "app:job:generate-texts", status: "failed" },
+    });
+    const raw = await res.text();
+    expect(res.status, raw).toBe(200);
+
+    // Both of tenant A's failures plus tenant B's — the tenant-visible record
+    // is an addition, it takes nothing away from the SystemAdmin view.
+    expect(JSON.parse(raw).data.rows).toHaveLength(3);
+    expect(raw).toContain(PROVIDER_MESSAGE);
+  });
+  test("a corrupt stored subject degrades to null instead of failing the list", async () => {
+    await insertOne(db, tenantJobFailuresTable, {
+      tenantId: tenantB,
+      jobName: "app:job:corrupt",
+      subject: '{"campaignId":"not-an-entry-array"}',
+      messageKey: DECLARED_KEY,
+      failedAt: Temporal.Now.instant(),
+    });
+
+    const rows = await failures(userB, { jobName: "app:job:corrupt" });
+    expect(rows).toHaveLength(1);
+    expect(rows[0]?.subject).toBeNull();
+  });
+});

package/src/jobs/changes.json

@@ -1,4 +1,11 @@
 [
+  {
+    "version": "0.291.0",
+    "type": "improvement",
+    "title": "Tenant-visible job failures: `r.job({ tenantVisibleFailure })` plus `jobs:query:failures` (fw#3079)",
+    "detail": "A fire-and-forget job that fails left the tenant's screen on a spinner that never ends — `jobs:query:list` is SystemAdmin and reads cross-tenant over `systemDb.unsafeRaw`, so a tenant could not see its own job failing. Apps worked around it with their own failure entity written in the job's catch.\nA job now opts in declaratively: `r.job(\"generateTexts\", { trigger: …, tenantVisibleFailure: { messageKey: \"app:errors.generationFailed\", subjectFields: [\"campaignId\"] } }, handler)`. When its last attempt fails, the run-logger records one row per tenant, job and subject in the new `store_tenant_job_failures` table, and the tenant reads it back through `jobs:query:failures` (every membership rank, own tenant only).\nOnly a translation key travels to the tenant: the thrown error's own `i18nKey` when it carries one, otherwise the declared `messageKey`. The provider's message stays on `store_job_runs.error` and in the run log, both SystemAdmin-only. Records are scoped to the run's tenant — a tenant-less run (cron, `SYSTEM_TENANT_ID`) records nothing.\nLifetime and retries: a record lives until the next successful run of the same job and subject deletes it; there is no acknowledgement step (tenant job administration stays out of scope). Only the final attempt records, so a job with `retries` that succeeds on a later attempt never shows the tenant a failure. The daily `retention-cleanup` job purges leftovers past `retentionDays`.\n`jobs:query:list`, `jobs:query:details` and `jobs:query:retry` are unchanged. `JobRunnerOptions.onJobComplete`/`onJobFailed` gained an optional fifth `outcome` argument — existing four-argument callbacks keep working.",
+    "migration": "New store table. Run `kumiko migrate generate` and apply the migration — `store_tenant_job_failures` is created empty and stays empty until a job declares `tenantVisibleFailure`. No change needed for apps that do not opt in."
+  },
   {
     "version": "0.241.0",
     "type": "breaking",

package/src/jobs/constants.ts

@@ -13,6 +13,7 @@ export const JobQueries = {
   list: "jobs:query:list",
   details: "jobs:query:details",
   catalog: "jobs:query:catalog",
+  failures: "jobs:query:failures",
 } as const;
 
 // Error codes

package/src/jobs/db/queries/retention.ts

@@ -13,12 +13,14 @@
 import { deleteManyBatched } from "@cosmicdrift/kumiko-framework/bun-db";
 import type { DbConnection } from "@cosmicdrift/kumiko-framework/db";
 import { jobRunLogsTable, jobRunsTable } from "../../job-run-table";
+import { tenantJobFailuresTable } from "../../tenant-job-failure-table";
 
 const RETENTION_DELETE_BATCH_SIZE = 500;
 
 export type JobRunRetentionResult = {
   readonly runsDeleted: number;
   readonly logsDeleted: number;
+  readonly tenantFailuresDeleted: number;
 };
 
 export async function deleteStaleJobRuns(
@@ -40,5 +42,19 @@ export async function deleteStaleJobRuns(
     { limit: RETENTION_DELETE_BATCH_SIZE },
   );
 
-  return { runsDeleted: runsResult.deleted, logsDeleted: logsResult.deleted };
+  // Same window for the tenant-visible failure records (fw#3079): the run
+  // they point at is gone by now, and a job whose subject never ran again
+  // would otherwise keep its record forever.
+  const tenantFailuresResult = await deleteManyBatched(
+    db,
+    tenantJobFailuresTable,
+    { failedAt: { lt: cutoff } },
+    { limit: RETENTION_DELETE_BATCH_SIZE },
+  );
+
+  return {
+    runsDeleted: runsResult.deleted,
+    logsDeleted: logsResult.deleted,
+    tenantFailuresDeleted: tenantFailuresResult.deleted,
+  };
 }

package/src/jobs/feature.ts

@@ -21,9 +21,11 @@ import {
   createStaleRunSweepJob,
   DEFAULT_JOB_RUN_STALE_TIMEOUT_HOURS,
 } from "./handlers/stale-run-sweep.job";
+import { tenantFailuresQuery } from "./handlers/tenant-failures.query";
 import { triggerWrite } from "./handlers/trigger.write";
 import { JOBS_I18N } from "./i18n";
 import { jobRunLogsTableMeta, jobRunsTableMeta } from "./job-run-table";
+import { tenantJobFailuresTableMeta } from "./tenant-job-failure-table";
 
 export type JobsFeatureOptions = {
   // How long a job run (and its logs) stays in store_job_runs/
@@ -42,7 +44,7 @@ export function createJobsFeature(options: JobsFeatureOptions = {}): FeatureDefi
   const staleRunTimeoutHours = options.staleRunTimeoutHours ?? DEFAULT_JOB_RUN_STALE_TIMEOUT_HOURS;
   return defineFeature("jobs", (r) => {
     r.describe(
-      "Persistence and operator tooling for background jobs registered via `r.job(...)`. Every job execution writes directly into `store_job_runs` (current status + duration) and `store_job_run_logs` (per-line log rows) from the BullMQ callbacks — no event stream in between (#2243). A daily `retention-cleanup` job deletes runs (and their logs) older than `retentionDays`; an hourly `stale-run-sweep` job marks runs stuck at status `running` past `staleRunTimeoutHours` as `failed` (#2246 — a crashed worker never fires the completion callback, so nothing else ever revisits the row). Exposes `jobs:write:trigger` (manual run) and `jobs:write:retry` (operator retry of a failed run), plus `jobs:query:list`, `jobs:query:details`, and `jobs:query:catalog` (manual jobs) for the operator UI.",
+      "Persistence and operator tooling for background jobs registered via `r.job(...)`. Every job execution writes directly into `store_job_runs` (current status + duration) and `store_job_run_logs` (per-line log rows) from the BullMQ callbacks — no event stream in between (#2243). A daily `retention-cleanup` job deletes runs (and their logs) older than `retentionDays`; an hourly `stale-run-sweep` job marks runs stuck at status `running` past `staleRunTimeoutHours` as `failed` (#2246 — a crashed worker never fires the completion callback, so nothing else ever revisits the row). Exposes `jobs:write:trigger` (manual run) and `jobs:write:retry` (operator retry of a failed run), plus `jobs:query:list`, `jobs:query:details`, and `jobs:query:catalog` (manual jobs) for the operator UI. A job that declares `tenantVisibleFailure` also records its last failed attempt per tenant and subject in `store_tenant_job_failures`, which the tenant itself reads through `jobs:query:failures` — a translation key only, never the provider's message (fw#3079).",
     );
     r.uiHints({
       displayLabel: "Jobs · Audit & Operator UI",
@@ -61,6 +63,9 @@ export function createJobsFeature(options: JobsFeatureOptions = {}): FeatureDefi
     r.storeTable(jobRunLogsTableMeta, {
       reason: "read_side.job_run_logs",
     });
+    r.storeTable(tenantJobFailuresTableMeta, {
+      reason: "direct_write.tenant_job_failures",
+    });
 
     // Framework-provided rebuild job — available whenever `jobs` is composed; enqueueProjectionRebuild dispatches it.
     r.job(
@@ -112,6 +117,7 @@ export function createJobsFeature(options: JobsFeatureOptions = {}): FeatureDefi
       list: r.queryHandler(listQuery),
       detail: r.queryHandler(detailQuery),
       catalog: r.queryHandler(catalogQuery),
+      failures: r.queryHandler(tenantFailuresQuery),
     };
 
     const systemAdminAccess = { roles: ["SystemAdmin"] as const };

package/src/jobs/handlers/tenant-failures.query.ts

@@ -0,0 +1,82 @@
+import { selectMany } from "@cosmicdrift/kumiko-framework/bun-db";
+import { access, defineQueryHandler } from "@cosmicdrift/kumiko-framework/engine";
+import { InternalError } from "@cosmicdrift/kumiko-framework/errors";
+import { parseJsonSafe } from "@cosmicdrift/kumiko-framework/utils";
+import { z } from "zod";
+import { tenantJobFailuresTable } from "../tenant-job-failure-table";
+
+type TenantJobFailureRow = {
+  readonly tenantId: string;
+  readonly jobName: string;
+  readonly subject: string | null;
+  readonly messageKey: string;
+  readonly failedAt: Temporal.Instant;
+};
+
+const DEFAULT_LIMIT = 50;
+
+// Mirrors job-runner.ts's jobSubjectKey: sorted field names, so a caller that
+// passes the same subject values gets the same string the writer stored.
+function subjectKey(subject: Record<string, string | number | boolean | null>): string {
+  return JSON.stringify(
+    Object.keys(subject)
+      .sort()
+      .map((field) => [field, subject[field] ?? null]),
+  );
+}
+
+function isSubjectEntry(value: unknown): value is [string, unknown] {
+  return Array.isArray(value) && typeof value[0] === "string";
+}
+
+function parseSubject(stored: string | null): Record<string, unknown> | null {
+  if (stored === null) return null;
+  // A corrupt subject must not fail the whole list — the record still tells
+  // the tenant which job failed and why. parseJsonSafe only survives a
+  // SyntaxError, so the shape needs its own check.
+  const parsed = parseJsonSafe<unknown>(stored, null);
+  if (!Array.isArray(parsed)) return null;
+  return Object.fromEntries(parsed.filter(isSubjectEntry));
+}
+
+export const tenantFailuresQuery = defineQueryHandler({
+  name: "failures",
+  description:
+    "Lists the calling tenant's own failed jobs — one record per job and subject, newest first, each carrying a translation key for the reason, never the provider's own error message; use it to tell a tenant that their asynchronous job failed instead of leaving the screen waiting.",
+  schema: z.object({
+    jobName: z.string().optional(),
+    subject: z.record(z.string(), z.union([z.string(), z.number(), z.boolean()])).optional(),
+    limit: z.number().min(1).max(200).optional(),
+  }),
+  // Every membership rank: a failure record carries a job name and a
+  // translation key, nothing a team member of the tenant may not see.
+  access: { roles: access.roles("User", "Editor", ...access.admin) },
+  handler: async (query, ctx) => {
+    if (!ctx.systemDb) {
+      throw new InternalError({
+        message: "jobs:query:failures requires ctx.systemDb (feature must declare r.systemScope())",
+      });
+    }
+    // assertTenantMatch is a self-check on the caller and returns the same
+    // unfiltered system-mode db (tenant-db.ts) — the explicit tenantId below
+    // is the filter, assertRowsTenant the second net. There is no
+    // cross-tenant mode here: SystemAdmin reads runs via jobs:query:list.
+    const db = ctx.systemDb.assertTenantMatch(query.user.tenantId);
+    const where: Record<string, unknown> = { tenantId: [query.user.tenantId] };
+    if (query.payload.jobName) where["jobName"] = query.payload.jobName;
+    if (query.payload.subject) where["subject"] = subjectKey(query.payload.subject);
+    const rows = await selectMany<TenantJobFailureRow>(db, tenantJobFailuresTable, where, {
+      orderBy: { col: "failedAt", direction: "desc" },
+      limit: query.payload.limit ?? DEFAULT_LIMIT,
+    });
+    return {
+      rows: ctx.systemDb.assertRowsTenant(rows, "tenantId").map((row) => ({
+        jobName: row.jobName,
+        subject: parseSubject(row.subject),
+        messageKey: row.messageKey,
+        failedAt: row.failedAt,
+      })),
+      nextCursor: null,
+    };
+  },
+});

package/src/jobs/index.ts

@@ -4,3 +4,4 @@ export type { JobRunLoggerCallbacks } from "./job-run-logger";
 export { createJobRunLogger } from "./job-run-logger";
 export type { JobLogLevel, JobRunStatus } from "./job-run-table";
 export { jobRunLogsTable, jobRunsTable } from "./job-run-table";
+export { tenantJobFailuresTable } from "./tenant-job-failure-table";

package/src/jobs/job-run-logger.ts

@@ -1,4 +1,10 @@
-import { fetchOne, insertMany, insertOne, updateMany } from "@cosmicdrift/kumiko-framework/bun-db";
+import {
+  deleteMany,
+  fetchOne,
+  insertMany,
+  insertOne,
+  updateMany,
+} from "@cosmicdrift/kumiko-framework/bun-db";
 import {
   configuredPiiSubjectKms,
   encryptPiiValueForSubject,
@@ -8,12 +14,18 @@ import {
 } from "@cosmicdrift/kumiko-framework/crypto";
 import type { DbConnection } from "@cosmicdrift/kumiko-framework/db";
 import { type Registry, SYSTEM_TENANT_ID } from "@cosmicdrift/kumiko-framework/engine";
-import type { JobLogEntry, JobMeta, JobRunnerOptions } from "@cosmicdrift/kumiko-framework/jobs";
+import type {
+  JobLogEntry,
+  JobMeta,
+  JobOutcomeMeta,
+  JobRunnerOptions,
+} from "@cosmicdrift/kumiko-framework/jobs";
 import { generateId } from "@cosmicdrift/kumiko-framework/utils";
 import { mapWithConcurrency } from "../shared";
 import { runCompletedSchema, runFailedSchema, runStartedSchema } from "./events";
 import { parseJobInstant } from "./job-instant";
 import { jobRunLogsTable, jobRunsTable } from "./job-run-table";
+import { tenantJobFailuresTable } from "./tenant-job-failure-table";
 
 // Matches PgKmsAdapter's default pool size (see tenant/handlers/*.query.ts) —
 // bounds concurrent getOrCreateDek calls so a large log batch doesn't claim
@@ -138,6 +150,54 @@ async function encryptStartedPayload(
   return encryptOrSentinel(kms, triggeredById, payload, "payload");
 }
 
+// fw#3079 — which row the tenant-visible failure record lives in: one per
+// (tenant, job, subject), so the next outcome of the same work replaces or
+// clears it. `subject` is null for a job that declares no subjectFields.
+// Null target = nothing to write or clear: the job did not opt in, the run
+// was tenant-less (cron resolves to SYSTEM_TENANT_ID, where no tenant-scoped
+// query could ever read the row), or the caller predates fw#3079 and passes
+// no outcome at all.
+function tenantJobFailureTarget(
+  jobName: string,
+  outcome: JobOutcomeMeta | undefined,
+): Record<string, unknown> | null {
+  if (!outcome?.tenantVisible || outcome.tenantId === SYSTEM_TENANT_ID) return null;
+  return { tenantId: outcome.tenantId, jobName, subject: outcome.tenantVisible.subject };
+}
+
+async function recordTenantJobFailure(
+  db: DbConnection,
+  jobName: string,
+  outcome: JobOutcomeMeta | undefined,
+): Promise<void> {
+  const where = tenantJobFailureTarget(jobName, outcome);
+  const messageKey = outcome?.tenantVisible?.messageKey;
+  // skip: no tenant-visible target, or an attempt BullMQ may still retry — a
+  // non-final failure must not show the tenant a failure the next attempt
+  // may still resolve.
+  if (!where || !messageKey || outcome?.finalAttempt !== true) return;
+  // ponytail: delete-then-insert instead of an upsert — two runs of the same
+  // key finishing at once can leave two rows, and the query returns the
+  // newest. Add a unique index + ON CONFLICT if that ever matters.
+  await deleteMany(db, tenantJobFailuresTable, where);
+  await insertOne(db, tenantJobFailuresTable, {
+    ...where,
+    messageKey,
+    failedAt: Temporal.Now.instant(),
+  });
+}
+
+async function clearTenantJobFailure(
+  db: DbConnection,
+  jobName: string,
+  outcome: JobOutcomeMeta | undefined,
+): Promise<void> {
+  const where = tenantJobFailureTarget(jobName, outcome);
+  // skip: no tenant-visible target — nothing was ever recorded
+  if (!where) return;
+  await deleteMany(db, tenantJobFailuresTable, where);
+}
+
 export function createJobRunLogger(opts: JobRunLoggerOptions): JobRunLoggerCallbacks {
   const { db } = opts;
 
@@ -241,11 +301,16 @@ export function createJobRunLogger(opts: JobRunLoggerOptions): JobRunLoggerCallb
     },
 
     onJobComplete: async (
-      _jobName: string,
+      jobName: string,
       bullJobId: string,
       duration: number,
       logs: JobLogEntry[],
+      outcome?: JobOutcomeMeta,
     ) => {
+      // Before the run-row write and independent of it: a successful run
+      // clears the tenant's failure record even when the run row itself is
+      // unreachable (the state-loss return below).
+      await clearTenantJobFailure(db, jobName, outcome);
       const resolved = await resolveRun(bullJobId);
       // skip: state loss between start + complete (worker restart, cache
       // evicted AND DB has no matching bull_job_id). Rare edge case; we
@@ -297,11 +362,15 @@ export function createJobRunLogger(opts: JobRunLoggerOptions): JobRunLoggerCallb
     },
 
     onJobFailed: async (
-      _jobName: string,
+      jobName: string,
       bullJobId: string,
       error: string,
       logs: JobLogEntry[],
+      outcome?: JobOutcomeMeta,
     ) => {
+      // Mirror of onJobComplete: recorded independently of the run row, so a
+      // tenant still learns their job failed if the row is unreachable.
+      await recordTenantJobFailure(db, jobName, outcome);
       const resolved = await resolveRun(bullJobId);
       // skip: same rare state-loss case as in onJobComplete — drop the
       // failure write rather than forge a run row from scratch.

package/src/jobs/tenant-job-failure-table.ts

@@ -0,0 +1,39 @@
+import { asEntityTableMeta } from "@cosmicdrift/kumiko-framework/bun-db";
+import {
+  type EntityTableMeta,
+  instant,
+  table as pgTable,
+  serial,
+  sql,
+  text,
+  uuid,
+} from "@cosmicdrift/kumiko-framework/db";
+
+// The tenant's own view of a failed job (fw#3079). Direct-write store like
+// store_job_runs: job-run-logger.ts writes it from the BullMQ callbacks,
+// outside any dispatcher transaction.
+//
+// Deliberately holds no error text. `message_key` is a translation key
+// (the thrown KumikoError's i18nKey or the one declared at the job), so a
+// provider message can never reach the tenant through this table — it stays
+// on store_job_runs.error and in store_job_run_logs, both SystemAdmin-only.
+//
+// One row per (tenant, job, subject): a later final-attempt failure replaces
+// it, a later successful run of the same key deletes it. `subject` is the
+// canonical JSON of the job's declared subjectFields, or NULL for a job that
+// declares none — stored in clear, so a job must not declare a PII field
+// as its subject.
+export const tenantJobFailuresTable = pgTable("store_tenant_job_failures", {
+  id: serial("id").primaryKey(),
+  tenantId: uuid("tenant_id").notNull(),
+  jobName: text("job_name").notNull(),
+  subject: text("subject"),
+  messageKey: text("message_key").notNull(),
+  failedAt: instant("failed_at").default(sql`now()`).notNull(),
+});
+
+const derivedTenantJobFailuresTableMeta = asEntityTableMeta(tenantJobFailuresTable);
+if (!derivedTenantJobFailuresTableMeta) {
+  throw new Error("tenantJobFailuresTable: table carries no EntityTableMeta — built via table()?");
+}
+export const tenantJobFailuresTableMeta: EntityTableMeta = derivedTenantJobFailuresTableMeta;