@cosmicdrift/kumiko-bundled-features 0.352.0 → 0.354.0

package/dist/jobs/db/queries/stale-run-sweep.js

@@ -11,11 +11,13 @@
 // schema/migration change, no web filter-dropdown/status-enum update, and
 // the run becomes retriable via the existing jobs:write:retry gate
 // (retry.write.ts only allows retry from "failed").
-import { selectMany, updateMany } from "@cosmicdrift/kumiko-framework/bun-db";
+import { deleteManyBatched, selectMany, updateMany } from "@cosmicdrift/kumiko-framework/bun-db";
 import { mapWithConcurrency } from "../../../shared/index.js";
 import { encryptFailureError } from "../../job-run-logger.js";
 import { jobRunsTable } from "../../job-run-table.js";
+import { tenantJobRunsTable } from "../../tenant-job-run-table.js";
 const KMS_POOL_CONCURRENCY = 4;
+const STALE_DELETE_BATCH_SIZE = 500;
 export const STALE_JOB_RUN_ERROR = "job run exceeded the stale-run timeout without a completion signal (likely a crashed worker process)";
 export async function markStaleJobRunsFailed(db, timeoutHours) {
     const cutoff = Temporal.Now.instant().subtract({ hours: timeoutHours });
@@ -57,5 +59,14 @@ export async function markStaleJobRunsFailed(db, timeoutHours) {
         return updated.length;
     });
     const runsMarkedFailed = updatedPerSubject.reduce((sum, count) => sum + count, 0);
-    return { runsMarkedFailed };
+    // Tenant-visible run state follows the same clock: a running row whose
+    // worker died is a failure, a queued row that never started (job lost from
+    // Redis, enqueue hook without a run) would otherwise show "waiting" forever.
+    const tenantRunsFailed = await updateMany(db, tenantJobRunsTable, { status: "failed", finishedAt: now, updatedAt: now }, { status: "running", startedAt: { lt: cutoff } });
+    const tenantQueuedStale = await deleteManyBatched(db, tenantJobRunsTable, { status: "queued", updatedAt: { lt: cutoff } }, { limit: STALE_DELETE_BATCH_SIZE });
+    return {
+        runsMarkedFailed,
+        tenantRunsMarkedFailed: tenantRunsFailed.length,
+        tenantQueuedDropped: tenantQueuedStale.deleted,
+    };
 }

package/dist/jobs/feature.js

@@ -9,15 +9,17 @@ import { createRetentionCleanupJob, DEFAULT_JOB_RUN_RETENTION_DAYS, } from "./ha
 import { retryWrite } from "./handlers/retry.write.js";
 import { createStaleRunSweepJob, DEFAULT_JOB_RUN_STALE_TIMEOUT_HOURS, } from "./handlers/stale-run-sweep.job.js";
 import { tenantFailuresQuery } from "./handlers/tenant-failures.query.js";
+import { tenantRunsQuery } from "./handlers/tenant-runs.query.js";
 import { triggerWrite } from "./handlers/trigger.write.js";
 import { JOBS_I18N } from "./i18n.js";
 import { jobRunLogsTableMeta, jobRunsTableMeta } from "./job-run-table.js";
 import { tenantJobFailuresTableMeta } from "./tenant-job-failure-table.js";
+import { tenantJobRunsTableMeta } from "./tenant-job-run-table.js";
 export function createJobsFeature(options = {}) {
     const retentionDays = options.retentionDays ?? DEFAULT_JOB_RUN_RETENTION_DAYS;
     const staleRunTimeoutHours = options.staleRunTimeoutHours ?? DEFAULT_JOB_RUN_STALE_TIMEOUT_HOURS;
     return defineFeature("jobs", (r) => {
-        r.describe("Persistence and operator tooling for background jobs registered via `r.job(...)`. Every job execution writes directly into `store_job_runs` (current status + duration) and `store_job_run_logs` (per-line log rows) from the BullMQ callbacks — no event stream in between (#2243). A daily `retention-cleanup` job deletes runs (and their logs) older than `retentionDays`; an hourly `stale-run-sweep` job marks runs stuck at status `running` past `staleRunTimeoutHours` as `failed` (#2246 — a crashed worker never fires the completion callback, so nothing else ever revisits the row). Exposes `jobs:write:trigger` (manual run) and `jobs:write:retry` (operator retry of a failed run), plus `jobs:query:list`, `jobs:query:details`, and `jobs:query:catalog` (manual jobs) for the operator UI. A job that declares `tenantVisibleFailure` also records its last failed attempt per tenant and subject in `store_tenant_job_failures`, which the tenant itself reads through `jobs:query:failures` — a translation key only, never the provider's message (fw#3079).");
+        r.describe("Persistence and operator tooling for background jobs registered via `r.job(...)`. Every job execution writes directly into `store_job_runs` (current status + duration) and `store_job_run_logs` (per-line log rows) from the BullMQ callbacks — no event stream in between (#2243). A daily `retention-cleanup` job deletes runs (and their logs) older than `retentionDays`; an hourly `stale-run-sweep` job marks runs stuck at status `running` past `staleRunTimeoutHours` as `failed` (#2246 — a crashed worker never fires the completion callback, so nothing else ever revisits the row). Exposes `jobs:write:trigger` (manual run) and `jobs:write:retry` (operator retry of a failed run), plus `jobs:query:list`, `jobs:query:details`, and `jobs:query:catalog` (manual jobs) for the operator UI. A job that declares `tenantVisibleFailure` also records its last failed attempt per tenant and subject in `store_tenant_job_failures`, which the tenant itself reads through `jobs:query:failures` — a translation key only, never the provider's message (fw#3079). A job that declares `tenantVisibleRun` also keeps its run state per tenant and subject in `store_tenant_job_runs` (queued, running, plus the latest completed or failed run, no payload, error text or logs), which the tenant reads through `jobs:query:tenant-runs` (fw#3616); the stale-run sweep fails its stuck `running` rows and drops its stuck `queued` rows.");
         r.uiHints({
             displayLabel: "Jobs · Audit & Operator UI",
             category: "operations",
@@ -38,6 +40,9 @@ export function createJobsFeature(options = {}) {
         r.storeTable(tenantJobFailuresTableMeta, {
             reason: "direct_write.tenant_job_failures",
         });
+        r.storeTable(tenantJobRunsTableMeta, {
+            reason: "direct_write.tenant_job_runs",
+        });
         // Framework-provided rebuild job — available whenever `jobs` is composed; enqueueProjectionRebuild dispatches it.
         r.job("projectionRebuild", { trigger: { manual: true }, concurrency: "skip", schema: projectionRebuildPayloadSchema }, projectionRebuildJob);
         // Retroactive search backfill (#1206/#1215) — manual + perTenant, so one
@@ -68,6 +73,7 @@ export function createJobsFeature(options = {}) {
             detail: r.queryHandler(detailQuery),
             catalog: r.queryHandler(catalogQuery),
             failures: r.queryHandler(tenantFailuresQuery),
+            tenantRuns: r.queryHandler(tenantRunsQuery),
         };
         const systemAdminAccess = { roles: ["SystemAdmin"] };
         r.translations({ keys: JOBS_I18N });

package/dist/jobs/handlers/list.query.d.ts

@@ -28,9 +28,10 @@ export declare const listQuery: import("@cosmicdrift/kumiko-types/define-handler
         desc: "desc";
     }>>;
     limit: z.ZodOptional<z.ZodNumber>;
+    cursor: z.ZodOptional<z.ZodString>;
     totalCount: z.ZodOptional<z.ZodBoolean>;
 }, z.core.$strip>, {
     rows: any[];
-    nextCursor: null;
+    nextCursor: string | null;
     total?: number | undefined;
 }, import("@cosmicdrift/kumiko-types/event-type-map").KumikoEventTypeMap>;

package/dist/jobs/handlers/list.query.js

@@ -1,10 +1,12 @@
 import { countWhere, selectMany } from "@cosmicdrift/kumiko-framework/bun-db";
+import { decodeCursor, encodeCursor } from "@cosmicdrift/kumiko-framework/db";
 import { defineQueryHandler } from "@cosmicdrift/kumiko-framework/engine";
-import { InternalError } from "@cosmicdrift/kumiko-framework/errors";
+import { InternalError, ValidationError } from "@cosmicdrift/kumiko-framework/errors";
 import * as z from "zod";
 import { decryptStoredPii, mapWithConcurrency } from "../../shared/index.js";
 import { jobRunsTable } from "../job-run-table.js";
 const KMS_POOL_CONCURRENCY = 4;
+const DEFAULT_PAGE_SIZE = 50;
 const jobRunStatusSchema = z.enum(["queued", "running", "completed", "failed"]);
 async function decryptRunRow(row) {
     let result = row;
@@ -19,6 +21,26 @@ async function decryptRunRow(row) {
     }
     return result;
 }
+function buildListWhere(payload) {
+    const where = {};
+    if (payload.jobName)
+        where["jobName"] = payload.jobName;
+    const statuses = payload.filters?.[0]?.value ?? (payload.status ? [payload.status] : []);
+    if (statuses.length === 1)
+        where["status"] = statuses[0];
+    else if (statuses.length > 1)
+        where["status"] = { in: statuses };
+    return where;
+}
+function parseOffsetCursor(cursor) {
+    const offset = cursor ? Number(decodeCursor(cursor)) : 0;
+    if (!Number.isInteger(offset) || offset < 0) {
+        throw new ValidationError({
+            fields: [{ path: "cursor", code: "invalid_cursor", i18nKey: "jobs.errors.invalidCursor" }],
+        });
+    }
+    return offset;
+}
 export const listQuery = defineQueryHandler({
     name: "list",
     description: "Lists job runs across all tenants newest-first, optionally filtered by job name and status; use it to check whether a job ran and whether it succeeded.",
@@ -34,7 +56,8 @@ export const listQuery = defineQueryHandler({
             .optional(),
         sort: z.enum(["jobName", "status", "startedAt", "duration"]).optional(),
         sortDirection: z.enum(["asc", "desc"]).optional(),
-        limit: z.number().optional(),
+        limit: z.number().int().min(1).optional(),
+        cursor: z.string().optional(),
         totalCount: z.boolean().optional(),
     }),
     access: { roles: ["SystemAdmin"] },
@@ -45,27 +68,29 @@ export const listQuery = defineQueryHandler({
             });
         }
         const db = ctx.systemDb.unsafeRaw("cross-tenant job monitoring: SystemAdmin lists job runs of every tenant");
-        const where = {};
-        if (query.payload.jobName)
-            where["jobName"] = query.payload.jobName;
-        const statusFilter = query.payload.filters?.find((filter) => filter.field === "status");
-        const statuses = statusFilter?.value ?? (query.payload.status ? [query.payload.status] : []);
-        if (statuses.length === 1)
-            where["status"] = statuses[0];
-        else if (statuses.length > 1)
-            where["status"] = { in: statuses };
+        const where = buildListWhere(query.payload);
         const sortColumn = query.payload.sort ?? "startedAt";
-        const rows = await selectMany(db, jobRunsTable, where, {
-            orderBy: { col: sortColumn, direction: query.payload.sortDirection ?? "desc" },
-            limit: query.payload.limit ?? 50,
+        const limit = query.payload.limit ?? DEFAULT_PAGE_SIZE;
+        const offset = parseOffsetCursor(query.payload.cursor);
+        const direction = query.payload.sortDirection ?? "desc";
+        // selectMany has no OFFSET: read through the end of the page plus one probe
+        // row, then slice. `id` as tiebreaker keeps pages stable for non-unique sorts.
+        const fetched = await selectMany(db, jobRunsTable, where, {
+            orderBy: [
+                { col: sortColumn, direction },
+                { col: "id", direction },
+            ],
+            limit: offset + limit + 1,
         });
+        const hasMore = fetched.length > offset + limit;
+        const rows = fetched.slice(offset, offset + limit);
         // countWhere reruns the SAME `where` without the limit — `rows.length` is
         // capped at the page size and would silently undercount the total.
         const total = query.payload.totalCount === true ? await countWhere(db, jobRunsTable, where) : undefined;
         // payload/error are stored encrypted under the triggering user's DEK (#799, #2307).
         return {
             rows: await mapWithConcurrency(rows, KMS_POOL_CONCURRENCY, decryptRunRow),
-            nextCursor: null,
+            nextCursor: hasMore ? encodeCursor(String(offset + limit)) : null,
             ...(total !== undefined && { total }),
         };
     },

package/dist/jobs/handlers/tenant-failures.query.js

@@ -26,7 +26,7 @@ export const tenantFailuresQuery = defineQueryHandler({
     schema: z.object({
         jobName: z.string().optional(),
         subject: z.record(z.string(), z.union([z.string(), z.number(), z.boolean()])).optional(),
-        limit: z.number().min(1).max(200).optional(),
+        limit: z.number().int().min(1).max(200).optional(),
     }),
     // Every membership rank: a failure record carries a job name and a
     // translation key, nothing a team member of the tenant may not see.

package/dist/jobs/handlers/tenant-runs.query.d.ts

@@ -0,0 +1,17 @@
+import * as z from "zod";
+import { type TenantJobRunStatus } from "../tenant-job-run-table.js";
+export declare const tenantRunsQuery: import("@cosmicdrift/kumiko-types/define-handler").QueryHandlerDefinition<"tenant-runs", z.ZodObject<{
+    jobName: z.ZodOptional<z.ZodString>;
+    subject: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnion<readonly [z.ZodString, z.ZodNumber, z.ZodBoolean]>>>;
+    limit: z.ZodOptional<z.ZodNumber>;
+}, z.core.$strip>, {
+    rows: {
+        jobName: string;
+        subject: Record<string, unknown> | null;
+        status: TenantJobRunStatus;
+        queuedAt: Temporal.Instant | null;
+        startedAt: Temporal.Instant | null;
+        finishedAt: Temporal.Instant | null;
+    }[];
+    nextCursor: null;
+}, import("@cosmicdrift/kumiko-types/event-type-map").KumikoEventTypeMap>;

package/dist/jobs/handlers/tenant-runs.query.js

@@ -0,0 +1,72 @@
+import { selectMany } from "@cosmicdrift/kumiko-framework/bun-db";
+import { access, defineQueryHandler } from "@cosmicdrift/kumiko-framework/engine";
+import { InternalError } from "@cosmicdrift/kumiko-framework/errors";
+import { serializeJobSubject } from "@cosmicdrift/kumiko-framework/jobs";
+import { parseJsonSafe } from "@cosmicdrift/kumiko-framework/utils";
+import * as z from "zod";
+import { tenantJobRunsTable } from "../tenant-job-run-table.js";
+const DEFAULT_LIMIT = 50;
+function isSubjectEntry(value) {
+    return Array.isArray(value) && typeof value[0] === "string";
+}
+function parseSubject(stored) {
+    if (stored === null)
+        return null;
+    // A corrupt subject must not fail the whole list; parseJsonSafe only
+    // survives a SyntaxError, so the shape needs its own check.
+    const parsed = parseJsonSafe(stored, null);
+    if (!Array.isArray(parsed))
+        return null;
+    return Object.fromEntries(parsed.filter(isSubjectEntry));
+}
+export const tenantRunsQuery = defineQueryHandler({
+    name: "tenant-runs",
+    description: "Lists the calling tenant's own job runs that opted in via tenantVisibleRun — per job and subject every queued or running run plus the latest completed or failed one, active runs first, then newest first; each row carries only status and queuedAt/startedAt/finishedAt, never payloads, error text or logs; the optional subject filter matches exactly: all declared subjectFields with the same JSON types as the job payload; use it to show whether work for a subject is waiting, running or done, and to tell a failed run from a finished one.",
+    schema: z.object({
+        jobName: z.string().optional(),
+        subject: z.record(z.string(), z.union([z.string(), z.number(), z.boolean()])).optional(),
+        limit: z.number().int().min(1).max(200).optional(),
+    }),
+    // Every membership rank: a run state carries a job name, a status and
+    // times, nothing a team member of the tenant may not see.
+    access: { roles: access.roles("User", "Editor", ...access.admin) },
+    handler: async (query, ctx) => {
+        if (!ctx.systemDb) {
+            throw new InternalError({
+                message: "jobs:query:tenant-runs requires ctx.systemDb (feature must declare r.systemScope())",
+            });
+        }
+        // assertTenantMatch returns the same unfiltered system-mode db — the
+        // explicit tenantId below is the filter, assertRowsTenant the second net.
+        const db = ctx.systemDb.assertTenantMatch(query.user.tenantId);
+        const where = { tenantId: [query.user.tenantId] };
+        if (query.payload.jobName)
+            where["jobName"] = query.payload.jobName;
+        if (query.payload.subject)
+            where["subject"] = serializeJobSubject(query.payload.subject);
+        const limit = query.payload.limit ?? DEFAULT_LIMIT;
+        const newestFirst = { col: "updatedAt", direction: "desc" };
+        const activeRows = await selectMany(db, tenantJobRunsTable, { ...where, status: { in: ["queued", "running"] } }, { orderBy: newestFirst, limit });
+        const finishedRows = await selectMany(db, tenantJobRunsTable, { ...where, status: { in: ["completed", "failed"] } }, { orderBy: newestFirst, limit });
+        const seenFinishedKeys = new Set();
+        const finishedNewestPerKey = finishedRows.filter((row) => {
+            const key = JSON.stringify([row.jobName, row.subject]);
+            if (seenFinishedKeys.has(key))
+                return false;
+            seenFinishedKeys.add(key);
+            return true;
+        });
+        const activeFirst = [...activeRows, ...finishedNewestPerKey];
+        return {
+            rows: ctx.systemDb.assertRowsTenant(activeFirst.slice(0, limit), "tenantId").map((row) => ({
+                jobName: row.jobName,
+                subject: parseSubject(row.subject),
+                status: row.status,
+                queuedAt: row.queuedAt,
+                startedAt: row.startedAt,
+                finishedAt: row.finishedAt,
+            })),
+            nextCursor: null,
+        };
+    },
+});

package/dist/jobs/i18n.js

@@ -33,6 +33,7 @@ export const JOBS_I18N = {
     "jobs.detail.logs": { en: "Logs" },
     "jobs.detail.retry": { en: "Retry" },
     "jobs.trigger.title": { en: "Run a job" },
+    "jobs.errors.invalidCursor": { en: "The pagination cursor is invalid." },
     "jobs.trigger.submit": { en: "Run" },
     "jobs.errors.unknownJob": { en: "Unknown job." },
     "jobs.errors.notManual": {

package/dist/jobs/index.d.ts

@@ -5,3 +5,5 @@ export { createJobRunLogger } from "./job-run-logger.js";
 export type { JobLogLevel, JobRunStatus } from "./job-run-table.js";
 export { jobRunLogsTable, jobRunsTable } from "./job-run-table.js";
 export { tenantJobFailuresTable } from "./tenant-job-failure-table.js";
+export type { TenantJobRunStatus } from "./tenant-job-run-table.js";
+export { tenantJobRunsTable } from "./tenant-job-run-table.js";

package/dist/jobs/index.js

@@ -3,3 +3,4 @@ export { createJobsFeature } from "./feature.js";
 export { createJobRunLogger } from "./job-run-logger.js";
 export { jobRunLogsTable, jobRunsTable } from "./job-run-table.js";
 export { tenantJobFailuresTable } from "./tenant-job-failure-table.js";
+export { tenantJobRunsTable } from "./tenant-job-run-table.js";

package/dist/jobs/job-run-logger.d.ts

@@ -1,10 +1,10 @@
-import type { DbConnection } from "@cosmicdrift/kumiko-framework/db";
+import { type DbConnection } from "@cosmicdrift/kumiko-framework/db";
 import { type Registry } from "@cosmicdrift/kumiko-framework/engine";
 import type { JobRunnerOptions } from "@cosmicdrift/kumiko-framework/jobs";
 export type JobRunLoggerOptions = {
     readonly db: DbConnection;
     readonly registry: Registry;
 };
-export type JobRunLoggerCallbacks = Pick<JobRunnerOptions, "onJobStart" | "onJobComplete" | "onJobFailed">;
+export type JobRunLoggerCallbacks = Pick<JobRunnerOptions, "onJobStart" | "onJobComplete" | "onJobFailed" | "onJobQueued" | "onJobDropped">;
 export declare function encryptFailureError(error: string, triggeredById: string | null): Promise<string>;
 export declare function createJobRunLogger(opts: JobRunLoggerOptions): JobRunLoggerCallbacks;

package/dist/jobs/job-run-logger.js

@@ -1,5 +1,6 @@
-import { deleteMany, fetchOne, insertMany, insertOne, updateMany, } from "@cosmicdrift/kumiko-framework/bun-db";
+import { deleteMany, fetchOne, insertMany, insertOnConflictDoNothing, insertOne, transaction, updateMany, upsertOnConflict, } from "@cosmicdrift/kumiko-framework/bun-db";
 import { configuredPiiSubjectKms, encryptPiiValueForSubject, KeyErasedError, PII_ERASED_SENTINEL, } from "@cosmicdrift/kumiko-framework/crypto";
+import { acquireNamespacedAdvisoryLock } from "@cosmicdrift/kumiko-framework/db";
 import { SYSTEM_TENANT_ID } from "@cosmicdrift/kumiko-framework/engine";
 import { createFallbackLogger } from "@cosmicdrift/kumiko-framework/logging";
 import { generateId } from "@cosmicdrift/kumiko-framework/utils";
@@ -8,6 +9,7 @@ import { runCompletedSchema, runFailedSchema, runStartedSchema } from "./events.
 import { parseJobInstant } from "./job-instant.js";
 import { jobRunLogsTable, jobRunsTable } from "./job-run-table.js";
 import { tenantJobFailuresTable } from "./tenant-job-failure-table.js";
+import { tenantJobRunsTable } from "./tenant-job-run-table.js";
 // Matches PgKmsAdapter's default pool size (see tenant/handlers/*.query.ts) —
 // bounds concurrent getOrCreateDek calls so a large log batch doesn't claim
 // every connection in the pool.
@@ -92,6 +94,9 @@ async function encryptStartedPayload(payload, triggeredById) {
         return payload;
     return encryptOrSentinel(kms, triggeredById, payload, "payload");
 }
+// pg_advisory_xact_lock namespace (int4): 'tjfl' as ASCII, disjoint from the
+// framework's other fixed advisory-lock keys.
+const TENANT_JOB_FAILURE_LOCK_NAMESPACE = 0x746a666c;
 // fw#3079 — which row the tenant-visible failure record lives in: one per
 // (tenant, job, subject), so the next outcome of the same work replaces or
 // clears it. `subject` is null for a job that declares no subjectFields.
@@ -112,14 +117,18 @@ async function recordTenantJobFailure(db, jobName, outcome) {
     // may still resolve.
     if (!where || !messageKey || outcome?.finalAttempt !== true)
         return;
-    // ponytail: delete-then-insert instead of an upsert — two runs of the same
-    // key finishing at once can leave two rows, and the query returns the
-    // newest. Add a unique index + ON CONFLICT if that ever matters.
-    await deleteMany(db, tenantJobFailuresTable, where);
-    await insertOne(db, tenantJobFailuresTable, {
-        ...where,
-        messageKey,
-        failedAt: Temporal.Now.instant(),
+    // Delete-then-insert under a per-key lock instead of an upsert: the two
+    // partial unique indexes (subject NULL vs. set) are no ON CONFLICT target.
+    // The lock makes the later of two racing final failures replace the earlier
+    // one; the indexes are the backstop against any writer that skips the lock.
+    await transaction(db, async (tx) => {
+        await acquireNamespacedAdvisoryLock(tx, TENANT_JOB_FAILURE_LOCK_NAMESPACE, JSON.stringify([where["tenantId"], where["jobName"], where["subject"]]));
+        await deleteMany(tx, tenantJobFailuresTable, where);
+        await insertOne(tx, tenantJobFailuresTable, {
+            ...where,
+            messageKey,
+            failedAt: Temporal.Now.instant(),
+        });
     });
 }
 async function clearTenantJobFailure(db, jobName, outcome) {
@@ -129,7 +138,78 @@ async function clearTenantJobFailure(db, jobName, outcome) {
         return;
     await deleteMany(db, tenantJobFailuresTable, where);
 }
+function tenantRunKey(jobName, tenantId, subject) {
+    // skip: tenant-less run (cron resolves to SYSTEM_TENANT_ID) — no tenant-scoped query could read it
+    if (tenantId === SYSTEM_TENANT_ID)
+        return null;
+    return { tenantId, jobName, subject };
+}
+async function recordTenantRunQueued(db, jobName, bullJobId, meta) {
+    const key = tenantRunKey(jobName, meta.tenantId, meta.subject);
+    // skip: system-tenant run, no tenant-scoped query reads it
+    if (!key)
+        return;
+    const now = Temporal.Now.instant();
+    // DO NOTHING: a fast worker can report the start before this enqueue hook
+    // returns, and a repeated jobId (debounce, fan-out dedup) is the same job.
+    await insertOnConflictDoNothing(db, tenantJobRunsTable, { ...key, bullJobId, status: "queued", queuedAt: now, updatedAt: now }, { conflictKeys: ["bullJobId"] });
+}
+async function recordTenantRunStarted(db, jobName, bullJobId, meta) {
+    // skip: job did not opt in to tenantVisibleRun
+    if (!meta.tenantVisibleRun)
+        return;
+    const key = tenantRunKey(jobName, meta.tenantVisibleRun.tenantId, meta.tenantVisibleRun.subject);
+    // skip: system-tenant run, no tenant-scoped query reads it
+    if (!key)
+        return;
+    const now = Temporal.Now.instant();
+    await upsertOnConflict(db, tenantJobRunsTable, { ...key, bullJobId, status: "running", startedAt: now, updatedAt: now }, {
+        conflictKeys: ["bullJobId"],
+        update: { status: "running", startedAt: now, finishedAt: null, updatedAt: now },
+    });
+}
+async function recordTenantRunOutcome(db, jobName, bullJobId, outcome, result) {
+    // skip: job did not opt in to tenantVisibleRun
+    if (!outcome?.tenantVisibleRun)
+        return;
+    const key = tenantRunKey(jobName, outcome.tenantId, outcome.tenantVisibleRun.subject);
+    // skip: system-tenant run, no tenant-scoped query reads it
+    if (!key)
+        return;
+    const now = Temporal.Now.instant();
+    // A failed attempt BullMQ will retry goes back to waiting: the tenant keeps
+    // seeing pending work, not a failure the next attempt may still resolve.
+    if (result === "failed" && !outcome.finalAttempt) {
+        await updateMany(db, tenantJobRunsTable, { status: "queued", startedAt: null, finishedAt: null, updatedAt: now }, { bullJobId });
+        // skip: retry pending, the row stays active until the final attempt
+        return;
+    }
+    await upsertOnConflict(db, tenantJobRunsTable, { ...key, bullJobId, status: result, finishedAt: now, updatedAt: now }, { conflictKeys: ["bullJobId"], update: { status: result, finishedAt: now, updatedAt: now } });
+    await deleteMany(db, tenantJobRunsTable, {
+        ...key,
+        status: { in: ["completed", "failed"] },
+        bullJobId: { ne: bullJobId },
+    });
+}
+async function dropTenantRunQueued(db, bullJobId) {
+    await deleteMany(db, tenantJobRunsTable, { bullJobId, status: "queued" });
+}
 const log = createFallbackLogger("job-run-logger");
+// The tenant run state is an optional side write: a throw here must not
+// fail the enqueue, skip the run-row write or turn a completed run into a
+// BullMQ retry.
+async function isolateTenantRunWrite(write, jobName, bullJobId) {
+    try {
+        await write();
+    }
+    catch (e) {
+        log.error("tenant job run state write failed", {
+            jobName,
+            bullJobId,
+            error: e instanceof Error ? e.message : String(e),
+        });
+    }
+}
 // The tenant failure record is an optional side write: a throw here must not
 // skip the run-row update or turn a completed run into a BullMQ retry.
 async function isolateTenantFailureWrite(write, jobName, bullJobId) {
@@ -191,7 +271,14 @@ export function createJobRunLogger(opts) {
         return { runId, triggeredById };
     }
     return {
+        onJobQueued: async (jobName, bullJobId, meta) => {
+            await isolateTenantRunWrite(() => recordTenantRunQueued(db, jobName, bullJobId, meta), jobName, bullJobId);
+        },
+        onJobDropped: async (jobName, bullJobId) => {
+            await isolateTenantRunWrite(() => dropTenantRunQueued(db, bullJobId), jobName, bullJobId);
+        },
         onJobStart: async (jobName, bullJobId, meta) => {
+            await isolateTenantRunWrite(() => recordTenantRunStarted(db, jobName, bullJobId, meta), jobName, bullJobId);
             const runId = generateId();
             const triggeredById = meta.triggeredById ?? null;
             cachePut(bullJobId, runId, triggeredById);
@@ -227,6 +314,7 @@ export function createJobRunLogger(opts) {
             // clears the tenant's failure record even when the run row itself is
             // unreachable (the state-loss return below).
             await isolateTenantFailureWrite(() => clearTenantJobFailure(db, jobName, outcome), jobName, bullJobId);
+            await isolateTenantRunWrite(() => recordTenantRunOutcome(db, jobName, bullJobId, outcome, "completed"), jobName, bullJobId);
             const resolved = await resolveRun(bullJobId);
             // skip: state loss between start + complete (worker restart, cache
             // evicted AND DB has no matching bull_job_id). Rare edge case; we
@@ -272,6 +360,7 @@ export function createJobRunLogger(opts) {
             // Mirror of onJobComplete: recorded independently of the run row, so a
             // tenant still learns their job failed if the row is unreachable.
             await isolateTenantFailureWrite(() => recordTenantJobFailure(db, jobName, outcome), jobName, bullJobId);
+            await isolateTenantRunWrite(() => recordTenantRunOutcome(db, jobName, bullJobId, outcome, "failed"), jobName, bullJobId);
             const resolved = await resolveRun(bullJobId);
             // skip: same rare state-loss case as in onJobComplete — drop the
             // failure write rather than forge a run row from scratch.

package/dist/jobs/job-run-table.d.ts

@@ -6,6 +6,7 @@ export declare const jobRunEntity: import("@cosmicdrift/kumiko-types/fields").En
         readonly type: "text";
         readonly description?: string;
         readonly maxLength?: number;
+        readonly minLength?: number;
         readonly required?: boolean;
         readonly searchable?: boolean;
         readonly sortable?: boolean;
@@ -27,6 +28,7 @@ export declare const jobRunEntity: import("@cosmicdrift/kumiko-types/fields").En
         readonly type: "text";
         readonly description?: string;
         readonly maxLength?: number;
+        readonly minLength?: number;
         readonly required?: boolean;
         readonly searchable?: boolean;
         readonly sortable?: boolean;
@@ -48,6 +50,7 @@ export declare const jobRunEntity: import("@cosmicdrift/kumiko-types/fields").En
         readonly type: "text";
         readonly description?: string;
         readonly maxLength?: number;
+        readonly minLength?: number;
         readonly required?: boolean;
         readonly searchable?: boolean;
         readonly sortable?: boolean;
@@ -69,6 +72,7 @@ export declare const jobRunEntity: import("@cosmicdrift/kumiko-types/fields").En
         readonly type: "text";
         readonly description?: string;
         readonly maxLength?: number;
+        readonly minLength?: number;
         readonly required?: boolean;
         readonly searchable?: boolean;
         readonly sortable?: boolean;
@@ -90,6 +94,7 @@ export declare const jobRunEntity: import("@cosmicdrift/kumiko-types/fields").En
         readonly type: "text";
         readonly description?: string;
         readonly maxLength?: number;
+        readonly minLength?: number;
         readonly required?: boolean;
         readonly searchable?: boolean;
         readonly sortable?: boolean;
@@ -179,6 +184,7 @@ export declare const jobRunEntity: import("@cosmicdrift/kumiko-types/fields").En
         readonly type: "text";
         readonly description?: string;
         readonly maxLength?: number;
+        readonly minLength?: number;
         readonly required?: boolean;
         readonly searchable?: boolean;
         readonly sortable?: boolean;

package/dist/jobs/tenant-job-failure-table.js

@@ -1,5 +1,5 @@
 import { requireEntityTableMeta } from "@cosmicdrift/kumiko-framework/bun-db";
-import { instant, table as pgTable, serial, sql, text, uuid, } from "@cosmicdrift/kumiko-framework/db";
+import { index, instant, table as pgTable, serial, sql, text, uniqueIndex, uuid, } from "@cosmicdrift/kumiko-framework/db";
 // The tenant's own view of a failed job (fw#3079). Direct-write store like
 // store_job_runs: job-run-logger.ts writes it from the BullMQ callbacks,
 // outside any dispatcher transaction.
@@ -21,5 +21,18 @@ export const tenantJobFailuresTable = pgTable("store_tenant_job_failures", {
     subject: text("subject"),
     messageKey: text("message_key").notNull(),
     failedAt: instant("failed_at").default(sql `now()`).notNull(),
-});
+}, (t) => [
+    // One row per key, also the lookup for the delete before every write;
+    // subject NULL needs its own partial index because NULLs
+    // are distinct in a plain unique index.
+    uniqueIndex("store_tenant_job_failures_tenant_job_subject_uidx")
+        .on(t.tenantId, t.jobName, t.subject)
+        .where(sql `subject IS NOT NULL`),
+    uniqueIndex("store_tenant_job_failures_tenant_job_null_subject_uidx")
+        .on(t.tenantId, t.jobName)
+        .where(sql `subject IS NULL`),
+    // The tenant list sorts by failed_at; retention filters on it alone.
+    index("store_tenant_job_failures_tenant_failed_at_idx").on(t.tenantId, t.failedAt),
+    index("store_tenant_job_failures_failed_at_idx").on(t.failedAt),
+]);
 export const tenantJobFailuresTableMeta = requireEntityTableMeta(tenantJobFailuresTable, "tenantJobFailuresTable");

package/dist/jobs/tenant-job-run-table.d.ts

@@ -0,0 +1,4 @@
+import { type EntityTableMeta } from "@cosmicdrift/kumiko-framework/db";
+export type TenantJobRunStatus = "queued" | "running" | "completed" | "failed";
+export declare const tenantJobRunsTable: import("@cosmicdrift/kumiko-types/schema-table-types").SchemaTable;
+export declare const tenantJobRunsTableMeta: EntityTableMeta;

package/dist/jobs/tenant-job-run-table.js

@@ -0,0 +1,34 @@
+import { requireEntityTableMeta } from "@cosmicdrift/kumiko-framework/bun-db";
+import { index, instant, table as pgTable, serial, sql, text, uniqueIndex, uuid, } from "@cosmicdrift/kumiko-framework/db";
+// The tenant's own view of a job's run state (fw#3616). Direct-write store
+// like store_job_runs, written by job-run-logger.ts from the runner callbacks.
+//
+// store_job_runs cannot serve the tenant: it anchors every run on the system
+// tenant, has no subject, and gets its row only when a run starts, so a queued
+// job is invisible there. This table holds no payload, error text or logs by
+// design — the tenant learns only that work is waiting, running or finished.
+//
+// One row per BullMQ job. Active rows (queued/running) are kept as they are;
+// a finished row replaces earlier finished rows of the same (tenant, job,
+// subject), so only the latest outcome stays. `subject` is the canonical JSON
+// of the job's declared subjectFields, or NULL for none — stored in clear, so
+// a job must not declare a PII field as its subject.
+export const tenantJobRunsTable = pgTable("store_tenant_job_runs", {
+    id: serial("id").primaryKey(),
+    tenantId: uuid("tenant_id").notNull(),
+    jobName: text("job_name").notNull(),
+    subject: text("subject"),
+    bullJobId: text("bull_job_id").notNull(),
+    status: text("status").notNull(),
+    queuedAt: instant("queued_at"),
+    startedAt: instant("started_at"),
+    finishedAt: instant("finished_at"),
+    updatedAt: instant("updated_at").default(sql `now()`).notNull(),
+}, (t) => [
+    uniqueIndex("store_tenant_job_runs_bull_job_id_unique").on(t.bullJobId),
+    // Finishing a run prunes older outcomes by this key; the tenant query reads by it.
+    index("store_tenant_job_runs_tenant_job_subject_idx").on(t.tenantId, t.jobName, t.subject),
+    // Stale sweep and retention filter on age alone.
+    index("store_tenant_job_runs_updated_at_idx").on(t.updatedAt),
+]);
+export const tenantJobRunsTableMeta = requireEntityTableMeta(tenantJobRunsTable, "tenantJobRunsTable");