@cosmicdrift/kumiko-samples 0.338.0

package/packages/bundled-features/src/jobs/feature.ts

@@ -0,0 +1,256 @@
+import {
+  defineFeature,
+  type FeatureDefinition,
+  i18nKey,
+} from "@cosmicdrift/kumiko-framework/engine";
+import {
+  JOB_RUN_DETAIL_SCREEN_ID,
+  JOB_RUNS_SCREEN_ID,
+  JobHandlers,
+  JobQueries,
+} from "./constants.js";
+import { catalogQuery } from "./handlers/catalog.query.js";
+import { detailQuery } from "./handlers/detail.query.js";
+import { listQuery } from "./handlers/list.query.js";
+import {
+  projectionRebuildJob,
+  projectionRebuildPayloadSchema,
+} from "./handlers/projection-rebuild.job.js";
+import { reindexEntityJob, reindexEntityPayloadSchema } from "./handlers/reindex-entity.job.js";
+import {
+  createRetentionCleanupJob,
+  DEFAULT_JOB_RUN_RETENTION_DAYS,
+} from "./handlers/retention-cleanup.job.js";
+import { retryWrite } from "./handlers/retry.write.js";
+import {
+  createStaleRunSweepJob,
+  DEFAULT_JOB_RUN_STALE_TIMEOUT_HOURS,
+} from "./handlers/stale-run-sweep.job.js";
+import { tenantFailuresQuery } from "./handlers/tenant-failures.query.js";
+import { triggerWrite } from "./handlers/trigger.write.js";
+import { JOBS_I18N } from "./i18n.js";
+import { jobRunLogsTableMeta, jobRunsTableMeta } from "./job-run-table.js";
+import { tenantJobFailuresTableMeta } from "./tenant-job-failure-table.js";
+
+export type JobsFeatureOptions = {
+  // How long a job run (and its logs) stays in store_job_runs/
+  // store_job_run_logs before the daily retention-cleanup job deletes it.
+  readonly retentionDays?: number;
+  // How long a run can sit at status "running" before the hourly
+  // stale-run-sweep job marks it "failed" (#2246 — a process kill mid-run
+  // never fires onJobComplete/onJobFailed, so nothing else ever revisits
+  // it). Must clear any legitimately long-running job (reindexEntity,
+  // projectionRebuild) — set high, not tight.
+  readonly staleRunTimeoutHours?: number;
+};
+
+export function createJobsFeature(options: JobsFeatureOptions = {}): FeatureDefinition {
+  const retentionDays = options.retentionDays ?? DEFAULT_JOB_RUN_RETENTION_DAYS;
+  const staleRunTimeoutHours = options.staleRunTimeoutHours ?? DEFAULT_JOB_RUN_STALE_TIMEOUT_HOURS;
+  return defineFeature("jobs", (r) => {
+    r.describe(
+      "Persistence and operator tooling for background jobs registered via `r.job(...)`. Every job execution writes directly into `store_job_runs` (current status + duration) and `store_job_run_logs` (per-line log rows) from the BullMQ callbacks — no event stream in between (#2243). A daily `retention-cleanup` job deletes runs (and their logs) older than `retentionDays`; an hourly `stale-run-sweep` job marks runs stuck at status `running` past `staleRunTimeoutHours` as `failed` (#2246 — a crashed worker never fires the completion callback, so nothing else ever revisits the row). Exposes `jobs:write:trigger` (manual run) and `jobs:write:retry` (operator retry of a failed run), plus `jobs:query:list`, `jobs:query:details`, and `jobs:query:catalog` (manual jobs) for the operator UI. A job that declares `tenantVisibleFailure` also records its last failed attempt per tenant and subject in `store_tenant_job_failures`, which the tenant itself reads through `jobs:query:failures` — a translation key only, never the provider's message (fw#3079).",
+    );
+    r.uiHints({
+      displayLabel: "Jobs · Audit & Operator UI",
+      category: "operations",
+      recommended: false,
+    });
+    r.systemScope();
+    r.storeTable(jobRunsTableMeta, {
+      reason: "direct_write.job_runs",
+      // payload/error carry personal:{of:"triggeredById"} (job-run-table.ts).
+      // job-run-logger.ts manually encrypts both before every insert/update
+      // (encryptStartedPayload/encryptFailureError) since this store bypasses
+      // the executor's automatic PII pipeline (#2243).
+      piiEncryptedOnWrite: true,
+    });
+    r.storeTable(jobRunLogsTableMeta, {
+      reason: "read_side.job_run_logs",
+    });
+    r.storeTable(tenantJobFailuresTableMeta, {
+      reason: "direct_write.tenant_job_failures",
+    });
+
+    // Framework-provided rebuild job — available whenever `jobs` is composed; enqueueProjectionRebuild dispatches it.
+    r.job(
+      "projectionRebuild",
+      { trigger: { manual: true }, concurrency: "skip", schema: projectionRebuildPayloadSchema },
+      projectionRebuildJob,
+    );
+
+    // Retroactive search backfill (#1206/#1215) — manual + perTenant, so one
+    // `jobs:write:trigger` call with { entity } fans out to every active
+    // tenant (job-runner.ts perTenant dispatch applies to manual triggers
+    // too, not just cron).
+    r.job(
+      "reindexEntity",
+      {
+        trigger: { manual: true },
+        perTenant: true,
+        concurrency: "skip",
+        schema: reindexEntityPayloadSchema,
+      },
+      reindexEntityJob,
+    );
+
+    // store_job_runs/store_job_run_logs are direct-write, unbounded-growth
+    // stores (#2243) — nothing else purges them.
+    r.job(
+      "retention-cleanup",
+      { trigger: { cron: "0 3 * * *" }, concurrency: "skip" },
+      createRetentionCleanupJob(retentionDays),
+    );
+
+    // Sweeps store_job_runs for rows stuck at status "running" past the
+    // timeout (#2246). Hourly matches the timeout's own granularity —
+    // frequent enough to catch a stuck run soon after it crosses the
+    // threshold, cheap enough that an (almost always empty) result set
+    // doesn't matter.
+    r.job(
+      "stale-run-sweep",
+      { trigger: { cron: "0 * * * *" }, concurrency: "skip" },
+      createStaleRunSweepJob(staleRunTimeoutHours),
+    );
+
+    const handlers = {
+      trigger: r.writeHandler(triggerWrite),
+      retry: r.writeHandler(retryWrite),
+    };
+
+    const queries = {
+      list: r.queryHandler(listQuery),
+      detail: r.queryHandler(detailQuery),
+      catalog: r.queryHandler(catalogQuery),
+      failures: r.queryHandler(tenantFailuresQuery),
+    };
+
+    const systemAdminAccess = { roles: ["SystemAdmin"] as const };
+
+    r.translations({ keys: JOBS_I18N });
+
+    r.screen({
+      id: JOB_RUNS_SCREEN_ID,
+      type: "projectionList",
+      query: JobQueries.list,
+      columns: [
+        { field: "jobName", label: i18nKey("jobs.runs.col.job") },
+        { field: "status", label: i18nKey("jobs.runs.col.status") },
+        {
+          field: "startedAt",
+          label: i18nKey("jobs.runs.col.started"),
+          renderer: { format: "timestamp" },
+        },
+        { field: "duration", label: i18nKey("jobs.runs.col.duration") },
+      ],
+      defaultSort: { field: "startedAt", dir: "desc" },
+      facets: [
+        {
+          field: "status",
+          type: "select",
+          label: i18nKey("jobs.runs.filter.status"),
+          options: [
+            { value: "queued", label: i18nKey("jobs.runs.filter.queued") },
+            { value: "running", label: i18nKey("jobs.runs.filter.running") },
+            { value: "completed", label: i18nKey("jobs.runs.filter.completed") },
+            { value: "failed", label: i18nKey("jobs.runs.filter.failed") },
+          ],
+        },
+      ],
+      rowActions: [
+        {
+          kind: "navigate",
+          id: "open",
+          label: i18nKey("jobs.runs.open"),
+          screen: JOB_RUN_DETAIL_SCREEN_ID,
+          entityId: "id",
+          rowClick: true,
+        },
+      ],
+      toolbarActions: [
+        {
+          kind: "drawer",
+          id: "trigger",
+          label: i18nKey("jobs.trigger.title"),
+          screen: "job-trigger",
+          style: "primary",
+        },
+      ],
+      pagination: "infinite",
+      access: systemAdminAccess,
+    });
+    r.screen({
+      id: "job-trigger",
+      type: "actionForm",
+      handler: JobHandlers.trigger,
+      fields: {
+        jobName: { type: "text", required: true },
+        payload: { type: "longText", multiline: true, default: "{}" },
+      },
+      layout: { sections: [{ fields: ["jobName", "payload"] }] },
+      submitLabel: i18nKey("jobs.trigger.submit"),
+      access: systemAdminAccess,
+      description: "jobs.screen.job-trigger.subtitle",
+    });
+    r.screen({
+      id: JOB_RUN_DETAIL_SCREEN_ID,
+      type: "projectionDetail",
+      query: JobQueries.details,
+      idParam: "runId",
+      fieldLabels: {
+        jobName: i18nKey("jobs.detail.field.job"),
+        status: i18nKey("jobs.detail.field.status"),
+        id: i18nKey("jobs.detail.field.id"),
+        startedAt: i18nKey("jobs.detail.field.started"),
+        finishedAt: i18nKey("jobs.detail.field.finished"),
+        duration: i18nKey("jobs.detail.field.duration"),
+        error: i18nKey("jobs.detail.field.error"),
+        logs: i18nKey("jobs.detail.logs"),
+      },
+      layout: {
+        sections: [
+          {
+            fields: [
+              "jobName",
+              "status",
+              "id",
+              { field: "startedAt", renderer: { format: "timestamp" } },
+              { field: "finishedAt", renderer: { format: "timestamp" } },
+              "duration",
+              "error",
+            ],
+          },
+          // No section `title` here (fw#2312 label-dedup fix): the section
+          // holds exactly one field, so a title would repeat the field's own
+          // label (rendered by RenderField as the Field's heading) verbatim.
+          {
+            fields: [{ field: "logs", renderer: { format: "json" } }],
+          },
+        ],
+      },
+      actions: [
+        {
+          kind: "writeHandler",
+          id: "retry",
+          label: i18nKey("jobs.detail.retry"),
+          handler: JobHandlers.retry,
+          payload: { map: { runId: "id" } },
+          visible: { field: "status", eq: "failed" },
+          style: "primary",
+        },
+      ],
+      description: "jobs.screen.job-run-detail.subtitle",
+      listScreenId: JOB_RUNS_SCREEN_ID,
+      access: systemAdminAccess,
+    });
+    r.nav({
+      id: "job-runs",
+      label: "jobs:nav.jobRuns",
+      icon: "list",
+      screen: "jobs:screen:job-runs",
+      order: 10,
+    });
+
+    return { handlers, queries };
+  });
+}

package/packages/bundled-features/src/jobs/handlers/catalog.query.ts

@@ -0,0 +1,42 @@
+import { defineQueryHandler, type JobDefinition } from "@cosmicdrift/kumiko-framework/engine";
+import * as z from "zod";
+import { isManualTrigger } from "../is-manual-trigger.js";
+
+export type ManualJobCatalogEntry = {
+  readonly jobName: string;
+  readonly perTenant: boolean;
+  /** Compact JSON Schema (or null when the job accepts any/empty payload). */
+  readonly payloadSchema: Record<string, unknown> | null;
+};
+
+function payloadSchemaJson(job: JobDefinition): Record<string, unknown> | null {
+  if (job.schema === undefined) return null;
+  try {
+    return z.toJSONSchema(job.schema) as Record<string, unknown>;
+  } catch {
+    // Non-JSON-Schema-able Zod types (transforms/effects) — UI still lists the job.
+    return null;
+  }
+}
+
+/** SystemAdmin catalog of jobs that may be started via `jobs:write:trigger`. */
+export const catalogQuery = defineQueryHandler({
+  name: "catalog",
+  description:
+    "Lists the jobs that may be started manually, each with its per-tenant flag and payload JSON Schema; use it to find out which job can be triggered and what payload it expects.",
+  schema: z.object({}),
+  access: { roles: ["SystemAdmin"] },
+  handler: async (_query, ctx): Promise<{ rows: readonly ManualJobCatalogEntry[] }> => {
+    const rows: ManualJobCatalogEntry[] = [];
+    for (const [jobName, job] of ctx.registry.getAllJobs()) {
+      if (!isManualTrigger(job.trigger)) continue;
+      rows.push({
+        jobName,
+        perTenant: job.perTenant === true,
+        payloadSchema: payloadSchemaJson(job),
+      });
+    }
+    rows.sort((a, b) => a.jobName.localeCompare(b.jobName));
+    return { rows };
+  },
+});

package/packages/bundled-features/src/jobs/handlers/detail.query.ts

@@ -0,0 +1,69 @@
+import { fetchOne, selectMany } from "@cosmicdrift/kumiko-framework/bun-db";
+import { defineQueryHandler } from "@cosmicdrift/kumiko-framework/engine";
+import { InternalError } from "@cosmicdrift/kumiko-framework/errors";
+import * as z from "zod";
+import { decryptStoredPii, mapWithConcurrency } from "../../shared/index.js";
+import { jobRunLogsTable, jobRunsTable } from "../job-run-table.js";
+
+// Matches job-run-logger.ts's KMS_POOL_CONCURRENCY / the tenant handlers'
+// KMS_POOL_CONCURRENCY — bounds concurrent decrypt calls against the KMS
+// adapter's connection pool.
+const KMS_POOL_CONCURRENCY = 4;
+
+export const detailQuery = defineQueryHandler({
+  name: "details",
+  description:
+    "Returns one job run across all tenants by its run id with status, timings, decrypted payload, error and its full log lines; use it to investigate why a run failed.",
+  // Post-ES: runId is the uuid aggregate-id of the jobRun event-stream.
+  // Pre-ES callers passed the serial row-id; the migration is breaking
+  // for API callers (intentional — jobs is framework-ops, no external
+  // contract). z.uuid() guards against accidental number-id passing.
+  schema: z.object({ runId: z.uuid() }),
+  access: { roles: ["SystemAdmin"] },
+  handler: async (query, ctx) => {
+    if (!ctx.systemDb) {
+      throw new InternalError({
+        message: "jobs:query:details requires ctx.systemDb (feature must declare r.systemScope())",
+      });
+    }
+    const db = ctx.systemDb.acknowledgeCrossTenant("cross-tenant job monitoring");
+
+    const row = await fetchOne(db, jobRunsTable, { id: query.payload.runId });
+
+    if (!row) return null;
+
+    // payload is stored encrypted under the triggering user's DEK (#799).
+    if (typeof row["payload"] === "string") {
+      row["payload"] = await decryptStoredPii(row["payload"], "payload", "job-run-detail");
+    }
+
+    // error is stored encrypted under the triggering user's DEK (#2307),
+    // same mechanism as row.payload above.
+    if (typeof row["error"] === "string") {
+      row["error"] = await decryptStoredPii(row["error"], "error", "job-run-detail");
+    }
+
+    const logs = await selectMany(
+      db,
+      jobRunLogsTable,
+      { runId: query.payload.runId },
+      {
+        orderBy: { col: "id", direction: "asc" },
+      },
+    );
+
+    // message is stored encrypted under the triggering user's DEK (#2247),
+    // same mechanism as row.payload above. Bounded concurrency (#2307) —
+    // unbounded Promise.all would fire one decrypt per log line at once
+    // against the KMS adapter's pool.
+    const decryptedLogs = await mapWithConcurrency(logs, KMS_POOL_CONCURRENCY, async (log) => {
+      if (typeof log["message"] !== "string") return log;
+      return {
+        ...log,
+        message: await decryptStoredPii(log["message"], "message", "job-run-detail-log"),
+      };
+    });
+
+    return { ...row, logs: decryptedLogs };
+  },
+});

package/packages/bundled-features/src/jobs/handlers/list.query.ts

@@ -0,0 +1,79 @@
+import { countWhere, selectMany, type WhereObject } from "@cosmicdrift/kumiko-framework/bun-db";
+import { defineQueryHandler } from "@cosmicdrift/kumiko-framework/engine";
+import { InternalError } from "@cosmicdrift/kumiko-framework/errors";
+import * as z from "zod";
+import { decryptStoredPii, mapWithConcurrency } from "../../shared/index.js";
+import { jobRunsTable } from "../job-run-table.js";
+
+const KMS_POOL_CONCURRENCY = 4;
+
+const jobRunStatusSchema = z.enum(["queued", "running", "completed", "failed"]);
+
+async function decryptRunRow<T extends Record<string, unknown>>(row: T): Promise<T> {
+  let result = row;
+  if (typeof result["payload"] === "string") {
+    result = {
+      ...result,
+      payload: await decryptStoredPii(result["payload"], "payload", "job-runs"),
+    };
+  }
+  if (typeof result["error"] === "string") {
+    result = { ...result, error: await decryptStoredPii(result["error"], "error", "job-runs") };
+  }
+  return result;
+}
+
+export const listQuery = defineQueryHandler({
+  name: "list",
+  description:
+    "Lists job runs across all tenants newest-first, optionally filtered by job name and status; use it to check whether a job ran and whether it succeeded.",
+  schema: z.object({
+    jobName: z.string().optional(),
+    status: jobRunStatusSchema.optional(),
+    filters: z
+      .array(
+        z.object({
+          field: z.literal("status"),
+          op: z.literal("in"),
+          value: z.array(jobRunStatusSchema),
+        }),
+      )
+      .optional(),
+    sort: z.enum(["jobName", "status", "startedAt", "duration"]).optional(),
+    sortDirection: z.enum(["asc", "desc"]).optional(),
+    limit: z.number().optional(),
+    totalCount: z.boolean().optional(),
+  }),
+  access: { roles: ["SystemAdmin"] },
+  handler: async (query, ctx) => {
+    if (!ctx.systemDb) {
+      throw new InternalError({
+        message: "jobs:query:list requires ctx.systemDb (feature must declare r.systemScope())",
+      });
+    }
+    const db = ctx.systemDb.unsafeRaw(
+      "cross-tenant job monitoring: SystemAdmin lists job runs of every tenant",
+    );
+    const where: WhereObject = {};
+    if (query.payload.jobName) where["jobName"] = query.payload.jobName;
+    const statusFilter = query.payload.filters?.find((filter) => filter.field === "status");
+    const statuses = statusFilter?.value ?? (query.payload.status ? [query.payload.status] : []);
+    if (statuses.length === 1) where["status"] = statuses[0];
+    else if (statuses.length > 1) where["status"] = { in: statuses };
+    const sortColumn = query.payload.sort ?? "startedAt";
+    const rows = await selectMany(db, jobRunsTable, where, {
+      orderBy: { col: sortColumn, direction: query.payload.sortDirection ?? "desc" },
+      limit: query.payload.limit ?? 50,
+    });
+    // countWhere reruns the SAME `where` without the limit — `rows.length` is
+    // capped at the page size and would silently undercount the total.
+    const total =
+      query.payload.totalCount === true ? await countWhere(db, jobRunsTable, where) : undefined;
+    // payload/error are stored encrypted under the triggering user's DEK (#799, #2307).
+    return {
+      rows: await mapWithConcurrency(rows, KMS_POOL_CONCURRENCY, decryptRunRow),
+      nextCursor: null,
+      ...(total !== undefined && { total }),
+    };
+  },
+});

package/packages/bundled-features/src/jobs/handlers/projection-rebuild.job.ts

@@ -0,0 +1,43 @@
+// Single-run projection-rebuild worker (`jobs:job:projection-rebuild`); manually triggered, typically via enqueueProjectionRebuild to refill an emptied projection.
+
+import type { DbConnection } from "@cosmicdrift/kumiko-framework/db";
+import type { JobHandlerFn } from "@cosmicdrift/kumiko-framework/engine";
+import { InternalError } from "@cosmicdrift/kumiko-framework/errors";
+import { rebuildProjectionOrMultiStream } from "@cosmicdrift/kumiko-framework/migrations";
+import * as z from "zod";
+
+export const projectionRebuildPayloadSchema = z.object({
+  projection: z.string().min(1),
+  // Quarantine mode (#760): skip + dead-letter poison events instead of
+  // failing the whole rebuild. Operator opt-in per run.
+  skipApplyErrors: z.boolean().optional(),
+});
+
+export const projectionRebuildJob: JobHandlerFn = async (rawPayload, ctx): Promise<void> => {
+  const { projection, skipApplyErrors } = projectionRebuildPayloadSchema.parse(rawPayload);
+  if (!ctx.systemDb) {
+    throw new InternalError({
+      message:
+        "[jobs:projection-rebuild] ctx.systemDb missing — is r.systemScope() still set on the jobs feature?",
+    });
+  }
+  if (!ctx.registry) {
+    throw new InternalError({
+      message:
+        "[jobs:projection-rebuild] ctx.registry missing — job context requires the registry.",
+    });
+  }
+  // Global rebuild by design (streams every tenant's events) — acknowledgeCrossTenant
+  // documents that instead of the previous implicit ctx.db cast.
+  ctx.systemDb.acknowledgeCrossTenant("global projection rebuild");
+  const db = ctx.systemDb.unsafeRaw("projection rebuild spans every tenant's rows") as DbConnection; // @cast-boundary db-operator — DbRunner narrows to DbConnection, jobs never run inside a DbTx
+  const result = await rebuildProjectionOrMultiStream(projection, {
+    db,
+    registry: ctx.registry,
+    ...(skipApplyErrors === true && { errorPolicy: { skipApplyErrors: true } }),
+  });
+  ctx.log?.info?.(
+    `[jobs:projection-rebuild] rebuilt ${projection}: ${result.eventsProcessed} events` +
+      `${result.eventsSkipped > 0 ? ` (${result.eventsSkipped} quarantined)` : ""} in ${result.durationMs}ms`,
+  );
+};

package/packages/bundled-features/src/jobs/handlers/reindex-entity.job.ts

@@ -0,0 +1,56 @@
+// Manual, perTenant backfill job (#1206/#1215) — indexes existing rows for
+// an entity that only got `searchable: true` after rows already existed.
+// perTenant fan-out (job-runner.ts) also applies to a manual dispatch, not
+// just cron: one `jobs:write:trigger` call with { entity } re-runs once per
+// active tenant.
+
+import type { JobHandlerFn } from "@cosmicdrift/kumiko-framework/engine";
+import { InternalError } from "@cosmicdrift/kumiko-framework/errors";
+import { reindexEntity } from "@cosmicdrift/kumiko-framework/search";
+import * as z from "zod";
+
+export const reindexEntityPayloadSchema = z.object({
+  entity: z.string().min(1),
+});
+
+export const reindexEntityJob: JobHandlerFn = async (rawPayload, ctx): Promise<void> => {
+  const { entity } = reindexEntityPayloadSchema.parse(rawPayload);
+  if (!ctx.registry) {
+    throw new InternalError({
+      message: "[jobs:reindex-entity] ctx.registry missing — job context requires the registry.",
+    });
+  }
+  if (!ctx.searchAdapter) {
+    throw new InternalError({
+      message:
+        "[jobs:reindex-entity] ctx.searchAdapter missing — mount a search adapter to run this job.",
+    });
+  }
+  // perTenant fan-out — the job-runner enqueues one child run per active
+  // tenant with ctx.systemUser scoped to it; a run without one (e.g.
+  // misconfigured manual dispatch outside the fan-out) has nothing scoped
+  // to reindex. ctx._tenantId is never populated per-dispatch (only
+  // ctx.systemUser is), so it is not a valid fallback here.
+  const tenantId = ctx.systemUser?.tenantId;
+  if (tenantId === undefined) {
+    // skip: fired without a perTenant fan-out tenant — nothing scoped to reindex
+    return;
+  }
+  if (!ctx.systemDb) {
+    throw new InternalError({
+      message:
+        "[jobs:reindex-entity] ctx.systemDb missing — the owning feature must declare r.systemScope().",
+    });
+  }
+  // unsafeRaw bypasses tenant filtering, safe here only because reindexEntity
+  // filters `WHERE tenant_id = $1` itself with this same tenantId.
+  ctx.systemDb.assertTenantMatch(tenantId);
+  const db = ctx.systemDb.unsafeRaw(
+    "reindexEntity filters WHERE tenant_id = $1 itself with this job's tenantId",
+  );
+  const result = await reindexEntity(db, ctx.registry, ctx.searchAdapter, entity, tenantId);
+  ctx.log?.info?.(
+    `[jobs:reindex-entity] tenant=${tenantId} reindexed ${entity}: ${result.indexedRows}/${result.scannedRows} rows indexed` +
+      `${result.failures.length > 0 ? ` (${result.failures.length} failed)` : ""}`,
+  );
+};

package/packages/bundled-features/src/jobs/handlers/retention-cleanup.job.ts

@@ -0,0 +1,23 @@
+import type { DbConnection } from "@cosmicdrift/kumiko-framework/db";
+import type { JobHandlerFn } from "@cosmicdrift/kumiko-framework/engine";
+import { InternalError } from "@cosmicdrift/kumiko-framework/errors";
+import { deleteStaleJobRuns } from "../db/queries/retention.js";
+
+// Single source for the retention window — change here, nowhere else.
+export const DEFAULT_JOB_RUN_RETENTION_DAYS = 30;
+
+export function createRetentionCleanupJob(retentionDays: number): JobHandlerFn {
+  return async (_payload, ctx) => {
+    if (!ctx.systemDb) {
+      throw new InternalError({
+        message:
+          "[jobs:retention-cleanup] ctx.systemDb missing — is r.systemScope() still set on the jobs feature?",
+      });
+    }
+    const db = ctx.systemDb.unsafeRaw(
+      "store_job_runs retention purges finished runs of every tenant",
+    ) as DbConnection; // @cast-boundary db-operator — DbRunner narrows to DbConnection, jobs never run inside a DbTx
+    const result = await deleteStaleJobRuns(db, retentionDays);
+    ctx.log?.info?.(`[jobs:retention-cleanup] complete: ${JSON.stringify(result)}`);
+  };
+}

package/packages/bundled-features/src/jobs/handlers/retry.write.ts

@@ -0,0 +1,100 @@
+import { fetchOne } from "@cosmicdrift/kumiko-framework/bun-db";
+import { isPiiCiphertext, PII_ERASED_SENTINEL } from "@cosmicdrift/kumiko-framework/crypto";
+import { defineWriteHandler } from "@cosmicdrift/kumiko-framework/engine";
+import {
+  InternalError,
+  NotFoundError,
+  UnprocessableError,
+  writeFailure,
+} from "@cosmicdrift/kumiko-framework/errors";
+import type { JobRunner } from "@cosmicdrift/kumiko-framework/jobs";
+import { parseJsonOrThrow } from "@cosmicdrift/kumiko-framework/utils";
+import * as z from "zod";
+import { decryptStoredPii } from "../../shared/index.js";
+import { JobErrors } from "../constants.js";
+import { jobRunsTable } from "../job-run-table.js";
+
+type JobRunRow = {
+  readonly status: string;
+  readonly jobName: string;
+  readonly payload: string | null;
+  readonly triggeredById: string | null;
+};
+
+export const retryWrite = defineWriteHandler({
+  name: "retry",
+  description:
+    "Re-dispatches a failed job run with its original payload and triggering user; use it to rerun a job that failed for a transient reason.",
+  // Post-ES: runId is the uuid aggregate-id. See detail.query for the
+  // rationale — jobs is framework-ops, callers are admin tooling only.
+  schema: z.object({ runId: z.uuid() }),
+  access: { roles: ["SystemAdmin"] },
+  handler: async (event, ctx) => {
+    if (!ctx.systemDb) {
+      throw new InternalError({
+        message: "jobs:write:retry requires ctx.systemDb (feature must declare r.systemScope())",
+      });
+    }
+    const db = ctx.systemDb.acknowledgeCrossTenant("cross-tenant job monitoring");
+    // @cast-boundary engine-payload — JobRunner attached by app-boot via ctx-extension
+    const jobRunner = ctx["jobRunner"] as JobRunner;
+
+    const run = await fetchOne<JobRunRow>(db, jobRunsTable, { id: event.payload.runId });
+
+    if (!run) {
+      return writeFailure(
+        new NotFoundError("jobRun", event.payload.runId, {
+          i18nKey: "jobs.errors.notFound",
+        }),
+      );
+    }
+
+    if (run.status !== "failed") {
+      return writeFailure(
+        new UnprocessableError(JobErrors.onlyFailedCanRetry, {
+          i18nKey: "jobs.errors.onlyFailedCanRetry",
+          details: { status: run.status },
+        }),
+      );
+    }
+
+    // payload is stored encrypted under the triggering user's DEK (#799),
+    // same mechanism as detail.query.ts/list.query.ts — decrypt before
+    // parsing it as JSON. A key erased since the run finished decrypts to
+    // PII_ERASED_SENTINEL, which is not JSON; reject the retry rather than
+    // dispatch it with a silently emptied payload.
+    let payload: Record<string, unknown> = {};
+    let decryptedPayloadJson: string | null = null;
+    if (run.payload) {
+      const decryptedPayload = await decryptStoredPii(
+        run.payload,
+        "payload",
+        `job run ${event.payload.runId} payload`,
+      );
+      // Still-ciphertext means no KMS could decrypt it (rollout mode); not JSON either.
+      if (decryptedPayload === PII_ERASED_SENTINEL || isPiiCiphertext(decryptedPayload)) {
+        return writeFailure(
+          new UnprocessableError(JobErrors.payloadErased, {
+            i18nKey: "jobs.errors.payloadErased",
+          }),
+        );
+      }
+      payload = parseJsonOrThrow<Record<string, unknown>>(
+        decryptedPayload,
+        `job run ${event.payload.runId} payload`,
+      );
+      decryptedPayloadJson = decryptedPayload;
+    }
+
+    // Retry re-uses the original run's DEK owner so the retry's logs/payload stay encrypted.
+    const bullJobId = await jobRunner.dispatch(run.jobName, payload, {
+      triggeredById: run.triggeredById ?? undefined,
+      ...(decryptedPayloadJson !== null && { payload: decryptedPayloadJson }),
+    });
+
+    return {
+      isSuccess: true,
+      data: { jobName: run.jobName, bullJobId, retriedFromRunId: event.payload.runId },
+    };
+  },
+});