@cosmicdrift/kumiko-bundled-features 0.211.0 → 0.213.0

package/src/jobs/__tests__/jobs-pii-kms.integration.test.ts

@@ -19,11 +19,15 @@ import { createEventsTable, eventsTable } from "@cosmicdrift/kumiko-framework/ev
 import {
   createTestDb,
   createTestRedis,
+  setupTestStack,
   type TestDb,
   type TestRedis,
+  type TestStack,
+  TestUsers,
   unsafePushTables,
 } from "@cosmicdrift/kumiko-framework/stack";
 import { resetPiiSubjectKmsForTests, resetTestTables } from "@cosmicdrift/kumiko-framework/testing";
+import { JobQueries } from "../constants";
 import { createJobsFeature } from "../feature";
 import { createJobRunLogger } from "../job-run-logger";
 import { jobRunLogsTable, jobRunsTable } from "../job-run-table";
@@ -120,3 +124,173 @@ describe("jobs run-started payload under KMS", () => {
     expect(row?.["payload"]).toBe(SECRET_PAYLOAD);
   });
 });
+
+// #2247: store_job_run_logs is the same unmanaged direct-write path as
+// jobRunsTable.payload above — onJobComplete/onJobFailed batch-insert log
+// lines straight from the BullMQ callback, with no event-PII catalog to
+// lean on. Same subject (triggeredById), same encrypt-under-DEK treatment,
+// same skip rules (null subject / no KMS stay plaintext).
+describe("jobs run-completed/-failed log messages under KMS (#2247)", () => {
+  const SECRET_LOG = "user export contained iban DE89370400440532013000";
+
+  async function runIdFor(bullJobId: string): Promise<string> {
+    const row = await fetchOne(testDb.db, jobRunsTable, { bullJobId });
+    return String(row?.["id"]);
+  }
+
+  test("onJobComplete: stored log message is ciphertext, decrypts to original", async () => {
+    await logger.onJobStart?.("app:job:export", "bull-c1", { triggeredById: USER_ID });
+    await logger.onJobComplete?.("app:job:export", "bull-c1", 42, [
+      { level: "info", message: SECRET_LOG, timestamp: Temporal.Now.instant() },
+    ]);
+
+    const logs = await selectMany(testDb.db, jobRunLogsTable, { runId: await runIdFor("bull-c1") });
+    expect(logs).toHaveLength(1);
+    expect(isPiiCiphertext(logs[0]?.["message"])).toBe(true);
+    expect(String(logs[0]?.["message"])).toContain(`user:${USER_ID}`);
+
+    const back = await decryptPiiFieldValues({ message: logs[0]?.["message"] }, ["message"], kms, {
+      requestId: "t",
+    });
+    expect(back["message"]).toBe(SECRET_LOG);
+  });
+
+  test("onJobFailed: stored log message is ciphertext, decrypts to original", async () => {
+    await logger.onJobStart?.("app:job:export", "bull-f1", { triggeredById: USER_ID });
+    await logger.onJobFailed?.("app:job:export", "bull-f1", "boom", [
+      { level: "error", message: SECRET_LOG, timestamp: Temporal.Now.instant() },
+    ]);
+
+    const logs = await selectMany(testDb.db, jobRunLogsTable, { runId: await runIdFor("bull-f1") });
+    expect(logs).toHaveLength(1);
+    expect(isPiiCiphertext(logs[0]?.["message"])).toBe(true);
+
+    const back = await decryptPiiFieldValues({ message: logs[0]?.["message"] }, ["message"], kms, {
+      requestId: "t",
+    });
+    expect(back["message"]).toBe(SECRET_LOG);
+  });
+
+  test("erase subject key after write → completed log message decrypts to [[erased]]", async () => {
+    await logger.onJobStart?.("app:job:export", "bull-c2", { triggeredById: USER_ID });
+    await logger.onJobComplete?.("app:job:export", "bull-c2", 10, [
+      { level: "info", message: SECRET_LOG, timestamp: Temporal.Now.instant() },
+    ]);
+    const logs = await selectMany(testDb.db, jobRunLogsTable, { runId: await runIdFor("bull-c2") });
+
+    await kms.eraseKey({ kind: "user", userId: USER_ID });
+    const after = await decryptPiiFieldValues({ message: logs[0]?.["message"] }, ["message"], kms, {
+      requestId: "t",
+    });
+    expect(after["message"]).toBe(PII_ERASED_SENTINEL);
+  });
+
+  test("system run (no triggeredById) → completed log messages stay plaintext", async () => {
+    await logger.onJobStart?.("app:job:cron-sweep", "bull-c3", {});
+    await logger.onJobComplete?.("app:job:cron-sweep", "bull-c3", 5, [
+      { level: "info", message: "plain sweep log", timestamp: Temporal.Now.instant() },
+    ]);
+    const logs = await selectMany(testDb.db, jobRunLogsTable, { runId: await runIdFor("bull-c3") });
+    expect(logs[0]?.["message"]).toBe("plain sweep log");
+  });
+
+  test("without a KMS the completed log message stays plaintext (rollout mode)", async () => {
+    await logger.onJobStart?.("app:job:export", "bull-c4", { triggeredById: USER_ID });
+    resetPiiSubjectKmsForTests();
+    await logger.onJobComplete?.("app:job:export", "bull-c4", 5, [
+      { level: "info", message: SECRET_LOG, timestamp: Temporal.Now.instant() },
+    ]);
+    const logs = await selectMany(testDb.db, jobRunLogsTable, { runId: await runIdFor("bull-c4") });
+    expect(logs[0]?.["message"]).toBe(SECRET_LOG);
+  });
+
+  // Documents a gap rather than fixing it: if the subject's key is erased
+  // between onJobStart and onJobComplete, encryptLogMessages' getOrCreateDek
+  // throws KeyErasedError — same propagate-on-erased-key behavior as
+  // onJobStart's own encryptStartedPayload (no catch there either). Here the
+  // status updateMany already ran before the log insert throws, so the run
+  // is left "completed" with its log batch dropped instead of forging a
+  // plaintext fallback. Same wedge-state class #2246 tracks (job runs stuck
+  // after abnormal termination) — not something this PR fixes.
+  test("erase subject key BEFORE onJobComplete: log encryption throws, run status already landed as completed", async () => {
+    await logger.onJobStart?.("app:job:export", "bull-c5", {
+      triggeredById: USER_ID,
+      payload: SECRET_PAYLOAD,
+    });
+    await kms.eraseKey({ kind: "user", userId: USER_ID });
+
+    let threw = false;
+    try {
+      await logger.onJobComplete?.("app:job:export", "bull-c5", 5, [
+        { level: "info", message: SECRET_LOG, timestamp: Temporal.Now.instant() },
+      ]);
+    } catch {
+      threw = true;
+    }
+    expect(threw).toBe(true);
+
+    const row = await fetchOne(testDb.db, jobRunsTable, { bullJobId: "bull-c5" });
+    expect(row?.["status"]).toBe("completed");
+    const logs = await selectMany(testDb.db, jobRunLogsTable, { runId: await runIdFor("bull-c5") });
+    expect(logs).toHaveLength(0);
+  });
+});
+
+// The suite above proves the write side (row is ciphertext) and decrypts by
+// calling decryptPiiFieldValues directly in the test body — neither touches
+// detail.query.ts's own decrypt wrapper. This block dispatches the real
+// jobs:query:details HTTP handler so a regression in those eleven lines
+// (wrong AAD field name, log-array key access, the row/log spread) fails a
+// test instead of shipping ciphertext to the job-run detail screen.
+describe("jobs:query:details decrypts log messages end-to-end (#2247)", () => {
+  let detailStack: TestStack;
+  let detailLogger: ReturnType<typeof createJobRunLogger>;
+
+  const DETAIL_USER_ID = "u-pii-detail-1";
+  const SECRET_DETAIL_LOG = "user export contained iban DE89370400440532013000";
+
+  beforeAll(async () => {
+    detailStack = await setupTestStack({
+      features: [createJobsFeature()],
+      jobs: {
+        consumerLane: "worker",
+        queueNamePrefix: `kumiko-jobs-detail-pii-test-${Date.now()}`,
+      },
+    });
+    await unsafePushTables(detailStack.db, { jobRunsTable, jobRunLogsTable });
+    detailLogger = createJobRunLogger({ db: detailStack.db, registry: detailStack.registry });
+  });
+
+  afterAll(async () => {
+    await detailStack.cleanup();
+  });
+
+  beforeEach(() => {
+    configurePiiSubjectKms(new InMemoryKmsAdapter());
+  });
+
+  afterEach(() => {
+    resetPiiSubjectKmsForTests();
+  });
+
+  test("jobs:query:details returns plaintext log message even though the stored row is ciphertext", async () => {
+    await detailLogger.onJobStart?.("app:job:export", "bull-detail-1", {
+      triggeredById: DETAIL_USER_ID,
+    });
+    await detailLogger.onJobComplete?.("app:job:export", "bull-detail-1", 7, [
+      { level: "info", message: SECRET_DETAIL_LOG, timestamp: Temporal.Now.instant() },
+    ]);
+
+    const row = await fetchOne(detailStack.db, jobRunsTable, { bullJobId: "bull-detail-1" });
+    const runId = String(row?.["id"]);
+    const storedLogs = await selectMany(detailStack.db, jobRunLogsTable, { runId });
+    expect(isPiiCiphertext(storedLogs[0]?.["message"])).toBe(true);
+
+    const result = await detailStack.http.queryOk<{
+      logs: readonly { message: string }[];
+    }>(JobQueries.details, { runId }, TestUsers.systemAdmin);
+
+    expect(result.logs).toHaveLength(1);
+    expect(result.logs[0]?.message).toBe(SECRET_DETAIL_LOG);
+  });
+});

package/src/jobs/__tests__/jobs-stale-run-sweep.integration.test.ts

@@ -0,0 +1,148 @@
+// Integration test for the stale-run-sweep job (#2246). Dispatches
+// jobs:job:stale-run-sweep through the real jobRunner (setupTestStack +
+// BullMQ worker), same pattern as jobs-retention.integration.test.ts — a
+// hand-built JobContext would only prove the handler function works in
+// isolation, not that the job is actually registered and dispatchable.
+
+import { afterEach, describe, expect, test } from "bun:test";
+import { fetchOne } from "@cosmicdrift/kumiko-framework/bun-db";
+import { sql } from "@cosmicdrift/kumiko-framework/db";
+import { SYSTEM_TENANT_ID } from "@cosmicdrift/kumiko-framework/engine";
+import {
+  setupTestStack,
+  type TestStack,
+  unsafePushTables,
+} from "@cosmicdrift/kumiko-framework/stack";
+import { seedRow, waitFor } from "@cosmicdrift/kumiko-framework/testing";
+import { STALE_JOB_RUN_ERROR } from "../db/queries/stale-run-sweep";
+import { createJobsFeature } from "../feature";
+import { DEFAULT_JOB_RUN_STALE_TIMEOUT_HOURS } from "../handlers/stale-run-sweep.job";
+import { jobRunLogsTable, jobRunsTable } from "../job-run-table";
+
+const SWEEP_JOB = "jobs:job:stale-run-sweep";
+
+let stack: TestStack | undefined;
+
+async function bootStack(staleRunTimeoutHours?: number): Promise<TestStack> {
+  const s = await setupTestStack({
+    features: [
+      createJobsFeature(staleRunTimeoutHours !== undefined ? { staleRunTimeoutHours } : {}),
+    ],
+    jobs: { consumerLane: "worker" },
+  });
+  await unsafePushTables(s.db, { jobRunsTable, jobRunLogsTable });
+  return s;
+}
+
+afterEach(async () => {
+  if (stack) await stack.cleanup();
+  stack = undefined;
+});
+
+function currentStack(): TestStack {
+  if (!stack) throw new Error("stack not booted");
+  return stack;
+}
+
+async function seedRun(opts: {
+  id: string;
+  status: "running" | "completed";
+  ageHours: number;
+}): Promise<void> {
+  const startedAt = sql`now() - ${sql.raw(`interval '${opts.ageHours} hours'`)}`;
+  await seedRow(currentStack().db, jobRunsTable, {
+    id: opts.id,
+    tenantId: SYSTEM_TENANT_ID,
+    jobName: "example:job:stale-run-probe",
+    bullJobId: `bull-${opts.id}`,
+    status: opts.status,
+    attempt: 1,
+    startedAt,
+  });
+}
+
+async function statusOf(id: string): Promise<string | undefined> {
+  const row = await fetchOne<{ status: string }>(currentStack().db, jobRunsTable, { id });
+  return row?.status;
+}
+
+describe("jobs:job:stale-run-sweep", () => {
+  test("a running row older than the timeout is marked failed with the sweeper error", async () => {
+    stack = await bootStack();
+    const staleId = "11111111-1111-4111-8111-111111111111";
+    await seedRun({
+      id: staleId,
+      status: "running",
+      ageHours: DEFAULT_JOB_RUN_STALE_TIMEOUT_HOURS + 1,
+    });
+
+    await currentStack().jobRunner?.dispatch(SWEEP_JOB);
+    await waitFor(async () => {
+      expect(await statusOf(staleId)).toBe("failed");
+    });
+
+    const row = await fetchOne<{ status: string; error: string | null }>(
+      currentStack().db,
+      jobRunsTable,
+      { id: staleId },
+    );
+    expect(row?.error).toBe(STALE_JOB_RUN_ERROR);
+  });
+
+  test("a running row still within the timeout is left running, untouched", async () => {
+    stack = await bootStack();
+    const staleId = "22222222-2222-4222-8222-222222222222";
+    const recentId = "33333333-3333-4333-8333-333333333333";
+    await seedRun({
+      id: staleId,
+      status: "running",
+      ageHours: DEFAULT_JOB_RUN_STALE_TIMEOUT_HOURS + 1,
+    });
+    await seedRun({ id: recentId, status: "running", ageHours: 1 });
+
+    await currentStack().jobRunner?.dispatch(SWEEP_JOB);
+    await waitFor(async () => {
+      expect(await statusOf(staleId)).toBe("failed");
+    });
+
+    expect(await statusOf(recentId)).toBe("running");
+  });
+
+  test("a completed row older than the timeout is left completed, untouched", async () => {
+    stack = await bootStack();
+    const staleRunningId = "44444444-4444-4444-8444-444444444444";
+    const oldCompletedId = "55555555-5555-4555-8555-555555555555";
+    await seedRun({
+      id: staleRunningId,
+      status: "running",
+      ageHours: DEFAULT_JOB_RUN_STALE_TIMEOUT_HOURS + 1,
+    });
+    await seedRun({
+      id: oldCompletedId,
+      status: "completed",
+      ageHours: DEFAULT_JOB_RUN_STALE_TIMEOUT_HOURS + 5,
+    });
+
+    await currentStack().jobRunner?.dispatch(SWEEP_JOB);
+    await waitFor(async () => {
+      expect(await statusOf(staleRunningId)).toBe("failed");
+    });
+
+    expect(await statusOf(oldCompletedId)).toBe("completed");
+  });
+
+  test("a custom staleRunTimeoutHours option is honored", async () => {
+    stack = await bootStack(2);
+    const staleId = "66666666-6666-4666-8666-666666666666";
+    const withinCustomWindowId = "77777777-7777-4777-8777-777777777777";
+    await seedRun({ id: staleId, status: "running", ageHours: 3 });
+    await seedRun({ id: withinCustomWindowId, status: "running", ageHours: 1 });
+
+    await currentStack().jobRunner?.dispatch(SWEEP_JOB);
+    await waitFor(async () => {
+      expect(await statusOf(staleId)).toBe("failed");
+    });
+
+    expect(await statusOf(withinCustomWindowId)).toBe("running");
+  });
+});

package/src/jobs/db/queries/stale-run-sweep.ts

@@ -0,0 +1,51 @@
+// Stale job-run sweep (#2246): store_job_runs is a direct-write store
+// (#2243) — status is set to "running" once at onJobStart and only ever
+// flipped to "completed"/"failed" by onJobComplete/onJobFailed. If the
+// process dies mid-run (crash, OOM, kill -9), those callbacks never fire and
+// the row is stuck on "running" forever — nothing else on the read side
+// (list.query.ts/detail.query.ts) or in job-runner.ts (no BullMQ 'stalled'
+// listener wired) ever revisits it.
+//
+// This sweep marks runs whose startedAt is older than the timeout as
+// "failed" instead of inventing a new status: reusing "failed" means no
+// schema/migration change, no web filter-dropdown/status-enum update, and
+// the run becomes retriable via the existing jobs:write:retry gate
+// (retry.write.ts only allows retry from "failed").
+
+import { updateMany } from "@cosmicdrift/kumiko-framework/bun-db";
+import type { DbConnection } from "@cosmicdrift/kumiko-framework/db";
+import { jobRunsTable } from "../../job-run-table";
+
+export const STALE_JOB_RUN_ERROR =
+  "job run exceeded the stale-run timeout without a completion signal (likely a crashed worker process)";
+
+export type StaleRunSweepResult = {
+  readonly runsMarkedFailed: number;
+};
+
+export async function markStaleJobRunsFailed(
+  db: DbConnection,
+  timeoutHours: number,
+): Promise<StaleRunSweepResult> {
+  const cutoff = Temporal.Now.instant().subtract({ hours: timeoutHours });
+  const now = Temporal.Now.instant();
+
+  // duration is deliberately left untouched: we don't know when the run
+  // actually died, only that it crossed the timeout, so recording a
+  // duration would misrepresent it as measured. detail-screen/list-screen
+  // both already render a null duration as "—".
+  const updated = await updateMany(
+    db,
+    jobRunsTable,
+    {
+      status: "failed",
+      error: STALE_JOB_RUN_ERROR,
+      finishedAt: now,
+      modifiedAt: now,
+      modifiedById: "system",
+    },
+    { status: "running", startedAt: { lt: cutoff } },
+  );
+
+  return { runsMarkedFailed: updated.length };
+}

package/src/jobs/feature.ts

@@ -13,6 +13,10 @@ import {
   DEFAULT_JOB_RUN_RETENTION_DAYS,
 } from "./handlers/retention-cleanup.job";
 import { retryWrite } from "./handlers/retry.write";
+import {
+  createStaleRunSweepJob,
+  DEFAULT_JOB_RUN_STALE_TIMEOUT_HOURS,
+} from "./handlers/stale-run-sweep.job";
 import { triggerWrite } from "./handlers/trigger.write";
 import { JOBS_I18N } from "./i18n";
 import { jobRunLogsTableMeta, jobRunsTableMeta } from "./job-run-table";
@@ -21,13 +25,20 @@ export type JobsFeatureOptions = {
   // How long a job run (and its logs) stays in store_job_runs/
   // store_job_run_logs before the daily retention-cleanup job deletes it.
   readonly retentionDays?: number;
+  // How long a run can sit at status "running" before the hourly
+  // stale-run-sweep job marks it "failed" (#2246 — a process kill mid-run
+  // never fires onJobComplete/onJobFailed, so nothing else ever revisits
+  // it). Must clear any legitimately long-running job (reindexEntity,
+  // projectionRebuild) — set high, not tight.
+  readonly staleRunTimeoutHours?: number;
 };
 
 export function createJobsFeature(options: JobsFeatureOptions = {}): FeatureDefinition {
   const retentionDays = options.retentionDays ?? DEFAULT_JOB_RUN_RETENTION_DAYS;
+  const staleRunTimeoutHours = options.staleRunTimeoutHours ?? DEFAULT_JOB_RUN_STALE_TIMEOUT_HOURS;
   return defineFeature("jobs", (r) => {
     r.describe(
-      "Persistence and operator tooling for background jobs registered via `r.job(...)`. Every job execution writes directly into `store_job_runs` (current status + duration) and `store_job_run_logs` (per-line log rows) from the BullMQ callbacks — no event stream in between (#2243). A daily `retention-cleanup` job deletes runs (and their logs) older than `retentionDays`. Exposes `jobs:write:trigger` (manual run) and `jobs:write:retry` (operator retry of a failed run), plus `jobs:query:list`, `jobs:query:details`, and `jobs:query:catalog` (manual jobs) for the operator UI.",
+      "Persistence and operator tooling for background jobs registered via `r.job(...)`. Every job execution writes directly into `store_job_runs` (current status + duration) and `store_job_run_logs` (per-line log rows) from the BullMQ callbacks — no event stream in between (#2243). A daily `retention-cleanup` job deletes runs (and their logs) older than `retentionDays`; an hourly `stale-run-sweep` job marks runs stuck at status `running` past `staleRunTimeoutHours` as `failed` (#2246 — a crashed worker never fires the completion callback, so nothing else ever revisits the row). Exposes `jobs:write:trigger` (manual run) and `jobs:write:retry` (operator retry of a failed run), plus `jobs:query:list`, `jobs:query:details`, and `jobs:query:catalog` (manual jobs) for the operator UI.",
     );
     r.uiHints({
       displayLabel: "Jobs · Audit & Operator UI",
@@ -67,6 +78,17 @@ export function createJobsFeature(options: JobsFeatureOptions = {}): FeatureDefi
       createRetentionCleanupJob(retentionDays),
     );
 
+    // Sweeps store_job_runs for rows stuck at status "running" past the
+    // timeout (#2246). Hourly matches the timeout's own granularity —
+    // frequent enough to catch a stuck run soon after it crosses the
+    // threshold, cheap enough that an (almost always empty) result set
+    // doesn't matter.
+    r.job(
+      "stale-run-sweep",
+      { trigger: { cron: "0 * * * *" }, concurrency: "skip" },
+      createStaleRunSweepJob(staleRunTimeoutHours),
+    );
+
     const handlers = {
       trigger: r.writeHandler(triggerWrite),
       retry: r.writeHandler(retryWrite),
@@ -82,12 +104,14 @@ export function createJobsFeature(options: JobsFeatureOptions = {}): FeatureDefi
 
     r.translations({ keys: JOBS_I18N });
 
+    // kumiko-lint-ignore app-feature-structure Phase-3 conversion tracked in #2312
     r.screen({
       id: JOB_RUNS_SCREEN_ID,
       type: "custom",
       renderer: { react: { __component: "JobRunsScreen" } },
       access: systemAdminAccess,
     });
+    // kumiko-lint-ignore app-feature-structure Phase-3 conversion tracked in #2312
     r.screen({
       id: JOB_RUN_DETAIL_SCREEN_ID,
       type: "custom",

package/src/jobs/handlers/detail.query.ts

@@ -39,6 +39,18 @@ export const detailQuery = defineQueryHandler({
       },
     );
 
-    return { ...row, logs };
+    // message is stored encrypted under the triggering user's DEK (#2247),
+    // same mechanism as row.payload above.
+    const decryptedLogs = await Promise.all(
+      logs.map(async (log) => {
+        if (typeof log["message"] !== "string") return log;
+        return {
+          ...log,
+          message: await decryptStoredPii(log["message"], "message", "job-run-detail-log"),
+        };
+      }),
+    );
+
+    return { ...row, logs: decryptedLogs };
   },
 });

package/src/jobs/handlers/stale-run-sweep.job.ts

@@ -0,0 +1,28 @@
+import type { DbConnection } from "@cosmicdrift/kumiko-framework/db";
+import type { JobHandlerFn } from "@cosmicdrift/kumiko-framework/engine";
+import { InternalError } from "@cosmicdrift/kumiko-framework/errors";
+import { markStaleJobRunsFailed } from "../db/queries/stale-run-sweep";
+
+// 24h, not e.g. the 1h cache-TTL used elsewhere in this feature (see
+// job-run-logger.ts) — a wrong guess there costs one extra DB lookup, a
+// wrong guess here marks a still-running job "failed" and opens
+// retry.write.ts's retry gate on it while the original run is still
+// executing (concurrent duplicate dispatch). reindexEntity/projectionRebuild
+// (registered in this same feature) can legitimately run for hours, so the
+// default has to clear any plausible real job, not just be "long".
+// Configurable per-app via JobsFeatureOptions.staleRunTimeoutHours.
+export const DEFAULT_JOB_RUN_STALE_TIMEOUT_HOURS = 24;
+
+export function createStaleRunSweepJob(timeoutHours: number): JobHandlerFn {
+  return async (_payload, ctx) => {
+    if (!ctx.db) {
+      throw new InternalError({
+        message:
+          "[jobs:stale-run-sweep] ctx.db missing — job context requires a database connection.",
+      });
+    }
+    const db = ctx.db as DbConnection; // @cast-boundary db-operator (matches sibling cron jobs)
+    const result = await markStaleJobRunsFailed(db, timeoutHours);
+    ctx.log?.info?.(`[jobs:stale-run-sweep] complete: ${JSON.stringify(result)}`);
+  };
+}