@cosmicdrift/kumiko-bundled-features 0.209.0 → 0.210.0

package/src/jobs/__tests__/jobs-events.integration.test.ts

@@ -1,16 +1,16 @@
-// Event-shape contract for jobRun aggregate. Pins the three domain
-// events (run-started / run-completed / run-failed) against their
-// registered schemas + the stable type-name constants. A silent rename
-// (event-type-string, aggregateType, or payload-shape) fails here
-// instead of breaking MSP consumers and audit exports.
+// Direct-write shape contract for job runs (#2243). Pins onJobStart/
+// -Complete/-Failed against the tables they write — jobRunsTable status
+// transitions and jobRunLogsTable batched log rows. A silent shape drift
+// (missing field, wrong status string) fails here instead of breaking the
+// operator UI silently.
 //
 // The jobs integration test (jobs-feature.integration.ts) covers the
-// projection side (list + detail queries). This file covers the event
-// side — complementary coverage, minimal overlap.
+// projection side (list + detail queries). This file covers the
+// write side — complementary coverage, minimal overlap.
 
 import { afterAll, beforeAll, beforeEach, describe, expect, test } from "bun:test";
 import { selectMany } from "@cosmicdrift/kumiko-framework/bun-db";
-import { createRegistry, SYSTEM_TENANT_ID } from "@cosmicdrift/kumiko-framework/engine";
+import { createRegistry } from "@cosmicdrift/kumiko-framework/engine";
 import { createEventsTable, eventsTable } from "@cosmicdrift/kumiko-framework/event-store";
 import {
   createTestDb,
@@ -20,14 +20,8 @@ import {
   unsafePushTables,
 } from "@cosmicdrift/kumiko-framework/stack";
 import { resetTestTables } from "@cosmicdrift/kumiko-framework/testing";
-import { runCompletedSchema, runFailedSchema, runStartedSchema } from "../events";
 import { createJobsFeature } from "../feature";
-import {
-  createJobRunLogger,
-  JOB_RUN_COMPLETED_EVENT,
-  JOB_RUN_FAILED_EVENT,
-  JOB_RUN_STARTED_EVENT,
-} from "../job-run-logger";
+import { createJobRunLogger } from "../job-run-logger";
 import { jobRunLogsTable, jobRunsTable } from "../job-run-table";
 
 let testDb: TestDb;
@@ -40,6 +34,8 @@ beforeAll(async () => {
   testRedis = await createTestRedis();
   registry = createRegistry([createJobsFeature()]);
   await unsafePushTables(testDb.db, { jobRunsTable, jobRunLogsTable });
+  // Kept only for the negative assertion below (no jobs:event:run-* rows) —
+  // the write path itself no longer touches the event store.
   await createEventsTable(testDb.db);
   logger = createJobRunLogger({ db: testDb.db, registry });
 });
@@ -53,114 +49,102 @@ beforeEach(async () => {
   await resetTestTables(testDb.db, [eventsTable, jobRunsTable, jobRunLogsTable]);
 });
 
-describe("jobRun event shapes", () => {
-  test("event-type constants are stable strings", () => {
-    // Guard against silent rename. Tests that subscribe via string-match
-    // (MSPs written in userland, audit export tools) break without this.
-    expect(JOB_RUN_STARTED_EVENT).toBe("jobs:event:run-started");
-    expect(JOB_RUN_COMPLETED_EVENT).toBe("jobs:event:run-completed");
-    expect(JOB_RUN_FAILED_EVENT).toBe("jobs:event:run-failed");
-  });
-
-  test("onJobStart writes a run-started event on the jobRun aggregate", async () => {
+describe("jobRun direct writes", () => {
+  test("onJobStart inserts a row into jobRunsTable", async () => {
     await logger.onJobStart?.("example:job:import", "bull-42", {
       triggeredById: "u-99",
       payload: JSON.stringify({ foo: 1 }),
       attempt: 1,
     });
 
-    const events = await selectMany(testDb.db, eventsTable, { type: JOB_RUN_STARTED_EVENT });
-
-    expect(events.length).toBe(1);
-    const e = events[0];
-    expect(e?.aggregateType).toBe("jobRun");
-    expect(e?.tenantId).toBe(SYSTEM_TENANT_ID);
-    // Payload round-trips through the registered schema — drift would
-    // fail parse here, not silently land on the stream.
-    expect(() => runStartedSchema.parse(e?.payload)).not.toThrow();
-    const p = runStartedSchema.parse(e?.payload);
-    expect(p.jobName).toBe("example:job:import");
-    expect(p.bullJobId).toBe("bull-42");
-    expect(p.triggeredById).toBe("u-99");
-    expect(p.attempt).toBe(1);
+    const runs = await selectMany(testDb.db, jobRunsTable, { bullJobId: "bull-42" });
+
+    expect(runs.length).toBe(1);
+    const run = runs[0];
+    expect(run?.jobName).toBe("example:job:import");
+    expect(run?.status).toBe("running");
+    expect(run?.triggeredById).toBe("u-99");
+    expect(run?.attempt).toBe(1);
   });
 
-  test("onJobComplete writes a run-completed event with batched logs", async () => {
+  test("onJobComplete updates the row to completed with batched logs", async () => {
     await logger.onJobStart?.("example:job:export", "bull-1", {});
     await logger.onJobComplete?.("example:job:export", "bull-1", 123, [
       { level: "info", message: "started", timestamp: Temporal.Now.instant() },
       { level: "info", message: "done", timestamp: Temporal.Now.instant() },
     ]);
 
-    const events = await selectMany(testDb.db, eventsTable, { type: JOB_RUN_COMPLETED_EVENT });
+    const runs = await selectMany(testDb.db, jobRunsTable, { bullJobId: "bull-1" });
+    expect(runs.length).toBe(1);
+    expect(runs[0]?.status).toBe("completed");
+    expect(runs[0]?.duration).toBe(123);
 
-    expect(events.length).toBe(1);
-    const p = runCompletedSchema.parse(events[0]?.payload);
-    expect(p.duration).toBe(123);
-    expect(p.logs).toHaveLength(2);
-    expect(p.logs[0]?.level).toBe("info");
+    const logs = await selectMany(testDb.db, jobRunLogsTable, { runId: runs[0]?.id as string });
+    expect(logs).toHaveLength(2);
+    expect(logs[0]?.level).toBe("info");
   });
 
-  test("onJobFailed writes a run-failed event with error + logs", async () => {
+  test("onJobFailed updates the row to failed with error + logs", async () => {
     await logger.onJobStart?.("example:job:fragile", "bull-9", {});
     await logger.onJobFailed?.("example:job:fragile", "bull-9", "boom", [
       { level: "error", message: "kaboom", timestamp: Temporal.Now.instant() },
     ]);
 
-    const events = await selectMany(testDb.db, eventsTable, { type: JOB_RUN_FAILED_EVENT });
+    const runs = await selectMany(testDb.db, jobRunsTable, { bullJobId: "bull-9" });
+    expect(runs.length).toBe(1);
+    expect(runs[0]?.status).toBe("failed");
+    expect(runs[0]?.error).toBe("boom");
 
-    expect(events.length).toBe(1);
-    const p = runFailedSchema.parse(events[0]?.payload);
-    expect(p.error).toBe("boom");
-    expect(p.logs).toHaveLength(1);
+    const logs = await selectMany(testDb.db, jobRunLogsTable, { runId: runs[0]?.id as string });
+    expect(logs).toHaveLength(1);
   });
 
-  test("start + complete both land on the SAME aggregate stream", async () => {
+  test("start + complete both act on the SAME row", async () => {
     await logger.onJobStart?.("example:job:stream", "bull-99", {});
     await logger.onJobComplete?.("example:job:stream", "bull-99", 10, []);
 
-    // Both events should share the same aggregateId — that's what makes
-    // the jobRun a single stream and lets ctx.loadAggregate() reduce
-    // them into a coherent state.
-    const events = await selectMany(testDb.db, eventsTable, { aggregateType: "jobRun" });
-
-    expect(events.length).toBe(2);
-    const ids = new Set(events.map((e) => e.aggregateId));
-    expect(ids.size).toBe(1);
+    // Exactly one row for this bullJobId — the complete-callback updated
+    // the row onJobStart created, it did not insert a second one.
+    const runs = await selectMany(testDb.db, jobRunsTable, { bullJobId: "bull-99" });
+    expect(runs.length).toBe(1);
+    expect(runs[0]?.status).toBe("completed");
   });
 
-  test("complete/fail without a prior start skips — does not forge a jobRun stream", async () => {
-    // State-loss path: worker restart with empty cache AND no projection row for
-    // this bullJobId. Dropping the terminal event is intentional — forging an
-    // aggregate from scratch would invent a run that never started.
+  test("complete/fail without a prior start skips — does not forge a run row", async () => {
+    // State-loss path: worker restart with empty cache AND no row for this
+    // bullJobId. Dropping the terminal write is intentional — forging a
+    // row from scratch would invent a run that never started.
     await logger.onJobComplete?.("example:job:orphan", "bull-orphan-complete", 50, []);
     await logger.onJobFailed?.("example:job:orphan", "bull-orphan-failed", "boom", []);
 
-    const completed = await selectMany(testDb.db, eventsTable, { type: JOB_RUN_COMPLETED_EVENT });
-    const failed = await selectMany(testDb.db, eventsTable, { type: JOB_RUN_FAILED_EVENT });
     const runs = await selectMany(testDb.db, jobRunsTable);
-
-    expect(completed).toHaveLength(0);
-    expect(failed).toHaveLength(0);
     expect(runs).toHaveLength(0);
   });
 
-  test("complete after cache loss recovers runId from the projection (same stream)", async () => {
-    // Simulates worker process restart: in-memory bullJobId→runId cache is gone,
-    // but the run-started projection row still has bull_job_id. A fresh logger
-    // must DB-lookup and append onto the original aggregate — not mint a second.
+  test("complete after cache loss recovers runId from jobRunsTable (same row)", async () => {
+    // Simulates worker process restart: in-memory bullJobId→runId cache is
+    // gone, but the run-started row still has bull_job_id. A fresh logger
+    // must DB-lookup and update the original row — not insert a second one.
     await logger.onJobStart?.("example:job:restart", "bull-restart-1", {});
-    const started = await selectMany(testDb.db, eventsTable, { type: JOB_RUN_STARTED_EVENT });
+    const started = await selectMany(testDb.db, jobRunsTable, { bullJobId: "bull-restart-1" });
     expect(started).toHaveLength(1);
-    const originalAggregateId = started[0]?.aggregateId;
-    expect(originalAggregateId).toBeTruthy();
+    const originalId = started[0]?.id;
+    expect(originalId).toBeTruthy();
 
     const coldLogger = createJobRunLogger({ db: testDb.db, registry });
     await coldLogger.onJobComplete?.("example:job:restart", "bull-restart-1", 42, []);
 
-    const all = await selectMany(testDb.db, eventsTable, { aggregateType: "jobRun" });
-    expect(all).toHaveLength(2);
-    expect(new Set(all.map((e) => e.aggregateId))).toEqual(new Set([originalAggregateId]));
-    expect(all.some((e) => e.type === JOB_RUN_COMPLETED_EVENT)).toBe(true);
+    const all = await selectMany(testDb.db, jobRunsTable, { bullJobId: "bull-restart-1" });
+    expect(all).toHaveLength(1);
+    expect(all[0]?.id).toBe(originalId);
+    expect(all[0]?.status).toBe("completed");
+  });
+
+  test("no jobs:event:run-* rows ever land in the event store", async () => {
+    await logger.onJobStart?.("example:job:no-events", "bull-no-events", {});
+    await logger.onJobComplete?.("example:job:no-events", "bull-no-events", 5, []);
+
+    const events = await selectMany(testDb.db, eventsTable, { aggregateType: "jobRun" });
+    expect(events).toHaveLength(0);
   });
 });

package/src/jobs/__tests__/jobs-pii-kms.integration.test.ts

@@ -1,9 +1,9 @@
-// Event-PII on the jobs run-logger (#799): runStarted.payload can carry
-// arbitrary user data and is written via LOW-LEVEL append() (not
-// ctx.appendEvent) — exactly the path the event-PII catalog must cover.
-// With a KMS active the stored event AND the projected read-row carry
-// ciphertext under the triggering user's DEK; erasing that key makes the
-// payload unreadable ([[erased]]) without touching the append-only stream.
+// PII on the jobs run-logger, direct-write path (#2243, formerly #799):
+// run-started payload can carry arbitrary user data and is written straight
+// into jobRunsTable (no event store, no event-PII catalog — #2243 removed
+// the jobRun r.defineEvent registrations). With a KMS active the stored row
+// carries ciphertext under the triggering user's DEK; erasing that key
+// makes the payload unreadable ([[erased]]) without touching the row.
 
 import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test";
 import { fetchOne, selectMany } from "@cosmicdrift/kumiko-framework/bun-db";
@@ -25,7 +25,7 @@ import {
 } from "@cosmicdrift/kumiko-framework/stack";
 import { resetPiiSubjectKmsForTests, resetTestTables } from "@cosmicdrift/kumiko-framework/testing";
 import { createJobsFeature } from "../feature";
-import { createJobRunLogger, JOB_RUN_STARTED_EVENT } from "../job-run-logger";
+import { createJobRunLogger } from "../job-run-logger";
 import { jobRunLogsTable, jobRunsTable } from "../job-run-table";
 
 let testDb: TestDb;
@@ -39,10 +39,10 @@ const SECRET_PAYLOAD = JSON.stringify({ iban: "DE89370400440532013000" });
 beforeAll(async () => {
   testDb = await createTestDb();
   testRedis = await createTestRedis();
-  // createRegistry publishes the event-PII catalog as a module singleton —
-  // the logger's low-level append() picks it up without further wiring.
   const registry = createRegistry([createJobsFeature()]);
   await unsafePushTables(testDb.db, { jobRunsTable, jobRunLogsTable });
+  // Kept only so the "no event store involved" assertion below has a table
+  // to assert against — the write path itself never touches it.
   await createEventsTable(testDb.db);
   logger = createJobRunLogger({ db: testDb.db, registry });
 });
@@ -63,25 +63,28 @@ afterEach(() => {
 });
 
 describe("jobs run-started payload under KMS", () => {
-  test("stored event carries ciphertext payload, plaintext triggeredById", async () => {
+  test("stored row carries ciphertext payload, plaintext triggeredById — no event-store write", async () => {
     await logger.onJobStart?.("app:job:export", "bull-1", {
       triggeredById: USER_ID,
       payload: SECRET_PAYLOAD,
       attempt: 1,
     });
 
-    const events = await selectMany(testDb.db, eventsTable, { type: JOB_RUN_STARTED_EVENT });
-    expect(events.length).toBe(1);
-    const payload = events[0]?.payload as Record<string, unknown>;
-    expect(isPiiCiphertext(payload["payload"])).toBe(true);
-    expect(String(payload["payload"])).toContain(`user:${USER_ID}`);
-    expect(payload["triggeredById"]).toBe(USER_ID);
+    const row = await fetchOne(testDb.db, jobRunsTable, { bullJobId: "bull-1" });
+    expect(isPiiCiphertext(row?.["payload"])).toBe(true);
+    expect(String(row?.["payload"])).toContain(`user:${USER_ID}`);
+    expect(row?.["triggeredById"]).toBe(USER_ID);
 
-    const back = await decryptPiiFieldValues(payload, ["payload"], kms, { requestId: "t" });
+    const back = await decryptPiiFieldValues({ payload: row?.["payload"] }, ["payload"], kms, {
+      requestId: "t",
+    });
     expect(back["payload"]).toBe(SECRET_PAYLOAD);
+
+    const events = await selectMany(testDb.db, eventsTable, { aggregateType: "jobRun" });
+    expect(events).toHaveLength(0);
   });
 
-  test("projected read-row carries the same ciphertext; erase → [[erased]]", async () => {
+  test("erase subject key → row payload decrypts to [[erased]]", async () => {
     await logger.onJobStart?.("app:job:export", "bull-2", {
       triggeredById: USER_ID,
       payload: SECRET_PAYLOAD,

package/src/jobs/__tests__/jobs-retention.integration.test.ts

@@ -0,0 +1,124 @@
+// Integration test for the job-run retention-cleanup job (#2243). Dispatches
+// jobs:job:retention-cleanup through the real jobRunner (setupTestStack +
+// BullMQ worker) — the same enqueue path production's cron trigger uses —
+// instead of hand-building a JobContext and casting into the handler. Each
+// test seeds a row past the active cutoff alongside one that must survive,
+// so waitFor's completion signal (the stale row's count dropping) is only
+// reached once the real dispatch has actually run; the survivor's fate is
+// decided by that same DELETE and can be asserted right after.
+
+import { afterEach, describe, expect, test } from "bun:test";
+import { selectMany } from "@cosmicdrift/kumiko-framework/bun-db";
+import { sql } from "@cosmicdrift/kumiko-framework/db";
+import { SYSTEM_TENANT_ID } from "@cosmicdrift/kumiko-framework/engine";
+import {
+  setupTestStack,
+  type TestStack,
+  unsafePushTables,
+} from "@cosmicdrift/kumiko-framework/stack";
+import { seedRow, waitFor } from "@cosmicdrift/kumiko-framework/testing";
+import { createJobsFeature } from "../feature";
+import { DEFAULT_JOB_RUN_RETENTION_DAYS } from "../handlers/retention-cleanup.job";
+import { jobRunLogsTable, jobRunsTable } from "../job-run-table";
+
+const RETENTION_JOB = "jobs:job:retention-cleanup";
+
+let stack: TestStack | undefined;
+
+async function bootStack(retentionDays?: number): Promise<TestStack> {
+  const s = await setupTestStack({
+    features: [createJobsFeature(retentionDays !== undefined ? { retentionDays } : {})],
+    jobs: { consumerLane: "worker" },
+  });
+  await unsafePushTables(s.db, { jobRunsTable, jobRunLogsTable });
+  return s;
+}
+
+afterEach(async () => {
+  if (stack) await stack.cleanup();
+  stack = undefined;
+});
+
+function currentStack(): TestStack {
+  if (!stack) throw new Error("stack not booted");
+  return stack;
+}
+
+async function seedRun(opts: { id: string; ageDays: number }): Promise<void> {
+  const insertedAt = sql`now() - ${sql.raw(`interval '${opts.ageDays} days'`)}`;
+  await seedRow(currentStack().db, jobRunsTable, {
+    id: opts.id,
+    tenantId: SYSTEM_TENANT_ID,
+    insertedAt,
+    jobName: "example:job:retention-probe",
+    bullJobId: `bull-${opts.id}`,
+    status: "completed",
+    attempt: 1,
+    startedAt: insertedAt,
+  });
+}
+
+async function countRuns(): Promise<number> {
+  return (await selectMany(currentStack().db, jobRunsTable)).length;
+}
+
+async function countLogs(): Promise<number> {
+  return (await selectMany(currentStack().db, jobRunLogsTable)).length;
+}
+
+// Dispatches the real job through jobRunner and blocks until the queued
+// worker has actually applied it, proven by the run count reaching
+// `expectedRuns` (not merely "some time has passed").
+async function dispatchAndWaitForRunCount(expectedRuns: number): Promise<void> {
+  await currentStack().jobRunner?.dispatch(RETENTION_JOB);
+  await waitFor(async () => {
+    expect(await countRuns()).toBe(expectedRuns);
+  });
+}
+
+describe("jobs:job:retention-cleanup", () => {
+  test("default window (30d): older-than-cutoff runs go, recent ones stay", async () => {
+    stack = await bootStack();
+    await seedRun({
+      id: "11111111-1111-4111-8111-111111111111",
+      ageDays: DEFAULT_JOB_RUN_RETENTION_DAYS + 1,
+    });
+    await seedRun({ id: "22222222-2222-4222-8222-222222222222", ageDays: 1 });
+    expect(await countRuns()).toBe(2);
+
+    await dispatchAndWaitForRunCount(1);
+
+    const remaining = await selectMany(currentStack().db, jobRunsTable);
+    expect(remaining[0]?.id).toBe("22222222-2222-4222-8222-222222222222");
+  });
+
+  test("associated log rows are deleted along with their run", async () => {
+    stack = await bootStack();
+    const runId = "33333333-3333-4333-8333-333333333333";
+    const ageDays = DEFAULT_JOB_RUN_RETENTION_DAYS + 5;
+    await seedRun({ id: runId, ageDays });
+    await seedRow(currentStack().db, jobRunLogsTable, {
+      runId,
+      level: "info",
+      message: "old log line",
+      timestamp: sql`now() - ${sql.raw(`interval '${ageDays} days'`)}`,
+    });
+    expect(await countLogs()).toBe(1);
+
+    await dispatchAndWaitForRunCount(0);
+
+    expect(await countLogs()).toBe(0);
+  });
+
+  test("a custom retentionDays option is honored: past-cutoff runs go, in-window runs stay", async () => {
+    stack = await bootStack(3);
+    await seedRun({ id: "44444444-4444-4444-8444-444444444444", ageDays: 5 });
+    await seedRun({ id: "55555555-5555-4555-8555-555555555555", ageDays: 1 });
+    expect(await countRuns()).toBe(2);
+
+    await dispatchAndWaitForRunCount(1);
+
+    const remaining = await selectMany(currentStack().db, jobRunsTable);
+    expect(remaining[0]?.id).toBe("55555555-5555-4555-8555-555555555555");
+  });
+});

package/src/jobs/db/queries/retention.ts

@@ -0,0 +1,44 @@
+// Job-run retention (#2243): store_job_runs/store_job_run_logs are
+// direct-write stores now, not event-sourced projections — nothing else
+// purges them, so without this query they grow forever with the run count
+// instead of the runtime. Not the retention-cleanup executor: that executor
+// is entity/tenant-scoped and jobRun has neither (system-scoped direct-write
+// store). Each table is purged by its own timestamp column independently —
+// run_id isn't a real FK constraint (see job-run-table.ts), so there is no
+// ordering requirement between the two deletes.
+//
+// deleteManyBatched (typed query API, no raw SQL in this file) chunks each
+// delete so a large backlog doesn't hold one lock for the whole sweep.
+
+import { deleteManyBatched } from "@cosmicdrift/kumiko-framework/bun-db";
+import type { DbConnection } from "@cosmicdrift/kumiko-framework/db";
+import { jobRunLogsTable, jobRunsTable } from "../../job-run-table";
+
+const RETENTION_DELETE_BATCH_SIZE = 500;
+
+export type JobRunRetentionResult = {
+  readonly runsDeleted: number;
+  readonly logsDeleted: number;
+};
+
+export async function deleteStaleJobRuns(
+  db: DbConnection,
+  retentionDays: number,
+): Promise<JobRunRetentionResult> {
+  const cutoff = Temporal.Now.instant().subtract({ hours: retentionDays * 24 });
+
+  const logsResult = await deleteManyBatched(
+    db,
+    jobRunLogsTable,
+    { timestamp: { lt: cutoff } },
+    { limit: RETENTION_DELETE_BATCH_SIZE },
+  );
+  const runsResult = await deleteManyBatched(
+    db,
+    jobRunsTable,
+    { insertedAt: { lt: cutoff } },
+    { limit: RETENTION_DELETE_BATCH_SIZE },
+  );
+
+  return { runsDeleted: runsResult.deleted, logsDeleted: logsResult.deleted };
+}

package/src/jobs/events.ts

@@ -1,11 +1,8 @@
-// Event-payload schemas for the jobRun aggregate. Shared between
-// jobs-feature.ts (registers them via r.defineEvent and consumes them
-// in the inline-projections) and job-run-logger.ts (parses payloads
-// before low-level append() so out-of-dispatcher writes stay as
-// type-safe as ctx.appendEvent writes).
-//
-// Keeping them in a separate module avoids the circular import between
-// jobs-feature.ts (imports the logger) and job-run-logger.ts.
+// Payload schemas for the jobRun direct-write path (#2243). Pre-#2243 these
+// backed jobRun's event-store payloads (r.defineEvent + inline projections);
+// now job-run-logger.ts parses against them before writing straight into
+// jobRunsTable/jobRunLogsTable, so an out-of-dispatcher write still gets the
+// same validation guarantee ctx.appendEvent used to give it.
 
 import { z } from "zod";