@cosmicdrift/kumiko-bundled-features 0.208.3 → 0.209.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/package.json +8 -8
  2. package/src/compliance-profiles/__tests__/profile-picker.boot.test.ts +10 -2
  3. package/src/compliance-profiles/__tests__/profile-picker.security.integration.test.ts +50 -0
  4. package/src/compliance-profiles/changes.json +8 -1
  5. package/src/compliance-profiles/constants.ts +5 -0
  6. package/src/compliance-profiles/feature.ts +28 -3
  7. package/src/compliance-profiles/i18n.ts +25 -0
  8. package/src/compliance-profiles/web/client-plugin.tsx +7 -4
  9. package/src/compliance-profiles/web/compliance-profile-catalog.tsx +55 -0
  10. package/src/compliance-profiles/web/i18n.ts +1 -7
  11. package/src/compliance-profiles/web/index.ts +1 -1
  12. package/src/delivery/__tests__/delivery-log-query.integration.test.ts +132 -0
  13. package/src/delivery/__tests__/delivery-pii-kms.integration.test.ts +5 -5
  14. package/src/delivery/__tests__/delivery-screens.boot.test.ts +2 -2
  15. package/src/delivery/__tests__/delivery.integration.test.ts +1 -1
  16. package/src/delivery/changes.json +6 -0
  17. package/src/delivery/constants.ts +1 -0
  18. package/src/delivery/feature.ts +28 -3
  19. package/src/delivery/handlers/log.query.ts +115 -18
  20. package/src/delivery/i18n.ts +4 -0
  21. package/src/delivery/public-names.ts +5 -0
  22. package/src/delivery/web/client-plugin.tsx +6 -7
  23. package/src/delivery/web/delivery-status-cell.tsx +26 -0
  24. package/src/delivery/web/index.ts +1 -1
  25. package/src/jobs/__tests__/jobs-events.integration.test.ts +65 -81
  26. package/src/jobs/__tests__/jobs-pii-kms.integration.test.ts +21 -18
  27. package/src/jobs/__tests__/jobs-retention.integration.test.ts +124 -0
  28. package/src/jobs/db/queries/retention.ts +44 -0
  29. package/src/jobs/events.ts +5 -8
  30. package/src/jobs/feature.ts +26 -146
  31. package/src/jobs/handlers/retention-cleanup.job.ts +21 -0
  32. package/src/jobs/index.ts +1 -1
  33. package/src/jobs/job-run-logger.ts +112 -59
  34. package/src/jobs/job-run-table.ts +36 -28
  35. package/src/sessions/__tests__/session-last-seen.integration.test.ts +165 -0
  36. package/src/sessions/changes.json +6 -0
  37. package/src/sessions/constants.ts +4 -0
  38. package/src/sessions/feature.ts +17 -1
  39. package/src/sessions/schema/user-session.ts +5 -0
  40. package/src/sessions/session-callbacks.ts +20 -1
  41. package/src/sessions/session-checker-fail-open.test.ts +38 -0
  42. package/src/user-data-rights/feature.ts +4 -4
  43. package/src/user-data-rights-defaults/__tests__/bundled-entities.userdata-hooks.integration.test.ts +1 -1
  44. package/src/compliance-profiles/web/compliance-profile-screen.tsx +0 -152
  45. package/src/delivery/web/delivery-log-screen.tsx +0 -103
  46. package/src/delivery/web/i18n.ts +0 -15
@@ -0,0 +1,124 @@
1
+ // Integration test for the job-run retention-cleanup job (#2243). Dispatches
2
+ // jobs:job:retention-cleanup through the real jobRunner (setupTestStack +
3
+ // BullMQ worker) — the same enqueue path production's cron trigger uses —
4
+ // instead of hand-building a JobContext and casting into the handler. Each
5
+ // test seeds a row past the active cutoff alongside one that must survive,
6
+ // so waitFor's completion signal (the stale row's count dropping) is only
7
+ // reached once the real dispatch has actually run; the survivor's fate is
8
+ // decided by that same DELETE and can be asserted right after.
9
+
10
+ import { afterEach, describe, expect, test } from "bun:test";
11
+ import { selectMany } from "@cosmicdrift/kumiko-framework/bun-db";
12
+ import { sql } from "@cosmicdrift/kumiko-framework/db";
13
+ import { SYSTEM_TENANT_ID } from "@cosmicdrift/kumiko-framework/engine";
14
+ import {
15
+ setupTestStack,
16
+ type TestStack,
17
+ unsafePushTables,
18
+ } from "@cosmicdrift/kumiko-framework/stack";
19
+ import { seedRow, waitFor } from "@cosmicdrift/kumiko-framework/testing";
20
+ import { createJobsFeature } from "../feature";
21
+ import { DEFAULT_JOB_RUN_RETENTION_DAYS } from "../handlers/retention-cleanup.job";
22
+ import { jobRunLogsTable, jobRunsTable } from "../job-run-table";
23
+
24
+ const RETENTION_JOB = "jobs:job:retention-cleanup";
25
+
26
+ let stack: TestStack | undefined;
27
+
28
+ async function bootStack(retentionDays?: number): Promise<TestStack> {
29
+ const s = await setupTestStack({
30
+ features: [createJobsFeature(retentionDays !== undefined ? { retentionDays } : {})],
31
+ jobs: { consumerLane: "worker" },
32
+ });
33
+ await unsafePushTables(s.db, { jobRunsTable, jobRunLogsTable });
34
+ return s;
35
+ }
36
+
37
+ afterEach(async () => {
38
+ if (stack) await stack.cleanup();
39
+ stack = undefined;
40
+ });
41
+
42
+ function currentStack(): TestStack {
43
+ if (!stack) throw new Error("stack not booted");
44
+ return stack;
45
+ }
46
+
47
+ async function seedRun(opts: { id: string; ageDays: number }): Promise<void> {
48
+ const insertedAt = sql`now() - ${sql.raw(`interval '${opts.ageDays} days'`)}`;
49
+ await seedRow(currentStack().db, jobRunsTable, {
50
+ id: opts.id,
51
+ tenantId: SYSTEM_TENANT_ID,
52
+ insertedAt,
53
+ jobName: "example:job:retention-probe",
54
+ bullJobId: `bull-${opts.id}`,
55
+ status: "completed",
56
+ attempt: 1,
57
+ startedAt: insertedAt,
58
+ });
59
+ }
60
+
61
+ async function countRuns(): Promise<number> {
62
+ return (await selectMany(currentStack().db, jobRunsTable)).length;
63
+ }
64
+
65
+ async function countLogs(): Promise<number> {
66
+ return (await selectMany(currentStack().db, jobRunLogsTable)).length;
67
+ }
68
+
69
+ // Dispatches the real job through jobRunner and blocks until the queued
70
+ // worker has actually applied it, proven by the run count reaching
71
+ // `expectedRuns` (not merely "some time has passed").
72
+ async function dispatchAndWaitForRunCount(expectedRuns: number): Promise<void> {
73
+ await currentStack().jobRunner?.dispatch(RETENTION_JOB);
74
+ await waitFor(async () => {
75
+ expect(await countRuns()).toBe(expectedRuns);
76
+ });
77
+ }
78
+
79
+ describe("jobs:job:retention-cleanup", () => {
80
+ test("default window (30d): older-than-cutoff runs go, recent ones stay", async () => {
81
+ stack = await bootStack();
82
+ await seedRun({
83
+ id: "11111111-1111-4111-8111-111111111111",
84
+ ageDays: DEFAULT_JOB_RUN_RETENTION_DAYS + 1,
85
+ });
86
+ await seedRun({ id: "22222222-2222-4222-8222-222222222222", ageDays: 1 });
87
+ expect(await countRuns()).toBe(2);
88
+
89
+ await dispatchAndWaitForRunCount(1);
90
+
91
+ const remaining = await selectMany(currentStack().db, jobRunsTable);
92
+ expect(remaining[0]?.id).toBe("22222222-2222-4222-8222-222222222222");
93
+ });
94
+
95
+ test("associated log rows are deleted along with their run", async () => {
96
+ stack = await bootStack();
97
+ const runId = "33333333-3333-4333-8333-333333333333";
98
+ const ageDays = DEFAULT_JOB_RUN_RETENTION_DAYS + 5;
99
+ await seedRun({ id: runId, ageDays });
100
+ await seedRow(currentStack().db, jobRunLogsTable, {
101
+ runId,
102
+ level: "info",
103
+ message: "old log line",
104
+ timestamp: sql`now() - ${sql.raw(`interval '${ageDays} days'`)}`,
105
+ });
106
+ expect(await countLogs()).toBe(1);
107
+
108
+ await dispatchAndWaitForRunCount(0);
109
+
110
+ expect(await countLogs()).toBe(0);
111
+ });
112
+
113
+ test("a custom retentionDays option is honored: past-cutoff runs go, in-window runs stay", async () => {
114
+ stack = await bootStack(3);
115
+ await seedRun({ id: "44444444-4444-4444-8444-444444444444", ageDays: 5 });
116
+ await seedRun({ id: "55555555-5555-4555-8555-555555555555", ageDays: 1 });
117
+ expect(await countRuns()).toBe(2);
118
+
119
+ await dispatchAndWaitForRunCount(1);
120
+
121
+ const remaining = await selectMany(currentStack().db, jobRunsTable);
122
+ expect(remaining[0]?.id).toBe("55555555-5555-4555-8555-555555555555");
123
+ });
124
+ });
@@ -0,0 +1,44 @@
1
+ // Job-run retention (#2243): store_job_runs/store_job_run_logs are
2
+ // direct-write stores now, not event-sourced projections — nothing else
3
+ // purges them, so without this query they grow forever with the run count
4
+ // instead of the runtime. Not the retention-cleanup executor: that executor
5
+ // is entity/tenant-scoped and jobRun has neither (system-scoped direct-write
6
+ // store). Each table is purged by its own timestamp column independently —
7
+ // run_id isn't a real FK constraint (see job-run-table.ts), so there is no
8
+ // ordering requirement between the two deletes.
9
+ //
10
+ // deleteManyBatched (typed query API, no raw SQL in this file) chunks each
11
+ // delete so a large backlog doesn't hold one lock for the whole sweep.
12
+
13
+ import { deleteManyBatched } from "@cosmicdrift/kumiko-framework/bun-db";
14
+ import type { DbConnection } from "@cosmicdrift/kumiko-framework/db";
15
+ import { jobRunLogsTable, jobRunsTable } from "../../job-run-table";
16
+
17
+ const RETENTION_DELETE_BATCH_SIZE = 500;
18
+
19
+ export type JobRunRetentionResult = {
20
+ readonly runsDeleted: number;
21
+ readonly logsDeleted: number;
22
+ };
23
+
24
+ export async function deleteStaleJobRuns(
25
+ db: DbConnection,
26
+ retentionDays: number,
27
+ ): Promise<JobRunRetentionResult> {
28
+ const cutoff = Temporal.Now.instant().subtract({ hours: retentionDays * 24 });
29
+
30
+ const logsResult = await deleteManyBatched(
31
+ db,
32
+ jobRunLogsTable,
33
+ { timestamp: { lt: cutoff } },
34
+ { limit: RETENTION_DELETE_BATCH_SIZE },
35
+ );
36
+ const runsResult = await deleteManyBatched(
37
+ db,
38
+ jobRunsTable,
39
+ { insertedAt: { lt: cutoff } },
40
+ { limit: RETENTION_DELETE_BATCH_SIZE },
41
+ );
42
+
43
+ return { runsDeleted: runsResult.deleted, logsDeleted: logsResult.deleted };
44
+ }
@@ -1,11 +1,8 @@
1
- // Event-payload schemas for the jobRun aggregate. Shared between
2
- // jobs-feature.ts (registers them via r.defineEvent and consumes them
3
- // in the inline-projections) and job-run-logger.ts (parses payloads
4
- // before low-level append() so out-of-dispatcher writes stay as
5
- // type-safe as ctx.appendEvent writes).
6
- //
7
- // Keeping them in a separate module avoids the circular import between
8
- // jobs-feature.ts (imports the logger) and job-run-logger.ts.
1
+ // Payload schemas for the jobRun direct-write path (#2243). Pre-#2243 these
2
+ // backed jobRun's event-store payloads (r.defineEvent + inline projections);
3
+ // now job-run-logger.ts parses against them before writing straight into
4
+ // jobRunsTable/jobRunLogsTable, so an out-of-dispatcher write still gets the
5
+ // same validation guarantee ctx.appendEvent used to give it.
9
6
 
10
7
  import { z } from "zod";
11
8
 
@@ -1,16 +1,5 @@
1
- import { insertMany, insertOne, updateMany } from "@cosmicdrift/kumiko-framework/bun-db";
2
- import {
3
- defineApply,
4
- defineFeature,
5
- type FeatureDefinition,
6
- } from "@cosmicdrift/kumiko-framework/engine";
7
- import type { z } from "zod";
1
+ import { defineFeature, type FeatureDefinition } from "@cosmicdrift/kumiko-framework/engine";
8
2
  import { JOB_RUN_DETAIL_SCREEN_ID, JOB_RUNS_SCREEN_ID } from "./constants";
9
- // Event-payload schemas live in a sibling module so the logger can import
10
- // them without the cycle jobs-feature ↔ job-run-logger. The logger parses
11
- // payloads against these schemas before low-level append() — that's what
12
- // keeps out-of-dispatcher writes as type-safe as ctx.appendEvent.
13
- import { runCompletedSchema, runFailedSchema, runStartedSchema } from "./events";
14
3
  import { catalogQuery } from "./handlers/catalog.query";
15
4
  import { detailQuery } from "./handlers/detail.query";
16
5
  import { listQuery } from "./handlers/list.query";
@@ -19,21 +8,26 @@ import {
19
8
  projectionRebuildPayloadSchema,
20
9
  } from "./handlers/projection-rebuild.job";
21
10
  import { reindexEntityJob, reindexEntityPayloadSchema } from "./handlers/reindex-entity.job";
11
+ import {
12
+ createRetentionCleanupJob,
13
+ DEFAULT_JOB_RUN_RETENTION_DAYS,
14
+ } from "./handlers/retention-cleanup.job";
22
15
  import { retryWrite } from "./handlers/retry.write";
23
16
  import { triggerWrite } from "./handlers/trigger.write";
24
17
  import { JOBS_I18N } from "./i18n";
25
- import { parseJobInstant } from "./job-instant";
26
- import {
27
- JOB_RUN_COMPLETED_EVENT,
28
- JOB_RUN_FAILED_EVENT,
29
- JOB_RUN_STARTED_EVENT,
30
- } from "./job-run-logger";
31
- import { jobRunLogsTable, jobRunLogsTableMeta, jobRunsTable } from "./job-run-table";
18
+ import { jobRunLogsTableMeta, jobRunsTableMeta } from "./job-run-table";
19
+
20
+ export type JobsFeatureOptions = {
21
+ // How long a job run (and its logs) stays in store_job_runs/
22
+ // store_job_run_logs before the daily retention-cleanup job deletes it.
23
+ readonly retentionDays?: number;
24
+ };
32
25
 
33
- export function createJobsFeature(): FeatureDefinition {
26
+ export function createJobsFeature(options: JobsFeatureOptions = {}): FeatureDefinition {
27
+ const retentionDays = options.retentionDays ?? DEFAULT_JOB_RUN_RETENTION_DAYS;
34
28
  return defineFeature("jobs", (r) => {
35
29
  r.describe(
36
- "Persistence and operator tooling for background jobs registered via `r.job(...)`. Every job execution appends `run-started`, `run-completed`, and `run-failed` events to the `jobRun` aggregate stream, which two inline projections materialize into `read_job_runs` (current status + duration) and `store_job_run_logs` (per-line log rows). Exposes `jobs:write:trigger` (manual run) and `jobs:write:retry` (operator retry of a failed run), plus `jobs:query:list`, `jobs:query:details`, and `jobs:query:catalog` (manual jobs) for the operator UI.",
30
+ "Persistence and operator tooling for background jobs registered via `r.job(...)`. Every job execution writes directly into `store_job_runs` (current status + duration) and `store_job_run_logs` (per-line log rows) from the BullMQ callbacks — no event stream in between (#2243). A daily `retention-cleanup` job deletes runs (and their logs) older than `retentionDays`. Exposes `jobs:write:trigger` (manual run) and `jobs:write:retry` (operator retry of a failed run), plus `jobs:query:list`, `jobs:query:details`, and `jobs:query:catalog` (manual jobs) for the operator UI.",
37
31
  );
38
32
  r.uiHints({
39
33
  displayLabel: "Jobs · Audit & Operator UI",
@@ -41,134 +35,12 @@ export function createJobsFeature(): FeatureDefinition {
41
35
  recommended: false,
42
36
  });
43
37
  r.systemScope();
38
+ r.storeTable(jobRunsTableMeta, {
39
+ reason: "direct_write.job_runs",
40
+ });
44
41
  r.storeTable(jobRunLogsTableMeta, {
45
42
  reason: "read_side.job_run_logs",
46
43
  });
47
- // Events-only aggregate: "jobRun" has no r.entity registration, because
48
- // the entire lifecycle is driven by BullMQ-callback → r.defineEvent
49
- // (no executor, no CRUD). The boot-validator accepts the two
50
- // projections below because every apply-key is a registered
51
- // domain-event.
52
- // payload can carry arbitrary user data; triggeredById stays plaintext
53
- // (pseudonymous fk). System runs (triggeredById null) stay plaintext —
54
- // no user subject to shred.
55
- r.defineEvent("run-started", runStartedSchema, {
56
- piiFields: { payload: { subjectField: "triggeredById" } },
57
- });
58
- r.defineEvent("run-completed", runCompletedSchema);
59
- r.defineEvent("run-failed", runFailedSchema);
60
-
61
- // Inline projection: status-row in jobRunsTable. Runs in same TX as
62
- // the event-append (the logger calls runProjectionsForEvent manually
63
- // because the BullMQ-callback path has no dispatcher-ctx).
64
- r.projection({
65
- name: "job-runs",
66
- source: "jobRun",
67
- table: jobRunsTable,
68
- apply: {
69
- [JOB_RUN_STARTED_EVENT]: defineApply<z.infer<typeof runStartedSchema>>(
70
- async (event, tx, table) => {
71
- const p = event.payload;
72
- await insertOne(tx, table, {
73
- id: event.aggregateId,
74
- tenantId: event.tenantId,
75
- version: event.version,
76
- insertedAt: event.createdAt,
77
- insertedById: event.metadata?.userId ?? "system",
78
- jobName: p.jobName,
79
- bullJobId: p.bullJobId,
80
- status: p.status,
81
- payload: p.payload,
82
- attempt: p.attempt,
83
- startedAt: parseJobInstant(p.startedAt),
84
- triggeredById: p.triggeredById,
85
- });
86
- },
87
- ),
88
- [JOB_RUN_COMPLETED_EVENT]: defineApply<z.infer<typeof runCompletedSchema>>(
89
- async (event, tx, table) => {
90
- const p = event.payload;
91
- await updateMany(
92
- tx,
93
- table,
94
- {
95
- status: "completed",
96
- duration: p.duration,
97
- finishedAt: parseJobInstant(p.finishedAt),
98
- version: event.version,
99
- modifiedAt: event.createdAt,
100
- modifiedById: event.metadata?.userId ?? "system",
101
- },
102
- { id: event.aggregateId },
103
- );
104
- },
105
- ),
106
- [JOB_RUN_FAILED_EVENT]: defineApply<z.infer<typeof runFailedSchema>>(
107
- async (event, tx, table) => {
108
- const p = event.payload;
109
- await updateMany(
110
- tx,
111
- table,
112
- {
113
- status: "failed",
114
- error: p.error,
115
- duration: p.duration,
116
- finishedAt: parseJobInstant(p.finishedAt),
117
- version: event.version,
118
- modifiedAt: event.createdAt,
119
- modifiedById: event.metadata?.userId ?? "system",
120
- },
121
- { id: event.aggregateId },
122
- );
123
- },
124
- ),
125
- },
126
- });
127
-
128
- // Second inline projection — same source, different table. Expands
129
- // the batched logs array from completed/failed events into N rows
130
- // per run in jobRunLogsTable.
131
- r.projection({
132
- name: "job-run-logs",
133
- source: "jobRun",
134
- table: jobRunLogsTable,
135
- apply: {
136
- [JOB_RUN_COMPLETED_EVENT]: defineApply<z.infer<typeof runCompletedSchema>>(
137
- async (event, tx) => {
138
- const p = event.payload;
139
- // skip: empty log batch — the worker ran silent. No child rows
140
- // to insert; the completed-event alone already updated the run's
141
- // status via the sibling job-runs projection.
142
- if (p.logs.length === 0) return;
143
- await insertMany(
144
- tx,
145
- jobRunLogsTable,
146
- p.logs.map((log) => ({
147
- runId: event.aggregateId,
148
- level: log.level,
149
- message: log.message,
150
- timestamp: parseJobInstant(log.timestamp),
151
- })),
152
- );
153
- },
154
- ),
155
- [JOB_RUN_FAILED_EVENT]: defineApply<z.infer<typeof runFailedSchema>>(async (event, tx) => {
156
- const p = event.payload;
157
- // skip: empty log batch — the worker ran silent (mirror of completed)
158
- if (p.logs.length === 0) return;
159
- await insertMany(
160
- tx,
161
- jobRunLogsTable,
162
- p.logs.map((log) => ({
163
- runId: event.aggregateId,
164
- level: log.level,
165
- message: log.message,
166
- timestamp: parseJobInstant(log.timestamp),
167
- })),
168
- );
169
- }),
170
- },
171
- });
172
44
 
173
45
  // Framework-provided rebuild job — available whenever `jobs` is composed; enqueueProjectionRebuild dispatches it.
174
46
  r.job(
@@ -187,6 +59,14 @@ export function createJobsFeature(): FeatureDefinition {
187
59
  reindexEntityJob,
188
60
  );
189
61
 
62
+ // store_job_runs/store_job_run_logs are direct-write, unbounded-growth
63
+ // stores (#2243) — nothing else purges them.
64
+ r.job(
65
+ "retention-cleanup",
66
+ { trigger: { cron: "0 3 * * *" }, concurrency: "skip" },
67
+ createRetentionCleanupJob(retentionDays),
68
+ );
69
+
190
70
  const handlers = {
191
71
  trigger: r.writeHandler(triggerWrite),
192
72
  retry: r.writeHandler(retryWrite),
@@ -0,0 +1,21 @@
1
+ import type { DbConnection } from "@cosmicdrift/kumiko-framework/db";
2
+ import type { JobHandlerFn } from "@cosmicdrift/kumiko-framework/engine";
3
+ import { InternalError } from "@cosmicdrift/kumiko-framework/errors";
4
+ import { deleteStaleJobRuns } from "../db/queries/retention";
5
+
6
+ // Single source for the retention window — change here, nowhere else.
7
+ export const DEFAULT_JOB_RUN_RETENTION_DAYS = 30;
8
+
9
+ export function createRetentionCleanupJob(retentionDays: number): JobHandlerFn {
10
+ return async (_payload, ctx) => {
11
+ if (!ctx.db) {
12
+ throw new InternalError({
13
+ message:
14
+ "[jobs:retention-cleanup] ctx.db missing — job context requires a database connection.",
15
+ });
16
+ }
17
+ const db = ctx.db as DbConnection; // @cast-boundary db-operator (matches sibling cron jobs)
18
+ const result = await deleteStaleJobRuns(db, retentionDays);
19
+ ctx.log?.info?.(`[jobs:retention-cleanup] complete: ${JSON.stringify(result)}`);
20
+ };
21
+ }
package/src/jobs/index.ts CHANGED
@@ -1,4 +1,4 @@
1
- export { createJobsFeature } from "./feature";
1
+ export { createJobsFeature, type JobsFeatureOptions } from "./feature";
2
2
  export type { JobRunLoggerCallbacks } from "./job-run-logger";
3
3
  export { createJobRunLogger } from "./job-run-logger";
4
4
  export type { JobLogLevel, JobRunStatus } from "./job-run-table";
@@ -1,27 +1,26 @@
1
- import { fetchOne } from "@cosmicdrift/kumiko-framework/bun-db";
1
+ import { fetchOne, insertMany, insertOne, updateMany } from "@cosmicdrift/kumiko-framework/bun-db";
2
+ import {
3
+ configuredPiiSubjectKms,
4
+ encryptPiiValueForSubject,
5
+ } from "@cosmicdrift/kumiko-framework/crypto";
2
6
  import type { DbConnection } from "@cosmicdrift/kumiko-framework/db";
3
7
  import { type Registry, SYSTEM_TENANT_ID } from "@cosmicdrift/kumiko-framework/engine";
4
- import { append, getStreamVersion } from "@cosmicdrift/kumiko-framework/event-store";
5
8
  import type { JobLogEntry, JobMeta, JobRunnerOptions } from "@cosmicdrift/kumiko-framework/jobs";
6
- import { runProjectionsForEvent } from "@cosmicdrift/kumiko-framework/pipeline";
7
9
  import { generateId } from "@cosmicdrift/kumiko-framework/utils";
8
10
  import { runCompletedSchema, runFailedSchema, runStartedSchema } from "./events";
9
- import { jobRunsTable } from "./job-run-table";
11
+ import { parseJobInstant } from "./job-instant";
12
+ import { jobRunLogsTable, jobRunsTable } from "./job-run-table";
10
13
 
11
- // ES job-run lifecycle:
12
- // - onJobStart → jobs:event:run-started (first append, version 0→1)
13
- // - onJobComplete → jobs:event:run-completed (append at current version,
14
- // payload carries the batched logs)
15
- // - onJobFailed → jobs:event:run-failed (same shape as completed + error)
14
+ // Direct-write job-run log (#2243): onJobStart/-Complete/-Failed write
15
+ // straight into jobRunsTable / jobRunLogsTable instead of appending to the
16
+ // event store and replaying through inline projections. Pre-#2243 every run
17
+ // left two permanent `kumiko_events` rows that nothing else ever replayed
18
+ // or MSP-subscribed to — in two production apps that was ~99% of all
19
+ // events. Same tables, same shape, no event stream in between.
16
20
  //
17
21
  // BullMQ callbacks don't carry a tenantId (jobs are cross-tenant). We
18
22
  // anchor every run on SYSTEM_TENANT_ID — mirrors how config system-scope
19
- // rows use the sentinel. The stream still works per-run because
20
- // aggregate_id is a fresh UUID per run.
21
-
22
- export const JOB_RUN_STARTED_EVENT = "jobs:event:run-started" as const;
23
- export const JOB_RUN_COMPLETED_EVENT = "jobs:event:run-completed" as const;
24
- export const JOB_RUN_FAILED_EVENT = "jobs:event:run-failed" as const;
23
+ // rows use the sentinel.
25
24
 
26
25
  export type JobRunLoggerOptions = {
27
26
  readonly db: DbConnection;
@@ -45,18 +44,40 @@ const DEFAULT_CACHE_MAX_ENTRIES = 10_000;
45
44
  // to DB-lookup if actually needed.
46
45
  const DEFAULT_CACHE_TTL_MS = 60 * 60 * 1000; // 1 hour
47
46
 
47
+ // The run-started payload can carry arbitrary user data; triggeredById
48
+ // names its owning user. No event-PII catalog involved (#2243 removed the
49
+ // jobRun r.defineEvent registrations, so there is nothing to catalog) —
50
+ // the subject is known statically, so we encrypt directly. A null subject
51
+ // (system cron runs, recipient-less triggers) stays plaintext: there is no
52
+ // user key to shred, mirroring the previous event-pii catalog's own skip
53
+ // rule. Absent KMS adapter stays plaintext too (rollout mode, unchanged).
54
+ async function encryptStartedPayload(
55
+ payload: string | null,
56
+ triggeredById: string | null,
57
+ ): Promise<string | null> {
58
+ if (payload === null || triggeredById === null) return payload;
59
+ const kms = configuredPiiSubjectKms();
60
+ if (!kms) return payload;
61
+ return encryptPiiValueForSubject(
62
+ kms,
63
+ { kind: "user", userId: triggeredById },
64
+ payload,
65
+ { requestId: "jobs:job-run-logger" },
66
+ "payload",
67
+ );
68
+ }
69
+
48
70
  export function createJobRunLogger(opts: JobRunLoggerOptions): JobRunLoggerCallbacks {
49
- const { db, registry } = opts;
71
+ const { db } = opts;
50
72
 
51
- // bullJobId → aggregate uuid. BullMQ hands us the bullJobId on every
52
- // callback, but our aggregate stream is keyed by a fresh UUID we mint
53
- // on start. The cache threads that UUID from onJobStart through to
54
- // onJobComplete/onJobFailed so the completion-event lands on the same
55
- // stream as the start-event.
73
+ // bullJobId → run uuid. BullMQ hands us the bullJobId on every callback,
74
+ // but the run row is keyed by a fresh UUID we mint on start. The cache
75
+ // threads that UUID from onJobStart through to onJobComplete/onJobFailed
76
+ // so the completion-write lands on the same row as the start-write.
56
77
  //
57
78
  // Bounded cache (LRU-ish with TTL) — worker-crash between start and
58
79
  // complete would otherwise leak entries. DB-lookup recovers evicted
59
- // entries via bull_job_id on the projection.
80
+ // entries via bull_job_id on jobRunsTable.
60
81
  type CacheEntry = { readonly runId: string; readonly expiresAt: number };
61
82
  const runIdByBullJobId = new Map<string, CacheEntry>();
62
83
 
@@ -106,7 +127,7 @@ export function createJobRunLogger(opts: JobRunLoggerOptions): JobRunLoggerCallb
106
127
  // Parse against the registered schema so out-of-dispatcher writes
107
128
  // get the same validation guarantee as ctx.appendEvent. A shape
108
129
  // drift between feature + logger fails loudly at the source
109
- // instead of silently landing on the events-table.
130
+ // instead of silently landing on the table.
110
131
  const payload = runStartedSchema.parse({
111
132
  jobName,
112
133
  bullJobId,
@@ -116,16 +137,19 @@ export function createJobRunLogger(opts: JobRunLoggerOptions): JobRunLoggerCallb
116
137
  startedAt: Temporal.Now.instant().toString(),
117
138
  attempt: meta.attempt ?? 1,
118
139
  });
119
- const event = await append(db, {
120
- aggregateId: runId,
121
- aggregateType: "jobRun",
140
+ const encryptedPayload = await encryptStartedPayload(payload.payload, payload.triggeredById);
141
+ await insertOne(db, jobRunsTable, {
142
+ id: runId,
122
143
  tenantId: SYSTEM_TENANT_ID,
123
- expectedVersion: 0,
124
- type: JOB_RUN_STARTED_EVENT,
125
- payload,
126
- metadata: { userId: "system" },
144
+ insertedById: "system",
145
+ jobName: payload.jobName,
146
+ bullJobId: payload.bullJobId,
147
+ status: payload.status,
148
+ payload: encryptedPayload,
149
+ attempt: payload.attempt,
150
+ startedAt: parseJobInstant(payload.startedAt),
151
+ triggeredById: payload.triggeredById,
127
152
  });
128
- await runProjectionsForEvent(event, registry, db);
129
153
  },
130
154
 
131
155
  onJobComplete: async (
@@ -137,10 +161,9 @@ export function createJobRunLogger(opts: JobRunLoggerOptions): JobRunLoggerCallb
137
161
  const runId = await resolveRunId(bullJobId);
138
162
  // skip: state loss between start + complete (worker restart, cache
139
163
  // evicted AND DB has no matching bull_job_id). Rare edge case; we
140
- // drop the completion event rather than forging a jobRun aggregate
141
- // from scratch — forensics still has the original BullMQ lifecycle.
164
+ // drop the completion write rather than forging a run row from
165
+ // scratch — forensics still has the original BullMQ lifecycle.
142
166
  if (!runId) return;
143
- const currentVersion = await getStreamVersion(db, runId, SYSTEM_TENANT_ID);
144
167
  const payload = runCompletedSchema.parse({
145
168
  duration,
146
169
  finishedAt: Temporal.Now.instant().toString(),
@@ -150,16 +173,32 @@ export function createJobRunLogger(opts: JobRunLoggerOptions): JobRunLoggerCallb
150
173
  timestamp: l.timestamp.toString(),
151
174
  })),
152
175
  });
153
- const event = await append(db, {
154
- aggregateId: runId,
155
- aggregateType: "jobRun",
156
- tenantId: SYSTEM_TENANT_ID,
157
- expectedVersion: currentVersion,
158
- type: JOB_RUN_COMPLETED_EVENT,
159
- payload,
160
- metadata: { userId: "system" },
161
- });
162
- await runProjectionsForEvent(event, registry, db);
176
+ await updateMany(
177
+ db,
178
+ jobRunsTable,
179
+ {
180
+ status: "completed",
181
+ duration: payload.duration,
182
+ finishedAt: parseJobInstant(payload.finishedAt),
183
+ modifiedAt: Temporal.Now.instant(),
184
+ modifiedById: "system",
185
+ },
186
+ { id: runId },
187
+ );
188
+ // skip: empty log batch — the worker ran silent. No child rows to
189
+ // insert; the status update above already recorded completion.
190
+ if (payload.logs.length > 0) {
191
+ await insertMany(
192
+ db,
193
+ jobRunLogsTable,
194
+ payload.logs.map((log) => ({
195
+ runId,
196
+ level: log.level,
197
+ message: log.message,
198
+ timestamp: parseJobInstant(log.timestamp),
199
+ })),
200
+ );
201
+ }
163
202
  runIdByBullJobId.delete(bullJobId); // immediate cleanup on terminal callback
164
203
  },
165
204
 
@@ -171,13 +210,11 @@ export function createJobRunLogger(opts: JobRunLoggerOptions): JobRunLoggerCallb
171
210
  ) => {
172
211
  const runId = await resolveRunId(bullJobId);
173
212
  // skip: same rare state-loss case as in onJobComplete — drop the
174
- // failure event rather than forge a jobRun aggregate from scratch.
213
+ // failure write rather than forge a run row from scratch.
175
214
  if (!runId) return;
176
- const currentVersion = await getStreamVersion(db, runId, SYSTEM_TENANT_ID);
177
- // Read started_at off the projection so we can compute duration
215
+ // Read started_at off the row so we can compute duration
178
216
  // symmetrically to onJobComplete (which gets duration from the
179
- // worker). The projection already has started_at from the
180
- // run-started inline-apply.
217
+ // worker). The row already has started_at from onJobStart.
181
218
  const row = await fetchOne<{ startedAt: Temporal.Instant }>(db, jobRunsTable, { id: runId });
182
219
  const now = Temporal.Now.instant();
183
220
  const duration = row ? Number(now.since(row.startedAt).total({ unit: "millisecond" })) : 0;
@@ -191,16 +228,32 @@ export function createJobRunLogger(opts: JobRunLoggerOptions): JobRunLoggerCallb
191
228
  timestamp: l.timestamp.toString(),
192
229
  })),
193
230
  });
194
- const event = await append(db, {
195
- aggregateId: runId,
196
- aggregateType: "jobRun",
197
- tenantId: SYSTEM_TENANT_ID,
198
- expectedVersion: currentVersion,
199
- type: JOB_RUN_FAILED_EVENT,
200
- payload,
201
- metadata: { userId: "system" },
202
- });
203
- await runProjectionsForEvent(event, registry, db);
231
+ await updateMany(
232
+ db,
233
+ jobRunsTable,
234
+ {
235
+ status: "failed",
236
+ error: payload.error,
237
+ duration: payload.duration,
238
+ finishedAt: parseJobInstant(payload.finishedAt),
239
+ modifiedAt: now,
240
+ modifiedById: "system",
241
+ },
242
+ { id: runId },
243
+ );
244
+ // skip: empty log batch — mirror of onJobComplete
245
+ if (payload.logs.length > 0) {
246
+ await insertMany(
247
+ db,
248
+ jobRunLogsTable,
249
+ payload.logs.map((log) => ({
250
+ runId,
251
+ level: log.level,
252
+ message: log.message,
253
+ timestamp: parseJobInstant(log.timestamp),
254
+ })),
255
+ );
256
+ }
204
257
  runIdByBullJobId.delete(bullJobId); // immediate cleanup on terminal callback
205
258
  },
206
259
  };