@cosmicdrift/kumiko-bundled-features 0.208.3 → 0.209.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +8 -8
- package/src/compliance-profiles/__tests__/profile-picker.boot.test.ts +10 -2
- package/src/compliance-profiles/__tests__/profile-picker.security.integration.test.ts +50 -0
- package/src/compliance-profiles/changes.json +8 -1
- package/src/compliance-profiles/constants.ts +5 -0
- package/src/compliance-profiles/feature.ts +28 -3
- package/src/compliance-profiles/i18n.ts +25 -0
- package/src/compliance-profiles/web/client-plugin.tsx +7 -4
- package/src/compliance-profiles/web/compliance-profile-catalog.tsx +55 -0
- package/src/compliance-profiles/web/i18n.ts +1 -7
- package/src/compliance-profiles/web/index.ts +1 -1
- package/src/delivery/__tests__/delivery-log-query.integration.test.ts +132 -0
- package/src/delivery/__tests__/delivery-pii-kms.integration.test.ts +5 -5
- package/src/delivery/__tests__/delivery-screens.boot.test.ts +2 -2
- package/src/delivery/__tests__/delivery.integration.test.ts +1 -1
- package/src/delivery/changes.json +6 -0
- package/src/delivery/constants.ts +1 -0
- package/src/delivery/feature.ts +28 -3
- package/src/delivery/handlers/log.query.ts +115 -18
- package/src/delivery/i18n.ts +4 -0
- package/src/delivery/public-names.ts +5 -0
- package/src/delivery/web/client-plugin.tsx +6 -7
- package/src/delivery/web/delivery-status-cell.tsx +26 -0
- package/src/delivery/web/index.ts +1 -1
- package/src/jobs/__tests__/jobs-events.integration.test.ts +65 -81
- package/src/jobs/__tests__/jobs-pii-kms.integration.test.ts +21 -18
- package/src/jobs/__tests__/jobs-retention.integration.test.ts +124 -0
- package/src/jobs/db/queries/retention.ts +44 -0
- package/src/jobs/events.ts +5 -8
- package/src/jobs/feature.ts +26 -146
- package/src/jobs/handlers/retention-cleanup.job.ts +21 -0
- package/src/jobs/index.ts +1 -1
- package/src/jobs/job-run-logger.ts +112 -59
- package/src/jobs/job-run-table.ts +36 -28
- package/src/sessions/__tests__/session-last-seen.integration.test.ts +165 -0
- package/src/sessions/changes.json +6 -0
- package/src/sessions/constants.ts +4 -0
- package/src/sessions/feature.ts +17 -1
- package/src/sessions/schema/user-session.ts +5 -0
- package/src/sessions/session-callbacks.ts +20 -1
- package/src/sessions/session-checker-fail-open.test.ts +38 -0
- package/src/user-data-rights/feature.ts +4 -4
- package/src/user-data-rights-defaults/__tests__/bundled-entities.userdata-hooks.integration.test.ts +1 -1
- package/src/compliance-profiles/web/compliance-profile-screen.tsx +0 -152
- package/src/delivery/web/delivery-log-screen.tsx +0 -103
- package/src/delivery/web/i18n.ts +0 -15
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
// Integration test for the job-run retention-cleanup job (#2243). Dispatches
|
|
2
|
+
// jobs:job:retention-cleanup through the real jobRunner (setupTestStack +
|
|
3
|
+
// BullMQ worker) — the same enqueue path production's cron trigger uses —
|
|
4
|
+
// instead of hand-building a JobContext and casting into the handler. Each
|
|
5
|
+
// test seeds a row past the active cutoff alongside one that must survive,
|
|
6
|
+
// so waitFor's completion signal (the stale row's count dropping) is only
|
|
7
|
+
// reached once the real dispatch has actually run; the survivor's fate is
|
|
8
|
+
// decided by that same DELETE and can be asserted right after.
|
|
9
|
+
|
|
10
|
+
import { afterEach, describe, expect, test } from "bun:test";
|
|
11
|
+
import { selectMany } from "@cosmicdrift/kumiko-framework/bun-db";
|
|
12
|
+
import { sql } from "@cosmicdrift/kumiko-framework/db";
|
|
13
|
+
import { SYSTEM_TENANT_ID } from "@cosmicdrift/kumiko-framework/engine";
|
|
14
|
+
import {
|
|
15
|
+
setupTestStack,
|
|
16
|
+
type TestStack,
|
|
17
|
+
unsafePushTables,
|
|
18
|
+
} from "@cosmicdrift/kumiko-framework/stack";
|
|
19
|
+
import { seedRow, waitFor } from "@cosmicdrift/kumiko-framework/testing";
|
|
20
|
+
import { createJobsFeature } from "../feature";
|
|
21
|
+
import { DEFAULT_JOB_RUN_RETENTION_DAYS } from "../handlers/retention-cleanup.job";
|
|
22
|
+
import { jobRunLogsTable, jobRunsTable } from "../job-run-table";
|
|
23
|
+
|
|
24
|
+
const RETENTION_JOB = "jobs:job:retention-cleanup";
|
|
25
|
+
|
|
26
|
+
let stack: TestStack | undefined;
|
|
27
|
+
|
|
28
|
+
async function bootStack(retentionDays?: number): Promise<TestStack> {
|
|
29
|
+
const s = await setupTestStack({
|
|
30
|
+
features: [createJobsFeature(retentionDays !== undefined ? { retentionDays } : {})],
|
|
31
|
+
jobs: { consumerLane: "worker" },
|
|
32
|
+
});
|
|
33
|
+
await unsafePushTables(s.db, { jobRunsTable, jobRunLogsTable });
|
|
34
|
+
return s;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
afterEach(async () => {
|
|
38
|
+
if (stack) await stack.cleanup();
|
|
39
|
+
stack = undefined;
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
function currentStack(): TestStack {
|
|
43
|
+
if (!stack) throw new Error("stack not booted");
|
|
44
|
+
return stack;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
async function seedRun(opts: { id: string; ageDays: number }): Promise<void> {
|
|
48
|
+
const insertedAt = sql`now() - ${sql.raw(`interval '${opts.ageDays} days'`)}`;
|
|
49
|
+
await seedRow(currentStack().db, jobRunsTable, {
|
|
50
|
+
id: opts.id,
|
|
51
|
+
tenantId: SYSTEM_TENANT_ID,
|
|
52
|
+
insertedAt,
|
|
53
|
+
jobName: "example:job:retention-probe",
|
|
54
|
+
bullJobId: `bull-${opts.id}`,
|
|
55
|
+
status: "completed",
|
|
56
|
+
attempt: 1,
|
|
57
|
+
startedAt: insertedAt,
|
|
58
|
+
});
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
async function countRuns(): Promise<number> {
|
|
62
|
+
return (await selectMany(currentStack().db, jobRunsTable)).length;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
async function countLogs(): Promise<number> {
|
|
66
|
+
return (await selectMany(currentStack().db, jobRunLogsTable)).length;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
// Dispatches the real job through jobRunner and blocks until the queued
|
|
70
|
+
// worker has actually applied it, proven by the run count reaching
|
|
71
|
+
// `expectedRuns` (not merely "some time has passed").
|
|
72
|
+
async function dispatchAndWaitForRunCount(expectedRuns: number): Promise<void> {
|
|
73
|
+
await currentStack().jobRunner?.dispatch(RETENTION_JOB);
|
|
74
|
+
await waitFor(async () => {
|
|
75
|
+
expect(await countRuns()).toBe(expectedRuns);
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
describe("jobs:job:retention-cleanup", () => {
|
|
80
|
+
test("default window (30d): older-than-cutoff runs go, recent ones stay", async () => {
|
|
81
|
+
stack = await bootStack();
|
|
82
|
+
await seedRun({
|
|
83
|
+
id: "11111111-1111-4111-8111-111111111111",
|
|
84
|
+
ageDays: DEFAULT_JOB_RUN_RETENTION_DAYS + 1,
|
|
85
|
+
});
|
|
86
|
+
await seedRun({ id: "22222222-2222-4222-8222-222222222222", ageDays: 1 });
|
|
87
|
+
expect(await countRuns()).toBe(2);
|
|
88
|
+
|
|
89
|
+
await dispatchAndWaitForRunCount(1);
|
|
90
|
+
|
|
91
|
+
const remaining = await selectMany(currentStack().db, jobRunsTable);
|
|
92
|
+
expect(remaining[0]?.id).toBe("22222222-2222-4222-8222-222222222222");
|
|
93
|
+
});
|
|
94
|
+
|
|
95
|
+
test("associated log rows are deleted along with their run", async () => {
|
|
96
|
+
stack = await bootStack();
|
|
97
|
+
const runId = "33333333-3333-4333-8333-333333333333";
|
|
98
|
+
const ageDays = DEFAULT_JOB_RUN_RETENTION_DAYS + 5;
|
|
99
|
+
await seedRun({ id: runId, ageDays });
|
|
100
|
+
await seedRow(currentStack().db, jobRunLogsTable, {
|
|
101
|
+
runId,
|
|
102
|
+
level: "info",
|
|
103
|
+
message: "old log line",
|
|
104
|
+
timestamp: sql`now() - ${sql.raw(`interval '${ageDays} days'`)}`,
|
|
105
|
+
});
|
|
106
|
+
expect(await countLogs()).toBe(1);
|
|
107
|
+
|
|
108
|
+
await dispatchAndWaitForRunCount(0);
|
|
109
|
+
|
|
110
|
+
expect(await countLogs()).toBe(0);
|
|
111
|
+
});
|
|
112
|
+
|
|
113
|
+
test("a custom retentionDays option is honored: past-cutoff runs go, in-window runs stay", async () => {
|
|
114
|
+
stack = await bootStack(3);
|
|
115
|
+
await seedRun({ id: "44444444-4444-4444-8444-444444444444", ageDays: 5 });
|
|
116
|
+
await seedRun({ id: "55555555-5555-4555-8555-555555555555", ageDays: 1 });
|
|
117
|
+
expect(await countRuns()).toBe(2);
|
|
118
|
+
|
|
119
|
+
await dispatchAndWaitForRunCount(1);
|
|
120
|
+
|
|
121
|
+
const remaining = await selectMany(currentStack().db, jobRunsTable);
|
|
122
|
+
expect(remaining[0]?.id).toBe("55555555-5555-4555-8555-555555555555");
|
|
123
|
+
});
|
|
124
|
+
});
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
// Job-run retention (#2243): store_job_runs/store_job_run_logs are
|
|
2
|
+
// direct-write stores now, not event-sourced projections — nothing else
|
|
3
|
+
// purges them, so without this query they grow forever with the run count
|
|
4
|
+
// instead of the runtime. Not the retention-cleanup executor: that executor
|
|
5
|
+
// is entity/tenant-scoped and jobRun has neither (system-scoped direct-write
|
|
6
|
+
// store). Each table is purged by its own timestamp column independently —
|
|
7
|
+
// run_id isn't a real FK constraint (see job-run-table.ts), so there is no
|
|
8
|
+
// ordering requirement between the two deletes.
|
|
9
|
+
//
|
|
10
|
+
// deleteManyBatched (typed query API, no raw SQL in this file) chunks each
|
|
11
|
+
// delete so a large backlog doesn't hold one lock for the whole sweep.
|
|
12
|
+
|
|
13
|
+
import { deleteManyBatched } from "@cosmicdrift/kumiko-framework/bun-db";
|
|
14
|
+
import type { DbConnection } from "@cosmicdrift/kumiko-framework/db";
|
|
15
|
+
import { jobRunLogsTable, jobRunsTable } from "../../job-run-table";
|
|
16
|
+
|
|
17
|
+
const RETENTION_DELETE_BATCH_SIZE = 500;
|
|
18
|
+
|
|
19
|
+
export type JobRunRetentionResult = {
|
|
20
|
+
readonly runsDeleted: number;
|
|
21
|
+
readonly logsDeleted: number;
|
|
22
|
+
};
|
|
23
|
+
|
|
24
|
+
export async function deleteStaleJobRuns(
|
|
25
|
+
db: DbConnection,
|
|
26
|
+
retentionDays: number,
|
|
27
|
+
): Promise<JobRunRetentionResult> {
|
|
28
|
+
const cutoff = Temporal.Now.instant().subtract({ hours: retentionDays * 24 });
|
|
29
|
+
|
|
30
|
+
const logsResult = await deleteManyBatched(
|
|
31
|
+
db,
|
|
32
|
+
jobRunLogsTable,
|
|
33
|
+
{ timestamp: { lt: cutoff } },
|
|
34
|
+
{ limit: RETENTION_DELETE_BATCH_SIZE },
|
|
35
|
+
);
|
|
36
|
+
const runsResult = await deleteManyBatched(
|
|
37
|
+
db,
|
|
38
|
+
jobRunsTable,
|
|
39
|
+
{ insertedAt: { lt: cutoff } },
|
|
40
|
+
{ limit: RETENTION_DELETE_BATCH_SIZE },
|
|
41
|
+
);
|
|
42
|
+
|
|
43
|
+
return { runsDeleted: runsResult.deleted, logsDeleted: logsResult.deleted };
|
|
44
|
+
}
|
package/src/jobs/events.ts
CHANGED
|
@@ -1,11 +1,8 @@
|
|
|
1
|
-
//
|
|
2
|
-
//
|
|
3
|
-
//
|
|
4
|
-
//
|
|
5
|
-
//
|
|
6
|
-
//
|
|
7
|
-
// Keeping them in a separate module avoids the circular import between
|
|
8
|
-
// jobs-feature.ts (imports the logger) and job-run-logger.ts.
|
|
1
|
+
// Payload schemas for the jobRun direct-write path (#2243). Pre-#2243 these
|
|
2
|
+
// backed jobRun's event-store payloads (r.defineEvent + inline projections);
|
|
3
|
+
// now job-run-logger.ts parses against them before writing straight into
|
|
4
|
+
// jobRunsTable/jobRunLogsTable, so an out-of-dispatcher write still gets the
|
|
5
|
+
// same validation guarantee ctx.appendEvent used to give it.
|
|
9
6
|
|
|
10
7
|
import { z } from "zod";
|
|
11
8
|
|
package/src/jobs/feature.ts
CHANGED
|
@@ -1,16 +1,5 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
defineApply,
|
|
4
|
-
defineFeature,
|
|
5
|
-
type FeatureDefinition,
|
|
6
|
-
} from "@cosmicdrift/kumiko-framework/engine";
|
|
7
|
-
import type { z } from "zod";
|
|
1
|
+
import { defineFeature, type FeatureDefinition } from "@cosmicdrift/kumiko-framework/engine";
|
|
8
2
|
import { JOB_RUN_DETAIL_SCREEN_ID, JOB_RUNS_SCREEN_ID } from "./constants";
|
|
9
|
-
// Event-payload schemas live in a sibling module so the logger can import
|
|
10
|
-
// them without the cycle jobs-feature ↔ job-run-logger. The logger parses
|
|
11
|
-
// payloads against these schemas before low-level append() — that's what
|
|
12
|
-
// keeps out-of-dispatcher writes as type-safe as ctx.appendEvent.
|
|
13
|
-
import { runCompletedSchema, runFailedSchema, runStartedSchema } from "./events";
|
|
14
3
|
import { catalogQuery } from "./handlers/catalog.query";
|
|
15
4
|
import { detailQuery } from "./handlers/detail.query";
|
|
16
5
|
import { listQuery } from "./handlers/list.query";
|
|
@@ -19,21 +8,26 @@ import {
|
|
|
19
8
|
projectionRebuildPayloadSchema,
|
|
20
9
|
} from "./handlers/projection-rebuild.job";
|
|
21
10
|
import { reindexEntityJob, reindexEntityPayloadSchema } from "./handlers/reindex-entity.job";
|
|
11
|
+
import {
|
|
12
|
+
createRetentionCleanupJob,
|
|
13
|
+
DEFAULT_JOB_RUN_RETENTION_DAYS,
|
|
14
|
+
} from "./handlers/retention-cleanup.job";
|
|
22
15
|
import { retryWrite } from "./handlers/retry.write";
|
|
23
16
|
import { triggerWrite } from "./handlers/trigger.write";
|
|
24
17
|
import { JOBS_I18N } from "./i18n";
|
|
25
|
-
import {
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
18
|
+
import { jobRunLogsTableMeta, jobRunsTableMeta } from "./job-run-table";
|
|
19
|
+
|
|
20
|
+
export type JobsFeatureOptions = {
|
|
21
|
+
// How long a job run (and its logs) stays in store_job_runs/
|
|
22
|
+
// store_job_run_logs before the daily retention-cleanup job deletes it.
|
|
23
|
+
readonly retentionDays?: number;
|
|
24
|
+
};
|
|
32
25
|
|
|
33
|
-
export function createJobsFeature(): FeatureDefinition {
|
|
26
|
+
export function createJobsFeature(options: JobsFeatureOptions = {}): FeatureDefinition {
|
|
27
|
+
const retentionDays = options.retentionDays ?? DEFAULT_JOB_RUN_RETENTION_DAYS;
|
|
34
28
|
return defineFeature("jobs", (r) => {
|
|
35
29
|
r.describe(
|
|
36
|
-
"Persistence and operator tooling for background jobs registered via `r.job(...)`. Every job execution
|
|
30
|
+
"Persistence and operator tooling for background jobs registered via `r.job(...)`. Every job execution writes directly into `store_job_runs` (current status + duration) and `store_job_run_logs` (per-line log rows) from the BullMQ callbacks — no event stream in between (#2243). A daily `retention-cleanup` job deletes runs (and their logs) older than `retentionDays`. Exposes `jobs:write:trigger` (manual run) and `jobs:write:retry` (operator retry of a failed run), plus `jobs:query:list`, `jobs:query:details`, and `jobs:query:catalog` (manual jobs) for the operator UI.",
|
|
37
31
|
);
|
|
38
32
|
r.uiHints({
|
|
39
33
|
displayLabel: "Jobs · Audit & Operator UI",
|
|
@@ -41,134 +35,12 @@ export function createJobsFeature(): FeatureDefinition {
|
|
|
41
35
|
recommended: false,
|
|
42
36
|
});
|
|
43
37
|
r.systemScope();
|
|
38
|
+
r.storeTable(jobRunsTableMeta, {
|
|
39
|
+
reason: "direct_write.job_runs",
|
|
40
|
+
});
|
|
44
41
|
r.storeTable(jobRunLogsTableMeta, {
|
|
45
42
|
reason: "read_side.job_run_logs",
|
|
46
43
|
});
|
|
47
|
-
// Events-only aggregate: "jobRun" has no r.entity registration, because
|
|
48
|
-
// the entire lifecycle is driven by BullMQ-callback → r.defineEvent
|
|
49
|
-
// (no executor, no CRUD). The boot-validator accepts the two
|
|
50
|
-
// projections below because every apply-key is a registered
|
|
51
|
-
// domain-event.
|
|
52
|
-
// payload can carry arbitrary user data; triggeredById stays plaintext
|
|
53
|
-
// (pseudonymous fk). System runs (triggeredById null) stay plaintext —
|
|
54
|
-
// no user subject to shred.
|
|
55
|
-
r.defineEvent("run-started", runStartedSchema, {
|
|
56
|
-
piiFields: { payload: { subjectField: "triggeredById" } },
|
|
57
|
-
});
|
|
58
|
-
r.defineEvent("run-completed", runCompletedSchema);
|
|
59
|
-
r.defineEvent("run-failed", runFailedSchema);
|
|
60
|
-
|
|
61
|
-
// Inline projection: status-row in jobRunsTable. Runs in same TX as
|
|
62
|
-
// the event-append (the logger calls runProjectionsForEvent manually
|
|
63
|
-
// because the BullMQ-callback path has no dispatcher-ctx).
|
|
64
|
-
r.projection({
|
|
65
|
-
name: "job-runs",
|
|
66
|
-
source: "jobRun",
|
|
67
|
-
table: jobRunsTable,
|
|
68
|
-
apply: {
|
|
69
|
-
[JOB_RUN_STARTED_EVENT]: defineApply<z.infer<typeof runStartedSchema>>(
|
|
70
|
-
async (event, tx, table) => {
|
|
71
|
-
const p = event.payload;
|
|
72
|
-
await insertOne(tx, table, {
|
|
73
|
-
id: event.aggregateId,
|
|
74
|
-
tenantId: event.tenantId,
|
|
75
|
-
version: event.version,
|
|
76
|
-
insertedAt: event.createdAt,
|
|
77
|
-
insertedById: event.metadata?.userId ?? "system",
|
|
78
|
-
jobName: p.jobName,
|
|
79
|
-
bullJobId: p.bullJobId,
|
|
80
|
-
status: p.status,
|
|
81
|
-
payload: p.payload,
|
|
82
|
-
attempt: p.attempt,
|
|
83
|
-
startedAt: parseJobInstant(p.startedAt),
|
|
84
|
-
triggeredById: p.triggeredById,
|
|
85
|
-
});
|
|
86
|
-
},
|
|
87
|
-
),
|
|
88
|
-
[JOB_RUN_COMPLETED_EVENT]: defineApply<z.infer<typeof runCompletedSchema>>(
|
|
89
|
-
async (event, tx, table) => {
|
|
90
|
-
const p = event.payload;
|
|
91
|
-
await updateMany(
|
|
92
|
-
tx,
|
|
93
|
-
table,
|
|
94
|
-
{
|
|
95
|
-
status: "completed",
|
|
96
|
-
duration: p.duration,
|
|
97
|
-
finishedAt: parseJobInstant(p.finishedAt),
|
|
98
|
-
version: event.version,
|
|
99
|
-
modifiedAt: event.createdAt,
|
|
100
|
-
modifiedById: event.metadata?.userId ?? "system",
|
|
101
|
-
},
|
|
102
|
-
{ id: event.aggregateId },
|
|
103
|
-
);
|
|
104
|
-
},
|
|
105
|
-
),
|
|
106
|
-
[JOB_RUN_FAILED_EVENT]: defineApply<z.infer<typeof runFailedSchema>>(
|
|
107
|
-
async (event, tx, table) => {
|
|
108
|
-
const p = event.payload;
|
|
109
|
-
await updateMany(
|
|
110
|
-
tx,
|
|
111
|
-
table,
|
|
112
|
-
{
|
|
113
|
-
status: "failed",
|
|
114
|
-
error: p.error,
|
|
115
|
-
duration: p.duration,
|
|
116
|
-
finishedAt: parseJobInstant(p.finishedAt),
|
|
117
|
-
version: event.version,
|
|
118
|
-
modifiedAt: event.createdAt,
|
|
119
|
-
modifiedById: event.metadata?.userId ?? "system",
|
|
120
|
-
},
|
|
121
|
-
{ id: event.aggregateId },
|
|
122
|
-
);
|
|
123
|
-
},
|
|
124
|
-
),
|
|
125
|
-
},
|
|
126
|
-
});
|
|
127
|
-
|
|
128
|
-
// Second inline projection — same source, different table. Expands
|
|
129
|
-
// the batched logs array from completed/failed events into N rows
|
|
130
|
-
// per run in jobRunLogsTable.
|
|
131
|
-
r.projection({
|
|
132
|
-
name: "job-run-logs",
|
|
133
|
-
source: "jobRun",
|
|
134
|
-
table: jobRunLogsTable,
|
|
135
|
-
apply: {
|
|
136
|
-
[JOB_RUN_COMPLETED_EVENT]: defineApply<z.infer<typeof runCompletedSchema>>(
|
|
137
|
-
async (event, tx) => {
|
|
138
|
-
const p = event.payload;
|
|
139
|
-
// skip: empty log batch — the worker ran silent. No child rows
|
|
140
|
-
// to insert; the completed-event alone already updated the run's
|
|
141
|
-
// status via the sibling job-runs projection.
|
|
142
|
-
if (p.logs.length === 0) return;
|
|
143
|
-
await insertMany(
|
|
144
|
-
tx,
|
|
145
|
-
jobRunLogsTable,
|
|
146
|
-
p.logs.map((log) => ({
|
|
147
|
-
runId: event.aggregateId,
|
|
148
|
-
level: log.level,
|
|
149
|
-
message: log.message,
|
|
150
|
-
timestamp: parseJobInstant(log.timestamp),
|
|
151
|
-
})),
|
|
152
|
-
);
|
|
153
|
-
},
|
|
154
|
-
),
|
|
155
|
-
[JOB_RUN_FAILED_EVENT]: defineApply<z.infer<typeof runFailedSchema>>(async (event, tx) => {
|
|
156
|
-
const p = event.payload;
|
|
157
|
-
// skip: empty log batch — the worker ran silent (mirror of completed)
|
|
158
|
-
if (p.logs.length === 0) return;
|
|
159
|
-
await insertMany(
|
|
160
|
-
tx,
|
|
161
|
-
jobRunLogsTable,
|
|
162
|
-
p.logs.map((log) => ({
|
|
163
|
-
runId: event.aggregateId,
|
|
164
|
-
level: log.level,
|
|
165
|
-
message: log.message,
|
|
166
|
-
timestamp: parseJobInstant(log.timestamp),
|
|
167
|
-
})),
|
|
168
|
-
);
|
|
169
|
-
}),
|
|
170
|
-
},
|
|
171
|
-
});
|
|
172
44
|
|
|
173
45
|
// Framework-provided rebuild job — available whenever `jobs` is composed; enqueueProjectionRebuild dispatches it.
|
|
174
46
|
r.job(
|
|
@@ -187,6 +59,14 @@ export function createJobsFeature(): FeatureDefinition {
|
|
|
187
59
|
reindexEntityJob,
|
|
188
60
|
);
|
|
189
61
|
|
|
62
|
+
// store_job_runs/store_job_run_logs are direct-write, unbounded-growth
|
|
63
|
+
// stores (#2243) — nothing else purges them.
|
|
64
|
+
r.job(
|
|
65
|
+
"retention-cleanup",
|
|
66
|
+
{ trigger: { cron: "0 3 * * *" }, concurrency: "skip" },
|
|
67
|
+
createRetentionCleanupJob(retentionDays),
|
|
68
|
+
);
|
|
69
|
+
|
|
190
70
|
const handlers = {
|
|
191
71
|
trigger: r.writeHandler(triggerWrite),
|
|
192
72
|
retry: r.writeHandler(retryWrite),
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import type { DbConnection } from "@cosmicdrift/kumiko-framework/db";
|
|
2
|
+
import type { JobHandlerFn } from "@cosmicdrift/kumiko-framework/engine";
|
|
3
|
+
import { InternalError } from "@cosmicdrift/kumiko-framework/errors";
|
|
4
|
+
import { deleteStaleJobRuns } from "../db/queries/retention";
|
|
5
|
+
|
|
6
|
+
// Single source for the retention window — change here, nowhere else.
|
|
7
|
+
export const DEFAULT_JOB_RUN_RETENTION_DAYS = 30;
|
|
8
|
+
|
|
9
|
+
export function createRetentionCleanupJob(retentionDays: number): JobHandlerFn {
|
|
10
|
+
return async (_payload, ctx) => {
|
|
11
|
+
if (!ctx.db) {
|
|
12
|
+
throw new InternalError({
|
|
13
|
+
message:
|
|
14
|
+
"[jobs:retention-cleanup] ctx.db missing — job context requires a database connection.",
|
|
15
|
+
});
|
|
16
|
+
}
|
|
17
|
+
const db = ctx.db as DbConnection; // @cast-boundary db-operator (matches sibling cron jobs)
|
|
18
|
+
const result = await deleteStaleJobRuns(db, retentionDays);
|
|
19
|
+
ctx.log?.info?.(`[jobs:retention-cleanup] complete: ${JSON.stringify(result)}`);
|
|
20
|
+
};
|
|
21
|
+
}
|
package/src/jobs/index.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export { createJobsFeature } from "./feature";
|
|
1
|
+
export { createJobsFeature, type JobsFeatureOptions } from "./feature";
|
|
2
2
|
export type { JobRunLoggerCallbacks } from "./job-run-logger";
|
|
3
3
|
export { createJobRunLogger } from "./job-run-logger";
|
|
4
4
|
export type { JobLogLevel, JobRunStatus } from "./job-run-table";
|
|
@@ -1,27 +1,26 @@
|
|
|
1
|
-
import { fetchOne } from "@cosmicdrift/kumiko-framework/bun-db";
|
|
1
|
+
import { fetchOne, insertMany, insertOne, updateMany } from "@cosmicdrift/kumiko-framework/bun-db";
|
|
2
|
+
import {
|
|
3
|
+
configuredPiiSubjectKms,
|
|
4
|
+
encryptPiiValueForSubject,
|
|
5
|
+
} from "@cosmicdrift/kumiko-framework/crypto";
|
|
2
6
|
import type { DbConnection } from "@cosmicdrift/kumiko-framework/db";
|
|
3
7
|
import { type Registry, SYSTEM_TENANT_ID } from "@cosmicdrift/kumiko-framework/engine";
|
|
4
|
-
import { append, getStreamVersion } from "@cosmicdrift/kumiko-framework/event-store";
|
|
5
8
|
import type { JobLogEntry, JobMeta, JobRunnerOptions } from "@cosmicdrift/kumiko-framework/jobs";
|
|
6
|
-
import { runProjectionsForEvent } from "@cosmicdrift/kumiko-framework/pipeline";
|
|
7
9
|
import { generateId } from "@cosmicdrift/kumiko-framework/utils";
|
|
8
10
|
import { runCompletedSchema, runFailedSchema, runStartedSchema } from "./events";
|
|
9
|
-
import {
|
|
11
|
+
import { parseJobInstant } from "./job-instant";
|
|
12
|
+
import { jobRunLogsTable, jobRunsTable } from "./job-run-table";
|
|
10
13
|
|
|
11
|
-
//
|
|
12
|
-
//
|
|
13
|
-
//
|
|
14
|
-
//
|
|
15
|
-
//
|
|
14
|
+
// Direct-write job-run log (#2243): onJobStart/-Complete/-Failed write
|
|
15
|
+
// straight into jobRunsTable / jobRunLogsTable instead of appending to the
|
|
16
|
+
// event store and replaying through inline projections. Pre-#2243 every run
|
|
17
|
+
// left two permanent `kumiko_events` rows that nothing else ever replayed
|
|
18
|
+
// or MSP-subscribed to — in two production apps that was ~99% of all
|
|
19
|
+
// events. Same tables, same shape, no event stream in between.
|
|
16
20
|
//
|
|
17
21
|
// BullMQ callbacks don't carry a tenantId (jobs are cross-tenant). We
|
|
18
22
|
// anchor every run on SYSTEM_TENANT_ID — mirrors how config system-scope
|
|
19
|
-
// rows use the sentinel.
|
|
20
|
-
// aggregate_id is a fresh UUID per run.
|
|
21
|
-
|
|
22
|
-
export const JOB_RUN_STARTED_EVENT = "jobs:event:run-started" as const;
|
|
23
|
-
export const JOB_RUN_COMPLETED_EVENT = "jobs:event:run-completed" as const;
|
|
24
|
-
export const JOB_RUN_FAILED_EVENT = "jobs:event:run-failed" as const;
|
|
23
|
+
// rows use the sentinel.
|
|
25
24
|
|
|
26
25
|
export type JobRunLoggerOptions = {
|
|
27
26
|
readonly db: DbConnection;
|
|
@@ -45,18 +44,40 @@ const DEFAULT_CACHE_MAX_ENTRIES = 10_000;
|
|
|
45
44
|
// to DB-lookup if actually needed.
|
|
46
45
|
const DEFAULT_CACHE_TTL_MS = 60 * 60 * 1000; // 1 hour
|
|
47
46
|
|
|
47
|
+
// The run-started payload can carry arbitrary user data; triggeredById
|
|
48
|
+
// names its owning user. No event-PII catalog involved (#2243 removed the
|
|
49
|
+
// jobRun r.defineEvent registrations, so there is nothing to catalog) —
|
|
50
|
+
// the subject is known statically, so we encrypt directly. A null subject
|
|
51
|
+
// (system cron runs, recipient-less triggers) stays plaintext: there is no
|
|
52
|
+
// user key to shred, mirroring the previous event-pii catalog's own skip
|
|
53
|
+
// rule. Absent KMS adapter stays plaintext too (rollout mode, unchanged).
|
|
54
|
+
async function encryptStartedPayload(
|
|
55
|
+
payload: string | null,
|
|
56
|
+
triggeredById: string | null,
|
|
57
|
+
): Promise<string | null> {
|
|
58
|
+
if (payload === null || triggeredById === null) return payload;
|
|
59
|
+
const kms = configuredPiiSubjectKms();
|
|
60
|
+
if (!kms) return payload;
|
|
61
|
+
return encryptPiiValueForSubject(
|
|
62
|
+
kms,
|
|
63
|
+
{ kind: "user", userId: triggeredById },
|
|
64
|
+
payload,
|
|
65
|
+
{ requestId: "jobs:job-run-logger" },
|
|
66
|
+
"payload",
|
|
67
|
+
);
|
|
68
|
+
}
|
|
69
|
+
|
|
48
70
|
export function createJobRunLogger(opts: JobRunLoggerOptions): JobRunLoggerCallbacks {
|
|
49
|
-
const { db
|
|
71
|
+
const { db } = opts;
|
|
50
72
|
|
|
51
|
-
// bullJobId →
|
|
52
|
-
//
|
|
53
|
-
//
|
|
54
|
-
//
|
|
55
|
-
// stream as the start-event.
|
|
73
|
+
// bullJobId → run uuid. BullMQ hands us the bullJobId on every callback,
|
|
74
|
+
// but the run row is keyed by a fresh UUID we mint on start. The cache
|
|
75
|
+
// threads that UUID from onJobStart through to onJobComplete/onJobFailed
|
|
76
|
+
// so the completion-write lands on the same row as the start-write.
|
|
56
77
|
//
|
|
57
78
|
// Bounded cache (LRU-ish with TTL) — worker-crash between start and
|
|
58
79
|
// complete would otherwise leak entries. DB-lookup recovers evicted
|
|
59
|
-
// entries via bull_job_id on
|
|
80
|
+
// entries via bull_job_id on jobRunsTable.
|
|
60
81
|
type CacheEntry = { readonly runId: string; readonly expiresAt: number };
|
|
61
82
|
const runIdByBullJobId = new Map<string, CacheEntry>();
|
|
62
83
|
|
|
@@ -106,7 +127,7 @@ export function createJobRunLogger(opts: JobRunLoggerOptions): JobRunLoggerCallb
|
|
|
106
127
|
// Parse against the registered schema so out-of-dispatcher writes
|
|
107
128
|
// get the same validation guarantee as ctx.appendEvent. A shape
|
|
108
129
|
// drift between feature + logger fails loudly at the source
|
|
109
|
-
// instead of silently landing on the
|
|
130
|
+
// instead of silently landing on the table.
|
|
110
131
|
const payload = runStartedSchema.parse({
|
|
111
132
|
jobName,
|
|
112
133
|
bullJobId,
|
|
@@ -116,16 +137,19 @@ export function createJobRunLogger(opts: JobRunLoggerOptions): JobRunLoggerCallb
|
|
|
116
137
|
startedAt: Temporal.Now.instant().toString(),
|
|
117
138
|
attempt: meta.attempt ?? 1,
|
|
118
139
|
});
|
|
119
|
-
const
|
|
120
|
-
|
|
121
|
-
|
|
140
|
+
const encryptedPayload = await encryptStartedPayload(payload.payload, payload.triggeredById);
|
|
141
|
+
await insertOne(db, jobRunsTable, {
|
|
142
|
+
id: runId,
|
|
122
143
|
tenantId: SYSTEM_TENANT_ID,
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
payload,
|
|
126
|
-
|
|
144
|
+
insertedById: "system",
|
|
145
|
+
jobName: payload.jobName,
|
|
146
|
+
bullJobId: payload.bullJobId,
|
|
147
|
+
status: payload.status,
|
|
148
|
+
payload: encryptedPayload,
|
|
149
|
+
attempt: payload.attempt,
|
|
150
|
+
startedAt: parseJobInstant(payload.startedAt),
|
|
151
|
+
triggeredById: payload.triggeredById,
|
|
127
152
|
});
|
|
128
|
-
await runProjectionsForEvent(event, registry, db);
|
|
129
153
|
},
|
|
130
154
|
|
|
131
155
|
onJobComplete: async (
|
|
@@ -137,10 +161,9 @@ export function createJobRunLogger(opts: JobRunLoggerOptions): JobRunLoggerCallb
|
|
|
137
161
|
const runId = await resolveRunId(bullJobId);
|
|
138
162
|
// skip: state loss between start + complete (worker restart, cache
|
|
139
163
|
// evicted AND DB has no matching bull_job_id). Rare edge case; we
|
|
140
|
-
// drop the completion
|
|
141
|
-
//
|
|
164
|
+
// drop the completion write rather than forging a run row from
|
|
165
|
+
// scratch — forensics still has the original BullMQ lifecycle.
|
|
142
166
|
if (!runId) return;
|
|
143
|
-
const currentVersion = await getStreamVersion(db, runId, SYSTEM_TENANT_ID);
|
|
144
167
|
const payload = runCompletedSchema.parse({
|
|
145
168
|
duration,
|
|
146
169
|
finishedAt: Temporal.Now.instant().toString(),
|
|
@@ -150,16 +173,32 @@ export function createJobRunLogger(opts: JobRunLoggerOptions): JobRunLoggerCallb
|
|
|
150
173
|
timestamp: l.timestamp.toString(),
|
|
151
174
|
})),
|
|
152
175
|
});
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
176
|
+
await updateMany(
|
|
177
|
+
db,
|
|
178
|
+
jobRunsTable,
|
|
179
|
+
{
|
|
180
|
+
status: "completed",
|
|
181
|
+
duration: payload.duration,
|
|
182
|
+
finishedAt: parseJobInstant(payload.finishedAt),
|
|
183
|
+
modifiedAt: Temporal.Now.instant(),
|
|
184
|
+
modifiedById: "system",
|
|
185
|
+
},
|
|
186
|
+
{ id: runId },
|
|
187
|
+
);
|
|
188
|
+
// skip: empty log batch — the worker ran silent. No child rows to
|
|
189
|
+
// insert; the status update above already recorded completion.
|
|
190
|
+
if (payload.logs.length > 0) {
|
|
191
|
+
await insertMany(
|
|
192
|
+
db,
|
|
193
|
+
jobRunLogsTable,
|
|
194
|
+
payload.logs.map((log) => ({
|
|
195
|
+
runId,
|
|
196
|
+
level: log.level,
|
|
197
|
+
message: log.message,
|
|
198
|
+
timestamp: parseJobInstant(log.timestamp),
|
|
199
|
+
})),
|
|
200
|
+
);
|
|
201
|
+
}
|
|
163
202
|
runIdByBullJobId.delete(bullJobId); // immediate cleanup on terminal callback
|
|
164
203
|
},
|
|
165
204
|
|
|
@@ -171,13 +210,11 @@ export function createJobRunLogger(opts: JobRunLoggerOptions): JobRunLoggerCallb
|
|
|
171
210
|
) => {
|
|
172
211
|
const runId = await resolveRunId(bullJobId);
|
|
173
212
|
// skip: same rare state-loss case as in onJobComplete — drop the
|
|
174
|
-
// failure
|
|
213
|
+
// failure write rather than forge a run row from scratch.
|
|
175
214
|
if (!runId) return;
|
|
176
|
-
|
|
177
|
-
// Read started_at off the projection so we can compute duration
|
|
215
|
+
// Read started_at off the row so we can compute duration
|
|
178
216
|
// symmetrically to onJobComplete (which gets duration from the
|
|
179
|
-
// worker). The
|
|
180
|
-
// run-started inline-apply.
|
|
217
|
+
// worker). The row already has started_at from onJobStart.
|
|
181
218
|
const row = await fetchOne<{ startedAt: Temporal.Instant }>(db, jobRunsTable, { id: runId });
|
|
182
219
|
const now = Temporal.Now.instant();
|
|
183
220
|
const duration = row ? Number(now.since(row.startedAt).total({ unit: "millisecond" })) : 0;
|
|
@@ -191,16 +228,32 @@ export function createJobRunLogger(opts: JobRunLoggerOptions): JobRunLoggerCallb
|
|
|
191
228
|
timestamp: l.timestamp.toString(),
|
|
192
229
|
})),
|
|
193
230
|
});
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
231
|
+
await updateMany(
|
|
232
|
+
db,
|
|
233
|
+
jobRunsTable,
|
|
234
|
+
{
|
|
235
|
+
status: "failed",
|
|
236
|
+
error: payload.error,
|
|
237
|
+
duration: payload.duration,
|
|
238
|
+
finishedAt: parseJobInstant(payload.finishedAt),
|
|
239
|
+
modifiedAt: now,
|
|
240
|
+
modifiedById: "system",
|
|
241
|
+
},
|
|
242
|
+
{ id: runId },
|
|
243
|
+
);
|
|
244
|
+
// skip: empty log batch — mirror of onJobComplete
|
|
245
|
+
if (payload.logs.length > 0) {
|
|
246
|
+
await insertMany(
|
|
247
|
+
db,
|
|
248
|
+
jobRunLogsTable,
|
|
249
|
+
payload.logs.map((log) => ({
|
|
250
|
+
runId,
|
|
251
|
+
level: log.level,
|
|
252
|
+
message: log.message,
|
|
253
|
+
timestamp: parseJobInstant(log.timestamp),
|
|
254
|
+
})),
|
|
255
|
+
);
|
|
256
|
+
}
|
|
204
257
|
runIdByBullJobId.delete(bullJobId); // immediate cleanup on terminal callback
|
|
205
258
|
},
|
|
206
259
|
};
|