@cosmicdrift/kumiko-bundled-features 0.209.0 → 0.209.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +8 -8
- package/src/jobs/__tests__/jobs-events.integration.test.ts +65 -81
- package/src/jobs/__tests__/jobs-pii-kms.integration.test.ts +21 -18
- package/src/jobs/__tests__/jobs-retention.integration.test.ts +124 -0
- package/src/jobs/db/queries/retention.ts +44 -0
- package/src/jobs/events.ts +5 -8
- package/src/jobs/feature.ts +26 -146
- package/src/jobs/handlers/retention-cleanup.job.ts +21 -0
- package/src/jobs/index.ts +1 -1
- package/src/jobs/job-run-logger.ts +112 -59
- package/src/jobs/job-run-table.ts +36 -28
- package/src/user-data-rights/feature.ts +4 -4
- package/src/user-data-rights-defaults/__tests__/bundled-entities.userdata-hooks.integration.test.ts +1 -1
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cosmicdrift/kumiko-bundled-features",
|
|
3
|
-
"version": "0.209.
|
|
3
|
+
"version": "0.209.1",
|
|
4
4
|
"description": "Built-in features — tenant, user, auth, delivery. The stuff you'd rewrite anyway, already typed.",
|
|
5
5
|
"license": "BUSL-1.1",
|
|
6
6
|
"author": "Marc Frost <marc@cosmicdriftgamestudio.com>",
|
|
@@ -126,12 +126,12 @@
|
|
|
126
126
|
"./step-dispatcher": "./src/step-dispatcher/index.ts"
|
|
127
127
|
},
|
|
128
128
|
"dependencies": {
|
|
129
|
-
"@cosmicdrift/kumiko-dispatcher-live": "0.209.
|
|
130
|
-
"@cosmicdrift/kumiko-framework": "0.209.
|
|
131
|
-
"@cosmicdrift/kumiko-headless": "0.209.
|
|
132
|
-
"@cosmicdrift/kumiko-renderer": "0.209.
|
|
133
|
-
"@cosmicdrift/kumiko-renderer-web": "0.209.
|
|
134
|
-
"@cosmicdrift/kumiko-types": "0.209.
|
|
129
|
+
"@cosmicdrift/kumiko-dispatcher-live": "0.209.1",
|
|
130
|
+
"@cosmicdrift/kumiko-framework": "0.209.1",
|
|
131
|
+
"@cosmicdrift/kumiko-headless": "0.209.1",
|
|
132
|
+
"@cosmicdrift/kumiko-renderer": "0.209.1",
|
|
133
|
+
"@cosmicdrift/kumiko-renderer-web": "0.209.1",
|
|
134
|
+
"@cosmicdrift/kumiko-types": "0.209.1",
|
|
135
135
|
"@mollie/api-client": "^4.5.0",
|
|
136
136
|
"@node-rs/argon2": "^2.0.2",
|
|
137
137
|
"@types/mailparser": "^3.4.6",
|
|
@@ -160,6 +160,6 @@
|
|
|
160
160
|
"devDependencies": {
|
|
161
161
|
"@testing-library/user-event": "^14.6.1",
|
|
162
162
|
"@types/qrcode": "^1.5.5",
|
|
163
|
-
"@cosmicdrift/kumiko-locale-de": "0.209.
|
|
163
|
+
"@cosmicdrift/kumiko-locale-de": "0.209.1"
|
|
164
164
|
}
|
|
165
165
|
}
|
|
@@ -1,16 +1,16 @@
|
|
|
1
|
-
//
|
|
2
|
-
//
|
|
3
|
-
//
|
|
4
|
-
// (
|
|
5
|
-
//
|
|
1
|
+
// Direct-write shape contract for job runs (#2243). Pins onJobStart/
|
|
2
|
+
// -Complete/-Failed against the tables they write — jobRunsTable status
|
|
3
|
+
// transitions and jobRunLogsTable batched log rows. A silent shape drift
|
|
4
|
+
// (missing field, wrong status string) fails here instead of breaking the
|
|
5
|
+
// operator UI silently.
|
|
6
6
|
//
|
|
7
7
|
// The jobs integration test (jobs-feature.integration.ts) covers the
|
|
8
|
-
// projection side (list + detail queries). This file covers the
|
|
9
|
-
// side — complementary coverage, minimal overlap.
|
|
8
|
+
// projection side (list + detail queries). This file covers the
|
|
9
|
+
// write side — complementary coverage, minimal overlap.
|
|
10
10
|
|
|
11
11
|
import { afterAll, beforeAll, beforeEach, describe, expect, test } from "bun:test";
|
|
12
12
|
import { selectMany } from "@cosmicdrift/kumiko-framework/bun-db";
|
|
13
|
-
import { createRegistry
|
|
13
|
+
import { createRegistry } from "@cosmicdrift/kumiko-framework/engine";
|
|
14
14
|
import { createEventsTable, eventsTable } from "@cosmicdrift/kumiko-framework/event-store";
|
|
15
15
|
import {
|
|
16
16
|
createTestDb,
|
|
@@ -20,14 +20,8 @@ import {
|
|
|
20
20
|
unsafePushTables,
|
|
21
21
|
} from "@cosmicdrift/kumiko-framework/stack";
|
|
22
22
|
import { resetTestTables } from "@cosmicdrift/kumiko-framework/testing";
|
|
23
|
-
import { runCompletedSchema, runFailedSchema, runStartedSchema } from "../events";
|
|
24
23
|
import { createJobsFeature } from "../feature";
|
|
25
|
-
import {
|
|
26
|
-
createJobRunLogger,
|
|
27
|
-
JOB_RUN_COMPLETED_EVENT,
|
|
28
|
-
JOB_RUN_FAILED_EVENT,
|
|
29
|
-
JOB_RUN_STARTED_EVENT,
|
|
30
|
-
} from "../job-run-logger";
|
|
24
|
+
import { createJobRunLogger } from "../job-run-logger";
|
|
31
25
|
import { jobRunLogsTable, jobRunsTable } from "../job-run-table";
|
|
32
26
|
|
|
33
27
|
let testDb: TestDb;
|
|
@@ -40,6 +34,8 @@ beforeAll(async () => {
|
|
|
40
34
|
testRedis = await createTestRedis();
|
|
41
35
|
registry = createRegistry([createJobsFeature()]);
|
|
42
36
|
await unsafePushTables(testDb.db, { jobRunsTable, jobRunLogsTable });
|
|
37
|
+
// Kept only for the negative assertion below (no jobs:event:run-* rows) —
|
|
38
|
+
// the write path itself no longer touches the event store.
|
|
43
39
|
await createEventsTable(testDb.db);
|
|
44
40
|
logger = createJobRunLogger({ db: testDb.db, registry });
|
|
45
41
|
});
|
|
@@ -53,114 +49,102 @@ beforeEach(async () => {
|
|
|
53
49
|
await resetTestTables(testDb.db, [eventsTable, jobRunsTable, jobRunLogsTable]);
|
|
54
50
|
});
|
|
55
51
|
|
|
56
|
-
describe("jobRun
|
|
57
|
-
test("
|
|
58
|
-
// Guard against silent rename. Tests that subscribe via string-match
|
|
59
|
-
// (MSPs written in userland, audit export tools) break without this.
|
|
60
|
-
expect(JOB_RUN_STARTED_EVENT).toBe("jobs:event:run-started");
|
|
61
|
-
expect(JOB_RUN_COMPLETED_EVENT).toBe("jobs:event:run-completed");
|
|
62
|
-
expect(JOB_RUN_FAILED_EVENT).toBe("jobs:event:run-failed");
|
|
63
|
-
});
|
|
64
|
-
|
|
65
|
-
test("onJobStart writes a run-started event on the jobRun aggregate", async () => {
|
|
52
|
+
describe("jobRun direct writes", () => {
|
|
53
|
+
test("onJobStart inserts a row into jobRunsTable", async () => {
|
|
66
54
|
await logger.onJobStart?.("example:job:import", "bull-42", {
|
|
67
55
|
triggeredById: "u-99",
|
|
68
56
|
payload: JSON.stringify({ foo: 1 }),
|
|
69
57
|
attempt: 1,
|
|
70
58
|
});
|
|
71
59
|
|
|
72
|
-
const
|
|
73
|
-
|
|
74
|
-
expect(
|
|
75
|
-
const
|
|
76
|
-
expect(
|
|
77
|
-
expect(
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
expect(() => runStartedSchema.parse(e?.payload)).not.toThrow();
|
|
81
|
-
const p = runStartedSchema.parse(e?.payload);
|
|
82
|
-
expect(p.jobName).toBe("example:job:import");
|
|
83
|
-
expect(p.bullJobId).toBe("bull-42");
|
|
84
|
-
expect(p.triggeredById).toBe("u-99");
|
|
85
|
-
expect(p.attempt).toBe(1);
|
|
60
|
+
const runs = await selectMany(testDb.db, jobRunsTable, { bullJobId: "bull-42" });
|
|
61
|
+
|
|
62
|
+
expect(runs.length).toBe(1);
|
|
63
|
+
const run = runs[0];
|
|
64
|
+
expect(run?.jobName).toBe("example:job:import");
|
|
65
|
+
expect(run?.status).toBe("running");
|
|
66
|
+
expect(run?.triggeredById).toBe("u-99");
|
|
67
|
+
expect(run?.attempt).toBe(1);
|
|
86
68
|
});
|
|
87
69
|
|
|
88
|
-
test("onJobComplete
|
|
70
|
+
test("onJobComplete updates the row to completed with batched logs", async () => {
|
|
89
71
|
await logger.onJobStart?.("example:job:export", "bull-1", {});
|
|
90
72
|
await logger.onJobComplete?.("example:job:export", "bull-1", 123, [
|
|
91
73
|
{ level: "info", message: "started", timestamp: Temporal.Now.instant() },
|
|
92
74
|
{ level: "info", message: "done", timestamp: Temporal.Now.instant() },
|
|
93
75
|
]);
|
|
94
76
|
|
|
95
|
-
const
|
|
77
|
+
const runs = await selectMany(testDb.db, jobRunsTable, { bullJobId: "bull-1" });
|
|
78
|
+
expect(runs.length).toBe(1);
|
|
79
|
+
expect(runs[0]?.status).toBe("completed");
|
|
80
|
+
expect(runs[0]?.duration).toBe(123);
|
|
96
81
|
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
expect(
|
|
100
|
-
expect(p.logs).toHaveLength(2);
|
|
101
|
-
expect(p.logs[0]?.level).toBe("info");
|
|
82
|
+
const logs = await selectMany(testDb.db, jobRunLogsTable, { runId: runs[0]?.id as string });
|
|
83
|
+
expect(logs).toHaveLength(2);
|
|
84
|
+
expect(logs[0]?.level).toBe("info");
|
|
102
85
|
});
|
|
103
86
|
|
|
104
|
-
test("onJobFailed
|
|
87
|
+
test("onJobFailed updates the row to failed with error + logs", async () => {
|
|
105
88
|
await logger.onJobStart?.("example:job:fragile", "bull-9", {});
|
|
106
89
|
await logger.onJobFailed?.("example:job:fragile", "bull-9", "boom", [
|
|
107
90
|
{ level: "error", message: "kaboom", timestamp: Temporal.Now.instant() },
|
|
108
91
|
]);
|
|
109
92
|
|
|
110
|
-
const
|
|
93
|
+
const runs = await selectMany(testDb.db, jobRunsTable, { bullJobId: "bull-9" });
|
|
94
|
+
expect(runs.length).toBe(1);
|
|
95
|
+
expect(runs[0]?.status).toBe("failed");
|
|
96
|
+
expect(runs[0]?.error).toBe("boom");
|
|
111
97
|
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
expect(p.error).toBe("boom");
|
|
115
|
-
expect(p.logs).toHaveLength(1);
|
|
98
|
+
const logs = await selectMany(testDb.db, jobRunLogsTable, { runId: runs[0]?.id as string });
|
|
99
|
+
expect(logs).toHaveLength(1);
|
|
116
100
|
});
|
|
117
101
|
|
|
118
|
-
test("start + complete both
|
|
102
|
+
test("start + complete both act on the SAME row", async () => {
|
|
119
103
|
await logger.onJobStart?.("example:job:stream", "bull-99", {});
|
|
120
104
|
await logger.onJobComplete?.("example:job:stream", "bull-99", 10, []);
|
|
121
105
|
|
|
122
|
-
//
|
|
123
|
-
// the
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
expect(events.length).toBe(2);
|
|
128
|
-
const ids = new Set(events.map((e) => e.aggregateId));
|
|
129
|
-
expect(ids.size).toBe(1);
|
|
106
|
+
// Exactly one row for this bullJobId — the complete-callback updated
|
|
107
|
+
// the row onJobStart created, it did not insert a second one.
|
|
108
|
+
const runs = await selectMany(testDb.db, jobRunsTable, { bullJobId: "bull-99" });
|
|
109
|
+
expect(runs.length).toBe(1);
|
|
110
|
+
expect(runs[0]?.status).toBe("completed");
|
|
130
111
|
});
|
|
131
112
|
|
|
132
|
-
test("complete/fail without a prior start skips — does not forge a
|
|
133
|
-
// State-loss path: worker restart with empty cache AND no
|
|
134
|
-
//
|
|
135
|
-
//
|
|
113
|
+
test("complete/fail without a prior start skips — does not forge a run row", async () => {
|
|
114
|
+
// State-loss path: worker restart with empty cache AND no row for this
|
|
115
|
+
// bullJobId. Dropping the terminal write is intentional — forging a
|
|
116
|
+
// row from scratch would invent a run that never started.
|
|
136
117
|
await logger.onJobComplete?.("example:job:orphan", "bull-orphan-complete", 50, []);
|
|
137
118
|
await logger.onJobFailed?.("example:job:orphan", "bull-orphan-failed", "boom", []);
|
|
138
119
|
|
|
139
|
-
const completed = await selectMany(testDb.db, eventsTable, { type: JOB_RUN_COMPLETED_EVENT });
|
|
140
|
-
const failed = await selectMany(testDb.db, eventsTable, { type: JOB_RUN_FAILED_EVENT });
|
|
141
120
|
const runs = await selectMany(testDb.db, jobRunsTable);
|
|
142
|
-
|
|
143
|
-
expect(completed).toHaveLength(0);
|
|
144
|
-
expect(failed).toHaveLength(0);
|
|
145
121
|
expect(runs).toHaveLength(0);
|
|
146
122
|
});
|
|
147
123
|
|
|
148
|
-
test("complete after cache loss recovers runId from
|
|
149
|
-
// Simulates worker process restart: in-memory bullJobId→runId cache is
|
|
150
|
-
// but the run-started
|
|
151
|
-
// must DB-lookup and
|
|
124
|
+
test("complete after cache loss recovers runId from jobRunsTable (same row)", async () => {
|
|
125
|
+
// Simulates worker process restart: in-memory bullJobId→runId cache is
|
|
126
|
+
// gone, but the run-started row still has bull_job_id. A fresh logger
|
|
127
|
+
// must DB-lookup and update the original row — not insert a second one.
|
|
152
128
|
await logger.onJobStart?.("example:job:restart", "bull-restart-1", {});
|
|
153
|
-
const started = await selectMany(testDb.db,
|
|
129
|
+
const started = await selectMany(testDb.db, jobRunsTable, { bullJobId: "bull-restart-1" });
|
|
154
130
|
expect(started).toHaveLength(1);
|
|
155
|
-
const
|
|
156
|
-
expect(
|
|
131
|
+
const originalId = started[0]?.id;
|
|
132
|
+
expect(originalId).toBeTruthy();
|
|
157
133
|
|
|
158
134
|
const coldLogger = createJobRunLogger({ db: testDb.db, registry });
|
|
159
135
|
await coldLogger.onJobComplete?.("example:job:restart", "bull-restart-1", 42, []);
|
|
160
136
|
|
|
161
|
-
const all = await selectMany(testDb.db,
|
|
162
|
-
expect(all).toHaveLength(
|
|
163
|
-
expect(
|
|
164
|
-
expect(all
|
|
137
|
+
const all = await selectMany(testDb.db, jobRunsTable, { bullJobId: "bull-restart-1" });
|
|
138
|
+
expect(all).toHaveLength(1);
|
|
139
|
+
expect(all[0]?.id).toBe(originalId);
|
|
140
|
+
expect(all[0]?.status).toBe("completed");
|
|
141
|
+
});
|
|
142
|
+
|
|
143
|
+
test("no jobs:event:run-* rows ever land in the event store", async () => {
|
|
144
|
+
await logger.onJobStart?.("example:job:no-events", "bull-no-events", {});
|
|
145
|
+
await logger.onJobComplete?.("example:job:no-events", "bull-no-events", 5, []);
|
|
146
|
+
|
|
147
|
+
const events = await selectMany(testDb.db, eventsTable, { aggregateType: "jobRun" });
|
|
148
|
+
expect(events).toHaveLength(0);
|
|
165
149
|
});
|
|
166
150
|
});
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
//
|
|
2
|
-
// arbitrary user data and is written
|
|
3
|
-
//
|
|
4
|
-
// With a KMS active the stored
|
|
5
|
-
// ciphertext under the triggering user's DEK; erasing that key
|
|
6
|
-
// payload unreadable ([[erased]]) without touching the
|
|
1
|
+
// PII on the jobs run-logger, direct-write path (#2243, formerly #799):
|
|
2
|
+
// run-started payload can carry arbitrary user data and is written straight
|
|
3
|
+
// into jobRunsTable (no event store, no event-PII catalog — #2243 removed
|
|
4
|
+
// the jobRun r.defineEvent registrations). With a KMS active the stored row
|
|
5
|
+
// carries ciphertext under the triggering user's DEK; erasing that key
|
|
6
|
+
// makes the payload unreadable ([[erased]]) without touching the row.
|
|
7
7
|
|
|
8
8
|
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test";
|
|
9
9
|
import { fetchOne, selectMany } from "@cosmicdrift/kumiko-framework/bun-db";
|
|
@@ -25,7 +25,7 @@ import {
|
|
|
25
25
|
} from "@cosmicdrift/kumiko-framework/stack";
|
|
26
26
|
import { resetPiiSubjectKmsForTests, resetTestTables } from "@cosmicdrift/kumiko-framework/testing";
|
|
27
27
|
import { createJobsFeature } from "../feature";
|
|
28
|
-
import { createJobRunLogger
|
|
28
|
+
import { createJobRunLogger } from "../job-run-logger";
|
|
29
29
|
import { jobRunLogsTable, jobRunsTable } from "../job-run-table";
|
|
30
30
|
|
|
31
31
|
let testDb: TestDb;
|
|
@@ -39,10 +39,10 @@ const SECRET_PAYLOAD = JSON.stringify({ iban: "DE89370400440532013000" });
|
|
|
39
39
|
beforeAll(async () => {
|
|
40
40
|
testDb = await createTestDb();
|
|
41
41
|
testRedis = await createTestRedis();
|
|
42
|
-
// createRegistry publishes the event-PII catalog as a module singleton —
|
|
43
|
-
// the logger's low-level append() picks it up without further wiring.
|
|
44
42
|
const registry = createRegistry([createJobsFeature()]);
|
|
45
43
|
await unsafePushTables(testDb.db, { jobRunsTable, jobRunLogsTable });
|
|
44
|
+
// Kept only so the "no event store involved" assertion below has a table
|
|
45
|
+
// to assert against — the write path itself never touches it.
|
|
46
46
|
await createEventsTable(testDb.db);
|
|
47
47
|
logger = createJobRunLogger({ db: testDb.db, registry });
|
|
48
48
|
});
|
|
@@ -63,25 +63,28 @@ afterEach(() => {
|
|
|
63
63
|
});
|
|
64
64
|
|
|
65
65
|
describe("jobs run-started payload under KMS", () => {
|
|
66
|
-
test("stored
|
|
66
|
+
test("stored row carries ciphertext payload, plaintext triggeredById — no event-store write", async () => {
|
|
67
67
|
await logger.onJobStart?.("app:job:export", "bull-1", {
|
|
68
68
|
triggeredById: USER_ID,
|
|
69
69
|
payload: SECRET_PAYLOAD,
|
|
70
70
|
attempt: 1,
|
|
71
71
|
});
|
|
72
72
|
|
|
73
|
-
const
|
|
74
|
-
expect(
|
|
75
|
-
|
|
76
|
-
expect(
|
|
77
|
-
expect(String(payload["payload"])).toContain(`user:${USER_ID}`);
|
|
78
|
-
expect(payload["triggeredById"]).toBe(USER_ID);
|
|
73
|
+
const row = await fetchOne(testDb.db, jobRunsTable, { bullJobId: "bull-1" });
|
|
74
|
+
expect(isPiiCiphertext(row?.["payload"])).toBe(true);
|
|
75
|
+
expect(String(row?.["payload"])).toContain(`user:${USER_ID}`);
|
|
76
|
+
expect(row?.["triggeredById"]).toBe(USER_ID);
|
|
79
77
|
|
|
80
|
-
const back = await decryptPiiFieldValues(payload, ["payload"], kms, {
|
|
78
|
+
const back = await decryptPiiFieldValues({ payload: row?.["payload"] }, ["payload"], kms, {
|
|
79
|
+
requestId: "t",
|
|
80
|
+
});
|
|
81
81
|
expect(back["payload"]).toBe(SECRET_PAYLOAD);
|
|
82
|
+
|
|
83
|
+
const events = await selectMany(testDb.db, eventsTable, { aggregateType: "jobRun" });
|
|
84
|
+
expect(events).toHaveLength(0);
|
|
82
85
|
});
|
|
83
86
|
|
|
84
|
-
test("
|
|
87
|
+
test("erase subject key → row payload decrypts to [[erased]]", async () => {
|
|
85
88
|
await logger.onJobStart?.("app:job:export", "bull-2", {
|
|
86
89
|
triggeredById: USER_ID,
|
|
87
90
|
payload: SECRET_PAYLOAD,
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
// Integration test for the job-run retention-cleanup job (#2243). Dispatches
|
|
2
|
+
// jobs:job:retention-cleanup through the real jobRunner (setupTestStack +
|
|
3
|
+
// BullMQ worker) — the same enqueue path production's cron trigger uses —
|
|
4
|
+
// instead of hand-building a JobContext and casting into the handler. Each
|
|
5
|
+
// test seeds a row past the active cutoff alongside one that must survive,
|
|
6
|
+
// so waitFor's completion signal (the stale row's count dropping) is only
|
|
7
|
+
// reached once the real dispatch has actually run; the survivor's fate is
|
|
8
|
+
// decided by that same DELETE and can be asserted right after.
|
|
9
|
+
|
|
10
|
+
import { afterEach, describe, expect, test } from "bun:test";
|
|
11
|
+
import { selectMany } from "@cosmicdrift/kumiko-framework/bun-db";
|
|
12
|
+
import { sql } from "@cosmicdrift/kumiko-framework/db";
|
|
13
|
+
import { SYSTEM_TENANT_ID } from "@cosmicdrift/kumiko-framework/engine";
|
|
14
|
+
import {
|
|
15
|
+
setupTestStack,
|
|
16
|
+
type TestStack,
|
|
17
|
+
unsafePushTables,
|
|
18
|
+
} from "@cosmicdrift/kumiko-framework/stack";
|
|
19
|
+
import { seedRow, waitFor } from "@cosmicdrift/kumiko-framework/testing";
|
|
20
|
+
import { createJobsFeature } from "../feature";
|
|
21
|
+
import { DEFAULT_JOB_RUN_RETENTION_DAYS } from "../handlers/retention-cleanup.job";
|
|
22
|
+
import { jobRunLogsTable, jobRunsTable } from "../job-run-table";
|
|
23
|
+
|
|
24
|
+
const RETENTION_JOB = "jobs:job:retention-cleanup";
|
|
25
|
+
|
|
26
|
+
let stack: TestStack | undefined;
|
|
27
|
+
|
|
28
|
+
async function bootStack(retentionDays?: number): Promise<TestStack> {
|
|
29
|
+
const s = await setupTestStack({
|
|
30
|
+
features: [createJobsFeature(retentionDays !== undefined ? { retentionDays } : {})],
|
|
31
|
+
jobs: { consumerLane: "worker" },
|
|
32
|
+
});
|
|
33
|
+
await unsafePushTables(s.db, { jobRunsTable, jobRunLogsTable });
|
|
34
|
+
return s;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
afterEach(async () => {
|
|
38
|
+
if (stack) await stack.cleanup();
|
|
39
|
+
stack = undefined;
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
function currentStack(): TestStack {
|
|
43
|
+
if (!stack) throw new Error("stack not booted");
|
|
44
|
+
return stack;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
async function seedRun(opts: { id: string; ageDays: number }): Promise<void> {
|
|
48
|
+
const insertedAt = sql`now() - ${sql.raw(`interval '${opts.ageDays} days'`)}`;
|
|
49
|
+
await seedRow(currentStack().db, jobRunsTable, {
|
|
50
|
+
id: opts.id,
|
|
51
|
+
tenantId: SYSTEM_TENANT_ID,
|
|
52
|
+
insertedAt,
|
|
53
|
+
jobName: "example:job:retention-probe",
|
|
54
|
+
bullJobId: `bull-${opts.id}`,
|
|
55
|
+
status: "completed",
|
|
56
|
+
attempt: 1,
|
|
57
|
+
startedAt: insertedAt,
|
|
58
|
+
});
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
async function countRuns(): Promise<number> {
|
|
62
|
+
return (await selectMany(currentStack().db, jobRunsTable)).length;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
async function countLogs(): Promise<number> {
|
|
66
|
+
return (await selectMany(currentStack().db, jobRunLogsTable)).length;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
// Dispatches the real job through jobRunner and blocks until the queued
|
|
70
|
+
// worker has actually applied it, proven by the run count reaching
|
|
71
|
+
// `expectedRuns` (not merely "some time has passed").
|
|
72
|
+
async function dispatchAndWaitForRunCount(expectedRuns: number): Promise<void> {
|
|
73
|
+
await currentStack().jobRunner?.dispatch(RETENTION_JOB);
|
|
74
|
+
await waitFor(async () => {
|
|
75
|
+
expect(await countRuns()).toBe(expectedRuns);
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
describe("jobs:job:retention-cleanup", () => {
|
|
80
|
+
test("default window (30d): older-than-cutoff runs go, recent ones stay", async () => {
|
|
81
|
+
stack = await bootStack();
|
|
82
|
+
await seedRun({
|
|
83
|
+
id: "11111111-1111-4111-8111-111111111111",
|
|
84
|
+
ageDays: DEFAULT_JOB_RUN_RETENTION_DAYS + 1,
|
|
85
|
+
});
|
|
86
|
+
await seedRun({ id: "22222222-2222-4222-8222-222222222222", ageDays: 1 });
|
|
87
|
+
expect(await countRuns()).toBe(2);
|
|
88
|
+
|
|
89
|
+
await dispatchAndWaitForRunCount(1);
|
|
90
|
+
|
|
91
|
+
const remaining = await selectMany(currentStack().db, jobRunsTable);
|
|
92
|
+
expect(remaining[0]?.id).toBe("22222222-2222-4222-8222-222222222222");
|
|
93
|
+
});
|
|
94
|
+
|
|
95
|
+
test("associated log rows are deleted along with their run", async () => {
|
|
96
|
+
stack = await bootStack();
|
|
97
|
+
const runId = "33333333-3333-4333-8333-333333333333";
|
|
98
|
+
const ageDays = DEFAULT_JOB_RUN_RETENTION_DAYS + 5;
|
|
99
|
+
await seedRun({ id: runId, ageDays });
|
|
100
|
+
await seedRow(currentStack().db, jobRunLogsTable, {
|
|
101
|
+
runId,
|
|
102
|
+
level: "info",
|
|
103
|
+
message: "old log line",
|
|
104
|
+
timestamp: sql`now() - ${sql.raw(`interval '${ageDays} days'`)}`,
|
|
105
|
+
});
|
|
106
|
+
expect(await countLogs()).toBe(1);
|
|
107
|
+
|
|
108
|
+
await dispatchAndWaitForRunCount(0);
|
|
109
|
+
|
|
110
|
+
expect(await countLogs()).toBe(0);
|
|
111
|
+
});
|
|
112
|
+
|
|
113
|
+
test("a custom retentionDays option is honored: past-cutoff runs go, in-window runs stay", async () => {
|
|
114
|
+
stack = await bootStack(3);
|
|
115
|
+
await seedRun({ id: "44444444-4444-4444-8444-444444444444", ageDays: 5 });
|
|
116
|
+
await seedRun({ id: "55555555-5555-4555-8555-555555555555", ageDays: 1 });
|
|
117
|
+
expect(await countRuns()).toBe(2);
|
|
118
|
+
|
|
119
|
+
await dispatchAndWaitForRunCount(1);
|
|
120
|
+
|
|
121
|
+
const remaining = await selectMany(currentStack().db, jobRunsTable);
|
|
122
|
+
expect(remaining[0]?.id).toBe("55555555-5555-4555-8555-555555555555");
|
|
123
|
+
});
|
|
124
|
+
});
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
// Job-run retention (#2243): store_job_runs/store_job_run_logs are
|
|
2
|
+
// direct-write stores now, not event-sourced projections — nothing else
|
|
3
|
+
// purges them, so without this query they grow forever with the run count
|
|
4
|
+
// instead of the runtime. Not the retention-cleanup executor: that executor
|
|
5
|
+
// is entity/tenant-scoped and jobRun has neither (system-scoped direct-write
|
|
6
|
+
// store). Each table is purged by its own timestamp column independently —
|
|
7
|
+
// run_id isn't a real FK constraint (see job-run-table.ts), so there is no
|
|
8
|
+
// ordering requirement between the two deletes.
|
|
9
|
+
//
|
|
10
|
+
// deleteManyBatched (typed query API, no raw SQL in this file) chunks each
|
|
11
|
+
// delete so a large backlog doesn't hold one lock for the whole sweep.
|
|
12
|
+
|
|
13
|
+
import { deleteManyBatched } from "@cosmicdrift/kumiko-framework/bun-db";
|
|
14
|
+
import type { DbConnection } from "@cosmicdrift/kumiko-framework/db";
|
|
15
|
+
import { jobRunLogsTable, jobRunsTable } from "../../job-run-table";
|
|
16
|
+
|
|
17
|
+
const RETENTION_DELETE_BATCH_SIZE = 500;
|
|
18
|
+
|
|
19
|
+
export type JobRunRetentionResult = {
|
|
20
|
+
readonly runsDeleted: number;
|
|
21
|
+
readonly logsDeleted: number;
|
|
22
|
+
};
|
|
23
|
+
|
|
24
|
+
export async function deleteStaleJobRuns(
|
|
25
|
+
db: DbConnection,
|
|
26
|
+
retentionDays: number,
|
|
27
|
+
): Promise<JobRunRetentionResult> {
|
|
28
|
+
const cutoff = Temporal.Now.instant().subtract({ hours: retentionDays * 24 });
|
|
29
|
+
|
|
30
|
+
const logsResult = await deleteManyBatched(
|
|
31
|
+
db,
|
|
32
|
+
jobRunLogsTable,
|
|
33
|
+
{ timestamp: { lt: cutoff } },
|
|
34
|
+
{ limit: RETENTION_DELETE_BATCH_SIZE },
|
|
35
|
+
);
|
|
36
|
+
const runsResult = await deleteManyBatched(
|
|
37
|
+
db,
|
|
38
|
+
jobRunsTable,
|
|
39
|
+
{ insertedAt: { lt: cutoff } },
|
|
40
|
+
{ limit: RETENTION_DELETE_BATCH_SIZE },
|
|
41
|
+
);
|
|
42
|
+
|
|
43
|
+
return { runsDeleted: runsResult.deleted, logsDeleted: logsResult.deleted };
|
|
44
|
+
}
|
package/src/jobs/events.ts
CHANGED
|
@@ -1,11 +1,8 @@
|
|
|
1
|
-
//
|
|
2
|
-
//
|
|
3
|
-
//
|
|
4
|
-
//
|
|
5
|
-
//
|
|
6
|
-
//
|
|
7
|
-
// Keeping them in a separate module avoids the circular import between
|
|
8
|
-
// jobs-feature.ts (imports the logger) and job-run-logger.ts.
|
|
1
|
+
// Payload schemas for the jobRun direct-write path (#2243). Pre-#2243 these
|
|
2
|
+
// backed jobRun's event-store payloads (r.defineEvent + inline projections);
|
|
3
|
+
// now job-run-logger.ts parses against them before writing straight into
|
|
4
|
+
// jobRunsTable/jobRunLogsTable, so an out-of-dispatcher write still gets the
|
|
5
|
+
// same validation guarantee ctx.appendEvent used to give it.
|
|
9
6
|
|
|
10
7
|
import { z } from "zod";
|
|
11
8
|
|
package/src/jobs/feature.ts
CHANGED
|
@@ -1,16 +1,5 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
defineApply,
|
|
4
|
-
defineFeature,
|
|
5
|
-
type FeatureDefinition,
|
|
6
|
-
} from "@cosmicdrift/kumiko-framework/engine";
|
|
7
|
-
import type { z } from "zod";
|
|
1
|
+
import { defineFeature, type FeatureDefinition } from "@cosmicdrift/kumiko-framework/engine";
|
|
8
2
|
import { JOB_RUN_DETAIL_SCREEN_ID, JOB_RUNS_SCREEN_ID } from "./constants";
|
|
9
|
-
// Event-payload schemas live in a sibling module so the logger can import
|
|
10
|
-
// them without the cycle jobs-feature ↔ job-run-logger. The logger parses
|
|
11
|
-
// payloads against these schemas before low-level append() — that's what
|
|
12
|
-
// keeps out-of-dispatcher writes as type-safe as ctx.appendEvent.
|
|
13
|
-
import { runCompletedSchema, runFailedSchema, runStartedSchema } from "./events";
|
|
14
3
|
import { catalogQuery } from "./handlers/catalog.query";
|
|
15
4
|
import { detailQuery } from "./handlers/detail.query";
|
|
16
5
|
import { listQuery } from "./handlers/list.query";
|
|
@@ -19,21 +8,26 @@ import {
|
|
|
19
8
|
projectionRebuildPayloadSchema,
|
|
20
9
|
} from "./handlers/projection-rebuild.job";
|
|
21
10
|
import { reindexEntityJob, reindexEntityPayloadSchema } from "./handlers/reindex-entity.job";
|
|
11
|
+
import {
|
|
12
|
+
createRetentionCleanupJob,
|
|
13
|
+
DEFAULT_JOB_RUN_RETENTION_DAYS,
|
|
14
|
+
} from "./handlers/retention-cleanup.job";
|
|
22
15
|
import { retryWrite } from "./handlers/retry.write";
|
|
23
16
|
import { triggerWrite } from "./handlers/trigger.write";
|
|
24
17
|
import { JOBS_I18N } from "./i18n";
|
|
25
|
-
import {
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
18
|
+
import { jobRunLogsTableMeta, jobRunsTableMeta } from "./job-run-table";
|
|
19
|
+
|
|
20
|
+
export type JobsFeatureOptions = {
|
|
21
|
+
// How long a job run (and its logs) stays in store_job_runs/
|
|
22
|
+
// store_job_run_logs before the daily retention-cleanup job deletes it.
|
|
23
|
+
readonly retentionDays?: number;
|
|
24
|
+
};
|
|
32
25
|
|
|
33
|
-
export function createJobsFeature(): FeatureDefinition {
|
|
26
|
+
export function createJobsFeature(options: JobsFeatureOptions = {}): FeatureDefinition {
|
|
27
|
+
const retentionDays = options.retentionDays ?? DEFAULT_JOB_RUN_RETENTION_DAYS;
|
|
34
28
|
return defineFeature("jobs", (r) => {
|
|
35
29
|
r.describe(
|
|
36
|
-
"Persistence and operator tooling for background jobs registered via `r.job(...)`. Every job execution
|
|
30
|
+
"Persistence and operator tooling for background jobs registered via `r.job(...)`. Every job execution writes directly into `store_job_runs` (current status + duration) and `store_job_run_logs` (per-line log rows) from the BullMQ callbacks — no event stream in between (#2243). A daily `retention-cleanup` job deletes runs (and their logs) older than `retentionDays`. Exposes `jobs:write:trigger` (manual run) and `jobs:write:retry` (operator retry of a failed run), plus `jobs:query:list`, `jobs:query:details`, and `jobs:query:catalog` (manual jobs) for the operator UI.",
|
|
37
31
|
);
|
|
38
32
|
r.uiHints({
|
|
39
33
|
displayLabel: "Jobs · Audit & Operator UI",
|
|
@@ -41,134 +35,12 @@ export function createJobsFeature(): FeatureDefinition {
|
|
|
41
35
|
recommended: false,
|
|
42
36
|
});
|
|
43
37
|
r.systemScope();
|
|
38
|
+
r.storeTable(jobRunsTableMeta, {
|
|
39
|
+
reason: "direct_write.job_runs",
|
|
40
|
+
});
|
|
44
41
|
r.storeTable(jobRunLogsTableMeta, {
|
|
45
42
|
reason: "read_side.job_run_logs",
|
|
46
43
|
});
|
|
47
|
-
// Events-only aggregate: "jobRun" has no r.entity registration, because
|
|
48
|
-
// the entire lifecycle is driven by BullMQ-callback → r.defineEvent
|
|
49
|
-
// (no executor, no CRUD). The boot-validator accepts the two
|
|
50
|
-
// projections below because every apply-key is a registered
|
|
51
|
-
// domain-event.
|
|
52
|
-
// payload can carry arbitrary user data; triggeredById stays plaintext
|
|
53
|
-
// (pseudonymous fk). System runs (triggeredById null) stay plaintext —
|
|
54
|
-
// no user subject to shred.
|
|
55
|
-
r.defineEvent("run-started", runStartedSchema, {
|
|
56
|
-
piiFields: { payload: { subjectField: "triggeredById" } },
|
|
57
|
-
});
|
|
58
|
-
r.defineEvent("run-completed", runCompletedSchema);
|
|
59
|
-
r.defineEvent("run-failed", runFailedSchema);
|
|
60
|
-
|
|
61
|
-
// Inline projection: status-row in jobRunsTable. Runs in same TX as
|
|
62
|
-
// the event-append (the logger calls runProjectionsForEvent manually
|
|
63
|
-
// because the BullMQ-callback path has no dispatcher-ctx).
|
|
64
|
-
r.projection({
|
|
65
|
-
name: "job-runs",
|
|
66
|
-
source: "jobRun",
|
|
67
|
-
table: jobRunsTable,
|
|
68
|
-
apply: {
|
|
69
|
-
[JOB_RUN_STARTED_EVENT]: defineApply<z.infer<typeof runStartedSchema>>(
|
|
70
|
-
async (event, tx, table) => {
|
|
71
|
-
const p = event.payload;
|
|
72
|
-
await insertOne(tx, table, {
|
|
73
|
-
id: event.aggregateId,
|
|
74
|
-
tenantId: event.tenantId,
|
|
75
|
-
version: event.version,
|
|
76
|
-
insertedAt: event.createdAt,
|
|
77
|
-
insertedById: event.metadata?.userId ?? "system",
|
|
78
|
-
jobName: p.jobName,
|
|
79
|
-
bullJobId: p.bullJobId,
|
|
80
|
-
status: p.status,
|
|
81
|
-
payload: p.payload,
|
|
82
|
-
attempt: p.attempt,
|
|
83
|
-
startedAt: parseJobInstant(p.startedAt),
|
|
84
|
-
triggeredById: p.triggeredById,
|
|
85
|
-
});
|
|
86
|
-
},
|
|
87
|
-
),
|
|
88
|
-
[JOB_RUN_COMPLETED_EVENT]: defineApply<z.infer<typeof runCompletedSchema>>(
|
|
89
|
-
async (event, tx, table) => {
|
|
90
|
-
const p = event.payload;
|
|
91
|
-
await updateMany(
|
|
92
|
-
tx,
|
|
93
|
-
table,
|
|
94
|
-
{
|
|
95
|
-
status: "completed",
|
|
96
|
-
duration: p.duration,
|
|
97
|
-
finishedAt: parseJobInstant(p.finishedAt),
|
|
98
|
-
version: event.version,
|
|
99
|
-
modifiedAt: event.createdAt,
|
|
100
|
-
modifiedById: event.metadata?.userId ?? "system",
|
|
101
|
-
},
|
|
102
|
-
{ id: event.aggregateId },
|
|
103
|
-
);
|
|
104
|
-
},
|
|
105
|
-
),
|
|
106
|
-
[JOB_RUN_FAILED_EVENT]: defineApply<z.infer<typeof runFailedSchema>>(
|
|
107
|
-
async (event, tx, table) => {
|
|
108
|
-
const p = event.payload;
|
|
109
|
-
await updateMany(
|
|
110
|
-
tx,
|
|
111
|
-
table,
|
|
112
|
-
{
|
|
113
|
-
status: "failed",
|
|
114
|
-
error: p.error,
|
|
115
|
-
duration: p.duration,
|
|
116
|
-
finishedAt: parseJobInstant(p.finishedAt),
|
|
117
|
-
version: event.version,
|
|
118
|
-
modifiedAt: event.createdAt,
|
|
119
|
-
modifiedById: event.metadata?.userId ?? "system",
|
|
120
|
-
},
|
|
121
|
-
{ id: event.aggregateId },
|
|
122
|
-
);
|
|
123
|
-
},
|
|
124
|
-
),
|
|
125
|
-
},
|
|
126
|
-
});
|
|
127
|
-
|
|
128
|
-
// Second inline projection — same source, different table. Expands
|
|
129
|
-
// the batched logs array from completed/failed events into N rows
|
|
130
|
-
// per run in jobRunLogsTable.
|
|
131
|
-
r.projection({
|
|
132
|
-
name: "job-run-logs",
|
|
133
|
-
source: "jobRun",
|
|
134
|
-
table: jobRunLogsTable,
|
|
135
|
-
apply: {
|
|
136
|
-
[JOB_RUN_COMPLETED_EVENT]: defineApply<z.infer<typeof runCompletedSchema>>(
|
|
137
|
-
async (event, tx) => {
|
|
138
|
-
const p = event.payload;
|
|
139
|
-
// skip: empty log batch — the worker ran silent. No child rows
|
|
140
|
-
// to insert; the completed-event alone already updated the run's
|
|
141
|
-
// status via the sibling job-runs projection.
|
|
142
|
-
if (p.logs.length === 0) return;
|
|
143
|
-
await insertMany(
|
|
144
|
-
tx,
|
|
145
|
-
jobRunLogsTable,
|
|
146
|
-
p.logs.map((log) => ({
|
|
147
|
-
runId: event.aggregateId,
|
|
148
|
-
level: log.level,
|
|
149
|
-
message: log.message,
|
|
150
|
-
timestamp: parseJobInstant(log.timestamp),
|
|
151
|
-
})),
|
|
152
|
-
);
|
|
153
|
-
},
|
|
154
|
-
),
|
|
155
|
-
[JOB_RUN_FAILED_EVENT]: defineApply<z.infer<typeof runFailedSchema>>(async (event, tx) => {
|
|
156
|
-
const p = event.payload;
|
|
157
|
-
// skip: empty log batch — the worker ran silent (mirror of completed)
|
|
158
|
-
if (p.logs.length === 0) return;
|
|
159
|
-
await insertMany(
|
|
160
|
-
tx,
|
|
161
|
-
jobRunLogsTable,
|
|
162
|
-
p.logs.map((log) => ({
|
|
163
|
-
runId: event.aggregateId,
|
|
164
|
-
level: log.level,
|
|
165
|
-
message: log.message,
|
|
166
|
-
timestamp: parseJobInstant(log.timestamp),
|
|
167
|
-
})),
|
|
168
|
-
);
|
|
169
|
-
}),
|
|
170
|
-
},
|
|
171
|
-
});
|
|
172
44
|
|
|
173
45
|
// Framework-provided rebuild job — available whenever `jobs` is composed; enqueueProjectionRebuild dispatches it.
|
|
174
46
|
r.job(
|
|
@@ -187,6 +59,14 @@ export function createJobsFeature(): FeatureDefinition {
|
|
|
187
59
|
reindexEntityJob,
|
|
188
60
|
);
|
|
189
61
|
|
|
62
|
+
// store_job_runs/store_job_run_logs are direct-write, unbounded-growth
|
|
63
|
+
// stores (#2243) — nothing else purges them.
|
|
64
|
+
r.job(
|
|
65
|
+
"retention-cleanup",
|
|
66
|
+
{ trigger: { cron: "0 3 * * *" }, concurrency: "skip" },
|
|
67
|
+
createRetentionCleanupJob(retentionDays),
|
|
68
|
+
);
|
|
69
|
+
|
|
190
70
|
const handlers = {
|
|
191
71
|
trigger: r.writeHandler(triggerWrite),
|
|
192
72
|
retry: r.writeHandler(retryWrite),
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import type { DbConnection } from "@cosmicdrift/kumiko-framework/db";
|
|
2
|
+
import type { JobHandlerFn } from "@cosmicdrift/kumiko-framework/engine";
|
|
3
|
+
import { InternalError } from "@cosmicdrift/kumiko-framework/errors";
|
|
4
|
+
import { deleteStaleJobRuns } from "../db/queries/retention";
|
|
5
|
+
|
|
6
|
+
// Single source for the retention window — change here, nowhere else.
|
|
7
|
+
export const DEFAULT_JOB_RUN_RETENTION_DAYS = 30;
|
|
8
|
+
|
|
9
|
+
export function createRetentionCleanupJob(retentionDays: number): JobHandlerFn {
|
|
10
|
+
return async (_payload, ctx) => {
|
|
11
|
+
if (!ctx.db) {
|
|
12
|
+
throw new InternalError({
|
|
13
|
+
message:
|
|
14
|
+
"[jobs:retention-cleanup] ctx.db missing — job context requires a database connection.",
|
|
15
|
+
});
|
|
16
|
+
}
|
|
17
|
+
const db = ctx.db as DbConnection; // @cast-boundary db-operator (matches sibling cron jobs)
|
|
18
|
+
const result = await deleteStaleJobRuns(db, retentionDays);
|
|
19
|
+
ctx.log?.info?.(`[jobs:retention-cleanup] complete: ${JSON.stringify(result)}`);
|
|
20
|
+
};
|
|
21
|
+
}
|
package/src/jobs/index.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export { createJobsFeature } from "./feature";
|
|
1
|
+
export { createJobsFeature, type JobsFeatureOptions } from "./feature";
|
|
2
2
|
export type { JobRunLoggerCallbacks } from "./job-run-logger";
|
|
3
3
|
export { createJobRunLogger } from "./job-run-logger";
|
|
4
4
|
export type { JobLogLevel, JobRunStatus } from "./job-run-table";
|
|
@@ -1,27 +1,26 @@
|
|
|
1
|
-
import { fetchOne } from "@cosmicdrift/kumiko-framework/bun-db";
|
|
1
|
+
import { fetchOne, insertMany, insertOne, updateMany } from "@cosmicdrift/kumiko-framework/bun-db";
|
|
2
|
+
import {
|
|
3
|
+
configuredPiiSubjectKms,
|
|
4
|
+
encryptPiiValueForSubject,
|
|
5
|
+
} from "@cosmicdrift/kumiko-framework/crypto";
|
|
2
6
|
import type { DbConnection } from "@cosmicdrift/kumiko-framework/db";
|
|
3
7
|
import { type Registry, SYSTEM_TENANT_ID } from "@cosmicdrift/kumiko-framework/engine";
|
|
4
|
-
import { append, getStreamVersion } from "@cosmicdrift/kumiko-framework/event-store";
|
|
5
8
|
import type { JobLogEntry, JobMeta, JobRunnerOptions } from "@cosmicdrift/kumiko-framework/jobs";
|
|
6
|
-
import { runProjectionsForEvent } from "@cosmicdrift/kumiko-framework/pipeline";
|
|
7
9
|
import { generateId } from "@cosmicdrift/kumiko-framework/utils";
|
|
8
10
|
import { runCompletedSchema, runFailedSchema, runStartedSchema } from "./events";
|
|
9
|
-
import {
|
|
11
|
+
import { parseJobInstant } from "./job-instant";
|
|
12
|
+
import { jobRunLogsTable, jobRunsTable } from "./job-run-table";
|
|
10
13
|
|
|
11
|
-
//
|
|
12
|
-
//
|
|
13
|
-
//
|
|
14
|
-
//
|
|
15
|
-
//
|
|
14
|
+
// Direct-write job-run log (#2243): onJobStart/-Complete/-Failed write
|
|
15
|
+
// straight into jobRunsTable / jobRunLogsTable instead of appending to the
|
|
16
|
+
// event store and replaying through inline projections. Pre-#2243 every run
|
|
17
|
+
// left two permanent `kumiko_events` rows that nothing else ever replayed
|
|
18
|
+
// or MSP-subscribed to — in two production apps that was ~99% of all
|
|
19
|
+
// events. Same tables, same shape, no event stream in between.
|
|
16
20
|
//
|
|
17
21
|
// BullMQ callbacks don't carry a tenantId (jobs are cross-tenant). We
|
|
18
22
|
// anchor every run on SYSTEM_TENANT_ID — mirrors how config system-scope
|
|
19
|
-
// rows use the sentinel.
|
|
20
|
-
// aggregate_id is a fresh UUID per run.
|
|
21
|
-
|
|
22
|
-
export const JOB_RUN_STARTED_EVENT = "jobs:event:run-started" as const;
|
|
23
|
-
export const JOB_RUN_COMPLETED_EVENT = "jobs:event:run-completed" as const;
|
|
24
|
-
export const JOB_RUN_FAILED_EVENT = "jobs:event:run-failed" as const;
|
|
23
|
+
// rows use the sentinel.
|
|
25
24
|
|
|
26
25
|
export type JobRunLoggerOptions = {
|
|
27
26
|
readonly db: DbConnection;
|
|
@@ -45,18 +44,40 @@ const DEFAULT_CACHE_MAX_ENTRIES = 10_000;
|
|
|
45
44
|
// to DB-lookup if actually needed.
|
|
46
45
|
const DEFAULT_CACHE_TTL_MS = 60 * 60 * 1000; // 1 hour
|
|
47
46
|
|
|
47
|
+
// The run-started payload can carry arbitrary user data; triggeredById
|
|
48
|
+
// names its owning user. No event-PII catalog involved (#2243 removed the
|
|
49
|
+
// jobRun r.defineEvent registrations, so there is nothing to catalog) —
|
|
50
|
+
// the subject is known statically, so we encrypt directly. A null subject
|
|
51
|
+
// (system cron runs, recipient-less triggers) stays plaintext: there is no
|
|
52
|
+
// user key to shred, mirroring the previous event-pii catalog's own skip
|
|
53
|
+
// rule. Absent KMS adapter stays plaintext too (rollout mode, unchanged).
|
|
54
|
+
async function encryptStartedPayload(
|
|
55
|
+
payload: string | null,
|
|
56
|
+
triggeredById: string | null,
|
|
57
|
+
): Promise<string | null> {
|
|
58
|
+
if (payload === null || triggeredById === null) return payload;
|
|
59
|
+
const kms = configuredPiiSubjectKms();
|
|
60
|
+
if (!kms) return payload;
|
|
61
|
+
return encryptPiiValueForSubject(
|
|
62
|
+
kms,
|
|
63
|
+
{ kind: "user", userId: triggeredById },
|
|
64
|
+
payload,
|
|
65
|
+
{ requestId: "jobs:job-run-logger" },
|
|
66
|
+
"payload",
|
|
67
|
+
);
|
|
68
|
+
}
|
|
69
|
+
|
|
48
70
|
export function createJobRunLogger(opts: JobRunLoggerOptions): JobRunLoggerCallbacks {
|
|
49
|
-
const { db
|
|
71
|
+
const { db } = opts;
|
|
50
72
|
|
|
51
|
-
// bullJobId →
|
|
52
|
-
//
|
|
53
|
-
//
|
|
54
|
-
//
|
|
55
|
-
// stream as the start-event.
|
|
73
|
+
// bullJobId → run uuid. BullMQ hands us the bullJobId on every callback,
|
|
74
|
+
// but the run row is keyed by a fresh UUID we mint on start. The cache
|
|
75
|
+
// threads that UUID from onJobStart through to onJobComplete/onJobFailed
|
|
76
|
+
// so the completion-write lands on the same row as the start-write.
|
|
56
77
|
//
|
|
57
78
|
// Bounded cache (LRU-ish with TTL) — worker-crash between start and
|
|
58
79
|
// complete would otherwise leak entries. DB-lookup recovers evicted
|
|
59
|
-
// entries via bull_job_id on
|
|
80
|
+
// entries via bull_job_id on jobRunsTable.
|
|
60
81
|
type CacheEntry = { readonly runId: string; readonly expiresAt: number };
|
|
61
82
|
const runIdByBullJobId = new Map<string, CacheEntry>();
|
|
62
83
|
|
|
@@ -106,7 +127,7 @@ export function createJobRunLogger(opts: JobRunLoggerOptions): JobRunLoggerCallb
|
|
|
106
127
|
// Parse against the registered schema so out-of-dispatcher writes
|
|
107
128
|
// get the same validation guarantee as ctx.appendEvent. A shape
|
|
108
129
|
// drift between feature + logger fails loudly at the source
|
|
109
|
-
// instead of silently landing on the
|
|
130
|
+
// instead of silently landing on the table.
|
|
110
131
|
const payload = runStartedSchema.parse({
|
|
111
132
|
jobName,
|
|
112
133
|
bullJobId,
|
|
@@ -116,16 +137,19 @@ export function createJobRunLogger(opts: JobRunLoggerOptions): JobRunLoggerCallb
|
|
|
116
137
|
startedAt: Temporal.Now.instant().toString(),
|
|
117
138
|
attempt: meta.attempt ?? 1,
|
|
118
139
|
});
|
|
119
|
-
const
|
|
120
|
-
|
|
121
|
-
|
|
140
|
+
const encryptedPayload = await encryptStartedPayload(payload.payload, payload.triggeredById);
|
|
141
|
+
await insertOne(db, jobRunsTable, {
|
|
142
|
+
id: runId,
|
|
122
143
|
tenantId: SYSTEM_TENANT_ID,
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
payload,
|
|
126
|
-
|
|
144
|
+
insertedById: "system",
|
|
145
|
+
jobName: payload.jobName,
|
|
146
|
+
bullJobId: payload.bullJobId,
|
|
147
|
+
status: payload.status,
|
|
148
|
+
payload: encryptedPayload,
|
|
149
|
+
attempt: payload.attempt,
|
|
150
|
+
startedAt: parseJobInstant(payload.startedAt),
|
|
151
|
+
triggeredById: payload.triggeredById,
|
|
127
152
|
});
|
|
128
|
-
await runProjectionsForEvent(event, registry, db);
|
|
129
153
|
},
|
|
130
154
|
|
|
131
155
|
onJobComplete: async (
|
|
@@ -137,10 +161,9 @@ export function createJobRunLogger(opts: JobRunLoggerOptions): JobRunLoggerCallb
|
|
|
137
161
|
const runId = await resolveRunId(bullJobId);
|
|
138
162
|
// skip: state loss between start + complete (worker restart, cache
|
|
139
163
|
// evicted AND DB has no matching bull_job_id). Rare edge case; we
|
|
140
|
-
// drop the completion
|
|
141
|
-
//
|
|
164
|
+
// drop the completion write rather than forging a run row from
|
|
165
|
+
// scratch — forensics still has the original BullMQ lifecycle.
|
|
142
166
|
if (!runId) return;
|
|
143
|
-
const currentVersion = await getStreamVersion(db, runId, SYSTEM_TENANT_ID);
|
|
144
167
|
const payload = runCompletedSchema.parse({
|
|
145
168
|
duration,
|
|
146
169
|
finishedAt: Temporal.Now.instant().toString(),
|
|
@@ -150,16 +173,32 @@ export function createJobRunLogger(opts: JobRunLoggerOptions): JobRunLoggerCallb
|
|
|
150
173
|
timestamp: l.timestamp.toString(),
|
|
151
174
|
})),
|
|
152
175
|
});
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
176
|
+
await updateMany(
|
|
177
|
+
db,
|
|
178
|
+
jobRunsTable,
|
|
179
|
+
{
|
|
180
|
+
status: "completed",
|
|
181
|
+
duration: payload.duration,
|
|
182
|
+
finishedAt: parseJobInstant(payload.finishedAt),
|
|
183
|
+
modifiedAt: Temporal.Now.instant(),
|
|
184
|
+
modifiedById: "system",
|
|
185
|
+
},
|
|
186
|
+
{ id: runId },
|
|
187
|
+
);
|
|
188
|
+
// skip: empty log batch — the worker ran silent. No child rows to
|
|
189
|
+
// insert; the status update above already recorded completion.
|
|
190
|
+
if (payload.logs.length > 0) {
|
|
191
|
+
await insertMany(
|
|
192
|
+
db,
|
|
193
|
+
jobRunLogsTable,
|
|
194
|
+
payload.logs.map((log) => ({
|
|
195
|
+
runId,
|
|
196
|
+
level: log.level,
|
|
197
|
+
message: log.message,
|
|
198
|
+
timestamp: parseJobInstant(log.timestamp),
|
|
199
|
+
})),
|
|
200
|
+
);
|
|
201
|
+
}
|
|
163
202
|
runIdByBullJobId.delete(bullJobId); // immediate cleanup on terminal callback
|
|
164
203
|
},
|
|
165
204
|
|
|
@@ -171,13 +210,11 @@ export function createJobRunLogger(opts: JobRunLoggerOptions): JobRunLoggerCallb
|
|
|
171
210
|
) => {
|
|
172
211
|
const runId = await resolveRunId(bullJobId);
|
|
173
212
|
// skip: same rare state-loss case as in onJobComplete — drop the
|
|
174
|
-
// failure
|
|
213
|
+
// failure write rather than forge a run row from scratch.
|
|
175
214
|
if (!runId) return;
|
|
176
|
-
|
|
177
|
-
// Read started_at off the projection so we can compute duration
|
|
215
|
+
// Read started_at off the row so we can compute duration
|
|
178
216
|
// symmetrically to onJobComplete (which gets duration from the
|
|
179
|
-
// worker). The
|
|
180
|
-
// run-started inline-apply.
|
|
217
|
+
// worker). The row already has started_at from onJobStart.
|
|
181
218
|
const row = await fetchOne<{ startedAt: Temporal.Instant }>(db, jobRunsTable, { id: runId });
|
|
182
219
|
const now = Temporal.Now.instant();
|
|
183
220
|
const duration = row ? Number(now.since(row.startedAt).total({ unit: "millisecond" })) : 0;
|
|
@@ -191,16 +228,32 @@ export function createJobRunLogger(opts: JobRunLoggerOptions): JobRunLoggerCallb
|
|
|
191
228
|
timestamp: l.timestamp.toString(),
|
|
192
229
|
})),
|
|
193
230
|
});
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
231
|
+
await updateMany(
|
|
232
|
+
db,
|
|
233
|
+
jobRunsTable,
|
|
234
|
+
{
|
|
235
|
+
status: "failed",
|
|
236
|
+
error: payload.error,
|
|
237
|
+
duration: payload.duration,
|
|
238
|
+
finishedAt: parseJobInstant(payload.finishedAt),
|
|
239
|
+
modifiedAt: now,
|
|
240
|
+
modifiedById: "system",
|
|
241
|
+
},
|
|
242
|
+
{ id: runId },
|
|
243
|
+
);
|
|
244
|
+
// skip: empty log batch — mirror of onJobComplete
|
|
245
|
+
if (payload.logs.length > 0) {
|
|
246
|
+
await insertMany(
|
|
247
|
+
db,
|
|
248
|
+
jobRunLogsTable,
|
|
249
|
+
payload.logs.map((log) => ({
|
|
250
|
+
runId,
|
|
251
|
+
level: log.level,
|
|
252
|
+
message: log.message,
|
|
253
|
+
timestamp: parseJobInstant(log.timestamp),
|
|
254
|
+
})),
|
|
255
|
+
);
|
|
256
|
+
}
|
|
204
257
|
runIdByBullJobId.delete(bullJobId); // immediate cleanup on terminal callback
|
|
205
258
|
},
|
|
206
259
|
};
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
|
-
buildEntityTable,
|
|
3
2
|
defineUnmanagedTable,
|
|
3
|
+
deriveEntityTableMeta,
|
|
4
4
|
type EntityTableMeta,
|
|
5
5
|
instant,
|
|
6
6
|
table as pgTable,
|
|
@@ -17,28 +17,29 @@ import {
|
|
|
17
17
|
export type JobRunStatus = "queued" | "running" | "completed" | "failed";
|
|
18
18
|
export type JobLogLevel = "info" | "warn" | "error";
|
|
19
19
|
|
|
20
|
-
// jobRun is a system-scoped
|
|
21
|
-
//
|
|
22
|
-
// -Complete / -Failed
|
|
23
|
-
//
|
|
24
|
-
//
|
|
25
|
-
//
|
|
26
|
-
//
|
|
20
|
+
// jobRun is a system-scoped direct-write store (#2243): every job execution
|
|
21
|
+
// writes straight into jobRunsTable / jobRunLogsTable from the BullMQ
|
|
22
|
+
// callbacks (onJobStart / -Complete / -Failed, see job-run-logger.ts) —
|
|
23
|
+
// no event-store detour. Pre-#2243 this was an events-only aggregate
|
|
24
|
+
// replayed through two inline projections; that generated two permanent
|
|
25
|
+
// `kumiko_events` rows per run for data that is itself already the
|
|
26
|
+
// system of record (no other consumer replays or MSP-subscribes to it).
|
|
27
27
|
//
|
|
28
|
-
// Logs
|
|
29
|
-
// design discussion: one
|
|
30
|
-
// no log duplication across status
|
|
31
|
-
// expands the batch into N rows in
|
|
32
|
-
//
|
|
28
|
+
// Logs are batched onto the completed/failed callback as an array —
|
|
29
|
+
// "Option B" from the original design discussion: one write per run
|
|
30
|
+
// instead of one write per log line, no log duplication across status
|
|
31
|
+
// transitions. job-run-logger.ts expands the batch into N rows in
|
|
32
|
+
// jobRunLogsTable.
|
|
33
33
|
//
|
|
34
|
-
// Entity-derived table
|
|
35
|
-
//
|
|
36
|
-
// the
|
|
37
|
-
//
|
|
38
|
-
//
|
|
39
|
-
//
|
|
34
|
+
// Entity-derived table (query API + migration meta share one field
|
|
35
|
+
// definition). status/$type<JobRunStatus> is not modeled in the entity
|
|
36
|
+
// schema — the column type is text, with the status union enforced at the
|
|
37
|
+
// app boundary (same pattern as template-resolver kind/scope).
|
|
38
|
+
// `table: "store_job_runs"` (not `read_*`) because this is no longer a
|
|
39
|
+
// rebuildable projection — `defineUnmanagedTable`/`deriveEntityTableMeta`
|
|
40
|
+
// reject the `read_` prefix for `source: "unmanaged"` (#1208/#1220).
|
|
40
41
|
export const jobRunEntity = createEntity({
|
|
41
|
-
table: "
|
|
42
|
+
table: "store_job_runs",
|
|
42
43
|
fields: {
|
|
43
44
|
jobName: createTextField({ required: true }),
|
|
44
45
|
bullJobId: createTextField({ required: true }),
|
|
@@ -53,7 +54,14 @@ export const jobRunEntity = createEntity({
|
|
|
53
54
|
},
|
|
54
55
|
});
|
|
55
56
|
|
|
56
|
-
|
|
57
|
+
// Plain EntityTableMeta, NOT a branded EntityTable (buildEntityTable would
|
|
58
|
+
// mark it executor-only): job-run is an unmanaged direct-write store, so
|
|
59
|
+
// onJobStart/-Complete/-Failed need to write via ctx.db/insertOne directly
|
|
60
|
+
// (same pattern as sessions/schema/user-session.ts).
|
|
61
|
+
export const jobRunsTable: EntityTableMeta = deriveEntityTableMeta("job-run", jobRunEntity, {
|
|
62
|
+
source: "unmanaged",
|
|
63
|
+
});
|
|
64
|
+
export const jobRunsTableMeta = jobRunsTable;
|
|
57
65
|
|
|
58
66
|
// Child projection keyed by the jobRun aggregate id. Pre-ES used a serial
|
|
59
67
|
// PK + integer runId; post-ES runId is still exposed but now holds the
|
|
@@ -68,13 +76,13 @@ export const jobRunLogsTable = pgTable("store_job_run_logs", {
|
|
|
68
76
|
timestamp: instant("timestamp").notNull(),
|
|
69
77
|
});
|
|
70
78
|
|
|
71
|
-
// **Unmanaged table** —
|
|
72
|
-
// - serial PK (
|
|
73
|
-
// -
|
|
74
|
-
// -
|
|
75
|
-
//
|
|
76
|
-
// pgTable
|
|
77
|
-
//
|
|
79
|
+
// **Unmanaged table** — deliberately no createEntity. Reasoning:
|
|
80
|
+
// - serial PK (not uuid) — pre-ES legacy, compatible with existing rows
|
|
81
|
+
// - no tenant_id — child table of jobRun, tenant context lives on the parent
|
|
82
|
+
// - no base columns (no version/inserted_at/inserted_by_id) — append-only
|
|
83
|
+
// log, no in-place update, no audit columns needed
|
|
84
|
+
// pgTable stays the source of truth for the query API; this meta mirrors it
|
|
85
|
+
// for migration generation.
|
|
78
86
|
export const jobRunLogsTableMeta: EntityTableMeta = defineUnmanagedTable({
|
|
79
87
|
tableName: "store_job_run_logs",
|
|
80
88
|
columns: [
|
|
@@ -134,8 +134,8 @@ export type UserDataRightsOptions = {
|
|
|
134
134
|
};
|
|
135
135
|
|
|
136
136
|
export function createUserDataRightsFeature(opts: UserDataRightsOptions = {}): FeatureDefinition {
|
|
137
|
-
// One-shot operator warning (the export cron fires
|
|
138
|
-
//
|
|
137
|
+
// One-shot operator warning (the export cron fires daily — warn once per
|
|
138
|
+
// process, not every run). Lives in the factory scope so the cron closure
|
|
139
139
|
// shares it across runs.
|
|
140
140
|
let warnedMissingExportUrl = false;
|
|
141
141
|
return defineFeature("user-data-rights", (r) => {
|
|
@@ -401,7 +401,7 @@ export function createUserDataRightsFeature(opts: UserDataRightsOptions = {}): F
|
|
|
401
401
|
// S2.U3 Atom 3b — Worker fuer Async Export-Pipeline. Cron-getriggert.
|
|
402
402
|
r.job(
|
|
403
403
|
"run-export-jobs",
|
|
404
|
-
{ trigger: { cron: "0
|
|
404
|
+
{ trigger: { cron: "0 3 * * *" }, concurrency: "skip" },
|
|
405
405
|
async (_payload, ctx) => {
|
|
406
406
|
if (!ctx.db || !ctx.registry) {
|
|
407
407
|
throw new Error(
|
|
@@ -488,7 +488,7 @@ export function createUserDataRightsFeature(opts: UserDataRightsOptions = {}): F
|
|
|
488
488
|
// — Art.17 wuerde nie ausgefuehrt.
|
|
489
489
|
r.job(
|
|
490
490
|
"run-forget-cleanup",
|
|
491
|
-
{ trigger: { cron: "0
|
|
491
|
+
{ trigger: { cron: "0 3 * * *" }, concurrency: "skip" },
|
|
492
492
|
async (_payload, ctx) => {
|
|
493
493
|
if (!ctx.db || !ctx.registry) {
|
|
494
494
|
throw new Error(
|
package/src/user-data-rights-defaults/__tests__/bundled-entities.userdata-hooks.integration.test.ts
CHANGED
|
@@ -401,7 +401,7 @@ describe("delivery-attempt userData-hooks (#799)", () => {
|
|
|
401
401
|
describe("job-run userData-hooks (#799)", () => {
|
|
402
402
|
async function seedRun(id: string, triggeredById: string | null): Promise<void> {
|
|
403
403
|
await asRawClient(full.db).unsafe(
|
|
404
|
-
`INSERT INTO
|
|
404
|
+
`INSERT INTO store_job_runs
|
|
405
405
|
(id, tenant_id, job_name, bull_job_id, status, payload, attempt, started_at, triggered_by_id)
|
|
406
406
|
VALUES ($1::uuid, $2, 'app:job:export', $3, 'completed', '{"scope":"mine"}', 1, now(), $4)`,
|
|
407
407
|
[id, TENANT_A, `bull-${id}`, triggeredById],
|