@cosmicdrift/kumiko-bundled-features 0.209.0 → 0.210.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +8 -8
- package/src/auth-mfa/schema/user-mfa.ts +8 -5
- package/src/auth-mfa-user-data/hooks.ts +9 -8
- package/src/billing-foundation/__tests__/billing-foundation.integration.test.ts +2 -2
- package/src/billing-foundation/entities.ts +5 -3
- package/src/billing-foundation/tenant-destroy-hook.ts +1 -1
- package/src/compliance-profiles/schema/profile-selection.ts +2 -1
- package/src/config/table.ts +4 -1
- package/src/crypto-shredding/__tests__/forget-subject.integration.test.ts +4 -4
- package/src/custom-fields/feature.ts +2 -2
- package/src/data-retention/__tests__/retention-cleanup.integration.test.ts +12 -4
- package/src/data-retention/schema/tenant-retention-override.ts +6 -3
- package/src/delivery/tables.ts +1 -1
- package/src/document-ingest-foundation/entity.ts +6 -3
- package/src/files/README.md +11 -11
- package/src/form-draft/entity.ts +6 -3
- package/src/inbound-mail-foundation/entities.ts +13 -11
- package/src/inbound-mail-foundation/feature.ts +5 -5
- package/src/jobs/__tests__/jobs-events.integration.test.ts +65 -81
- package/src/jobs/__tests__/jobs-pii-kms.integration.test.ts +21 -18
- package/src/jobs/__tests__/jobs-retention.integration.test.ts +124 -0
- package/src/jobs/db/queries/retention.ts +44 -0
- package/src/jobs/events.ts +5 -8
- package/src/jobs/feature.ts +26 -146
- package/src/jobs/handlers/retention-cleanup.job.ts +21 -0
- package/src/jobs/index.ts +1 -1
- package/src/jobs/job-run-logger.ts +112 -59
- package/src/jobs/job-run-table.ts +36 -28
- package/src/ledger/entity.ts +5 -3
- package/src/managed-pages/screens/branding-screen.ts +2 -1
- package/src/managed-pages/table.ts +3 -2
- package/src/notes-history/entity.ts +8 -4
- package/src/notes-history-user-data/hooks.ts +6 -5
- package/src/personal-access-tokens/schema/api-token.ts +2 -1
- package/src/sessions/feature.ts +0 -4
- package/src/sessions/schema/user-session.ts +4 -2
- package/src/tags/entity.ts +11 -5
- package/src/template-resolver/__tests__/collection-ownership.integration.test.ts +1 -1
- package/src/template-resolver/table.ts +4 -1
- package/src/template-resolver/user-content-table.ts +5 -2
- package/src/template-resolver-user-data/hooks.ts +6 -5
- package/src/tenant/invitation-table.ts +3 -3
- package/src/tenant-lifecycle/boot-checks.ts +2 -2
- package/src/tenant-settings/__tests__/settings-hub-i18n.test.ts +58 -0
- package/src/tenant-settings/feature.ts +12 -0
- package/src/user/schema/user.ts +5 -5
- package/src/user-data-rights/__tests__/boot-checks.test.ts +73 -9
- package/src/user-data-rights/__tests__/forget-cleanup-search-purge-ordering.integration.test.ts +2 -2
- package/src/user-data-rights/boot-checks.ts +2 -2
- package/src/user-data-rights/feature.ts +4 -4
- package/src/user-data-rights/schema/download-token.ts +12 -10
- package/src/user-data-rights/schema/export-job.ts +5 -4
- package/src/user-data-rights-defaults/__tests__/bundled-entities.userdata-hooks.integration.test.ts +1 -1
package/src/jobs/feature.ts
CHANGED
|
@@ -1,16 +1,5 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
defineApply,
|
|
4
|
-
defineFeature,
|
|
5
|
-
type FeatureDefinition,
|
|
6
|
-
} from "@cosmicdrift/kumiko-framework/engine";
|
|
7
|
-
import type { z } from "zod";
|
|
1
|
+
import { defineFeature, type FeatureDefinition } from "@cosmicdrift/kumiko-framework/engine";
|
|
8
2
|
import { JOB_RUN_DETAIL_SCREEN_ID, JOB_RUNS_SCREEN_ID } from "./constants";
|
|
9
|
-
// Event-payload schemas live in a sibling module so the logger can import
|
|
10
|
-
// them without the cycle jobs-feature ↔ job-run-logger. The logger parses
|
|
11
|
-
// payloads against these schemas before low-level append() — that's what
|
|
12
|
-
// keeps out-of-dispatcher writes as type-safe as ctx.appendEvent.
|
|
13
|
-
import { runCompletedSchema, runFailedSchema, runStartedSchema } from "./events";
|
|
14
3
|
import { catalogQuery } from "./handlers/catalog.query";
|
|
15
4
|
import { detailQuery } from "./handlers/detail.query";
|
|
16
5
|
import { listQuery } from "./handlers/list.query";
|
|
@@ -19,21 +8,26 @@ import {
|
|
|
19
8
|
projectionRebuildPayloadSchema,
|
|
20
9
|
} from "./handlers/projection-rebuild.job";
|
|
21
10
|
import { reindexEntityJob, reindexEntityPayloadSchema } from "./handlers/reindex-entity.job";
|
|
11
|
+
import {
|
|
12
|
+
createRetentionCleanupJob,
|
|
13
|
+
DEFAULT_JOB_RUN_RETENTION_DAYS,
|
|
14
|
+
} from "./handlers/retention-cleanup.job";
|
|
22
15
|
import { retryWrite } from "./handlers/retry.write";
|
|
23
16
|
import { triggerWrite } from "./handlers/trigger.write";
|
|
24
17
|
import { JOBS_I18N } from "./i18n";
|
|
25
|
-
import {
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
18
|
+
import { jobRunLogsTableMeta, jobRunsTableMeta } from "./job-run-table";
|
|
19
|
+
|
|
20
|
+
export type JobsFeatureOptions = {
|
|
21
|
+
// How long a job run (and its logs) stays in store_job_runs/
|
|
22
|
+
// store_job_run_logs before the daily retention-cleanup job deletes it.
|
|
23
|
+
readonly retentionDays?: number;
|
|
24
|
+
};
|
|
32
25
|
|
|
33
|
-
export function createJobsFeature(): FeatureDefinition {
|
|
26
|
+
export function createJobsFeature(options: JobsFeatureOptions = {}): FeatureDefinition {
|
|
27
|
+
const retentionDays = options.retentionDays ?? DEFAULT_JOB_RUN_RETENTION_DAYS;
|
|
34
28
|
return defineFeature("jobs", (r) => {
|
|
35
29
|
r.describe(
|
|
36
|
-
"Persistence and operator tooling for background jobs registered via `r.job(...)`. Every job execution
|
|
30
|
+
"Persistence and operator tooling for background jobs registered via `r.job(...)`. Every job execution writes directly into `store_job_runs` (current status + duration) and `store_job_run_logs` (per-line log rows) from the BullMQ callbacks — no event stream in between (#2243). A daily `retention-cleanup` job deletes runs (and their logs) older than `retentionDays`. Exposes `jobs:write:trigger` (manual run) and `jobs:write:retry` (operator retry of a failed run), plus `jobs:query:list`, `jobs:query:details`, and `jobs:query:catalog` (manual jobs) for the operator UI.",
|
|
37
31
|
);
|
|
38
32
|
r.uiHints({
|
|
39
33
|
displayLabel: "Jobs · Audit & Operator UI",
|
|
@@ -41,134 +35,12 @@ export function createJobsFeature(): FeatureDefinition {
|
|
|
41
35
|
recommended: false,
|
|
42
36
|
});
|
|
43
37
|
r.systemScope();
|
|
38
|
+
r.storeTable(jobRunsTableMeta, {
|
|
39
|
+
reason: "direct_write.job_runs",
|
|
40
|
+
});
|
|
44
41
|
r.storeTable(jobRunLogsTableMeta, {
|
|
45
42
|
reason: "read_side.job_run_logs",
|
|
46
43
|
});
|
|
47
|
-
// Events-only aggregate: "jobRun" has no r.entity registration, because
|
|
48
|
-
// the entire lifecycle is driven by BullMQ-callback → r.defineEvent
|
|
49
|
-
// (no executor, no CRUD). The boot-validator accepts the two
|
|
50
|
-
// projections below because every apply-key is a registered
|
|
51
|
-
// domain-event.
|
|
52
|
-
// payload can carry arbitrary user data; triggeredById stays plaintext
|
|
53
|
-
// (pseudonymous fk). System runs (triggeredById null) stay plaintext —
|
|
54
|
-
// no user subject to shred.
|
|
55
|
-
r.defineEvent("run-started", runStartedSchema, {
|
|
56
|
-
piiFields: { payload: { subjectField: "triggeredById" } },
|
|
57
|
-
});
|
|
58
|
-
r.defineEvent("run-completed", runCompletedSchema);
|
|
59
|
-
r.defineEvent("run-failed", runFailedSchema);
|
|
60
|
-
|
|
61
|
-
// Inline projection: status-row in jobRunsTable. Runs in same TX as
|
|
62
|
-
// the event-append (the logger calls runProjectionsForEvent manually
|
|
63
|
-
// because the BullMQ-callback path has no dispatcher-ctx).
|
|
64
|
-
r.projection({
|
|
65
|
-
name: "job-runs",
|
|
66
|
-
source: "jobRun",
|
|
67
|
-
table: jobRunsTable,
|
|
68
|
-
apply: {
|
|
69
|
-
[JOB_RUN_STARTED_EVENT]: defineApply<z.infer<typeof runStartedSchema>>(
|
|
70
|
-
async (event, tx, table) => {
|
|
71
|
-
const p = event.payload;
|
|
72
|
-
await insertOne(tx, table, {
|
|
73
|
-
id: event.aggregateId,
|
|
74
|
-
tenantId: event.tenantId,
|
|
75
|
-
version: event.version,
|
|
76
|
-
insertedAt: event.createdAt,
|
|
77
|
-
insertedById: event.metadata?.userId ?? "system",
|
|
78
|
-
jobName: p.jobName,
|
|
79
|
-
bullJobId: p.bullJobId,
|
|
80
|
-
status: p.status,
|
|
81
|
-
payload: p.payload,
|
|
82
|
-
attempt: p.attempt,
|
|
83
|
-
startedAt: parseJobInstant(p.startedAt),
|
|
84
|
-
triggeredById: p.triggeredById,
|
|
85
|
-
});
|
|
86
|
-
},
|
|
87
|
-
),
|
|
88
|
-
[JOB_RUN_COMPLETED_EVENT]: defineApply<z.infer<typeof runCompletedSchema>>(
|
|
89
|
-
async (event, tx, table) => {
|
|
90
|
-
const p = event.payload;
|
|
91
|
-
await updateMany(
|
|
92
|
-
tx,
|
|
93
|
-
table,
|
|
94
|
-
{
|
|
95
|
-
status: "completed",
|
|
96
|
-
duration: p.duration,
|
|
97
|
-
finishedAt: parseJobInstant(p.finishedAt),
|
|
98
|
-
version: event.version,
|
|
99
|
-
modifiedAt: event.createdAt,
|
|
100
|
-
modifiedById: event.metadata?.userId ?? "system",
|
|
101
|
-
},
|
|
102
|
-
{ id: event.aggregateId },
|
|
103
|
-
);
|
|
104
|
-
},
|
|
105
|
-
),
|
|
106
|
-
[JOB_RUN_FAILED_EVENT]: defineApply<z.infer<typeof runFailedSchema>>(
|
|
107
|
-
async (event, tx, table) => {
|
|
108
|
-
const p = event.payload;
|
|
109
|
-
await updateMany(
|
|
110
|
-
tx,
|
|
111
|
-
table,
|
|
112
|
-
{
|
|
113
|
-
status: "failed",
|
|
114
|
-
error: p.error,
|
|
115
|
-
duration: p.duration,
|
|
116
|
-
finishedAt: parseJobInstant(p.finishedAt),
|
|
117
|
-
version: event.version,
|
|
118
|
-
modifiedAt: event.createdAt,
|
|
119
|
-
modifiedById: event.metadata?.userId ?? "system",
|
|
120
|
-
},
|
|
121
|
-
{ id: event.aggregateId },
|
|
122
|
-
);
|
|
123
|
-
},
|
|
124
|
-
),
|
|
125
|
-
},
|
|
126
|
-
});
|
|
127
|
-
|
|
128
|
-
// Second inline projection — same source, different table. Expands
|
|
129
|
-
// the batched logs array from completed/failed events into N rows
|
|
130
|
-
// per run in jobRunLogsTable.
|
|
131
|
-
r.projection({
|
|
132
|
-
name: "job-run-logs",
|
|
133
|
-
source: "jobRun",
|
|
134
|
-
table: jobRunLogsTable,
|
|
135
|
-
apply: {
|
|
136
|
-
[JOB_RUN_COMPLETED_EVENT]: defineApply<z.infer<typeof runCompletedSchema>>(
|
|
137
|
-
async (event, tx) => {
|
|
138
|
-
const p = event.payload;
|
|
139
|
-
// skip: empty log batch — the worker ran silent. No child rows
|
|
140
|
-
// to insert; the completed-event alone already updated the run's
|
|
141
|
-
// status via the sibling job-runs projection.
|
|
142
|
-
if (p.logs.length === 0) return;
|
|
143
|
-
await insertMany(
|
|
144
|
-
tx,
|
|
145
|
-
jobRunLogsTable,
|
|
146
|
-
p.logs.map((log) => ({
|
|
147
|
-
runId: event.aggregateId,
|
|
148
|
-
level: log.level,
|
|
149
|
-
message: log.message,
|
|
150
|
-
timestamp: parseJobInstant(log.timestamp),
|
|
151
|
-
})),
|
|
152
|
-
);
|
|
153
|
-
},
|
|
154
|
-
),
|
|
155
|
-
[JOB_RUN_FAILED_EVENT]: defineApply<z.infer<typeof runFailedSchema>>(async (event, tx) => {
|
|
156
|
-
const p = event.payload;
|
|
157
|
-
// skip: empty log batch — the worker ran silent (mirror of completed)
|
|
158
|
-
if (p.logs.length === 0) return;
|
|
159
|
-
await insertMany(
|
|
160
|
-
tx,
|
|
161
|
-
jobRunLogsTable,
|
|
162
|
-
p.logs.map((log) => ({
|
|
163
|
-
runId: event.aggregateId,
|
|
164
|
-
level: log.level,
|
|
165
|
-
message: log.message,
|
|
166
|
-
timestamp: parseJobInstant(log.timestamp),
|
|
167
|
-
})),
|
|
168
|
-
);
|
|
169
|
-
}),
|
|
170
|
-
},
|
|
171
|
-
});
|
|
172
44
|
|
|
173
45
|
// Framework-provided rebuild job — available whenever `jobs` is composed; enqueueProjectionRebuild dispatches it.
|
|
174
46
|
r.job(
|
|
@@ -187,6 +59,14 @@ export function createJobsFeature(): FeatureDefinition {
|
|
|
187
59
|
reindexEntityJob,
|
|
188
60
|
);
|
|
189
61
|
|
|
62
|
+
// store_job_runs/store_job_run_logs are direct-write, unbounded-growth
|
|
63
|
+
// stores (#2243) — nothing else purges them.
|
|
64
|
+
r.job(
|
|
65
|
+
"retention-cleanup",
|
|
66
|
+
{ trigger: { cron: "0 3 * * *" }, concurrency: "skip" },
|
|
67
|
+
createRetentionCleanupJob(retentionDays),
|
|
68
|
+
);
|
|
69
|
+
|
|
190
70
|
const handlers = {
|
|
191
71
|
trigger: r.writeHandler(triggerWrite),
|
|
192
72
|
retry: r.writeHandler(retryWrite),
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import type { DbConnection } from "@cosmicdrift/kumiko-framework/db";
|
|
2
|
+
import type { JobHandlerFn } from "@cosmicdrift/kumiko-framework/engine";
|
|
3
|
+
import { InternalError } from "@cosmicdrift/kumiko-framework/errors";
|
|
4
|
+
import { deleteStaleJobRuns } from "../db/queries/retention";
|
|
5
|
+
|
|
6
|
+
// Single source for the retention window — change here, nowhere else.
|
|
7
|
+
export const DEFAULT_JOB_RUN_RETENTION_DAYS = 30;
|
|
8
|
+
|
|
9
|
+
export function createRetentionCleanupJob(retentionDays: number): JobHandlerFn {
|
|
10
|
+
return async (_payload, ctx) => {
|
|
11
|
+
if (!ctx.db) {
|
|
12
|
+
throw new InternalError({
|
|
13
|
+
message:
|
|
14
|
+
"[jobs:retention-cleanup] ctx.db missing — job context requires a database connection.",
|
|
15
|
+
});
|
|
16
|
+
}
|
|
17
|
+
const db = ctx.db as DbConnection; // @cast-boundary db-operator (matches sibling cron jobs)
|
|
18
|
+
const result = await deleteStaleJobRuns(db, retentionDays);
|
|
19
|
+
ctx.log?.info?.(`[jobs:retention-cleanup] complete: ${JSON.stringify(result)}`);
|
|
20
|
+
};
|
|
21
|
+
}
|
package/src/jobs/index.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export { createJobsFeature } from "./feature";
|
|
1
|
+
export { createJobsFeature, type JobsFeatureOptions } from "./feature";
|
|
2
2
|
export type { JobRunLoggerCallbacks } from "./job-run-logger";
|
|
3
3
|
export { createJobRunLogger } from "./job-run-logger";
|
|
4
4
|
export type { JobLogLevel, JobRunStatus } from "./job-run-table";
|
|
@@ -1,27 +1,26 @@
|
|
|
1
|
-
import { fetchOne } from "@cosmicdrift/kumiko-framework/bun-db";
|
|
1
|
+
import { fetchOne, insertMany, insertOne, updateMany } from "@cosmicdrift/kumiko-framework/bun-db";
|
|
2
|
+
import {
|
|
3
|
+
configuredPiiSubjectKms,
|
|
4
|
+
encryptPiiValueForSubject,
|
|
5
|
+
} from "@cosmicdrift/kumiko-framework/crypto";
|
|
2
6
|
import type { DbConnection } from "@cosmicdrift/kumiko-framework/db";
|
|
3
7
|
import { type Registry, SYSTEM_TENANT_ID } from "@cosmicdrift/kumiko-framework/engine";
|
|
4
|
-
import { append, getStreamVersion } from "@cosmicdrift/kumiko-framework/event-store";
|
|
5
8
|
import type { JobLogEntry, JobMeta, JobRunnerOptions } from "@cosmicdrift/kumiko-framework/jobs";
|
|
6
|
-
import { runProjectionsForEvent } from "@cosmicdrift/kumiko-framework/pipeline";
|
|
7
9
|
import { generateId } from "@cosmicdrift/kumiko-framework/utils";
|
|
8
10
|
import { runCompletedSchema, runFailedSchema, runStartedSchema } from "./events";
|
|
9
|
-
import {
|
|
11
|
+
import { parseJobInstant } from "./job-instant";
|
|
12
|
+
import { jobRunLogsTable, jobRunsTable } from "./job-run-table";
|
|
10
13
|
|
|
11
|
-
//
|
|
12
|
-
//
|
|
13
|
-
//
|
|
14
|
-
//
|
|
15
|
-
//
|
|
14
|
+
// Direct-write job-run log (#2243): onJobStart/-Complete/-Failed write
|
|
15
|
+
// straight into jobRunsTable / jobRunLogsTable instead of appending to the
|
|
16
|
+
// event store and replaying through inline projections. Pre-#2243 every run
|
|
17
|
+
// left two permanent `kumiko_events` rows that nothing else ever replayed
|
|
18
|
+
// or MSP-subscribed to — in two production apps that was ~99% of all
|
|
19
|
+
// events. Same tables, same shape, no event stream in between.
|
|
16
20
|
//
|
|
17
21
|
// BullMQ callbacks don't carry a tenantId (jobs are cross-tenant). We
|
|
18
22
|
// anchor every run on SYSTEM_TENANT_ID — mirrors how config system-scope
|
|
19
|
-
// rows use the sentinel.
|
|
20
|
-
// aggregate_id is a fresh UUID per run.
|
|
21
|
-
|
|
22
|
-
export const JOB_RUN_STARTED_EVENT = "jobs:event:run-started" as const;
|
|
23
|
-
export const JOB_RUN_COMPLETED_EVENT = "jobs:event:run-completed" as const;
|
|
24
|
-
export const JOB_RUN_FAILED_EVENT = "jobs:event:run-failed" as const;
|
|
23
|
+
// rows use the sentinel.
|
|
25
24
|
|
|
26
25
|
export type JobRunLoggerOptions = {
|
|
27
26
|
readonly db: DbConnection;
|
|
@@ -45,18 +44,40 @@ const DEFAULT_CACHE_MAX_ENTRIES = 10_000;
|
|
|
45
44
|
// to DB-lookup if actually needed.
|
|
46
45
|
const DEFAULT_CACHE_TTL_MS = 60 * 60 * 1000; // 1 hour
|
|
47
46
|
|
|
47
|
+
// The run-started payload can carry arbitrary user data; triggeredById
|
|
48
|
+
// names its owning user. No event-PII catalog involved (#2243 removed the
|
|
49
|
+
// jobRun r.defineEvent registrations, so there is nothing to catalog) —
|
|
50
|
+
// the subject is known statically, so we encrypt directly. A null subject
|
|
51
|
+
// (system cron runs, recipient-less triggers) stays plaintext: there is no
|
|
52
|
+
// user key to shred, mirroring the previous event-pii catalog's own skip
|
|
53
|
+
// rule. Absent KMS adapter stays plaintext too (rollout mode, unchanged).
|
|
54
|
+
async function encryptStartedPayload(
|
|
55
|
+
payload: string | null,
|
|
56
|
+
triggeredById: string | null,
|
|
57
|
+
): Promise<string | null> {
|
|
58
|
+
if (payload === null || triggeredById === null) return payload;
|
|
59
|
+
const kms = configuredPiiSubjectKms();
|
|
60
|
+
if (!kms) return payload;
|
|
61
|
+
return encryptPiiValueForSubject(
|
|
62
|
+
kms,
|
|
63
|
+
{ kind: "user", userId: triggeredById },
|
|
64
|
+
payload,
|
|
65
|
+
{ requestId: "jobs:job-run-logger" },
|
|
66
|
+
"payload",
|
|
67
|
+
);
|
|
68
|
+
}
|
|
69
|
+
|
|
48
70
|
export function createJobRunLogger(opts: JobRunLoggerOptions): JobRunLoggerCallbacks {
|
|
49
|
-
const { db
|
|
71
|
+
const { db } = opts;
|
|
50
72
|
|
|
51
|
-
// bullJobId →
|
|
52
|
-
//
|
|
53
|
-
//
|
|
54
|
-
//
|
|
55
|
-
// stream as the start-event.
|
|
73
|
+
// bullJobId → run uuid. BullMQ hands us the bullJobId on every callback,
|
|
74
|
+
// but the run row is keyed by a fresh UUID we mint on start. The cache
|
|
75
|
+
// threads that UUID from onJobStart through to onJobComplete/onJobFailed
|
|
76
|
+
// so the completion-write lands on the same row as the start-write.
|
|
56
77
|
//
|
|
57
78
|
// Bounded cache (LRU-ish with TTL) — worker-crash between start and
|
|
58
79
|
// complete would otherwise leak entries. DB-lookup recovers evicted
|
|
59
|
-
// entries via bull_job_id on
|
|
80
|
+
// entries via bull_job_id on jobRunsTable.
|
|
60
81
|
type CacheEntry = { readonly runId: string; readonly expiresAt: number };
|
|
61
82
|
const runIdByBullJobId = new Map<string, CacheEntry>();
|
|
62
83
|
|
|
@@ -106,7 +127,7 @@ export function createJobRunLogger(opts: JobRunLoggerOptions): JobRunLoggerCallb
|
|
|
106
127
|
// Parse against the registered schema so out-of-dispatcher writes
|
|
107
128
|
// get the same validation guarantee as ctx.appendEvent. A shape
|
|
108
129
|
// drift between feature + logger fails loudly at the source
|
|
109
|
-
// instead of silently landing on the
|
|
130
|
+
// instead of silently landing on the table.
|
|
110
131
|
const payload = runStartedSchema.parse({
|
|
111
132
|
jobName,
|
|
112
133
|
bullJobId,
|
|
@@ -116,16 +137,19 @@ export function createJobRunLogger(opts: JobRunLoggerOptions): JobRunLoggerCallb
|
|
|
116
137
|
startedAt: Temporal.Now.instant().toString(),
|
|
117
138
|
attempt: meta.attempt ?? 1,
|
|
118
139
|
});
|
|
119
|
-
const
|
|
120
|
-
|
|
121
|
-
|
|
140
|
+
const encryptedPayload = await encryptStartedPayload(payload.payload, payload.triggeredById);
|
|
141
|
+
await insertOne(db, jobRunsTable, {
|
|
142
|
+
id: runId,
|
|
122
143
|
tenantId: SYSTEM_TENANT_ID,
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
payload,
|
|
126
|
-
|
|
144
|
+
insertedById: "system",
|
|
145
|
+
jobName: payload.jobName,
|
|
146
|
+
bullJobId: payload.bullJobId,
|
|
147
|
+
status: payload.status,
|
|
148
|
+
payload: encryptedPayload,
|
|
149
|
+
attempt: payload.attempt,
|
|
150
|
+
startedAt: parseJobInstant(payload.startedAt),
|
|
151
|
+
triggeredById: payload.triggeredById,
|
|
127
152
|
});
|
|
128
|
-
await runProjectionsForEvent(event, registry, db);
|
|
129
153
|
},
|
|
130
154
|
|
|
131
155
|
onJobComplete: async (
|
|
@@ -137,10 +161,9 @@ export function createJobRunLogger(opts: JobRunLoggerOptions): JobRunLoggerCallb
|
|
|
137
161
|
const runId = await resolveRunId(bullJobId);
|
|
138
162
|
// skip: state loss between start + complete (worker restart, cache
|
|
139
163
|
// evicted AND DB has no matching bull_job_id). Rare edge case; we
|
|
140
|
-
// drop the completion
|
|
141
|
-
//
|
|
164
|
+
// drop the completion write rather than forging a run row from
|
|
165
|
+
// scratch — forensics still has the original BullMQ lifecycle.
|
|
142
166
|
if (!runId) return;
|
|
143
|
-
const currentVersion = await getStreamVersion(db, runId, SYSTEM_TENANT_ID);
|
|
144
167
|
const payload = runCompletedSchema.parse({
|
|
145
168
|
duration,
|
|
146
169
|
finishedAt: Temporal.Now.instant().toString(),
|
|
@@ -150,16 +173,32 @@ export function createJobRunLogger(opts: JobRunLoggerOptions): JobRunLoggerCallb
|
|
|
150
173
|
timestamp: l.timestamp.toString(),
|
|
151
174
|
})),
|
|
152
175
|
});
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
176
|
+
await updateMany(
|
|
177
|
+
db,
|
|
178
|
+
jobRunsTable,
|
|
179
|
+
{
|
|
180
|
+
status: "completed",
|
|
181
|
+
duration: payload.duration,
|
|
182
|
+
finishedAt: parseJobInstant(payload.finishedAt),
|
|
183
|
+
modifiedAt: Temporal.Now.instant(),
|
|
184
|
+
modifiedById: "system",
|
|
185
|
+
},
|
|
186
|
+
{ id: runId },
|
|
187
|
+
);
|
|
188
|
+
// skip: empty log batch — the worker ran silent. No child rows to
|
|
189
|
+
// insert; the status update above already recorded completion.
|
|
190
|
+
if (payload.logs.length > 0) {
|
|
191
|
+
await insertMany(
|
|
192
|
+
db,
|
|
193
|
+
jobRunLogsTable,
|
|
194
|
+
payload.logs.map((log) => ({
|
|
195
|
+
runId,
|
|
196
|
+
level: log.level,
|
|
197
|
+
message: log.message,
|
|
198
|
+
timestamp: parseJobInstant(log.timestamp),
|
|
199
|
+
})),
|
|
200
|
+
);
|
|
201
|
+
}
|
|
163
202
|
runIdByBullJobId.delete(bullJobId); // immediate cleanup on terminal callback
|
|
164
203
|
},
|
|
165
204
|
|
|
@@ -171,13 +210,11 @@ export function createJobRunLogger(opts: JobRunLoggerOptions): JobRunLoggerCallb
|
|
|
171
210
|
) => {
|
|
172
211
|
const runId = await resolveRunId(bullJobId);
|
|
173
212
|
// skip: same rare state-loss case as in onJobComplete — drop the
|
|
174
|
-
// failure
|
|
213
|
+
// failure write rather than forge a run row from scratch.
|
|
175
214
|
if (!runId) return;
|
|
176
|
-
|
|
177
|
-
// Read started_at off the projection so we can compute duration
|
|
215
|
+
// Read started_at off the row so we can compute duration
|
|
178
216
|
// symmetrically to onJobComplete (which gets duration from the
|
|
179
|
-
// worker). The
|
|
180
|
-
// run-started inline-apply.
|
|
217
|
+
// worker). The row already has started_at from onJobStart.
|
|
181
218
|
const row = await fetchOne<{ startedAt: Temporal.Instant }>(db, jobRunsTable, { id: runId });
|
|
182
219
|
const now = Temporal.Now.instant();
|
|
183
220
|
const duration = row ? Number(now.since(row.startedAt).total({ unit: "millisecond" })) : 0;
|
|
@@ -191,16 +228,32 @@ export function createJobRunLogger(opts: JobRunLoggerOptions): JobRunLoggerCallb
|
|
|
191
228
|
timestamp: l.timestamp.toString(),
|
|
192
229
|
})),
|
|
193
230
|
});
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
231
|
+
await updateMany(
|
|
232
|
+
db,
|
|
233
|
+
jobRunsTable,
|
|
234
|
+
{
|
|
235
|
+
status: "failed",
|
|
236
|
+
error: payload.error,
|
|
237
|
+
duration: payload.duration,
|
|
238
|
+
finishedAt: parseJobInstant(payload.finishedAt),
|
|
239
|
+
modifiedAt: now,
|
|
240
|
+
modifiedById: "system",
|
|
241
|
+
},
|
|
242
|
+
{ id: runId },
|
|
243
|
+
);
|
|
244
|
+
// skip: empty log batch — mirror of onJobComplete
|
|
245
|
+
if (payload.logs.length > 0) {
|
|
246
|
+
await insertMany(
|
|
247
|
+
db,
|
|
248
|
+
jobRunLogsTable,
|
|
249
|
+
payload.logs.map((log) => ({
|
|
250
|
+
runId,
|
|
251
|
+
level: log.level,
|
|
252
|
+
message: log.message,
|
|
253
|
+
timestamp: parseJobInstant(log.timestamp),
|
|
254
|
+
})),
|
|
255
|
+
);
|
|
256
|
+
}
|
|
204
257
|
runIdByBullJobId.delete(bullJobId); // immediate cleanup on terminal callback
|
|
205
258
|
},
|
|
206
259
|
};
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
|
-
buildEntityTable,
|
|
3
2
|
defineUnmanagedTable,
|
|
3
|
+
deriveEntityTableMeta,
|
|
4
4
|
type EntityTableMeta,
|
|
5
5
|
instant,
|
|
6
6
|
table as pgTable,
|
|
@@ -17,28 +17,29 @@ import {
|
|
|
17
17
|
export type JobRunStatus = "queued" | "running" | "completed" | "failed";
|
|
18
18
|
export type JobLogLevel = "info" | "warn" | "error";
|
|
19
19
|
|
|
20
|
-
// jobRun is a system-scoped
|
|
21
|
-
//
|
|
22
|
-
// -Complete / -Failed
|
|
23
|
-
//
|
|
24
|
-
//
|
|
25
|
-
//
|
|
26
|
-
//
|
|
20
|
+
// jobRun is a system-scoped direct-write store (#2243): every job execution
|
|
21
|
+
// writes straight into jobRunsTable / jobRunLogsTable from the BullMQ
|
|
22
|
+
// callbacks (onJobStart / -Complete / -Failed, see job-run-logger.ts) —
|
|
23
|
+
// no event-store detour. Pre-#2243 this was an events-only aggregate
|
|
24
|
+
// replayed through two inline projections; that generated two permanent
|
|
25
|
+
// `kumiko_events` rows per run for data that is itself already the
|
|
26
|
+
// system of record (no other consumer replays or MSP-subscribes to it).
|
|
27
27
|
//
|
|
28
|
-
// Logs
|
|
29
|
-
// design discussion: one
|
|
30
|
-
// no log duplication across status
|
|
31
|
-
// expands the batch into N rows in
|
|
32
|
-
//
|
|
28
|
+
// Logs are batched onto the completed/failed callback as an array —
|
|
29
|
+
// "Option B" from the original design discussion: one write per run
|
|
30
|
+
// instead of one write per log line, no log duplication across status
|
|
31
|
+
// transitions. job-run-logger.ts expands the batch into N rows in
|
|
32
|
+
// jobRunLogsTable.
|
|
33
33
|
//
|
|
34
|
-
// Entity-derived table
|
|
35
|
-
//
|
|
36
|
-
// the
|
|
37
|
-
//
|
|
38
|
-
//
|
|
39
|
-
//
|
|
34
|
+
// Entity-derived table (query API + migration meta share one field
|
|
35
|
+
// definition). status/$type<JobRunStatus> is not modeled in the entity
|
|
36
|
+
// schema — the column type is text, with the status union enforced at the
|
|
37
|
+
// app boundary (same pattern as template-resolver kind/scope).
|
|
38
|
+
// `table: "store_job_runs"` (not `read_*`) because this is no longer a
|
|
39
|
+
// rebuildable projection — `defineUnmanagedTable`/`deriveEntityTableMeta`
|
|
40
|
+
// reject the `read_` prefix for `source: "unmanaged"` (#1208/#1220).
|
|
40
41
|
export const jobRunEntity = createEntity({
|
|
41
|
-
table: "
|
|
42
|
+
table: "store_job_runs",
|
|
42
43
|
fields: {
|
|
43
44
|
jobName: createTextField({ required: true }),
|
|
44
45
|
bullJobId: createTextField({ required: true }),
|
|
@@ -53,7 +54,14 @@ export const jobRunEntity = createEntity({
|
|
|
53
54
|
},
|
|
54
55
|
});
|
|
55
56
|
|
|
56
|
-
|
|
57
|
+
// Plain EntityTableMeta, NOT a branded EntityTable (buildEntityTable would
|
|
58
|
+
// mark it executor-only): job-run is an unmanaged direct-write store, so
|
|
59
|
+
// onJobStart/-Complete/-Failed need to write via ctx.db/insertOne directly
|
|
60
|
+
// (same pattern as sessions/schema/user-session.ts).
|
|
61
|
+
export const jobRunsTable: EntityTableMeta = deriveEntityTableMeta("job-run", jobRunEntity, {
|
|
62
|
+
source: "unmanaged",
|
|
63
|
+
});
|
|
64
|
+
export const jobRunsTableMeta = jobRunsTable;
|
|
57
65
|
|
|
58
66
|
// Child projection keyed by the jobRun aggregate id. Pre-ES used a serial
|
|
59
67
|
// PK + integer runId; post-ES runId is still exposed but now holds the
|
|
@@ -68,13 +76,13 @@ export const jobRunLogsTable = pgTable("store_job_run_logs", {
|
|
|
68
76
|
timestamp: instant("timestamp").notNull(),
|
|
69
77
|
});
|
|
70
78
|
|
|
71
|
-
// **Unmanaged table** —
|
|
72
|
-
// - serial PK (
|
|
73
|
-
// -
|
|
74
|
-
// -
|
|
75
|
-
//
|
|
76
|
-
// pgTable
|
|
77
|
-
//
|
|
79
|
+
// **Unmanaged table** — deliberately no createEntity. Reasoning:
|
|
80
|
+
// - serial PK (not uuid) — pre-ES legacy, compatible with existing rows
|
|
81
|
+
// - no tenant_id — child table of jobRun, tenant context lives on the parent
|
|
82
|
+
// - no base columns (no version/inserted_at/inserted_by_id) — append-only
|
|
83
|
+
// log, no in-place update, no audit columns needed
|
|
84
|
+
// pgTable stays the source of truth for the query API; this meta mirrors it
|
|
85
|
+
// for migration generation.
|
|
78
86
|
export const jobRunLogsTableMeta: EntityTableMeta = defineUnmanagedTable({
|
|
79
87
|
tableName: "store_job_run_logs",
|
|
80
88
|
columns: [
|
package/src/ledger/entity.ts
CHANGED
|
@@ -41,11 +41,12 @@ export const transactionEntity = createEntity({
|
|
|
41
41
|
fields: {
|
|
42
42
|
date: createDateField({ required: true }),
|
|
43
43
|
// Journal narration ("Miete Januar", "Storno: …") is accounting data, not
|
|
44
|
-
// user-generated PII →
|
|
44
|
+
// user-generated PII → `personal: false` silences the user-content heuristic.
|
|
45
45
|
description: createTextField({
|
|
46
46
|
required: true,
|
|
47
47
|
maxLength: 200,
|
|
48
|
-
|
|
48
|
+
personal: false,
|
|
49
|
+
reason: "is_business_data",
|
|
49
50
|
}),
|
|
50
51
|
// For a Storno entry this points at the reversed transaction's id.
|
|
51
52
|
reference: createTextField({ maxLength: 120 }),
|
|
@@ -79,7 +80,8 @@ export const scheduleEntity = createEntity({
|
|
|
79
80
|
description: createTextField({
|
|
80
81
|
required: true,
|
|
81
82
|
maxLength: 200,
|
|
82
|
-
|
|
83
|
+
personal: false,
|
|
84
|
+
reason: "is_business_data",
|
|
83
85
|
}),
|
|
84
86
|
startDate: createDateField({ required: true }),
|
|
85
87
|
// Absent → open-ended (projects to the window's end).
|
|
@@ -42,7 +42,8 @@ export function createBrandingSettingsScreen(opts: {
|
|
|
42
42
|
description: createTextField({
|
|
43
43
|
maxLength: 500,
|
|
44
44
|
multiline: { rows: 3 },
|
|
45
|
-
|
|
45
|
+
personal: false,
|
|
46
|
+
reason: "is_business_data",
|
|
46
47
|
}),
|
|
47
48
|
siteUrl: createTextField({ maxLength: 2000, format: "url" }),
|
|
48
49
|
accentColor: createTextField({ maxLength: 9 }),
|
|
@@ -26,9 +26,10 @@ export const pageEntity = createEntity({
|
|
|
26
26
|
body: createTextField({
|
|
27
27
|
multiline: { rows: 16 },
|
|
28
28
|
maxLength: 100_000,
|
|
29
|
-
|
|
29
|
+
personal: false,
|
|
30
|
+
reason: "is_business_data",
|
|
30
31
|
}),
|
|
31
|
-
description: createTextField({ maxLength: 500,
|
|
32
|
+
description: createTextField({ maxLength: 500, personal: false, reason: "is_business_data" }),
|
|
32
33
|
ogImage: createTextField({ maxLength: 2000 }),
|
|
33
34
|
published: createBooleanField({ default: false }),
|
|
34
35
|
},
|
|
@@ -15,8 +15,9 @@ import {
|
|
|
15
15
|
// registered (see feature.ts). A correction is a new entry, not an edit —
|
|
16
16
|
// that is the whole point of a note *history* instead of the single
|
|
17
17
|
// overwritable textarea this bundle replaces (solon#13). GDPR erasure of the
|
|
18
|
-
// author still works without a delete path: `body` is `
|
|
19
|
-
// shredding on the author's subject key), not
|
|
18
|
+
// author still works without a delete path: `body` is `personal: { of:
|
|
19
|
+
// "authorId" }` (crypto-shredding on the author's subject key), not
|
|
20
|
+
// `personal: "self"` on the entity itself.
|
|
20
21
|
export const noteEntryEntity = createEntity({
|
|
21
22
|
table: "read_note_entries",
|
|
22
23
|
fields: {
|
|
@@ -27,11 +28,14 @@ export const noteEntryEntity = createEntity({
|
|
|
27
28
|
// ctx.user.id (see feature.ts), so a note can't be authored as someone
|
|
28
29
|
// else. subjectRef feeds the GDPR-hook-coverage boot guard (it's a plain
|
|
29
30
|
// FK into `user`, not content of its own).
|
|
30
|
-
authorId: createTextField({
|
|
31
|
+
authorId: createTextField({
|
|
32
|
+
personal: "ref",
|
|
33
|
+
}),
|
|
31
34
|
body: createLongTextField({
|
|
32
35
|
required: true,
|
|
33
36
|
maxLength: 20_000,
|
|
34
|
-
|
|
37
|
+
personal: { of: "authorId" },
|
|
38
|
+
find: "none",
|
|
35
39
|
}),
|
|
36
40
|
},
|
|
37
41
|
});
|