@cosmicdrift/kumiko-framework 0.306.0 → 0.307.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +4 -4
- package/src/api/__tests__/redis-sse-broker.integration.test.ts +66 -0
- package/src/api/__tests__/sse-broker.test.ts +49 -0
- package/src/api/redis-sse-broker.ts +17 -3
- package/src/api/request-context.ts +24 -0
- package/src/api/sse-broker.ts +29 -11
- package/src/changes.json +26 -0
- package/src/db/queries/event-consumer.ts +57 -3
- package/src/db/queries/event-store.ts +69 -0
- package/src/db/tenant-db.ts +43 -5
- package/src/event-store/__tests__/event-attribution.integration.test.ts +53 -3
- package/src/event-store/admin-api.ts +5 -0
- package/src/event-store/event-store.ts +16 -7
- package/src/jobs/__tests__/job-public-intake-origin.integration.test.ts +536 -0
- package/src/jobs/job-runner.ts +61 -6
- package/src/pipeline/__tests__/dispatcher-utils.test.ts +8 -0
- package/src/pipeline/__tests__/event-dispatcher-commit-order.integration.test.ts +278 -0
- package/src/pipeline/__tests__/event-dispatcher-delivery-max-attempts.test.ts +1 -0
- package/src/pipeline/__tests__/event-dispatcher-lifecycle.integration.test.ts +6 -6
- package/src/pipeline/__tests__/event-dispatcher-per-consumer-turns.integration.test.ts +126 -0
- package/src/pipeline/__tests__/public-intake-runtime-gate.integration.test.ts +261 -2
- package/src/pipeline/dispatch-batch.ts +49 -19
- package/src/pipeline/dispatch-stream.ts +7 -3
- package/src/pipeline/dispatcher-utils.ts +21 -2
- package/src/pipeline/dispatcher.ts +71 -6
- package/src/pipeline/event-consumer-state.ts +26 -0
- package/src/pipeline/event-dispatcher-admin.ts +32 -5
- package/src/pipeline/event-dispatcher-delivery.ts +109 -57
- package/src/pipeline/event-dispatcher.ts +167 -50
- package/src/pipeline/pending-gap-ranges.ts +72 -0
- package/src/pipeline/system-hooks.ts +8 -1
- package/src/pipeline/write-origin.ts +31 -10
- package/src/stack/test-stack.ts +1 -1
|
@@ -0,0 +1,536 @@
|
|
|
1
|
+
// Every job here is retries:0 so a gated failure surfaces on the first attempt.
|
|
2
|
+
|
|
3
|
+
import { afterAll, beforeAll, beforeEach, describe, expect, test } from "bun:test";
|
|
4
|
+
import { Queue } from "bullmq";
|
|
5
|
+
import { z } from "zod";
|
|
6
|
+
import type { SchemaTable } from "../../db";
|
|
7
|
+
import { createEventStoreExecutor } from "../../db/event-store-executor";
|
|
8
|
+
import { asRawClient, selectMany } from "../../db/query";
|
|
9
|
+
import { buildEntityTable } from "../../db/table-builder";
|
|
10
|
+
import { createTenantDb } from "../../db/tenant-db";
|
|
11
|
+
import { createEntity, createTextField, defineFeature } from "../../engine";
|
|
12
|
+
import { SYSTEM_ROLE } from "../../engine/system-user";
|
|
13
|
+
import type { TenantId } from "../../engine/types";
|
|
14
|
+
import { AccessDeniedError, FrameworkReasons } from "../../errors";
|
|
15
|
+
import { setupTestStack, type TestStack, TestUsers, unsafeCreateEntityTable } from "../../stack";
|
|
16
|
+
import { waitFor } from "../../testing";
|
|
17
|
+
|
|
18
|
+
// #983: ctx.jobRunner is typed as the narrow JobRunnerRef (handleEvent only);
|
|
19
|
+
// manual dispatch from a handler/job is a dynamic context extension, same
|
|
20
|
+
// cast as setup-test-stack-jobs.integration.test.ts.
|
|
21
|
+
type ManualDispatchRef = {
|
|
22
|
+
dispatch: (name: string, payload: Record<string, unknown>) => Promise<string>;
|
|
23
|
+
};
|
|
24
|
+
function manualDispatch(ctx: { jobRunner?: unknown }): ManualDispatchRef | undefined {
|
|
25
|
+
return ctx.jobRunner as ManualDispatchRef | undefined; // @cast-boundary dynamic-key
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
// Records every gated failure a PII-writing job actually raised — the only
|
|
29
|
+
// way a "blocked" test proves the job ran and was denied, instead of just
|
|
30
|
+
// observing the row count that would also be 0 before the job ever fires.
|
|
31
|
+
type GatedFailure = { job: string; reason: unknown; details: unknown; message: string };
|
|
32
|
+
const gatedFailures: GatedFailure[] = [];
|
|
33
|
+
|
|
34
|
+
function isAccessDeniedError(err: unknown): err is AccessDeniedError {
|
|
35
|
+
return err instanceof AccessDeniedError;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
function recordGatedFailure(job: string, err: unknown): never {
|
|
39
|
+
if (isAccessDeniedError(err)) {
|
|
40
|
+
const details = err.details;
|
|
41
|
+
const reason =
|
|
42
|
+
typeof details === "object" && details !== null && "reason" in details
|
|
43
|
+
? (details as { reason: unknown }).reason
|
|
44
|
+
: undefined;
|
|
45
|
+
gatedFailures.push({ job, reason, details, message: err.message });
|
|
46
|
+
}
|
|
47
|
+
throw err;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
const TENANT_ID = "00000000-0000-4000-8000-000000000002" as TenantId;
|
|
51
|
+
const RATE_LIMIT = { per: "ip", limit: 1000, windowSeconds: 60 } as const;
|
|
52
|
+
const QUEUE_NAME_PREFIX = `kumiko-write-origin-test-${Date.now()}`;
|
|
53
|
+
const UNSAFE_RAW_REASON =
|
|
54
|
+
"test: proves ctx.systemDb.unsafeRaw()-derived createTenantDb inherits the job's gate";
|
|
55
|
+
|
|
56
|
+
// --- Feature holding the PII entity + the job's "foreign handler" target ---
|
|
57
|
+
|
|
58
|
+
const secretEntity = createEntity({
|
|
59
|
+
table: "job_origin_secrets",
|
|
60
|
+
fields: {
|
|
61
|
+
value: createTextField({ personal: "self", find: "none", default: "" }),
|
|
62
|
+
},
|
|
63
|
+
});
|
|
64
|
+
const secretTable = buildEntityTable("secret", secretEntity);
|
|
65
|
+
|
|
66
|
+
const secretsFeature = defineFeature("secrets", (r) => {
|
|
67
|
+
r.entity("secret", secretEntity);
|
|
68
|
+
r.writeHandler(
|
|
69
|
+
"create",
|
|
70
|
+
z.object({ value: z.string() }),
|
|
71
|
+
async (event, ctx) => {
|
|
72
|
+
const crud = createEventStoreExecutor(secretTable, secretEntity, { entityName: "secret" });
|
|
73
|
+
return crud.create({ value: event.payload.value }, event.user, ctx.db);
|
|
74
|
+
},
|
|
75
|
+
{ access: { roles: [SYSTEM_ROLE] } },
|
|
76
|
+
);
|
|
77
|
+
});
|
|
78
|
+
|
|
79
|
+
// --- Feature owning the jobs under test ---
|
|
80
|
+
|
|
81
|
+
const jobsFeature = defineFeature("jobsx", (r) => {
|
|
82
|
+
r.defineEvent("leaked", z.object({ value: z.string() }), { piiFields: "none" });
|
|
83
|
+
|
|
84
|
+
r.job("write-direct", { trigger: { manual: true }, retries: 0 }, async (payload, ctx) => {
|
|
85
|
+
try {
|
|
86
|
+
await ctx.db.insertOne(secretTable as unknown as SchemaTable, {
|
|
87
|
+
// @cast-boundary test-fixture
|
|
88
|
+
value: payload["value"] as string, // @cast-boundary test-fixture
|
|
89
|
+
});
|
|
90
|
+
} catch (err) {
|
|
91
|
+
recordGatedFailure("jobsx:job:write-direct", err);
|
|
92
|
+
}
|
|
93
|
+
});
|
|
94
|
+
|
|
95
|
+
r.job("write-foreign", { trigger: { manual: true }, retries: 0 }, async (payload, ctx) => {
|
|
96
|
+
const result = await ctx.write("secrets:write:create", {
|
|
97
|
+
value: payload["value"] as string, // @cast-boundary test-fixture
|
|
98
|
+
});
|
|
99
|
+
if (!result.isSuccess) {
|
|
100
|
+
recordGatedFailure(
|
|
101
|
+
"jobsx:job:write-foreign",
|
|
102
|
+
new AccessDeniedError({ message: result.error.message, details: result.error.details }),
|
|
103
|
+
);
|
|
104
|
+
}
|
|
105
|
+
});
|
|
106
|
+
|
|
107
|
+
r.job("chain-dispatch", { trigger: { manual: true }, retries: 0 }, async (payload, ctx) => {
|
|
108
|
+
await manualDispatch(ctx)?.dispatch("jobsx:job:write-direct", { value: payload["value"] });
|
|
109
|
+
});
|
|
110
|
+
|
|
111
|
+
r.job(
|
|
112
|
+
"on-trigger-no-declare",
|
|
113
|
+
{ trigger: { on: "jobsx:write:trigger-source-no-declare" }, retries: 0 },
|
|
114
|
+
async (payload, ctx) => {
|
|
115
|
+
try {
|
|
116
|
+
await ctx.db.insertOne(secretTable as unknown as SchemaTable, {
|
|
117
|
+
// @cast-boundary test-fixture
|
|
118
|
+
value: payload["value"] as string, // @cast-boundary test-fixture
|
|
119
|
+
});
|
|
120
|
+
} catch (err) {
|
|
121
|
+
recordGatedFailure("jobsx:job:on-trigger-no-declare", err);
|
|
122
|
+
}
|
|
123
|
+
},
|
|
124
|
+
);
|
|
125
|
+
r.job(
|
|
126
|
+
"on-trigger-declare",
|
|
127
|
+
{ trigger: { on: "jobsx:write:trigger-source-declare" }, retries: 0 },
|
|
128
|
+
async (payload, ctx) => {
|
|
129
|
+
try {
|
|
130
|
+
await ctx.db.insertOne(secretTable as unknown as SchemaTable, {
|
|
131
|
+
// @cast-boundary test-fixture
|
|
132
|
+
value: payload["value"] as string, // @cast-boundary test-fixture
|
|
133
|
+
});
|
|
134
|
+
} catch (err) {
|
|
135
|
+
recordGatedFailure("jobsx:job:on-trigger-declare", err);
|
|
136
|
+
}
|
|
137
|
+
},
|
|
138
|
+
);
|
|
139
|
+
r.job(
|
|
140
|
+
"on-defined-event",
|
|
141
|
+
{ trigger: { on: "jobsx:event:leaked" }, retries: 0 },
|
|
142
|
+
async (payload, ctx) => {
|
|
143
|
+
try {
|
|
144
|
+
await ctx.db.insertOne(secretTable as unknown as SchemaTable, {
|
|
145
|
+
// @cast-boundary test-fixture
|
|
146
|
+
value: payload["value"] as string, // @cast-boundary test-fixture
|
|
147
|
+
});
|
|
148
|
+
} catch (err) {
|
|
149
|
+
recordGatedFailure("jobsx:job:on-defined-event", err);
|
|
150
|
+
}
|
|
151
|
+
},
|
|
152
|
+
);
|
|
153
|
+
|
|
154
|
+
r.writeHandler(
|
|
155
|
+
"dispatch-no-declare",
|
|
156
|
+
z.object({ value: z.string() }),
|
|
157
|
+
async (event, ctx) => {
|
|
158
|
+
await manualDispatch(ctx)?.dispatch("jobsx:job:write-direct", { value: event.payload.value });
|
|
159
|
+
return { isSuccess: true as const, data: { ok: true as const } };
|
|
160
|
+
},
|
|
161
|
+
{ access: { roles: ["anonymous"] }, rateLimit: RATE_LIMIT },
|
|
162
|
+
);
|
|
163
|
+
r.writeHandler(
|
|
164
|
+
"dispatch-declare",
|
|
165
|
+
z.object({ value: z.string() }),
|
|
166
|
+
async (event, ctx) => {
|
|
167
|
+
await manualDispatch(ctx)?.dispatch("jobsx:job:write-direct", { value: event.payload.value });
|
|
168
|
+
return { isSuccess: true as const, data: { ok: true as const } };
|
|
169
|
+
},
|
|
170
|
+
{ access: { roles: ["anonymous"], personalData: "public-intake" }, rateLimit: RATE_LIMIT },
|
|
171
|
+
);
|
|
172
|
+
|
|
173
|
+
r.writeHandler(
|
|
174
|
+
"foreign-no-declare",
|
|
175
|
+
z.object({ value: z.string() }),
|
|
176
|
+
async (event, ctx) => {
|
|
177
|
+
await manualDispatch(ctx)?.dispatch("jobsx:job:write-foreign", {
|
|
178
|
+
value: event.payload.value,
|
|
179
|
+
});
|
|
180
|
+
return { isSuccess: true as const, data: { ok: true as const } };
|
|
181
|
+
},
|
|
182
|
+
{ access: { roles: ["anonymous"] }, rateLimit: RATE_LIMIT },
|
|
183
|
+
);
|
|
184
|
+
r.writeHandler(
|
|
185
|
+
"foreign-declare",
|
|
186
|
+
z.object({ value: z.string() }),
|
|
187
|
+
async (event, ctx) => {
|
|
188
|
+
await manualDispatch(ctx)?.dispatch("jobsx:job:write-foreign", {
|
|
189
|
+
value: event.payload.value,
|
|
190
|
+
});
|
|
191
|
+
return { isSuccess: true as const, data: { ok: true as const } };
|
|
192
|
+
},
|
|
193
|
+
{ access: { roles: ["anonymous"], personalData: "public-intake" }, rateLimit: RATE_LIMIT },
|
|
194
|
+
);
|
|
195
|
+
|
|
196
|
+
r.writeHandler(
|
|
197
|
+
"chain-no-declare",
|
|
198
|
+
z.object({ value: z.string() }),
|
|
199
|
+
async (event, ctx) => {
|
|
200
|
+
await manualDispatch(ctx)?.dispatch("jobsx:job:chain-dispatch", {
|
|
201
|
+
value: event.payload.value,
|
|
202
|
+
});
|
|
203
|
+
return { isSuccess: true as const, data: { ok: true as const } };
|
|
204
|
+
},
|
|
205
|
+
{ access: { roles: ["anonymous"] }, rateLimit: RATE_LIMIT },
|
|
206
|
+
);
|
|
207
|
+
r.writeHandler(
|
|
208
|
+
"chain-declare",
|
|
209
|
+
z.object({ value: z.string() }),
|
|
210
|
+
async (event, ctx) => {
|
|
211
|
+
await manualDispatch(ctx)?.dispatch("jobsx:job:chain-dispatch", {
|
|
212
|
+
value: event.payload.value,
|
|
213
|
+
});
|
|
214
|
+
return { isSuccess: true as const, data: { ok: true as const } };
|
|
215
|
+
},
|
|
216
|
+
{ access: { roles: ["anonymous"], personalData: "public-intake" }, rateLimit: RATE_LIMIT },
|
|
217
|
+
);
|
|
218
|
+
|
|
219
|
+
r.writeHandler(
|
|
220
|
+
"trigger-source-no-declare",
|
|
221
|
+
z.object({ value: z.string() }),
|
|
222
|
+
async (event) => ({ isSuccess: true as const, data: { value: event.payload.value } }),
|
|
223
|
+
{ access: { roles: ["anonymous"] }, rateLimit: RATE_LIMIT },
|
|
224
|
+
);
|
|
225
|
+
r.writeHandler(
|
|
226
|
+
"trigger-source-declare",
|
|
227
|
+
z.object({ value: z.string() }),
|
|
228
|
+
async (event) => ({ isSuccess: true as const, data: { value: event.payload.value } }),
|
|
229
|
+
{ access: { roles: ["anonymous"], personalData: "public-intake" }, rateLimit: RATE_LIMIT },
|
|
230
|
+
);
|
|
231
|
+
|
|
232
|
+
r.writeHandler(
|
|
233
|
+
"definedevent-no-declare",
|
|
234
|
+
z.object({ value: z.string() }),
|
|
235
|
+
async (event, ctx) => {
|
|
236
|
+
await ctx.unsafeAppendEvent({
|
|
237
|
+
aggregateId: crypto.randomUUID(),
|
|
238
|
+
aggregateType: "jobsx-leak",
|
|
239
|
+
type: "jobsx:event:leaked",
|
|
240
|
+
payload: { value: event.payload.value },
|
|
241
|
+
});
|
|
242
|
+
return { isSuccess: true as const, data: { ok: true as const } };
|
|
243
|
+
},
|
|
244
|
+
{ access: { roles: ["anonymous"] }, rateLimit: RATE_LIMIT },
|
|
245
|
+
);
|
|
246
|
+
r.writeHandler(
|
|
247
|
+
"definedevent-declare",
|
|
248
|
+
z.object({ value: z.string() }),
|
|
249
|
+
async (event, ctx) => {
|
|
250
|
+
await ctx.unsafeAppendEvent({
|
|
251
|
+
aggregateId: crypto.randomUUID(),
|
|
252
|
+
aggregateType: "jobsx-leak",
|
|
253
|
+
type: "jobsx:event:leaked",
|
|
254
|
+
payload: { value: event.payload.value },
|
|
255
|
+
});
|
|
256
|
+
return { isSuccess: true as const, data: { ok: true as const } };
|
|
257
|
+
},
|
|
258
|
+
{ access: { roles: ["anonymous"], personalData: "public-intake" }, rateLimit: RATE_LIMIT },
|
|
259
|
+
);
|
|
260
|
+
|
|
261
|
+
r.queryHandler(
|
|
262
|
+
"query-dispatch-no-declare",
|
|
263
|
+
z.object({ value: z.string() }),
|
|
264
|
+
async (event, ctx) => {
|
|
265
|
+
await manualDispatch(ctx)?.dispatch("jobsx:job:write-direct", { value: event.payload.value });
|
|
266
|
+
return { ok: true as const };
|
|
267
|
+
},
|
|
268
|
+
{ access: { roles: ["anonymous"] }, rateLimit: RATE_LIMIT },
|
|
269
|
+
);
|
|
270
|
+
|
|
271
|
+
r.writeHandler(
|
|
272
|
+
"authenticated-dispatch",
|
|
273
|
+
z.object({ value: z.string() }),
|
|
274
|
+
async (event, ctx) => {
|
|
275
|
+
await manualDispatch(ctx)?.dispatch("jobsx:job:write-direct", { value: event.payload.value });
|
|
276
|
+
return { isSuccess: true as const, data: { ok: true as const } };
|
|
277
|
+
},
|
|
278
|
+
{ access: { roles: ["Admin"] } },
|
|
279
|
+
);
|
|
280
|
+
});
|
|
281
|
+
|
|
282
|
+
// --- Feature-level r.systemScope() job: proves ctx.systemDb.unsafeRaw()'s
|
|
283
|
+
// createTenantDb-derived TenantDb also inherits the job's gate (tenant-db.ts's
|
|
284
|
+
// runnerPersonalDataGates, same mechanism pass 1 pinned for handlers). ---
|
|
285
|
+
|
|
286
|
+
const systemJobFeature = defineFeature("systemjob", (r) => {
|
|
287
|
+
r.systemScope();
|
|
288
|
+
r.job("write-systemdb", { trigger: { manual: true }, retries: 0 }, async (payload, ctx) => {
|
|
289
|
+
const raw = ctx.systemDb?.unsafeRaw(UNSAFE_RAW_REASON);
|
|
290
|
+
if (!raw) throw new Error("test setup error: ctx.systemDb missing on a systemScope() job");
|
|
291
|
+
try {
|
|
292
|
+
await createTenantDb(raw, ctx.systemUser.tenantId, "system").insertOne(
|
|
293
|
+
secretTable as unknown as SchemaTable, // @cast-boundary test-fixture
|
|
294
|
+
{ value: payload["value"] as string },
|
|
295
|
+
);
|
|
296
|
+
} catch (err) {
|
|
297
|
+
recordGatedFailure("systemjob:job:write-systemdb", err);
|
|
298
|
+
}
|
|
299
|
+
});
|
|
300
|
+
});
|
|
301
|
+
|
|
302
|
+
const dispatchSystemDbFeature = defineFeature("systemjobdispatch", (r) => {
|
|
303
|
+
r.writeHandler(
|
|
304
|
+
"no-declare",
|
|
305
|
+
z.object({ value: z.string() }),
|
|
306
|
+
async (event, ctx) => {
|
|
307
|
+
await manualDispatch(ctx)?.dispatch("systemjob:job:write-systemdb", {
|
|
308
|
+
value: event.payload.value,
|
|
309
|
+
});
|
|
310
|
+
return { isSuccess: true as const, data: { ok: true as const } };
|
|
311
|
+
},
|
|
312
|
+
{ access: { roles: ["anonymous"] }, rateLimit: RATE_LIMIT },
|
|
313
|
+
);
|
|
314
|
+
r.writeHandler(
|
|
315
|
+
"declare",
|
|
316
|
+
z.object({ value: z.string() }),
|
|
317
|
+
async (event, ctx) => {
|
|
318
|
+
await manualDispatch(ctx)?.dispatch("systemjob:job:write-systemdb", {
|
|
319
|
+
value: event.payload.value,
|
|
320
|
+
});
|
|
321
|
+
return { isSuccess: true as const, data: { ok: true as const } };
|
|
322
|
+
},
|
|
323
|
+
{ access: { roles: ["anonymous"], personalData: "public-intake" }, rateLimit: RATE_LIMIT },
|
|
324
|
+
);
|
|
325
|
+
});
|
|
326
|
+
|
|
327
|
+
describe("job/event write-origin inheritance", () => {
|
|
328
|
+
let stack: TestStack;
|
|
329
|
+
|
|
330
|
+
beforeAll(async () => {
|
|
331
|
+
stack = await setupTestStack({
|
|
332
|
+
features: [secretsFeature, jobsFeature, systemJobFeature, dispatchSystemDbFeature],
|
|
333
|
+
anonymousAccess: { defaultTenantId: TENANT_ID },
|
|
334
|
+
jobs: { consumerLane: "worker", queueNamePrefix: QUEUE_NAME_PREFIX },
|
|
335
|
+
});
|
|
336
|
+
await unsafeCreateEntityTable(stack.db, secretEntity, "secret");
|
|
337
|
+
});
|
|
338
|
+
|
|
339
|
+
afterAll(() => stack.cleanup());
|
|
340
|
+
|
|
341
|
+
beforeEach(async () => {
|
|
342
|
+
await asRawClient(stack.db).unsafe(`DELETE FROM "${secretTable.tableName}"`);
|
|
343
|
+
gatedFailures.length = 0;
|
|
344
|
+
});
|
|
345
|
+
|
|
346
|
+
async function rowCount(): Promise<number> {
|
|
347
|
+
const rows = await selectMany(stack.db, secretTable as unknown as SchemaTable); // @cast-boundary test-fixture
|
|
348
|
+
return rows.length;
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
async function post(type: string, value: string, endpoint = "/api/write"): Promise<void> {
|
|
352
|
+
const res = await stack.http.raw("POST", endpoint, { type, payload: { value } });
|
|
353
|
+
expect(res.status).toBe(200);
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
// Waits for the job's own gated failure, not just an unchanged row count —
|
|
357
|
+
// rowCount()==0 also holds before the job has run at all, so asserting on
|
|
358
|
+
// it alone would pass even if the gate never fired.
|
|
359
|
+
async function expectBlocked(
|
|
360
|
+
type: string,
|
|
361
|
+
value: string,
|
|
362
|
+
job: string,
|
|
363
|
+
rootHandler: string,
|
|
364
|
+
endpoint = "/api/write",
|
|
365
|
+
): Promise<void> {
|
|
366
|
+
await post(type, value, endpoint);
|
|
367
|
+
await waitFor(async () => {
|
|
368
|
+
await stack.eventDispatcher?.runOnce();
|
|
369
|
+
expect(gatedFailures.some((f) => f.job === job)).toBe(true);
|
|
370
|
+
});
|
|
371
|
+
const failure = gatedFailures.find((f) => f.job === job);
|
|
372
|
+
expect(failure?.reason).toBe(FrameworkReasons.publicIntakeRequired);
|
|
373
|
+
expect((failure?.details as { job?: string } | undefined)?.job).toBe(job);
|
|
374
|
+
expect((failure?.details as { rootHandler?: string } | undefined)?.rootHandler).toBe(
|
|
375
|
+
rootHandler,
|
|
376
|
+
);
|
|
377
|
+
expect((failure?.details as { fields?: readonly string[] } | undefined)?.fields).toContain(
|
|
378
|
+
"value",
|
|
379
|
+
);
|
|
380
|
+
expect(failure?.message).toContain(`via job "${job}"`);
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
async function expectAllowed(type: string, value: string): Promise<void> {
|
|
384
|
+
await post(type, value);
|
|
385
|
+
await waitFor(async () => {
|
|
386
|
+
await stack.eventDispatcher?.runOnce();
|
|
387
|
+
expect(await rowCount()).toBe(1);
|
|
388
|
+
});
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
test("ctx.jobRunner.dispatch from an anonymous root — blocked without declaration", async () => {
|
|
392
|
+
await expectBlocked(
|
|
393
|
+
"jobsx:write:dispatch-no-declare",
|
|
394
|
+
"leak-dispatch",
|
|
395
|
+
"jobsx:job:write-direct",
|
|
396
|
+
"jobsx:write:dispatch-no-declare",
|
|
397
|
+
);
|
|
398
|
+
expect(await rowCount()).toBe(0);
|
|
399
|
+
});
|
|
400
|
+
|
|
401
|
+
test("ctx.jobRunner.dispatch from an anonymous root — allowed once declared", async () => {
|
|
402
|
+
await expectAllowed("jobsx:write:dispatch-declare", "leak-dispatch-declared");
|
|
403
|
+
expect(gatedFailures).toHaveLength(0);
|
|
404
|
+
});
|
|
405
|
+
|
|
406
|
+
test("anonymous query root dispatching a PII job — blocked", async () => {
|
|
407
|
+
await expectBlocked(
|
|
408
|
+
"jobsx:query:query-dispatch-no-declare",
|
|
409
|
+
"leak-query-dispatch",
|
|
410
|
+
"jobsx:job:write-direct",
|
|
411
|
+
"jobsx:query:query-dispatch-no-declare",
|
|
412
|
+
"/api/query",
|
|
413
|
+
);
|
|
414
|
+
expect(await rowCount()).toBe(0);
|
|
415
|
+
});
|
|
416
|
+
|
|
417
|
+
test("sync handler-QN job trigger (afterCommit) — blocked without declaration", async () => {
|
|
418
|
+
await expectBlocked(
|
|
419
|
+
"jobsx:write:trigger-source-no-declare",
|
|
420
|
+
"leak-trigger",
|
|
421
|
+
"jobsx:job:on-trigger-no-declare",
|
|
422
|
+
"jobsx:write:trigger-source-no-declare",
|
|
423
|
+
);
|
|
424
|
+
expect(await rowCount()).toBe(0);
|
|
425
|
+
});
|
|
426
|
+
|
|
427
|
+
test("sync handler-QN job trigger (afterCommit) — allowed once declared", async () => {
|
|
428
|
+
await expectAllowed("jobsx:write:trigger-source-declare", "leak-trigger-declared");
|
|
429
|
+
expect(gatedFailures).toHaveLength(0);
|
|
430
|
+
});
|
|
431
|
+
|
|
432
|
+
test("job's ctx.write into a foreign handler — blocked without declaration", async () => {
|
|
433
|
+
await expectBlocked(
|
|
434
|
+
"jobsx:write:foreign-no-declare",
|
|
435
|
+
"leak-foreign",
|
|
436
|
+
"jobsx:job:write-foreign",
|
|
437
|
+
"jobsx:write:foreign-no-declare",
|
|
438
|
+
);
|
|
439
|
+
expect(await rowCount()).toBe(0);
|
|
440
|
+
});
|
|
441
|
+
|
|
442
|
+
test("job's ctx.write into a foreign handler — allowed once declared", async () => {
|
|
443
|
+
await expectAllowed("jobsx:write:foreign-declare", "leak-foreign-declared");
|
|
444
|
+
expect(gatedFailures).toHaveLength(0);
|
|
445
|
+
});
|
|
446
|
+
|
|
447
|
+
test("job's ctx.systemDb.unsafeRaw()-derived createTenantDb — blocked without declaration", async () => {
|
|
448
|
+
await expectBlocked(
|
|
449
|
+
"systemjobdispatch:write:no-declare",
|
|
450
|
+
"leak-systemdb",
|
|
451
|
+
"systemjob:job:write-systemdb",
|
|
452
|
+
"systemjobdispatch:write:no-declare",
|
|
453
|
+
);
|
|
454
|
+
expect(await rowCount()).toBe(0);
|
|
455
|
+
});
|
|
456
|
+
|
|
457
|
+
test("job's ctx.systemDb.unsafeRaw()-derived createTenantDb — allowed once declared", async () => {
|
|
458
|
+
await expectAllowed("systemjobdispatch:write:declare", "leak-systemdb-declared");
|
|
459
|
+
expect(gatedFailures).toHaveLength(0);
|
|
460
|
+
});
|
|
461
|
+
|
|
462
|
+
test("job chaining (A dispatches B, B writes PII) — blocked without declaration", async () => {
|
|
463
|
+
await expectBlocked(
|
|
464
|
+
"jobsx:write:chain-no-declare",
|
|
465
|
+
"leak-chain",
|
|
466
|
+
"jobsx:job:write-direct",
|
|
467
|
+
"jobsx:write:chain-no-declare",
|
|
468
|
+
);
|
|
469
|
+
expect(await rowCount()).toBe(0);
|
|
470
|
+
});
|
|
471
|
+
|
|
472
|
+
test("job chaining (A dispatches B, B writes PII) — allowed once declared", async () => {
|
|
473
|
+
await expectAllowed("jobsx:write:chain-declare", "leak-chain-declared");
|
|
474
|
+
expect(gatedFailures).toHaveLength(0);
|
|
475
|
+
});
|
|
476
|
+
|
|
477
|
+
test("r.defineEvent job trigger via ctx.appendEvent — blocked without declaration", async () => {
|
|
478
|
+
await expectBlocked(
|
|
479
|
+
"jobsx:write:definedevent-no-declare",
|
|
480
|
+
"leak-event",
|
|
481
|
+
"jobsx:job:on-defined-event",
|
|
482
|
+
"jobsx:write:definedevent-no-declare",
|
|
483
|
+
);
|
|
484
|
+
expect(await rowCount()).toBe(0);
|
|
485
|
+
});
|
|
486
|
+
|
|
487
|
+
test("r.defineEvent job trigger via ctx.appendEvent — allowed once declared", async () => {
|
|
488
|
+
await expectAllowed("jobsx:write:definedevent-declare", "leak-event-declared");
|
|
489
|
+
expect(gatedFailures).toHaveLength(0);
|
|
490
|
+
});
|
|
491
|
+
|
|
492
|
+
test("authenticated root dispatching the same job — unaffected", async () => {
|
|
493
|
+
const token = await stack.jwt.sign(TestUsers.admin);
|
|
494
|
+
const res = await stack.app.request("/api/write", {
|
|
495
|
+
method: "POST",
|
|
496
|
+
headers: { "Content-Type": "application/json", Authorization: `Bearer ${token}` },
|
|
497
|
+
body: JSON.stringify({
|
|
498
|
+
type: "jobsx:write:authenticated-dispatch",
|
|
499
|
+
payload: { value: "leak-authenticated" },
|
|
500
|
+
}),
|
|
501
|
+
});
|
|
502
|
+
expect(res.status).toBe(200);
|
|
503
|
+
await waitFor(async () => {
|
|
504
|
+
expect(await rowCount()).toBe(1);
|
|
505
|
+
});
|
|
506
|
+
});
|
|
507
|
+
|
|
508
|
+
test("a legacy job dispatched directly (no request context, no _writeOrigin) runs ungated", async () => {
|
|
509
|
+
if (!stack.jobRunner) throw new Error("test setup error: stack.jobRunner missing");
|
|
510
|
+
await stack.jobRunner.dispatch("jobsx:job:write-direct", { value: "leak-legacy" });
|
|
511
|
+
await waitFor(async () => {
|
|
512
|
+
expect(await rowCount()).toBe(1);
|
|
513
|
+
});
|
|
514
|
+
});
|
|
515
|
+
|
|
516
|
+
test("a tampered _writeOrigin fails the job closed instead of running ungated", async () => {
|
|
517
|
+
// bullmq bundles its own ioredis; a shared Redis instance's class type
|
|
518
|
+
// doesn't structurally match its ConnectionOptions, but plain
|
|
519
|
+
// host/port/db data does (mirrors job-runner.ts's parseRedisOpts).
|
|
520
|
+
const { host, port, db } = stack.redis.redis.options;
|
|
521
|
+
const queue = new Queue(`${QUEUE_NAME_PREFIX}-worker`, { connection: { host, port, db } });
|
|
522
|
+
try {
|
|
523
|
+
const job = await queue.add("jobsx:job:write-direct", {
|
|
524
|
+
value: "leak-tampered",
|
|
525
|
+
_writeOrigin: "not-an-object",
|
|
526
|
+
});
|
|
527
|
+
await waitFor(async () => {
|
|
528
|
+
const state = await job.getState();
|
|
529
|
+
expect(state).toBe("failed");
|
|
530
|
+
});
|
|
531
|
+
expect(await rowCount()).toBe(0);
|
|
532
|
+
} finally {
|
|
533
|
+
await queue.close();
|
|
534
|
+
}
|
|
535
|
+
});
|
|
536
|
+
});
|
package/src/jobs/job-runner.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { WriteOrigin } from "@cosmicdrift/kumiko-types/event-store-types";
|
|
1
2
|
import { type JobsOptions, Queue, Worker } from "bullmq";
|
|
2
3
|
import { Redis } from "ioredis";
|
|
3
4
|
import { requestContext } from "../api/request-context";
|
|
@@ -18,6 +19,7 @@ import {
|
|
|
18
19
|
SYSTEM_TENANT_ID,
|
|
19
20
|
type TenantId,
|
|
20
21
|
} from "../engine/types";
|
|
22
|
+
import { InternalError } from "../errors";
|
|
21
23
|
import { isKumikoError } from "../errors/kumiko-error";
|
|
22
24
|
import { createFileContext } from "../files/file-handle";
|
|
23
25
|
import { createFallbackLogger } from "../logging";
|
|
@@ -33,8 +35,22 @@ import {
|
|
|
33
35
|
import { createEscapeHatchReporter } from "../observability/escape-hatch-report";
|
|
34
36
|
import { createDistributedLock, type DistributedLock } from "../pipeline/distributed-lock";
|
|
35
37
|
import { RedisKeys } from "../pipeline/redis-keys";
|
|
38
|
+
import {
|
|
39
|
+
buildPersonalDataGate,
|
|
40
|
+
isPersonalDataGated,
|
|
41
|
+
parseWriteOrigin,
|
|
42
|
+
} from "../pipeline/write-origin";
|
|
36
43
|
import { bridgeStub } from "../testing/handler-context";
|
|
37
44
|
|
|
45
|
+
// A payload's own `_writeOrigin` is never trusted; only the ambient gated origin is stamped.
|
|
46
|
+
function stampGatedWriteOrigin(data: Record<string, unknown>): void {
|
|
47
|
+
delete data["_writeOrigin"];
|
|
48
|
+
const origin = requestContext.get()?.writeOrigin;
|
|
49
|
+
if (origin && isPersonalDataGated(origin)) {
|
|
50
|
+
data["_writeOrigin"] = origin;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
|
|
38
54
|
// Queue-name convention: <prefix>-<lane>. The prefix is fixed in prod
|
|
39
55
|
// ("kumiko-jobs") — it must match between enqueuers and consumers, and an
|
|
40
56
|
// accidental drift would silently drop jobs. Tests override via
|
|
@@ -731,6 +747,21 @@ export function createJobRunner(options: JobRunnerOptions): JobRunner {
|
|
|
731
747
|
// multi-trigger dispatch; exposed as jobContext.triggerName so handlers
|
|
732
748
|
// don't dig through the raw payload themselves.
|
|
733
749
|
const triggerName = rawData["_triggerName"] as string | undefined; // @cast-boundary dynamic-key
|
|
750
|
+
|
|
751
|
+
// Absent = legacy or ungated root; present but invalid fails the run closed.
|
|
752
|
+
const rawWriteOrigin = rawData["_writeOrigin"]; // @cast-boundary dynamic-key
|
|
753
|
+
let jobOrigin: WriteOrigin | undefined;
|
|
754
|
+
let writeOriginInvalid = false;
|
|
755
|
+
if (rawWriteOrigin !== undefined) {
|
|
756
|
+
const parsed = parseWriteOrigin(rawWriteOrigin);
|
|
757
|
+
if (parsed) {
|
|
758
|
+
jobOrigin = { ...parsed, viaJob: jobName };
|
|
759
|
+
} else {
|
|
760
|
+
writeOriginInvalid = true;
|
|
761
|
+
}
|
|
762
|
+
}
|
|
763
|
+
const jobPersonalDataGate = jobOrigin ? buildPersonalDataGate(registry, jobOrigin) : undefined;
|
|
764
|
+
|
|
734
765
|
// Mirror dispatch-shared.ts buildHandlerContext: ctx.files must resolve
|
|
735
766
|
// through the same _fileProviderResolver for jobs as for write-handlers,
|
|
736
767
|
// otherwise event-triggered jobs silently get an unresolved ctx.files.
|
|
@@ -749,8 +780,19 @@ export function createJobRunner(options: JobRunnerOptions): JobRunner {
|
|
|
749
780
|
// systemScope() status (pre-existing, not something this change alters)
|
|
750
781
|
// — isSystemJob below is what actually keeps ctx.systemDb off a
|
|
751
782
|
// non-system job; it is the only thing standing between this db and an
|
|
752
|
-
// unchecked cross-tenant escape hatch for such a job.
|
|
753
|
-
|
|
783
|
+
// unchecked cross-tenant escape hatch for such a job. Gated like jobDb so
|
|
784
|
+
// ctx.systemDb, built from it, is gated too.
|
|
785
|
+
const tenantScopedDb = configDb
|
|
786
|
+
? createTenantDb(
|
|
787
|
+
configDb,
|
|
788
|
+
tenantId,
|
|
789
|
+
"system",
|
|
790
|
+
undefined,
|
|
791
|
+
undefined,
|
|
792
|
+
undefined,
|
|
793
|
+
jobPersonalDataGate ? { personalDataGate: jobPersonalDataGate } : undefined,
|
|
794
|
+
)
|
|
795
|
+
: undefined;
|
|
754
796
|
const isSystemJob = registry.isJobSystemScoped(jobName);
|
|
755
797
|
// One reporter for ctx.systemDb and ctx.db.unsafeRaw() so both dedupe in the same window.
|
|
756
798
|
const reportEscapeHatch = createEscapeHatchReporter({
|
|
@@ -768,6 +810,7 @@ export function createJobRunner(options: JobRunnerOptions): JobRunner {
|
|
|
768
810
|
? createTenantDb(configDb, tenantId, "tenant", context.tracer, context.meter, undefined, {
|
|
769
811
|
unsafeRaw: jobDef.escapeHatch,
|
|
770
812
|
report: reportEscapeHatch,
|
|
813
|
+
...(jobPersonalDataGate && { personalDataGate: jobPersonalDataGate }),
|
|
771
814
|
})
|
|
772
815
|
: undefined;
|
|
773
816
|
const config =
|
|
@@ -818,7 +861,7 @@ export function createJobRunner(options: JobRunnerOptions): JobRunner {
|
|
|
818
861
|
"JobContext.write called before dispatcher attached — call attachDispatcher() first",
|
|
819
862
|
);
|
|
820
863
|
}
|
|
821
|
-
return dispatchWriteRef.write(jobSystemUser, qn, payload);
|
|
864
|
+
return dispatchWriteRef.write(jobSystemUser, qn, payload, jobOrigin);
|
|
822
865
|
},
|
|
823
866
|
writeAs: (user: SessionUser, qn: string, payload: unknown) => {
|
|
824
867
|
if (!dispatchWriteRef) {
|
|
@@ -826,7 +869,7 @@ export function createJobRunner(options: JobRunnerOptions): JobRunner {
|
|
|
826
869
|
"JobContext.writeAs called before dispatcher attached — call attachDispatcher() first",
|
|
827
870
|
);
|
|
828
871
|
}
|
|
829
|
-
return dispatchWriteRef.write(user, qn, payload);
|
|
872
|
+
return dispatchWriteRef.write(user, qn, payload, jobOrigin);
|
|
830
873
|
},
|
|
831
874
|
queryAs: (user: SessionUser, qn: string, payload: unknown) => {
|
|
832
875
|
if (!dispatchWriteRef) {
|
|
@@ -834,7 +877,7 @@ export function createJobRunner(options: JobRunnerOptions): JobRunner {
|
|
|
834
877
|
"JobContext.queryAs called before dispatcher attached — call attachDispatcher() first",
|
|
835
878
|
);
|
|
836
879
|
}
|
|
837
|
-
return dispatchWriteRef.queryAs(user, qn, payload);
|
|
880
|
+
return dispatchWriteRef.queryAs(user, qn, payload, jobOrigin);
|
|
838
881
|
},
|
|
839
882
|
queryAsMember: (userId: string, qn: string, payload: unknown) => {
|
|
840
883
|
if (!dispatchWriteRef) {
|
|
@@ -866,6 +909,13 @@ export function createJobRunner(options: JobRunnerOptions): JobRunner {
|
|
|
866
909
|
|
|
867
910
|
const runInSpan = async (): Promise<void> => {
|
|
868
911
|
try {
|
|
912
|
+
if (writeOriginInvalid) {
|
|
913
|
+
throw new InternalError({
|
|
914
|
+
message:
|
|
915
|
+
`Job "${jobName}" received an unparseable _writeOrigin — refusing to run without ` +
|
|
916
|
+
"a trustworthy anonymous-root gate.",
|
|
917
|
+
});
|
|
918
|
+
}
|
|
869
919
|
await requestContext.run(
|
|
870
920
|
{
|
|
871
921
|
requestId: jobRequestId,
|
|
@@ -873,6 +923,7 @@ export function createJobRunner(options: JobRunnerOptions): JobRunner {
|
|
|
873
923
|
// #3043 — events a job writes carry the job as their origin.
|
|
874
924
|
handler: jobName,
|
|
875
925
|
feature: qnScope(jobName),
|
|
926
|
+
writeOrigin: jobOrigin,
|
|
876
927
|
},
|
|
877
928
|
() => jobDef.handler(payload, jobContext),
|
|
878
929
|
);
|
|
@@ -1107,9 +1158,11 @@ export function createJobRunner(options: JobRunnerOptions): JobRunner {
|
|
|
1107
1158
|
|
|
1108
1159
|
// perTenant: dispatch the fan-out wrapper instead
|
|
1109
1160
|
if (jobDef.perTenant) {
|
|
1161
|
+
const perTenantData: Record<string, unknown> = { ...(payload ?? {}) };
|
|
1162
|
+
stampGatedWriteOrigin(perTenantData);
|
|
1110
1163
|
const job = await targetQueue.add(
|
|
1111
1164
|
`_perTenant:${jobName}`,
|
|
1112
|
-
|
|
1165
|
+
perTenantData,
|
|
1113
1166
|
buildRetryBullOpts(jobDef),
|
|
1114
1167
|
);
|
|
1115
1168
|
return job.id ?? "unknown";
|
|
@@ -1184,6 +1237,7 @@ export function createJobRunner(options: JobRunnerOptions): JobRunner {
|
|
|
1184
1237
|
// stamp the same correlation as the HTTP request that scheduled it.
|
|
1185
1238
|
const reqCtx = requestContext.get();
|
|
1186
1239
|
if (reqCtx?.correlationId) data["_correlationId"] = reqCtx.correlationId;
|
|
1240
|
+
stampGatedWriteOrigin(data);
|
|
1187
1241
|
|
|
1188
1242
|
const job = await targetQueue.add(jobName, data, bullOpts);
|
|
1189
1243
|
return job.id ?? "unknown";
|
|
@@ -1228,6 +1282,7 @@ export function createJobRunner(options: JobRunnerOptions): JobRunner {
|
|
|
1228
1282
|
continue;
|
|
1229
1283
|
}
|
|
1230
1284
|
}
|
|
1285
|
+
stampGatedWriteOrigin(data);
|
|
1231
1286
|
// Route to the job's declared lane, not a fixed queue — that's
|
|
1232
1287
|
// the whole reason both queues are held.
|
|
1233
1288
|
await queues[laneForJob(jobDef)].add(name, data, buildRetryBullOpts(jobDef));
|