@cosmicdrift/kumiko-framework 0.311.0 → 0.312.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +4 -4
- package/src/changes.json +18 -0
- package/src/derivatives/derivatives-context.ts +1 -8
- package/src/engine/feature-config-events-jobs.ts +3 -0
- package/src/files/__tests__/files.integration.test.ts +115 -1
- package/src/files/file-routes.ts +13 -2
- package/src/files/index.ts +2 -0
- package/src/files/types.ts +44 -2
- package/src/jobs/job-runner.ts +12 -0
- package/src/pipeline/__tests__/job-trigger-consumer.integration.test.ts +100 -1
- package/src/pipeline/msp-rebuild.ts +1 -0
- package/src/pipeline/multi-stream-apply-context.ts +1 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cosmicdrift/kumiko-framework",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.312.0",
|
|
4
4
|
"description": "Framework core — engine, pipeline, API, DB, and every other bit that makes Kumiko go.",
|
|
5
5
|
"license": "BUSL-1.1",
|
|
6
6
|
"author": "Marc Frost <marc@cosmicdriftgamestudio.com>",
|
|
@@ -198,8 +198,8 @@
|
|
|
198
198
|
"./package.json": "./package.json"
|
|
199
199
|
},
|
|
200
200
|
"dependencies": {
|
|
201
|
-
"@cosmicdrift/kumiko-http": "0.
|
|
202
|
-
"@cosmicdrift/kumiko-types": "0.
|
|
201
|
+
"@cosmicdrift/kumiko-http": "0.312.0",
|
|
202
|
+
"@cosmicdrift/kumiko-types": "0.312.0",
|
|
203
203
|
"bullmq": "^5.76.7",
|
|
204
204
|
"bun-types": "^1.3.13",
|
|
205
205
|
"hono": "^4.13.1",
|
|
@@ -215,7 +215,7 @@
|
|
|
215
215
|
"zod": "^4.4.3"
|
|
216
216
|
},
|
|
217
217
|
"devDependencies": {
|
|
218
|
-
"@cosmicdrift/kumiko-dispatcher-live": "0.
|
|
218
|
+
"@cosmicdrift/kumiko-dispatcher-live": "0.312.0",
|
|
219
219
|
"bun-types": "^1.3.13",
|
|
220
220
|
"pino-pretty": "^13.1.3"
|
|
221
221
|
},
|
package/src/changes.json
CHANGED
|
@@ -1,4 +1,22 @@
|
|
|
1
1
|
[
|
|
2
|
+
{
|
|
3
|
+
"version": "0.312.0",
|
|
4
|
+
"type": "improvement",
|
|
5
|
+
"title": "files: doc/docx accepted, with magic-byte content verification",
|
|
6
|
+
"detail": "EXTENSION_MIME_WHITELIST and MAGIC_BYTE_SIGNATURES now cover .doc (application/msword, OLE signature) and .docx (application/vnd.openxmlformats-officedocument.wordprocessingml.document, ZIP signature). New exported validateFileContent(mimeType, content) checks, for doc/docx only, the declared mimeType against the file's magic bytes independent of options.accept — validateFile's own signature is unchanged (metadata-only); file-routes.ts calls validateFile first, then validateFileContent once after reading the upload body. normalizeMimeType is exported from files/types.ts as the single case/parameter-normalization helper (derivatives-context.ts's local copy was removed in favor of it)."
|
|
7
|
+
},
|
|
8
|
+
{
|
|
9
|
+
"version": "0.312.0",
|
|
10
|
+
"type": "breaking",
|
|
11
|
+
"title": "MultiStreamApplyContext gains a required registry field",
|
|
12
|
+
"migration": "Any hand-built MultiStreamApplyContext (e.g. in an MSP test that constructs the context object literal instead of using createMultiStreamApplyContext/setupTestStack) must add registry: <the app Registry instance>, the same instance HandlerContext/JobContext already expose. Lets an apply resolve extension-point usages (registry.getExtensionUsages) to pick behavior by payload discriminant, e.g. a provider-routed MSP."
|
|
13
|
+
},
|
|
14
|
+
{
|
|
15
|
+
"version": "0.312.0",
|
|
16
|
+
"type": "improvement",
|
|
17
|
+
"title": "r.job triggers can filter on payload fields via JobTrigger.where",
|
|
18
|
+
"detail": "A job's {on} trigger can now carry an optional where: equality filter on top-level event-payload fields, checked in job-runner's handleEvent before the job is enqueued (both the sync write-handler dispatch path and the async event-consumer path share this check). Lets N jobs share one broad event QN, partitioned by a payload discriminant, instead of each needing its own narrow event type."
|
|
19
|
+
},
|
|
2
20
|
{
|
|
3
21
|
"version": "0.310.0",
|
|
4
22
|
"type": "improvement",
|
|
@@ -14,7 +14,7 @@ import type { Registry, TenantId } from "../engine/types";
|
|
|
14
14
|
import { InternalError, NotFoundError } from "../errors";
|
|
15
15
|
import type { FileContext } from "../files/file-handle";
|
|
16
16
|
import { fileRefsTable } from "../files/file-ref-table";
|
|
17
|
-
import { assertSafeStorageKey } from "../files/types";
|
|
17
|
+
import { assertSafeStorageKey, normalizeMimeType } from "../files/types";
|
|
18
18
|
import { variantSuffix } from "./variant-key";
|
|
19
19
|
|
|
20
20
|
export type DerivativesContextDeps = {
|
|
@@ -30,13 +30,6 @@ function isDerivativeRendererPlugin(o: unknown): o is DerivativeRendererPlugin {
|
|
|
30
30
|
return typeof o === "object" && o !== null && "render" in o && typeof o.render === "function";
|
|
31
31
|
}
|
|
32
32
|
|
|
33
|
-
// Mirrors validateFile's mime-normalization: strip a `; charset=…` suffix,
|
|
34
|
-
// trim, lowercase — so a provider-supplied `image/jpeg; charset=binary`
|
|
35
|
-
// still resolves.
|
|
36
|
-
function normalizeMimeType(mimeType: string): string {
|
|
37
|
-
return mimeType.toLowerCase().split(";")[0]?.trim() ?? "";
|
|
38
|
-
}
|
|
39
|
-
|
|
40
33
|
export function resolveRenderer(
|
|
41
34
|
registry: Registry,
|
|
42
35
|
mimeType: string,
|
|
@@ -222,6 +222,9 @@ export function buildConfigEventsJobsMethods<TName extends string>(
|
|
|
222
222
|
on: Array.isArray(jobOptions.trigger.on)
|
|
223
223
|
? jobOptions.trigger.on.map(resolveName)
|
|
224
224
|
: resolveName(jobOptions.trigger.on as NameOrRef), // @cast-boundary engine-bridge
|
|
225
|
+
...(jobOptions.trigger.where !== undefined
|
|
226
|
+
? { where: jobOptions.trigger.where }
|
|
227
|
+
: {}),
|
|
225
228
|
}
|
|
226
229
|
: jobOptions.trigger;
|
|
227
230
|
state.jobs[jobName] = { ...jobOptions, trigger, name: jobName, handler: jobHandler };
|
|
@@ -35,7 +35,7 @@ import type { FileRoutesOptions } from "../file-routes";
|
|
|
35
35
|
import { createInMemoryFileProvider } from "../in-memory-provider";
|
|
36
36
|
import { createLocalProvider } from "../local-provider";
|
|
37
37
|
import type { FileStorageProvider, SignedUrlOptions } from "../types";
|
|
38
|
-
import { parseMaxSize, sniffMimeType, validateFile } from "../types";
|
|
38
|
+
import { parseMaxSize, sniffMimeType, validateFile, validateFileContent } from "../types";
|
|
39
39
|
|
|
40
40
|
// UUID for "this row doesn't exist" assertions. Valid v4 format so PG accepts
|
|
41
41
|
// the query — the row just isn't there. Pre-v1 files-feature tests used
|
|
@@ -222,6 +222,80 @@ describe("file validation", () => {
|
|
|
222
222
|
expect(sniffMimeType(new Uint8Array([0x00, 0x01, 0x02]))).toBeNull();
|
|
223
223
|
expect(sniffMimeType(new TextEncoder().encode("hello world"))).toBeNull();
|
|
224
224
|
});
|
|
225
|
+
|
|
226
|
+
// --- doc/docx magic-byte content verification ---
|
|
227
|
+
|
|
228
|
+
const docBytes = new Uint8Array([
|
|
229
|
+
0xd0,
|
|
230
|
+
0xcf,
|
|
231
|
+
0x11,
|
|
232
|
+
0xe0,
|
|
233
|
+
0xa1,
|
|
234
|
+
0xb1,
|
|
235
|
+
0x1a,
|
|
236
|
+
0xe1,
|
|
237
|
+
...Array(20).fill(0),
|
|
238
|
+
]);
|
|
239
|
+
const docxBytes = new Uint8Array([0x50, 0x4b, 0x03, 0x04, ...Array(20).fill(0)]);
|
|
240
|
+
|
|
241
|
+
test("validateFile accepts a .doc/.docx upload on extension/mimeType alone", () => {
|
|
242
|
+
expect(
|
|
243
|
+
validateFile(
|
|
244
|
+
{ fileName: "report.doc", mimeType: "application/msword", size: docBytes.length },
|
|
245
|
+
{ accept: ["doc", "docx"] },
|
|
246
|
+
),
|
|
247
|
+
).toBeNull();
|
|
248
|
+
expect(
|
|
249
|
+
validateFile(
|
|
250
|
+
{
|
|
251
|
+
fileName: "report.docx",
|
|
252
|
+
mimeType: "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
|
253
|
+
size: docxBytes.length,
|
|
254
|
+
},
|
|
255
|
+
{ accept: ["doc", "docx"] },
|
|
256
|
+
),
|
|
257
|
+
).toBeNull();
|
|
258
|
+
});
|
|
259
|
+
|
|
260
|
+
test("validateFileContent accepts .doc/.docx bytes matching their declared mimeType", () => {
|
|
261
|
+
expect(validateFileContent("application/msword", docBytes)).toBeNull();
|
|
262
|
+
expect(
|
|
263
|
+
validateFileContent(
|
|
264
|
+
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
|
265
|
+
docxBytes,
|
|
266
|
+
),
|
|
267
|
+
).toBeNull();
|
|
268
|
+
});
|
|
269
|
+
|
|
270
|
+
test("validateFileContent rejects .doc/.docx whose content doesn't match the declared mimeType", () => {
|
|
271
|
+
const fakeDoc = validateFileContent(
|
|
272
|
+
"application/msword",
|
|
273
|
+
new TextEncoder().encode("not actually a Word document, just text"),
|
|
274
|
+
);
|
|
275
|
+
expectErrorIncludes(fakeDoc, "content_mismatch");
|
|
276
|
+
|
|
277
|
+
const fakeDocx = validateFileContent(
|
|
278
|
+
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
|
279
|
+
new TextEncoder().encode("not actually a docx, just text"),
|
|
280
|
+
);
|
|
281
|
+
expectErrorIncludes(fakeDocx, "content_mismatch");
|
|
282
|
+
});
|
|
283
|
+
|
|
284
|
+
test("validateFileContent does not content-verify pre-existing types (e.g. pdf) even when content mismatches", () => {
|
|
285
|
+
expect(
|
|
286
|
+
validateFileContent(
|
|
287
|
+
"application/pdf",
|
|
288
|
+
new TextEncoder().encode("not a PDF at all, just text"),
|
|
289
|
+
),
|
|
290
|
+
).toBeNull();
|
|
291
|
+
});
|
|
292
|
+
|
|
293
|
+
test("sniffMimeType recognizes doc (OLE) and docx (ZIP) signatures", () => {
|
|
294
|
+
expect(sniffMimeType(docBytes)).toBe("application/msword");
|
|
295
|
+
expect(sniffMimeType(docxBytes)).toBe(
|
|
296
|
+
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
|
297
|
+
);
|
|
298
|
+
});
|
|
225
299
|
});
|
|
226
300
|
|
|
227
301
|
// --- Integration: Upload → Download → Delete via real HTTP API ---
|
|
@@ -409,6 +483,46 @@ describe("download Content-Type is sniffed from bytes, not trusted from the decl
|
|
|
409
483
|
});
|
|
410
484
|
});
|
|
411
485
|
|
|
486
|
+
// --- doc/docx upload via API: content-verified independent of `accept` ---
|
|
487
|
+
|
|
488
|
+
describe("doc/docx upload is content-verified against magic bytes", () => {
|
|
489
|
+
const docBytes = new Uint8Array([
|
|
490
|
+
0xd0,
|
|
491
|
+
0xcf,
|
|
492
|
+
0x11,
|
|
493
|
+
0xe0,
|
|
494
|
+
0xa1,
|
|
495
|
+
0xb1,
|
|
496
|
+
0x1a,
|
|
497
|
+
0xe1,
|
|
498
|
+
...Array(20).fill(0),
|
|
499
|
+
]);
|
|
500
|
+
const docxBytes = new Uint8Array([0x50, 0x4b, 0x03, 0x04, ...Array(20).fill(0)]);
|
|
501
|
+
|
|
502
|
+
test("a real .doc upload succeeds", async () => {
|
|
503
|
+
const res = await uploadFile(adminUser, "report.doc", docBytes, "application/msword");
|
|
504
|
+
expect(res.status).toBe(201);
|
|
505
|
+
});
|
|
506
|
+
|
|
507
|
+
test("a real .docx upload succeeds", async () => {
|
|
508
|
+
const res = await uploadFile(
|
|
509
|
+
adminUser,
|
|
510
|
+
"report.docx",
|
|
511
|
+
docxBytes,
|
|
512
|
+
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
|
513
|
+
);
|
|
514
|
+
expect(res.status).toBe(201);
|
|
515
|
+
});
|
|
516
|
+
|
|
517
|
+
test("a .doc upload whose bytes don't match the declared mimeType is rejected with content_mismatch, even unattached (no `accept` list)", async () => {
|
|
518
|
+
const fakeDocBytes = new TextEncoder().encode("plain text pretending to be a Word document");
|
|
519
|
+
const res = await uploadFile(adminUser, "fake.doc", fakeDocBytes, "application/msword");
|
|
520
|
+
expect(res.status).toBe(400);
|
|
521
|
+
const body = await res.json();
|
|
522
|
+
expect(body.error).toContain("content_mismatch");
|
|
523
|
+
});
|
|
524
|
+
});
|
|
525
|
+
|
|
412
526
|
// --- Cross-user access within a tenant (attached file owner-scope) ---
|
|
413
527
|
|
|
414
528
|
describe("attached file owner-scope", () => {
|
package/src/files/file-routes.ts
CHANGED
|
@@ -27,7 +27,12 @@ import { createFileContext } from "./file-handle";
|
|
|
27
27
|
import { fileRefEntity } from "./file-ref-entity";
|
|
28
28
|
import { fileRefsTable } from "./file-ref-table";
|
|
29
29
|
import type { FileProviderResolver } from "./provider-resolver";
|
|
30
|
-
import {
|
|
30
|
+
import {
|
|
31
|
+
buildStorageKey,
|
|
32
|
+
resolveServedContentType,
|
|
33
|
+
validateFile,
|
|
34
|
+
validateFileContent,
|
|
35
|
+
} from "./types";
|
|
31
36
|
|
|
32
37
|
// Decision returned by a FileAccessGuard — distinct from boolean so callers
|
|
33
38
|
// can't accidentally negate or default it.
|
|
@@ -228,13 +233,19 @@ export function createFileRoutes(options: FileRoutesOptions): Hono {
|
|
|
228
233
|
generateId(),
|
|
229
234
|
);
|
|
230
235
|
|
|
236
|
+
const data = new Uint8Array(await file.arrayBuffer());
|
|
237
|
+
|
|
238
|
+
const contentValidationError = validateFileContent(file.type, data);
|
|
239
|
+
if (contentValidationError) {
|
|
240
|
+
return c.json({ error: contentValidationError }, 400);
|
|
241
|
+
}
|
|
242
|
+
|
|
231
243
|
// Write binary FIRST (outside the tx — network/disk I/O doesn't belong
|
|
232
244
|
// inside a PG connection's tx window). On DB-tx rollback below the bytes
|
|
233
245
|
// are orphaned in the provider; cleanup-jobs sweep those later. Losing a
|
|
234
246
|
// row on append-failure is acceptable; corrupting a committed row with a
|
|
235
247
|
// missing binary is not.
|
|
236
248
|
const storageProvider = await options.resolveProvider(user.tenantId);
|
|
237
|
-
const data = new Uint8Array(await file.arrayBuffer());
|
|
238
249
|
await storageProvider.write(storageKey, data, file.type);
|
|
239
250
|
|
|
240
251
|
// Create via the standard entity executor: emits fileRef.created +
|
package/src/files/index.ts
CHANGED
|
@@ -37,10 +37,12 @@ export type {
|
|
|
37
37
|
export {
|
|
38
38
|
assertSafeStorageKey,
|
|
39
39
|
buildStorageKey,
|
|
40
|
+
normalizeMimeType,
|
|
40
41
|
parseMaxSize,
|
|
41
42
|
tenantExportPrefix,
|
|
42
43
|
tenantStoragePrefixes,
|
|
43
44
|
validateFile,
|
|
45
|
+
validateFileContent,
|
|
44
46
|
} from "./types";
|
|
45
47
|
export type { ZipEntry } from "./zip-stream";
|
|
46
48
|
export { createZipStream } from "./zip-stream";
|
package/src/files/types.ts
CHANGED
|
@@ -50,6 +50,8 @@ const EXTENSION_MIME_WHITELIST: Record<string, readonly string[]> = {
|
|
|
50
50
|
csv: ["text/csv", "application/csv", "text/plain"],
|
|
51
51
|
json: ["application/json", "text/json"],
|
|
52
52
|
md: ["text/markdown", "text/plain"],
|
|
53
|
+
doc: ["application/msword"],
|
|
54
|
+
docx: ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"],
|
|
53
55
|
} satisfies Record<string, readonly string[]>;
|
|
54
56
|
|
|
55
57
|
// Magic-byte signatures for the subset of EXTENSION_MIME_WHITELIST that has
|
|
@@ -78,6 +80,19 @@ const MAGIC_BYTE_SIGNATURES: ReadonlyArray<{
|
|
|
78
80
|
matches: (bytes) => startsWithAscii(bytes, "RIFF") && startsWithAscii(bytes, "WEBP", 8),
|
|
79
81
|
},
|
|
80
82
|
{ mimeType: "application/pdf", matches: (bytes) => startsWithAscii(bytes, "%PDF-") },
|
|
83
|
+
// Full 8-byte OLE Compound File signature (legacy .doc/.xls/.ppt share it —
|
|
84
|
+
// sniffing can't tell them apart by bytes alone, only content-verification
|
|
85
|
+
// below narrows to msword specifically).
|
|
86
|
+
{
|
|
87
|
+
mimeType: "application/msword",
|
|
88
|
+
matches: (bytes) => startsWithBytes(bytes, [0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1]),
|
|
89
|
+
},
|
|
90
|
+
// ZIP local-file-header signature — .docx/.xlsx/.pptx/plain .zip all share
|
|
91
|
+
// it; same caveat as OLE above.
|
|
92
|
+
{
|
|
93
|
+
mimeType: "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
|
94
|
+
matches: (bytes) => startsWithBytes(bytes, [0x50, 0x4b, 0x03, 0x04]),
|
|
95
|
+
},
|
|
81
96
|
];
|
|
82
97
|
|
|
83
98
|
function startsWithBytes(bytes: Uint8Array, signature: readonly number[]): boolean {
|
|
@@ -100,6 +115,12 @@ export function sniffMimeType(bytes: Uint8Array): string | null {
|
|
|
100
115
|
return null;
|
|
101
116
|
}
|
|
102
117
|
|
|
118
|
+
// Strips a `; charset=…`-style suffix and trims/lowercases — the one
|
|
119
|
+
// normalization every mimeType comparison needs before comparing values.
|
|
120
|
+
export function normalizeMimeType(mimeType: string): string {
|
|
121
|
+
return mimeType.toLowerCase().split(";")[0]?.trim() ?? "";
|
|
122
|
+
}
|
|
123
|
+
|
|
103
124
|
// image/jpg is not a registered IANA type but EXTENSION_MIME_WHITELIST above
|
|
104
125
|
// accepts it as a jpg alias — sniffMimeType only ever returns the canonical
|
|
105
126
|
// image/jpeg, so without this a legitimately-declared "image/jpg" upload
|
|
@@ -119,11 +140,19 @@ const DECLARED_MIME_ALIASES: Readonly<Record<string, string>> = {
|
|
|
119
140
|
export function resolveServedContentType(bytes: Uint8Array, declaredMimeType: string): string {
|
|
120
141
|
const sniffed = sniffMimeType(bytes);
|
|
121
142
|
if (!sniffed) return "application/octet-stream";
|
|
122
|
-
const normalizedDeclared = declaredMimeType
|
|
143
|
+
const normalizedDeclared = normalizeMimeType(declaredMimeType);
|
|
123
144
|
const declared = DECLARED_MIME_ALIASES[normalizedDeclared] ?? normalizedDeclared;
|
|
124
145
|
return sniffed === declared ? sniffed : "application/octet-stream";
|
|
125
146
|
}
|
|
126
147
|
|
|
148
|
+
// Signature-checked against the declared mimeType regardless of
|
|
149
|
+
// `options.accept`. Deliberately narrow: most of the whitelist has no magic
|
|
150
|
+
// bytes at all and would reject honestly-mislabeled uploads if checked.
|
|
151
|
+
const CONTENT_VERIFIED_MIME_TYPES = new Set([
|
|
152
|
+
"application/msword",
|
|
153
|
+
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
|
154
|
+
]);
|
|
155
|
+
|
|
127
156
|
export function validateFile(
|
|
128
157
|
metadata: FileMetadata,
|
|
129
158
|
options: FileValidationOptions,
|
|
@@ -146,7 +175,7 @@ export function validateFile(
|
|
|
146
175
|
// and having the mimeType reflect that.
|
|
147
176
|
const allowedMimes = EXTENSION_MIME_WHITELIST[ext];
|
|
148
177
|
if (allowedMimes && metadata.mimeType) {
|
|
149
|
-
const normalized = metadata.mimeType
|
|
178
|
+
const normalized = normalizeMimeType(metadata.mimeType);
|
|
150
179
|
if (!allowedMimes.includes(normalized)) {
|
|
151
180
|
return `mime_mismatch: extension ".${ext}" does not match mimeType "${metadata.mimeType}"`;
|
|
152
181
|
}
|
|
@@ -156,6 +185,19 @@ export function validateFile(
|
|
|
156
185
|
return null;
|
|
157
186
|
}
|
|
158
187
|
|
|
188
|
+
// Separate from validateFile: metadata is known before the upload body is
|
|
189
|
+
// fully read, content-verification only after — callers reject early on
|
|
190
|
+
// metadata without buffering bytes they'll then have to reject anyway.
|
|
191
|
+
export function validateFileContent(mimeType: string, content: Uint8Array): string | null {
|
|
192
|
+
const normalizedDeclared = normalizeMimeType(mimeType);
|
|
193
|
+
if (!CONTENT_VERIFIED_MIME_TYPES.has(normalizedDeclared)) return null;
|
|
194
|
+
const signature = MAGIC_BYTE_SIGNATURES.find((s) => s.mimeType === normalizedDeclared);
|
|
195
|
+
if (signature && !signature.matches(content)) {
|
|
196
|
+
return `content_mismatch: bytes do not match declared mimeType "${mimeType}"`;
|
|
197
|
+
}
|
|
198
|
+
return null;
|
|
199
|
+
}
|
|
200
|
+
|
|
159
201
|
export function assertSafeStorageKey(key: string): void {
|
|
160
202
|
if (key.split("/").some((segment) => segment === "." || segment === "..")) {
|
|
161
203
|
throw new Error(`storage key contains a path-traversal segment: "${key}"`);
|
package/src/jobs/job-runner.ts
CHANGED
|
@@ -367,6 +367,17 @@ function buildRetryBullOpts(jobDef: JobDefinition): Pick<JobsOptions, "attempts"
|
|
|
367
367
|
return opts;
|
|
368
368
|
}
|
|
369
369
|
|
|
370
|
+
// Equality filter on top-level payload fields for a job's trigger.where —
|
|
371
|
+
// lets N jobs share one broad event QN, partitioned by a payload
|
|
372
|
+
// discriminant instead of needing N distinct event types.
|
|
373
|
+
function payloadMatchesTriggerFilter(
|
|
374
|
+
where: Readonly<Record<string, string | number | boolean>> | undefined,
|
|
375
|
+
payload: Record<string, unknown>,
|
|
376
|
+
): boolean {
|
|
377
|
+
if (!where) return true;
|
|
378
|
+
return Object.entries(where).every(([key, value]) => payload[key] === value);
|
|
379
|
+
}
|
|
380
|
+
|
|
370
381
|
// BullMQ sweeps queue-wide: any job finishing with keepJobs set can evict
|
|
371
382
|
// OTHER jobs in the same completed/failed zset (moveToFinished lua,
|
|
372
383
|
// removeJobsByMaxAge/removeJobsByMaxCount), not just the finishing job, and
|
|
@@ -1263,6 +1274,7 @@ export function createJobRunner(options: JobRunnerOptions): JobRunner {
|
|
|
1263
1274
|
? triggerOn.includes(eventName)
|
|
1264
1275
|
: triggerOn === eventName;
|
|
1265
1276
|
if (!matches) continue;
|
|
1277
|
+
if (!payloadMatchesTriggerFilter(jobDef.trigger.where, payload)) continue;
|
|
1266
1278
|
const data: Record<string, unknown> = { ...payload };
|
|
1267
1279
|
if (user) {
|
|
1268
1280
|
data["_tenantId"] = user.tenantId;
|
|
@@ -23,6 +23,7 @@ import { defineFeature } from "../../engine";
|
|
|
23
23
|
import { createInMemoryFileProvider, type InMemoryFileProvider } from "../../files";
|
|
24
24
|
import { setupTestStack, type TestStack, TestUsers } from "../../stack";
|
|
25
25
|
import { waitFor } from "../../testing";
|
|
26
|
+
import { generateId } from "../../utils";
|
|
26
27
|
|
|
27
28
|
const ITEM_REQUESTED_EVENT_QN = "job-trigger-fixture:event:item-requested";
|
|
28
29
|
const FILE_REF_CREATED = entityEventName("fileRef", "created");
|
|
@@ -62,25 +63,90 @@ const jobTriggerFixtureFeature = defineFeature("job-trigger-fixture", (r) => {
|
|
|
62
63
|
);
|
|
63
64
|
});
|
|
64
65
|
|
|
66
|
+
// trigger.where partition fixture — two jobs share one broad QN, each
|
|
67
|
+
// filtered to a disjoint payload.kind, mirroring document-ingest-foundation's
|
|
68
|
+
// N-providers-one-QN shape without pulling in the whole feature.
|
|
69
|
+
const WHERE_ITEM_REQUESTED_EVENT_QN = "job-trigger-where-fixture:event:item-requested";
|
|
70
|
+
|
|
71
|
+
const alphaProcessed: Array<{ readonly fileRefId: string }> = [];
|
|
72
|
+
const betaProcessed: Array<{ readonly fileRefId: string }> = [];
|
|
73
|
+
|
|
74
|
+
const jobTriggerWhereFixtureFeature = defineFeature("job-trigger-where-fixture", (r) => {
|
|
75
|
+
r.defineEvent(
|
|
76
|
+
"item-requested",
|
|
77
|
+
z.object({ fileRefId: z.string().min(1), kind: z.enum(["alpha", "beta"]) }),
|
|
78
|
+
{ piiFields: "none" },
|
|
79
|
+
);
|
|
80
|
+
|
|
81
|
+
r.multiStreamProjection({
|
|
82
|
+
name: "request-item",
|
|
83
|
+
apply: {
|
|
84
|
+
[FILE_REF_CREATED]: async (event, _tx, ctx) => {
|
|
85
|
+
const fileName = event.payload["fileName"];
|
|
86
|
+
const kind =
|
|
87
|
+
typeof fileName === "string" && fileName.startsWith("alpha-") ? "alpha" : "beta";
|
|
88
|
+
await ctx.unsafeAppendEvent({
|
|
89
|
+
aggregateId: event.aggregateId,
|
|
90
|
+
aggregateType: "job-trigger-where-fixture-request",
|
|
91
|
+
type: WHERE_ITEM_REQUESTED_EVENT_QN,
|
|
92
|
+
payload: { fileRefId: event.aggregateId, kind },
|
|
93
|
+
});
|
|
94
|
+
},
|
|
95
|
+
},
|
|
96
|
+
});
|
|
97
|
+
|
|
98
|
+
// Under test: both jobs trigger on the SAME QN; only the one whose
|
|
99
|
+
// `where` matches the appended payload's `kind` may run.
|
|
100
|
+
r.job(
|
|
101
|
+
"process-alpha",
|
|
102
|
+
{ trigger: { on: WHERE_ITEM_REQUESTED_EVENT_QN, where: { kind: "alpha" } }, runIn: "worker" },
|
|
103
|
+
async (payload) => {
|
|
104
|
+
alphaProcessed.push({ fileRefId: payload["fileRefId"] as string });
|
|
105
|
+
},
|
|
106
|
+
);
|
|
107
|
+
r.job(
|
|
108
|
+
"process-beta",
|
|
109
|
+
{ trigger: { on: WHERE_ITEM_REQUESTED_EVENT_QN, where: { kind: "beta" } }, runIn: "worker" },
|
|
110
|
+
async (payload) => {
|
|
111
|
+
betaProcessed.push({ fileRefId: payload["fileRefId"] as string });
|
|
112
|
+
},
|
|
113
|
+
);
|
|
114
|
+
});
|
|
115
|
+
|
|
65
116
|
let stack: TestStack;
|
|
66
117
|
let provider: InMemoryFileProvider;
|
|
118
|
+
let whereStack: TestStack;
|
|
119
|
+
let whereProvider: InMemoryFileProvider;
|
|
67
120
|
|
|
68
121
|
beforeAll(async () => {
|
|
69
122
|
provider = createInMemoryFileProvider();
|
|
70
123
|
stack = await setupTestStack({
|
|
71
124
|
features: [jobTriggerFixtureFeature],
|
|
72
125
|
files: { storageProvider: provider },
|
|
73
|
-
jobs: { consumerLane: "worker" },
|
|
126
|
+
jobs: { consumerLane: "worker", queueNamePrefix: `job-trigger-fixture-${generateId()}` },
|
|
127
|
+
});
|
|
128
|
+
whereProvider = createInMemoryFileProvider();
|
|
129
|
+
whereStack = await setupTestStack({
|
|
130
|
+
features: [jobTriggerWhereFixtureFeature],
|
|
131
|
+
files: { storageProvider: whereProvider },
|
|
132
|
+
// Distinct prefix from `stack` above — createTestRedis's keyPrefix
|
|
133
|
+
// isolation doesn't cover BullMQ's own connections, so without this both
|
|
134
|
+
// JobRunners would race each other on the same default queue name.
|
|
135
|
+
jobs: { consumerLane: "worker", queueNamePrefix: `job-trigger-where-fixture-${generateId()}` },
|
|
74
136
|
});
|
|
75
137
|
});
|
|
76
138
|
|
|
77
139
|
afterAll(async () => {
|
|
78
140
|
await stack.cleanup();
|
|
141
|
+
await whereStack.cleanup();
|
|
79
142
|
});
|
|
80
143
|
|
|
81
144
|
beforeEach(() => {
|
|
82
145
|
processedItems.length = 0;
|
|
83
146
|
provider.clear();
|
|
147
|
+
alphaProcessed.length = 0;
|
|
148
|
+
betaProcessed.length = 0;
|
|
149
|
+
whereProvider.clear();
|
|
84
150
|
});
|
|
85
151
|
|
|
86
152
|
describe("job-trigger event consumer", () => {
|
|
@@ -106,3 +172,36 @@ describe("job-trigger event consumer", () => {
|
|
|
106
172
|
expect(processedItems[0]?.fileRefId).toBeTruthy();
|
|
107
173
|
});
|
|
108
174
|
});
|
|
175
|
+
|
|
176
|
+
describe("job-trigger event consumer — trigger.where partitions one QN across jobs", () => {
|
|
177
|
+
test("only the job whose where matches the payload runs — both partitions checked, not just the absence of the other", async () => {
|
|
178
|
+
const token = await whereStack.jwt.sign(TestUsers.admin);
|
|
179
|
+
async function uploadKind(fileName: string): Promise<string> {
|
|
180
|
+
const formData = new FormData();
|
|
181
|
+
formData.append("file", new File([Buffer.from("hello")], fileName, { type: "text/plain" }));
|
|
182
|
+
const res = await whereStack.app.request("/api/files", {
|
|
183
|
+
method: "POST",
|
|
184
|
+
headers: { Authorization: `Bearer ${token}` },
|
|
185
|
+
body: formData,
|
|
186
|
+
});
|
|
187
|
+
expect(res.status).toBe(201);
|
|
188
|
+
const body = (await res.json()) as { id: string };
|
|
189
|
+
return body.id;
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
const alphaFileRefId = await uploadKind("alpha-note.txt");
|
|
193
|
+
const betaFileRefId = await uploadKind("beta-note.txt");
|
|
194
|
+
|
|
195
|
+
await waitFor(async () => {
|
|
196
|
+
// Drives both jobs' consumers in the same pass — asserting both lists
|
|
197
|
+
// are non-empty before checking membership rules out "beta simply
|
|
198
|
+
// hasn't run yet" as an explanation for an empty betaProcessed.
|
|
199
|
+
await whereStack.eventDispatcher?.runOnce();
|
|
200
|
+
expect(alphaProcessed).toHaveLength(1);
|
|
201
|
+
expect(betaProcessed).toHaveLength(1);
|
|
202
|
+
});
|
|
203
|
+
|
|
204
|
+
expect(alphaProcessed).toEqual([{ fileRefId: alphaFileRefId }]);
|
|
205
|
+
expect(betaProcessed).toEqual([{ fileRefId: betaFileRefId }]);
|
|
206
|
+
});
|
|
207
|
+
});
|
|
@@ -115,6 +115,7 @@ function createRebuildCtx(
|
|
|
115
115
|
});
|
|
116
116
|
};
|
|
117
117
|
return {
|
|
118
|
+
registry,
|
|
118
119
|
appendEvent: refuseAppend as MultiStreamApplyContext["appendEvent"], // @cast-boundary engine-bridge
|
|
119
120
|
unsafeAppendEvent: refuseAppend,
|
|
120
121
|
loadAggregate: async (aggregateId, options) => {
|
|
@@ -38,6 +38,7 @@ export function createMultiStreamApplyContext(
|
|
|
38
38
|
deps: MultiStreamApplyContextDeps,
|
|
39
39
|
): MultiStreamApplyContext {
|
|
40
40
|
return {
|
|
41
|
+
registry: deps.registry,
|
|
41
42
|
...(deps.files ? { files: deps.files } : {}),
|
|
42
43
|
...(deps.derivatives ? { derivatives: deps.derivatives } : {}),
|
|
43
44
|
appendEvent: (async (args: AppendEventArgs) => {
|