@cosmicdrift/kumiko-framework 0.313.0 → 0.315.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +4 -4
- package/src/__tests__/upgrade-cli.test.ts +28 -0
- package/src/api/__tests__/origin-middleware.integration.test.ts +142 -1
- package/src/api/auth-middleware.ts +30 -1
- package/src/api/origin-middleware.ts +23 -20
- package/src/api/server.ts +10 -1
- package/src/changes.json +36 -0
- package/src/derivatives/__tests__/derivatives-context.integration.test.ts +4 -2
- package/src/derivatives/__tests__/variant-route.integration.test.ts +10 -2
- package/src/files/__tests__/file-field-pipeline.integration.test.ts +24 -10
- package/src/files/__tests__/files.integration.test.ts +252 -20
- package/src/files/__tests__/storage-tracking.integration.test.ts +5 -3
- package/src/files/file-routes.ts +11 -4
- package/src/files/index.ts +3 -0
- package/src/files/types.ts +89 -30
- package/src/jobs/__tests__/job-backoff.integration.test.ts +1 -5
- package/src/pipeline/system-hooks.ts +2 -1
- package/src/stack/__tests__/drain-event-consumers.integration.test.ts +166 -0
- package/src/stack/__tests__/setup-test-stack-jobs.integration.test.ts +60 -0
- package/src/stack/drain-event-consumers.ts +74 -0
- package/src/stack/index.ts +1 -0
- package/src/stack/redis.ts +3 -0
- package/src/stack/test-stack.ts +13 -4
- package/src/upgrade-cli.ts +17 -3
|
@@ -8,6 +8,7 @@ import { buildServer } from "../../api/server";
|
|
|
8
8
|
import { configurePiiSubjectKms, InMemoryKmsAdapter } from "../../crypto";
|
|
9
9
|
import {
|
|
10
10
|
createEntity,
|
|
11
|
+
createFileField,
|
|
11
12
|
createImageField,
|
|
12
13
|
createRegistry,
|
|
13
14
|
createTextField,
|
|
@@ -35,7 +36,13 @@ import type { FileRoutesOptions } from "../file-routes";
|
|
|
35
36
|
import { createInMemoryFileProvider } from "../in-memory-provider";
|
|
36
37
|
import { createLocalProvider } from "../local-provider";
|
|
37
38
|
import type { FileStorageProvider, SignedUrlOptions } from "../types";
|
|
38
|
-
import {
|
|
39
|
+
import {
|
|
40
|
+
parseMaxSize,
|
|
41
|
+
resolveUploadMimeType,
|
|
42
|
+
sniffMimeType,
|
|
43
|
+
validateFile,
|
|
44
|
+
validateFileContent,
|
|
45
|
+
} from "../types";
|
|
39
46
|
|
|
40
47
|
// UUID for "this row doesn't exist" assertions. Valid v4 format so PG accepts
|
|
41
48
|
// the query — the row just isn't there. Pre-v1 files-feature tests used
|
|
@@ -59,6 +66,7 @@ const testTenantEntity = createEntity({
|
|
|
59
66
|
fields: {
|
|
60
67
|
name: createTextField({ personal: false, reason: "test_fixture", required: true }),
|
|
61
68
|
logo: createImageField({ maxSize: "2mb", accept: ["png", "jpg"] }),
|
|
69
|
+
importFile: createFileField({ maxSize: "2mb", accept: ["csv"] }),
|
|
62
70
|
},
|
|
63
71
|
});
|
|
64
72
|
|
|
@@ -257,37 +265,58 @@ describe("file validation", () => {
|
|
|
257
265
|
).toBeNull();
|
|
258
266
|
});
|
|
259
267
|
|
|
260
|
-
test("validateFileContent accepts .doc/.docx bytes matching their declared
|
|
261
|
-
expect(validateFileContent("
|
|
262
|
-
expect(
|
|
263
|
-
validateFileContent(
|
|
264
|
-
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
|
265
|
-
docxBytes,
|
|
266
|
-
),
|
|
267
|
-
).toBeNull();
|
|
268
|
+
test("validateFileContent accepts .doc/.docx bytes matching their declared extension", () => {
|
|
269
|
+
expect(validateFileContent("report.doc", docBytes)).toBeNull();
|
|
270
|
+
expect(validateFileContent("report.docx", docxBytes)).toBeNull();
|
|
268
271
|
});
|
|
269
272
|
|
|
270
|
-
test("validateFileContent rejects .doc/.docx whose content doesn't match the
|
|
273
|
+
test("validateFileContent rejects .doc/.docx whose content doesn't match the extension", () => {
|
|
271
274
|
const fakeDoc = validateFileContent(
|
|
272
|
-
"
|
|
275
|
+
"fake.doc",
|
|
273
276
|
new TextEncoder().encode("not actually a Word document, just text"),
|
|
274
277
|
);
|
|
275
278
|
expectErrorIncludes(fakeDoc, "content_mismatch");
|
|
276
279
|
|
|
277
280
|
const fakeDocx = validateFileContent(
|
|
278
|
-
"
|
|
281
|
+
"fake.docx",
|
|
279
282
|
new TextEncoder().encode("not actually a docx, just text"),
|
|
280
283
|
);
|
|
281
284
|
expectErrorIncludes(fakeDocx, "content_mismatch");
|
|
282
285
|
});
|
|
283
286
|
|
|
284
|
-
test("validateFileContent
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
287
|
+
test("validateFileContent now content-verifies every signature-bearing extension (e.g. pdf), not just doc/docx", () => {
|
|
288
|
+
const mismatch = validateFileContent(
|
|
289
|
+
"fake.pdf",
|
|
290
|
+
new TextEncoder().encode("not a PDF at all, just text"),
|
|
291
|
+
);
|
|
292
|
+
expectErrorIncludes(mismatch, "content_mismatch");
|
|
293
|
+
});
|
|
294
|
+
|
|
295
|
+
test("validateFileContent leaves extensions without a magic-byte signature unchecked (csv/txt/svg/md/json/unknown)", () => {
|
|
296
|
+
const text = new TextEncoder().encode("just plain text, not any binary format");
|
|
297
|
+
expect(validateFileContent("data.csv", text)).toBeNull();
|
|
298
|
+
expect(validateFileContent("notes.txt", text)).toBeNull();
|
|
299
|
+
expect(validateFileContent("readme.md", text)).toBeNull();
|
|
300
|
+
expect(validateFileContent("data.json", text)).toBeNull();
|
|
301
|
+
expect(validateFileContent("logo.svg", text)).toBeNull();
|
|
302
|
+
expect(validateFileContent("file.unknownext", text)).toBeNull();
|
|
303
|
+
});
|
|
304
|
+
|
|
305
|
+
test("validateFileContent treats a prototype-property extension as unknown instead of crashing", () => {
|
|
306
|
+
expect(validateFileContent("evil.constructor", docBytes)).toBeNull();
|
|
307
|
+
expect(validateFileContent("evil.__proto__", docBytes)).toBeNull();
|
|
308
|
+
expect(validateFileContent("evil.toString", docBytes)).toBeNull();
|
|
309
|
+
});
|
|
310
|
+
|
|
311
|
+
test("validateFile with accept: ['constructor'] and a matching filename never throws", () => {
|
|
312
|
+
let result: string | null = null;
|
|
313
|
+
expect(() => {
|
|
314
|
+
result = validateFile(
|
|
315
|
+
{ fileName: "x.constructor", mimeType: "application/octet-stream", size: 10 },
|
|
316
|
+
{ accept: ["constructor"] },
|
|
317
|
+
);
|
|
318
|
+
}).not.toThrow();
|
|
319
|
+
expect(result === null || typeof result === "string").toBe(true);
|
|
291
320
|
});
|
|
292
321
|
|
|
293
322
|
test("sniffMimeType recognizes doc (OLE) and docx (ZIP) signatures", () => {
|
|
@@ -296,6 +325,91 @@ describe("file validation", () => {
|
|
|
296
325
|
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
|
297
326
|
);
|
|
298
327
|
});
|
|
328
|
+
|
|
329
|
+
// --- .csv declared as application/vnd.ms-excel (Windows browsers) ---
|
|
330
|
+
|
|
331
|
+
test("validateFile accepts .csv declared as application/vnd.ms-excel when csv is in accept", () => {
|
|
332
|
+
expect(
|
|
333
|
+
validateFile(
|
|
334
|
+
{ fileName: "data.csv", mimeType: "application/vnd.ms-excel", size: 100 },
|
|
335
|
+
{ accept: ["csv"] },
|
|
336
|
+
),
|
|
337
|
+
).toBeNull();
|
|
338
|
+
});
|
|
339
|
+
|
|
340
|
+
test("validateFile does not extend the vnd.ms-excel alias to extensions other than csv", () => {
|
|
341
|
+
const error = validateFile(
|
|
342
|
+
{ fileName: "photo.jpg", mimeType: "application/vnd.ms-excel", size: 100 },
|
|
343
|
+
{ accept: ["jpg"] },
|
|
344
|
+
);
|
|
345
|
+
expectErrorIncludes(error, "mime_mismatch");
|
|
346
|
+
});
|
|
347
|
+
|
|
348
|
+
test("validateFile rejects a prototype-property mimeType instead of treating it as an alias match", () => {
|
|
349
|
+
const error = validateFile(
|
|
350
|
+
{ fileName: "photo.jpg", mimeType: "constructor", size: 100 },
|
|
351
|
+
{ accept: ["jpg"] },
|
|
352
|
+
);
|
|
353
|
+
expectErrorIncludes(error, "mime_mismatch");
|
|
354
|
+
});
|
|
355
|
+
|
|
356
|
+
describe("resolveUploadMimeType", () => {
|
|
357
|
+
const textCsvBytes = new TextEncoder().encode("name,age\nAda,36\n");
|
|
358
|
+
|
|
359
|
+
test("rewrites .csv + application/vnd.ms-excel to text/csv when bytes are text", () => {
|
|
360
|
+
expect(resolveUploadMimeType("data.csv", "application/vnd.ms-excel", textCsvBytes)).toEqual({
|
|
361
|
+
kind: "ok",
|
|
362
|
+
mimeType: "text/csv",
|
|
363
|
+
});
|
|
364
|
+
});
|
|
365
|
+
|
|
366
|
+
test("rewrites regardless of charset suffix or case", () => {
|
|
367
|
+
expect(
|
|
368
|
+
resolveUploadMimeType("data.csv", "Application/VND.MS-EXCEL; charset=utf-8", textCsvBytes),
|
|
369
|
+
).toEqual({ kind: "ok", mimeType: "text/csv" });
|
|
370
|
+
});
|
|
371
|
+
|
|
372
|
+
test("rejects .csv + application/vnd.ms-excel when bytes are a real OLE binary", () => {
|
|
373
|
+
const result = resolveUploadMimeType("data.csv", "application/vnd.ms-excel", docBytes);
|
|
374
|
+
expect(result.kind).toBe("rejected");
|
|
375
|
+
if (result.kind === "rejected") {
|
|
376
|
+
expectErrorIncludes(result.error, "content_mismatch");
|
|
377
|
+
}
|
|
378
|
+
});
|
|
379
|
+
|
|
380
|
+
test("rejects .csv + application/vnd.ms-excel when bytes are a ZIP container (.xlsx)", () => {
|
|
381
|
+
const result = resolveUploadMimeType("data.csv", "application/vnd.ms-excel", docxBytes);
|
|
382
|
+
expect(result.kind).toBe("rejected");
|
|
383
|
+
if (result.kind === "rejected") {
|
|
384
|
+
expectErrorIncludes(result.error, "content_mismatch");
|
|
385
|
+
}
|
|
386
|
+
});
|
|
387
|
+
|
|
388
|
+
test("leaves .csv + text/plain unchanged", () => {
|
|
389
|
+
expect(resolveUploadMimeType("data.csv", "text/plain", textCsvBytes)).toEqual({
|
|
390
|
+
kind: "ok",
|
|
391
|
+
mimeType: "text/plain",
|
|
392
|
+
});
|
|
393
|
+
});
|
|
394
|
+
|
|
395
|
+
test("leaves .xls + application/vnd.ms-excel unchanged (alias is csv-only)", () => {
|
|
396
|
+
expect(resolveUploadMimeType("data.xls", "application/vnd.ms-excel", docBytes)).toEqual({
|
|
397
|
+
kind: "ok",
|
|
398
|
+
mimeType: "application/vnd.ms-excel",
|
|
399
|
+
});
|
|
400
|
+
});
|
|
401
|
+
|
|
402
|
+
test("a prototype-property extension or mimeType never resolves as an alias", () => {
|
|
403
|
+
expect(resolveUploadMimeType("x.constructor", "name", textCsvBytes)).toEqual({
|
|
404
|
+
kind: "ok",
|
|
405
|
+
mimeType: "name",
|
|
406
|
+
});
|
|
407
|
+
expect(resolveUploadMimeType("data.csv", "constructor", textCsvBytes)).toEqual({
|
|
408
|
+
kind: "ok",
|
|
409
|
+
mimeType: "constructor",
|
|
410
|
+
});
|
|
411
|
+
});
|
|
412
|
+
});
|
|
299
413
|
});
|
|
300
414
|
|
|
301
415
|
// --- Integration: Upload → Download → Delete via real HTTP API ---
|
|
@@ -523,6 +637,124 @@ describe("doc/docx upload is content-verified against magic bytes", () => {
|
|
|
523
637
|
});
|
|
524
638
|
});
|
|
525
639
|
|
|
640
|
+
// --- Content-verification is keyed on the filename extension, not the ---
|
|
641
|
+
// --- declared/runtime mimeType (Option A, kumiko-framework#3287) ---
|
|
642
|
+
|
|
643
|
+
describe("content_mismatch is enforced by filename extension for every signature-bearing type", () => {
|
|
644
|
+
const jpegBytes = new Uint8Array([0xff, 0xd8, 0xff, 0xe0, ...Array(20).fill(0)]);
|
|
645
|
+
const pngBytes = new Uint8Array([
|
|
646
|
+
0x89,
|
|
647
|
+
0x50,
|
|
648
|
+
0x4e,
|
|
649
|
+
0x47,
|
|
650
|
+
0x0d,
|
|
651
|
+
0x0a,
|
|
652
|
+
0x1a,
|
|
653
|
+
0x0a,
|
|
654
|
+
...Array(20).fill(0),
|
|
655
|
+
]);
|
|
656
|
+
const pdfBytes = new TextEncoder().encode("%PDF-1.4 minimal");
|
|
657
|
+
const docBytes = new Uint8Array([
|
|
658
|
+
0xd0,
|
|
659
|
+
0xcf,
|
|
660
|
+
0x11,
|
|
661
|
+
0xe0,
|
|
662
|
+
0xa1,
|
|
663
|
+
0xb1,
|
|
664
|
+
0x1a,
|
|
665
|
+
0xe1,
|
|
666
|
+
...Array(20).fill(0),
|
|
667
|
+
]);
|
|
668
|
+
const docxBytes = new Uint8Array([0x50, 0x4b, 0x03, 0x04, ...Array(20).fill(0)]);
|
|
669
|
+
|
|
670
|
+
test("PDF bytes uploaded as x.jpg against an accept: [jpg] field are rejected with content_mismatch", async () => {
|
|
671
|
+
const res = await uploadFile(adminUser, "x.jpg", pdfBytes, "image/jpeg", {
|
|
672
|
+
entityType: "tenant",
|
|
673
|
+
entityId: "1",
|
|
674
|
+
fieldName: "logo",
|
|
675
|
+
});
|
|
676
|
+
expect(res.status).toBe(400);
|
|
677
|
+
const body = await res.json();
|
|
678
|
+
expect(body.error).toContain("content_mismatch");
|
|
679
|
+
});
|
|
680
|
+
|
|
681
|
+
test("real signature bytes with a matching extension upload successfully", async () => {
|
|
682
|
+
for (const [fileName, bytes, mimeType] of [
|
|
683
|
+
["real.jpg", jpegBytes, "image/jpeg"],
|
|
684
|
+
["real.png", pngBytes, "image/png"],
|
|
685
|
+
["real.pdf", pdfBytes, "application/pdf"],
|
|
686
|
+
["real.doc", docBytes, "application/msword"],
|
|
687
|
+
[
|
|
688
|
+
"real.docx",
|
|
689
|
+
docxBytes,
|
|
690
|
+
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
|
691
|
+
],
|
|
692
|
+
] as const) {
|
|
693
|
+
const res = await uploadFile(adminUser, fileName, bytes, mimeType);
|
|
694
|
+
expect(res.status).toBe(201);
|
|
695
|
+
}
|
|
696
|
+
});
|
|
697
|
+
|
|
698
|
+
test(".csv and .txt plain text upload successfully (no magic-byte signature to check)", async () => {
|
|
699
|
+
const text = new TextEncoder().encode("name,age\nAda,36\n");
|
|
700
|
+
const csvRes = await uploadFile(adminUser, "data.csv", text, "text/csv");
|
|
701
|
+
expect(csvRes.status).toBe(201);
|
|
702
|
+
const txtRes = await uploadFile(adminUser, "notes.txt", text, "text/plain");
|
|
703
|
+
expect(txtRes.status).toBe(201);
|
|
704
|
+
});
|
|
705
|
+
|
|
706
|
+
test("an unattached upload with a prototype-property extension is handled as unknown, not a 500", async () => {
|
|
707
|
+
const constructorRes = await uploadFile(adminUser, "evil.constructor", pngBytes, "image/png");
|
|
708
|
+
expect(constructorRes.status).toBe(201);
|
|
709
|
+
const protoRes = await uploadFile(adminUser, "evil.__proto__", pngBytes, "image/png");
|
|
710
|
+
expect(protoRes.status).toBe(201);
|
|
711
|
+
});
|
|
712
|
+
|
|
713
|
+
test("a too-short .png upload (fewer bytes than the PNG signature) is rejected with content_mismatch", async () => {
|
|
714
|
+
// A zero-byte file part is dropped by Bun's multipart parser before it
|
|
715
|
+
// ever reaches the route handler as `file`, so this uses one byte
|
|
716
|
+
// instead — still far short of the 8-byte PNG signature.
|
|
717
|
+
const res = await uploadFile(adminUser, "empty.png", new Uint8Array([0x00]), "image/png");
|
|
718
|
+
expect(res.status).toBe(400);
|
|
719
|
+
const body = await res.json();
|
|
720
|
+
expect(body.error).toContain("content_mismatch");
|
|
721
|
+
});
|
|
722
|
+
});
|
|
723
|
+
|
|
724
|
+
// Bun's formData() derives File#type from the filename, so .csv never
|
|
725
|
+
// arrives as vnd.ms-excel here; the csv alias is covered by the unit tests above.
|
|
726
|
+
describe("vnd.ms-excel via HTTP (.xls)", () => {
|
|
727
|
+
const docBytes = new Uint8Array([
|
|
728
|
+
0xd0,
|
|
729
|
+
0xcf,
|
|
730
|
+
0x11,
|
|
731
|
+
0xe0,
|
|
732
|
+
0xa1,
|
|
733
|
+
0xb1,
|
|
734
|
+
0x1a,
|
|
735
|
+
0xe1,
|
|
736
|
+
...Array(20).fill(0),
|
|
737
|
+
]);
|
|
738
|
+
|
|
739
|
+
test("unattached .xls declared as application/vnd.ms-excel keeps its declared mimeType unchanged", async () => {
|
|
740
|
+
const res = await uploadFile(adminUser, "legacy.xls", docBytes, "application/vnd.ms-excel");
|
|
741
|
+
expect(res.status).toBe(201);
|
|
742
|
+
const body = await res.json();
|
|
743
|
+
expect(body.mimeType).toBe("application/vnd.ms-excel");
|
|
744
|
+
});
|
|
745
|
+
|
|
746
|
+
test("attached .xls upload is rejected by the importFile field's csv-only accept list", async () => {
|
|
747
|
+
const res = await uploadFile(adminUser, "legacy.xls", docBytes, "application/vnd.ms-excel", {
|
|
748
|
+
entityType: "tenant",
|
|
749
|
+
entityId: "1",
|
|
750
|
+
fieldName: "importFile",
|
|
751
|
+
});
|
|
752
|
+
expect(res.status).toBe(400);
|
|
753
|
+
const body = await res.json();
|
|
754
|
+
expect(body.error).toContain("invalid_file_type");
|
|
755
|
+
});
|
|
756
|
+
});
|
|
757
|
+
|
|
526
758
|
// --- Cross-user access within a tenant (attached file owner-scope) ---
|
|
527
759
|
|
|
528
760
|
describe("attached file owner-scope", () => {
|
|
@@ -746,7 +978,7 @@ describe("custom file access guard", () => {
|
|
|
746
978
|
|
|
747
979
|
describe("tenant isolation", () => {
|
|
748
980
|
test("tenant 2 uploads a file, tenant 1 cannot see it", async () => {
|
|
749
|
-
const content = new TextEncoder().encode("tenant2-secret-file");
|
|
981
|
+
const content = new TextEncoder().encode("%PDF-1.4 tenant2-secret-file");
|
|
750
982
|
const uploadRes = await uploadFile(otherTenantUser, "secret.pdf", content, "application/pdf");
|
|
751
983
|
expect(uploadRes.status).toBe(201);
|
|
752
984
|
const { id } = await uploadRes.json();
|
|
@@ -38,9 +38,11 @@ const otherAdmin = createTestUser({
|
|
|
38
38
|
});
|
|
39
39
|
|
|
40
40
|
// Two tiny payloads with distinct lengths so the sum assertion can tell
|
|
41
|
-
// them apart without relying on the underlying bytes.
|
|
42
|
-
|
|
43
|
-
const
|
|
41
|
+
// them apart without relying on the underlying bytes. Both carry the full
|
|
42
|
+
// 8-byte PNG signature so validateFileContent accepts them.
|
|
43
|
+
const PNG_SIGNATURE = [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a];
|
|
44
|
+
const SMALL = new Uint8Array([...PNG_SIGNATURE, ...Array(12).fill(0)]); // 20 bytes
|
|
45
|
+
const LARGE = new Uint8Array([...PNG_SIGNATURE, ...Array(92).fill(0)]); // 100 bytes
|
|
44
46
|
|
|
45
47
|
beforeAll(async () => {
|
|
46
48
|
patchFileInstanceofForBunTest();
|
package/src/files/file-routes.ts
CHANGED
|
@@ -30,6 +30,7 @@ import type { FileProviderResolver } from "./provider-resolver";
|
|
|
30
30
|
import {
|
|
31
31
|
buildStorageKey,
|
|
32
32
|
resolveServedContentType,
|
|
33
|
+
resolveUploadMimeType,
|
|
33
34
|
validateFile,
|
|
34
35
|
validateFileContent,
|
|
35
36
|
} from "./types";
|
|
@@ -235,18 +236,24 @@ export function createFileRoutes(options: FileRoutesOptions): Hono {
|
|
|
235
236
|
|
|
236
237
|
const data = new Uint8Array(await file.arrayBuffer());
|
|
237
238
|
|
|
238
|
-
const contentValidationError = validateFileContent(file.
|
|
239
|
+
const contentValidationError = validateFileContent(file.name, data);
|
|
239
240
|
if (contentValidationError) {
|
|
240
241
|
return c.json({ error: contentValidationError }, 400);
|
|
241
242
|
}
|
|
242
243
|
|
|
244
|
+
const mimeTypeResolution = resolveUploadMimeType(file.name, file.type, data);
|
|
245
|
+
if (mimeTypeResolution.kind === "rejected") {
|
|
246
|
+
return c.json({ error: mimeTypeResolution.error }, 400);
|
|
247
|
+
}
|
|
248
|
+
const mimeType = mimeTypeResolution.mimeType;
|
|
249
|
+
|
|
243
250
|
// Write binary FIRST (outside the tx — network/disk I/O doesn't belong
|
|
244
251
|
// inside a PG connection's tx window). On DB-tx rollback below the bytes
|
|
245
252
|
// are orphaned in the provider; cleanup-jobs sweep those later. Losing a
|
|
246
253
|
// row on append-failure is acceptable; corrupting a committed row with a
|
|
247
254
|
// missing binary is not.
|
|
248
255
|
const storageProvider = await options.resolveProvider(user.tenantId);
|
|
249
|
-
await storageProvider.write(storageKey, data,
|
|
256
|
+
await storageProvider.write(storageKey, data, mimeType);
|
|
250
257
|
|
|
251
258
|
// Create via the standard entity executor: emits fileRef.created +
|
|
252
259
|
// materialises the file_refs row in one tx (read-your-own-write). id is
|
|
@@ -260,7 +267,7 @@ export function createFileRoutes(options: FileRoutesOptions): Hono {
|
|
|
260
267
|
id: fileRefId,
|
|
261
268
|
storageKey,
|
|
262
269
|
fileName: file.name,
|
|
263
|
-
mimeType
|
|
270
|
+
mimeType,
|
|
264
271
|
size: file.size,
|
|
265
272
|
entityType: entityType ?? null,
|
|
266
273
|
entityId: entityId ?? null,
|
|
@@ -277,7 +284,7 @@ export function createFileRoutes(options: FileRoutesOptions): Hono {
|
|
|
277
284
|
{
|
|
278
285
|
id: fileRefId,
|
|
279
286
|
fileName: file.name,
|
|
280
|
-
mimeType
|
|
287
|
+
mimeType,
|
|
281
288
|
size: file.size,
|
|
282
289
|
storageKey,
|
|
283
290
|
},
|
package/src/files/index.ts
CHANGED
|
@@ -37,6 +37,7 @@ export type {
|
|
|
37
37
|
FileStorageProvider,
|
|
38
38
|
FileValidationOptions,
|
|
39
39
|
SignedUrlOptions,
|
|
40
|
+
UploadMimeTypeResolution,
|
|
40
41
|
WriteStreamOptions,
|
|
41
42
|
} from "./types";
|
|
42
43
|
export {
|
|
@@ -44,6 +45,8 @@ export {
|
|
|
44
45
|
buildStorageKey,
|
|
45
46
|
normalizeMimeType,
|
|
46
47
|
parseMaxSize,
|
|
48
|
+
resolveUploadMimeType,
|
|
49
|
+
sniffMimeType,
|
|
47
50
|
tenantExportPrefix,
|
|
48
51
|
tenantStoragePrefixes,
|
|
49
52
|
validateFile,
|
package/src/files/types.ts
CHANGED
|
@@ -34,10 +34,14 @@ export function parseMaxSize(maxSize: string): number {
|
|
|
34
34
|
}
|
|
35
35
|
}
|
|
36
36
|
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
37
|
+
function extractExtension(fileName: string): string | undefined {
|
|
38
|
+
return fileName.split(".").pop()?.toLowerCase();
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
// Extension → acceptable MIME-type whitelist. Backs the cheap metadata
|
|
42
|
+
// pre-check in validateFile AND the signature lookup in validateFileContent.
|
|
43
|
+
// Kept small & conservative; add entries on demand rather than importing a
|
|
44
|
+
// heavyweight mime DB.
|
|
41
45
|
const EXTENSION_MIME_WHITELIST: Record<string, readonly string[]> = {
|
|
42
46
|
jpg: ["image/jpeg", "image/jpg"],
|
|
43
47
|
jpeg: ["image/jpeg", "image/jpg"],
|
|
@@ -54,8 +58,30 @@ const EXTENSION_MIME_WHITELIST: Record<string, readonly string[]> = {
|
|
|
54
58
|
docx: ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"],
|
|
55
59
|
} satisfies Record<string, readonly string[]>;
|
|
56
60
|
|
|
61
|
+
// Extension → declared MIME → canonical MIME. Windows resolves .csv's upload
|
|
62
|
+
// MIME from Explorer's Excel file association, sending "application/vnd.ms-excel" for plain-text CSV.
|
|
63
|
+
const EXTENSION_MIME_ALIASES: Readonly<Record<string, Readonly<Record<string, string>>>> = {
|
|
64
|
+
csv: { "application/vnd.ms-excel": "text/csv" },
|
|
65
|
+
};
|
|
66
|
+
|
|
67
|
+
// Object.hasOwn at both levels: a plain `in` check on `{}` resolves to
|
|
68
|
+
// Object.prototype members ("constructor", …) for any ext/mimeType.
|
|
69
|
+
function lookupMimeAlias(ext: string, normalizedMimeType: string): string | undefined {
|
|
70
|
+
if (!Object.hasOwn(EXTENSION_MIME_ALIASES, ext)) return undefined;
|
|
71
|
+
const aliasesForExt = EXTENSION_MIME_ALIASES[ext];
|
|
72
|
+
if (!aliasesForExt || !Object.hasOwn(aliasesForExt, normalizedMimeType)) return undefined;
|
|
73
|
+
return aliasesForExt[normalizedMimeType];
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
// Same hasOwn guard as lookupMimeAlias above, for the ext-keyed whitelist.
|
|
77
|
+
function lookupExtensionWhitelist(ext: string): readonly string[] | undefined {
|
|
78
|
+
if (!Object.hasOwn(EXTENSION_MIME_WHITELIST, ext)) return undefined;
|
|
79
|
+
return EXTENSION_MIME_WHITELIST[ext];
|
|
80
|
+
}
|
|
81
|
+
|
|
57
82
|
// Magic-byte signatures for the subset of EXTENSION_MIME_WHITELIST that has
|
|
58
|
-
// a reliable binary signature. Used at
|
|
83
|
+
// a reliable binary signature. Used both at upload time (validateFileContent)
|
|
84
|
+
// and at serve time (resolveServedContentType) — never trust the stored/
|
|
59
85
|
// client-declared mimeType for Content-Type on its own, sniff the actual
|
|
60
86
|
// bytes instead. Types without a stable signature (svg, txt, csv, json, md)
|
|
61
87
|
// and anything that matches none of these fall back to
|
|
@@ -80,9 +106,8 @@ const MAGIC_BYTE_SIGNATURES: ReadonlyArray<{
|
|
|
80
106
|
matches: (bytes) => startsWithAscii(bytes, "RIFF") && startsWithAscii(bytes, "WEBP", 8),
|
|
81
107
|
},
|
|
82
108
|
{ mimeType: "application/pdf", matches: (bytes) => startsWithAscii(bytes, "%PDF-") },
|
|
83
|
-
// Full 8-byte OLE Compound File signature
|
|
84
|
-
//
|
|
85
|
-
// below narrows to msword specifically).
|
|
109
|
+
// Full 8-byte OLE Compound File signature — legacy .doc/.xls/.ppt share it,
|
|
110
|
+
// so any OLE file sniffs as msword regardless of which one it really is.
|
|
86
111
|
{
|
|
87
112
|
mimeType: "application/msword",
|
|
88
113
|
matches: (bytes) => startsWithBytes(bytes, [0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1]),
|
|
@@ -108,6 +133,14 @@ function startsWithAscii(bytes: Uint8Array, ascii: string, offset = 0): boolean
|
|
|
108
133
|
return true;
|
|
109
134
|
}
|
|
110
135
|
|
|
136
|
+
/**
|
|
137
|
+
* Canonical MIME type for a known binary magic-byte signature (png, jpeg,
|
|
138
|
+
* gif, webp, pdf, OLE, ZIP), or null for everything else — text formats
|
|
139
|
+
* (csv, txt, json, svg, md) have no signature and always return null. Every
|
|
140
|
+
* OLE Compound File (.doc/.xls/.ppt) returns "application/msword" and every
|
|
141
|
+
* ZIP (.docx/.xlsx/.pptx/.zip) returns the docx MIME — callers can't tell
|
|
142
|
+
* those apart by bytes alone.
|
|
143
|
+
*/
|
|
111
144
|
export function sniffMimeType(bytes: Uint8Array): string | null {
|
|
112
145
|
for (const signature of MAGIC_BYTE_SIGNATURES) {
|
|
113
146
|
if (signature.matches(bytes)) return signature.mimeType;
|
|
@@ -145,14 +178,6 @@ export function resolveServedContentType(bytes: Uint8Array, declaredMimeType: st
|
|
|
145
178
|
return sniffed === declared ? sniffed : "application/octet-stream";
|
|
146
179
|
}
|
|
147
180
|
|
|
148
|
-
// Signature-checked against the declared mimeType regardless of
|
|
149
|
-
// `options.accept`. Deliberately narrow: most of the whitelist has no magic
|
|
150
|
-
// bytes at all and would reject honestly-mislabeled uploads if checked.
|
|
151
|
-
const CONTENT_VERIFIED_MIME_TYPES = new Set([
|
|
152
|
-
"application/msword",
|
|
153
|
-
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
|
154
|
-
]);
|
|
155
|
-
|
|
156
181
|
export function validateFile(
|
|
157
182
|
metadata: FileMetadata,
|
|
158
183
|
options: FileValidationOptions,
|
|
@@ -165,18 +190,17 @@ export function validateFile(
|
|
|
165
190
|
}
|
|
166
191
|
|
|
167
192
|
if (options.accept && options.accept.length > 0) {
|
|
168
|
-
const ext = metadata.fileName
|
|
193
|
+
const ext = extractExtension(metadata.fileName);
|
|
169
194
|
if (!ext || !options.accept.includes(ext)) {
|
|
170
195
|
return `invalid_file_type: ".${ext}" is not in [${options.accept.join(", ")}]`;
|
|
171
196
|
}
|
|
172
|
-
//
|
|
173
|
-
//
|
|
174
|
-
|
|
175
|
-
// and having the mimeType reflect that.
|
|
176
|
-
const allowedMimes = EXTENSION_MIME_WHITELIST[ext];
|
|
197
|
+
// Cheap metadata pre-check only: the client/runtime mimeType is no
|
|
198
|
+
// evidence of file content (Bun derives it from the filename itself).
|
|
199
|
+
const allowedMimes = lookupExtensionWhitelist(ext);
|
|
177
200
|
if (allowedMimes && metadata.mimeType) {
|
|
178
201
|
const normalized = normalizeMimeType(metadata.mimeType);
|
|
179
|
-
|
|
202
|
+
const isAliasedMime = lookupMimeAlias(ext, normalized) !== undefined;
|
|
203
|
+
if (!allowedMimes.includes(normalized) && !isAliasedMime) {
|
|
180
204
|
return `mime_mismatch: extension ".${ext}" does not match mimeType "${metadata.mimeType}"`;
|
|
181
205
|
}
|
|
182
206
|
}
|
|
@@ -185,17 +209,52 @@ export function validateFile(
|
|
|
185
209
|
return null;
|
|
186
210
|
}
|
|
187
211
|
|
|
212
|
+
function signatureMimeTypesForExtension(ext: string): readonly string[] {
|
|
213
|
+
const whitelist = lookupExtensionWhitelist(ext);
|
|
214
|
+
if (!whitelist) return [];
|
|
215
|
+
return whitelist.filter((candidate) =>
|
|
216
|
+
MAGIC_BYTE_SIGNATURES.some((signature) => signature.mimeType === candidate),
|
|
217
|
+
);
|
|
218
|
+
}
|
|
219
|
+
|
|
188
220
|
// Separate from validateFile: metadata is known before the upload body is
|
|
189
221
|
// fully read, content-verification only after — callers reject early on
|
|
190
222
|
// metadata without buffering bytes they'll then have to reject anyway.
|
|
191
|
-
export function validateFileContent(
|
|
192
|
-
const
|
|
193
|
-
if (!
|
|
194
|
-
const
|
|
195
|
-
if (
|
|
196
|
-
|
|
223
|
+
export function validateFileContent(fileName: string, content: Uint8Array): string | null {
|
|
224
|
+
const ext = extractExtension(fileName);
|
|
225
|
+
if (!ext) return null;
|
|
226
|
+
const signatureMimeTypes = signatureMimeTypesForExtension(ext);
|
|
227
|
+
if (signatureMimeTypes.length === 0) return null;
|
|
228
|
+
const sniffed = sniffMimeType(content);
|
|
229
|
+
if (sniffed && signatureMimeTypes.includes(sniffed)) return null;
|
|
230
|
+
return `content_mismatch: ".${ext}" upload bytes are ${sniffed ?? "unrecognized"}, expected ${signatureMimeTypes.join(" or ")}`;
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
export type UploadMimeTypeResolution =
|
|
234
|
+
| { readonly kind: "ok"; readonly mimeType: string }
|
|
235
|
+
| { readonly kind: "rejected"; readonly error: string };
|
|
236
|
+
|
|
237
|
+
/**
|
|
238
|
+
* Rewrites an aliased declared mimeType to its canonical form and, in the
|
|
239
|
+
* same call, sniffs the bytes — a renamed binary (a real .xls/.xlsx saved
|
|
240
|
+
* as .csv) must fail here rather than through an optional, skippable step.
|
|
241
|
+
*/
|
|
242
|
+
export function resolveUploadMimeType(
|
|
243
|
+
fileName: string,
|
|
244
|
+
declaredMimeType: string,
|
|
245
|
+
content: Uint8Array,
|
|
246
|
+
): UploadMimeTypeResolution {
|
|
247
|
+
const ext = extractExtension(fileName);
|
|
248
|
+
const normalizedDeclared = normalizeMimeType(declaredMimeType);
|
|
249
|
+
const canonical = ext ? lookupMimeAlias(ext, normalizedDeclared) : undefined;
|
|
250
|
+
if (!canonical) return { kind: "ok", mimeType: declaredMimeType };
|
|
251
|
+
if (sniffMimeType(content) !== null) {
|
|
252
|
+
return {
|
|
253
|
+
kind: "rejected",
|
|
254
|
+
error: `content_mismatch: bytes of ".${ext}" upload declared as "${declaredMimeType}" are a binary format`,
|
|
255
|
+
};
|
|
197
256
|
}
|
|
198
|
-
return
|
|
257
|
+
return { kind: "ok", mimeType: canonical };
|
|
199
258
|
}
|
|
200
259
|
|
|
201
260
|
export function assertSafeStorageKey(key: string): void {
|
|
@@ -14,7 +14,6 @@ import * as z from "zod";
|
|
|
14
14
|
import { defineFeature } from "../../engine";
|
|
15
15
|
import { setupTestStack, type TestStack, TestUsers } from "../../stack";
|
|
16
16
|
import { waitFor } from "../../testing";
|
|
17
|
-
import { generateId } from "../../utils";
|
|
18
17
|
|
|
19
18
|
const exponentialDefaultStarts: number[] = [];
|
|
20
19
|
const exponentialCustomStarts: number[] = [];
|
|
@@ -92,10 +91,7 @@ let stack: TestStack;
|
|
|
92
91
|
beforeAll(async () => {
|
|
93
92
|
stack = await setupTestStack({
|
|
94
93
|
features: [backoffFixtureFeature],
|
|
95
|
-
|
|
96
|
-
// Redis is consumed by every parallel test process, and a foreign worker
|
|
97
|
-
// grabbing a promoted delayed retry fails it as "Unknown job".
|
|
98
|
-
jobs: { consumerLane: "worker", queueNamePrefix: `backoff-test-${generateId()}` },
|
|
94
|
+
jobs: { consumerLane: "worker" },
|
|
99
95
|
});
|
|
100
96
|
});
|
|
101
97
|
|
|
@@ -468,7 +468,8 @@ export async function buildSearchDocument(
|
|
|
468
468
|
// die auf optimistic-update setzen merken das nicht; strictly-waiting
|
|
469
469
|
// Clients müssten poll-after-write.
|
|
470
470
|
//
|
|
471
|
-
// Tests drain
|
|
471
|
+
// Tests drain deterministically via `await stack.eventDispatcher.runOnce()`,
|
|
472
|
+
// or via `drainEventConsumers` for this consumer name specifically.
|
|
472
473
|
export const SSE_BROADCAST_CONSUMER_NAME = "system:consumer:sse-broadcast";
|
|
473
474
|
|
|
474
475
|
export function createSseBroadcastEventConsumer(sseBroker: SseBroker): EventConsumer {
|