@cosmicdrift/kumiko-framework 0.311.0 → 0.313.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +4 -4
- package/src/api/auth-routes.ts +27 -2
- package/src/changes.json +36 -0
- package/src/crypto/pg-kms-adapter.ts +2 -1
- package/src/db/__tests__/connection-close-outage.integration.test.ts +110 -0
- package/src/db/api.ts +6 -0
- package/src/db/bun-provider.ts +8 -3
- package/src/db/connection.ts +3 -3
- package/src/db/eagerload.ts +2 -9
- package/src/db/index.ts +1 -1
- package/src/db/postgres-provider.ts +7 -4
- package/src/db/queries/event-consumer.ts +4 -3
- package/src/derivatives/derivatives-context.ts +1 -8
- package/src/engine/boot-validator/__tests__/transfer-graph.test.ts +12 -0
- package/src/engine/boot-validator/transfer-graph.ts +2 -1
- package/src/engine/feature-config-events-jobs.ts +3 -0
- package/src/engine/index.ts +3 -0
- package/src/engine/parse-ref-target.ts +6 -0
- package/src/engine/registry-validate.ts +3 -12
- package/src/engine/types/index.ts +1 -0
- package/src/files/__tests__/files.integration.test.ts +115 -1
- package/src/files/__tests__/storage-tracking.test.ts +46 -0
- package/src/files/file-routes.ts +13 -2
- package/src/files/index.ts +8 -1
- package/src/files/storage-tracking.ts +160 -15
- package/src/files/types.ts +44 -2
- package/src/jobs/job-runner.ts +39 -0
- package/src/pipeline/__tests__/event-dispatcher-db-outage.integration.test.ts +27 -0
- package/src/pipeline/__tests__/job-trigger-consumer.integration.test.ts +133 -6
- package/src/pipeline/event-dispatcher-delivery.ts +22 -1
- package/src/pipeline/event-dispatcher.ts +25 -2
- package/src/pipeline/index.ts +4 -1
- package/src/pipeline/msp-rebuild.ts +1 -0
- package/src/pipeline/multi-stream-apply-context.ts +1 -0
- package/src/pipeline/system-hooks.ts +4 -1
|
@@ -35,7 +35,7 @@ import type { FileRoutesOptions } from "../file-routes";
|
|
|
35
35
|
import { createInMemoryFileProvider } from "../in-memory-provider";
|
|
36
36
|
import { createLocalProvider } from "../local-provider";
|
|
37
37
|
import type { FileStorageProvider, SignedUrlOptions } from "../types";
|
|
38
|
-
import { parseMaxSize, sniffMimeType, validateFile } from "../types";
|
|
38
|
+
import { parseMaxSize, sniffMimeType, validateFile, validateFileContent } from "../types";
|
|
39
39
|
|
|
40
40
|
// UUID for "this row doesn't exist" assertions. Valid v4 format so PG accepts
|
|
41
41
|
// the query — the row just isn't there. Pre-v1 files-feature tests used
|
|
@@ -222,6 +222,80 @@ describe("file validation", () => {
|
|
|
222
222
|
expect(sniffMimeType(new Uint8Array([0x00, 0x01, 0x02]))).toBeNull();
|
|
223
223
|
expect(sniffMimeType(new TextEncoder().encode("hello world"))).toBeNull();
|
|
224
224
|
});
|
|
225
|
+
|
|
226
|
+
// --- doc/docx magic-byte content verification ---
|
|
227
|
+
|
|
228
|
+
const docBytes = new Uint8Array([
|
|
229
|
+
0xd0,
|
|
230
|
+
0xcf,
|
|
231
|
+
0x11,
|
|
232
|
+
0xe0,
|
|
233
|
+
0xa1,
|
|
234
|
+
0xb1,
|
|
235
|
+
0x1a,
|
|
236
|
+
0xe1,
|
|
237
|
+
...Array(20).fill(0),
|
|
238
|
+
]);
|
|
239
|
+
const docxBytes = new Uint8Array([0x50, 0x4b, 0x03, 0x04, ...Array(20).fill(0)]);
|
|
240
|
+
|
|
241
|
+
test("validateFile accepts a .doc/.docx upload on extension/mimeType alone", () => {
|
|
242
|
+
expect(
|
|
243
|
+
validateFile(
|
|
244
|
+
{ fileName: "report.doc", mimeType: "application/msword", size: docBytes.length },
|
|
245
|
+
{ accept: ["doc", "docx"] },
|
|
246
|
+
),
|
|
247
|
+
).toBeNull();
|
|
248
|
+
expect(
|
|
249
|
+
validateFile(
|
|
250
|
+
{
|
|
251
|
+
fileName: "report.docx",
|
|
252
|
+
mimeType: "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
|
253
|
+
size: docxBytes.length,
|
|
254
|
+
},
|
|
255
|
+
{ accept: ["doc", "docx"] },
|
|
256
|
+
),
|
|
257
|
+
).toBeNull();
|
|
258
|
+
});
|
|
259
|
+
|
|
260
|
+
test("validateFileContent accepts .doc/.docx bytes matching their declared mimeType", () => {
|
|
261
|
+
expect(validateFileContent("application/msword", docBytes)).toBeNull();
|
|
262
|
+
expect(
|
|
263
|
+
validateFileContent(
|
|
264
|
+
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
|
265
|
+
docxBytes,
|
|
266
|
+
),
|
|
267
|
+
).toBeNull();
|
|
268
|
+
});
|
|
269
|
+
|
|
270
|
+
test("validateFileContent rejects .doc/.docx whose content doesn't match the declared mimeType", () => {
|
|
271
|
+
const fakeDoc = validateFileContent(
|
|
272
|
+
"application/msword",
|
|
273
|
+
new TextEncoder().encode("not actually a Word document, just text"),
|
|
274
|
+
);
|
|
275
|
+
expectErrorIncludes(fakeDoc, "content_mismatch");
|
|
276
|
+
|
|
277
|
+
const fakeDocx = validateFileContent(
|
|
278
|
+
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
|
279
|
+
new TextEncoder().encode("not actually a docx, just text"),
|
|
280
|
+
);
|
|
281
|
+
expectErrorIncludes(fakeDocx, "content_mismatch");
|
|
282
|
+
});
|
|
283
|
+
|
|
284
|
+
test("validateFileContent does not content-verify pre-existing types (e.g. pdf) even when content mismatches", () => {
|
|
285
|
+
expect(
|
|
286
|
+
validateFileContent(
|
|
287
|
+
"application/pdf",
|
|
288
|
+
new TextEncoder().encode("not a PDF at all, just text"),
|
|
289
|
+
),
|
|
290
|
+
).toBeNull();
|
|
291
|
+
});
|
|
292
|
+
|
|
293
|
+
test("sniffMimeType recognizes doc (OLE) and docx (ZIP) signatures", () => {
|
|
294
|
+
expect(sniffMimeType(docBytes)).toBe("application/msword");
|
|
295
|
+
expect(sniffMimeType(docxBytes)).toBe(
|
|
296
|
+
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
|
297
|
+
);
|
|
298
|
+
});
|
|
225
299
|
});
|
|
226
300
|
|
|
227
301
|
// --- Integration: Upload → Download → Delete via real HTTP API ---
|
|
@@ -409,6 +483,46 @@ describe("download Content-Type is sniffed from bytes, not trusted from the decl
|
|
|
409
483
|
});
|
|
410
484
|
});
|
|
411
485
|
|
|
486
|
+
// --- doc/docx upload via API: content-verified independent of `accept` ---
|
|
487
|
+
|
|
488
|
+
describe("doc/docx upload is content-verified against magic bytes", () => {
|
|
489
|
+
const docBytes = new Uint8Array([
|
|
490
|
+
0xd0,
|
|
491
|
+
0xcf,
|
|
492
|
+
0x11,
|
|
493
|
+
0xe0,
|
|
494
|
+
0xa1,
|
|
495
|
+
0xb1,
|
|
496
|
+
0x1a,
|
|
497
|
+
0xe1,
|
|
498
|
+
...Array(20).fill(0),
|
|
499
|
+
]);
|
|
500
|
+
const docxBytes = new Uint8Array([0x50, 0x4b, 0x03, 0x04, ...Array(20).fill(0)]);
|
|
501
|
+
|
|
502
|
+
test("a real .doc upload succeeds", async () => {
|
|
503
|
+
const res = await uploadFile(adminUser, "report.doc", docBytes, "application/msword");
|
|
504
|
+
expect(res.status).toBe(201);
|
|
505
|
+
});
|
|
506
|
+
|
|
507
|
+
test("a real .docx upload succeeds", async () => {
|
|
508
|
+
const res = await uploadFile(
|
|
509
|
+
adminUser,
|
|
510
|
+
"report.docx",
|
|
511
|
+
docxBytes,
|
|
512
|
+
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
|
513
|
+
);
|
|
514
|
+
expect(res.status).toBe(201);
|
|
515
|
+
});
|
|
516
|
+
|
|
517
|
+
test("a .doc upload whose bytes don't match the declared mimeType is rejected with content_mismatch, even unattached (no `accept` list)", async () => {
|
|
518
|
+
const fakeDocBytes = new TextEncoder().encode("plain text pretending to be a Word document");
|
|
519
|
+
const res = await uploadFile(adminUser, "fake.doc", fakeDocBytes, "application/msword");
|
|
520
|
+
expect(res.status).toBe(400);
|
|
521
|
+
const body = await res.json();
|
|
522
|
+
expect(body.error).toContain("content_mismatch");
|
|
523
|
+
});
|
|
524
|
+
});
|
|
525
|
+
|
|
412
526
|
// --- Cross-user access within a tenant (attached file owner-scope) ---
|
|
413
527
|
|
|
414
528
|
describe("attached file owner-scope", () => {
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
// fileRefStorageDelta — the single source of truth the tenant-storage-usage
|
|
2
|
+
// MSP's apply handlers and transferTenantStorageUsage's replay both use to
|
|
3
|
+
// turn one fileRef event into a signed (bytes, files) delta.
|
|
4
|
+
|
|
5
|
+
import { describe, expect, test } from "bun:test";
|
|
6
|
+
import { entityEventName } from "../../db";
|
|
7
|
+
import { fileRefStorageDelta } from "../storage-tracking";
|
|
8
|
+
|
|
9
|
+
const CREATED = entityEventName("fileRef", "created");
|
|
10
|
+
const DELETED = entityEventName("fileRef", "deleted");
|
|
11
|
+
const RESTORED = entityEventName("fileRef", "restored");
|
|
12
|
+
|
|
13
|
+
describe("fileRefStorageDelta", () => {
|
|
14
|
+
test("created: positive delta from payload.size", () => {
|
|
15
|
+
expect(fileRefStorageDelta({ type: CREATED, payload: { size: 42 } })).toEqual({
|
|
16
|
+
bytes: 42,
|
|
17
|
+
files: 1,
|
|
18
|
+
});
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
test("created: missing/non-numeric size reads as 0 bytes, still counts a file", () => {
|
|
22
|
+
expect(fileRefStorageDelta({ type: CREATED, payload: {} })).toEqual({ bytes: 0, files: 1 });
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
test("deleted: negative delta from payload.previous.size", () => {
|
|
26
|
+
expect(fileRefStorageDelta({ type: DELETED, payload: { previous: { size: 42 } } })).toEqual({
|
|
27
|
+
bytes: -42,
|
|
28
|
+
files: -1,
|
|
29
|
+
});
|
|
30
|
+
});
|
|
31
|
+
|
|
32
|
+
test("deleted: missing previous reads as 0 bytes, still counts -1 file", () => {
|
|
33
|
+
expect(fileRefStorageDelta({ type: DELETED, payload: {} })).toEqual({ bytes: 0, files: -1 });
|
|
34
|
+
});
|
|
35
|
+
|
|
36
|
+
test("restored: positive delta from payload.previous.size, symmetric to delete", () => {
|
|
37
|
+
expect(fileRefStorageDelta({ type: RESTORED, payload: { previous: { size: 42 } } })).toEqual({
|
|
38
|
+
bytes: 42,
|
|
39
|
+
files: 1,
|
|
40
|
+
});
|
|
41
|
+
});
|
|
42
|
+
|
|
43
|
+
test("an unrelated event type returns null", () => {
|
|
44
|
+
expect(fileRefStorageDelta({ type: "fileRef.renamed", payload: { size: 42 } })).toBeNull();
|
|
45
|
+
});
|
|
46
|
+
});
|
package/src/files/file-routes.ts
CHANGED
|
@@ -27,7 +27,12 @@ import { createFileContext } from "./file-handle";
|
|
|
27
27
|
import { fileRefEntity } from "./file-ref-entity";
|
|
28
28
|
import { fileRefsTable } from "./file-ref-table";
|
|
29
29
|
import type { FileProviderResolver } from "./provider-resolver";
|
|
30
|
-
import {
|
|
30
|
+
import {
|
|
31
|
+
buildStorageKey,
|
|
32
|
+
resolveServedContentType,
|
|
33
|
+
validateFile,
|
|
34
|
+
validateFileContent,
|
|
35
|
+
} from "./types";
|
|
31
36
|
|
|
32
37
|
// Decision returned by a FileAccessGuard — distinct from boolean so callers
|
|
33
38
|
// can't accidentally negate or default it.
|
|
@@ -228,13 +233,19 @@ export function createFileRoutes(options: FileRoutesOptions): Hono {
|
|
|
228
233
|
generateId(),
|
|
229
234
|
);
|
|
230
235
|
|
|
236
|
+
const data = new Uint8Array(await file.arrayBuffer());
|
|
237
|
+
|
|
238
|
+
const contentValidationError = validateFileContent(file.type, data);
|
|
239
|
+
if (contentValidationError) {
|
|
240
|
+
return c.json({ error: contentValidationError }, 400);
|
|
241
|
+
}
|
|
242
|
+
|
|
231
243
|
// Write binary FIRST (outside the tx — network/disk I/O doesn't belong
|
|
232
244
|
// inside a PG connection's tx window). On DB-tx rollback below the bytes
|
|
233
245
|
// are orphaned in the provider; cleanup-jobs sweep those later. Losing a
|
|
234
246
|
// row on append-failure is acceptable; corrupting a committed row with a
|
|
235
247
|
// missing binary is not.
|
|
236
248
|
const storageProvider = await options.resolveProvider(user.tenantId);
|
|
237
|
-
const data = new Uint8Array(await file.arrayBuffer());
|
|
238
249
|
await storageProvider.write(storageKey, data, file.type);
|
|
239
250
|
|
|
240
251
|
// Create via the standard entity executor: emits fileRef.created +
|
package/src/files/index.ts
CHANGED
|
@@ -26,7 +26,12 @@ export {
|
|
|
26
26
|
isFileProviderPlugin,
|
|
27
27
|
makeFileProviderResolver,
|
|
28
28
|
} from "./provider-resolver";
|
|
29
|
-
export {
|
|
29
|
+
export {
|
|
30
|
+
fileRefStorageDelta,
|
|
31
|
+
filesStorageTrackingFeature,
|
|
32
|
+
tenantStorageUsageTable,
|
|
33
|
+
transferTenantStorageUsage,
|
|
34
|
+
} from "./storage-tracking";
|
|
30
35
|
export type {
|
|
31
36
|
FileMetadata,
|
|
32
37
|
FileStorageProvider,
|
|
@@ -37,10 +42,12 @@ export type {
|
|
|
37
42
|
export {
|
|
38
43
|
assertSafeStorageKey,
|
|
39
44
|
buildStorageKey,
|
|
45
|
+
normalizeMimeType,
|
|
40
46
|
parseMaxSize,
|
|
41
47
|
tenantExportPrefix,
|
|
42
48
|
tenantStoragePrefixes,
|
|
43
49
|
validateFile,
|
|
50
|
+
validateFileContent,
|
|
44
51
|
} from "./types";
|
|
45
52
|
export type { ZipEntry } from "./zip-stream";
|
|
46
53
|
export { createZipStream } from "./zip-stream";
|
|
@@ -11,18 +11,28 @@
|
|
|
11
11
|
// consumer-cursor row. Apps that want it pass filesStorageTrackingFeature
|
|
12
12
|
// into createApp / setupTestStack alongside their domain features.
|
|
13
13
|
|
|
14
|
-
import { entityEventName } from "../db";
|
|
14
|
+
import { type DbRunner, entityEventName, executeRawQueryRead } from "../db";
|
|
15
15
|
import { bigint, instant, integer, table as pgTable, sql, uuid } from "../db/dialect";
|
|
16
16
|
import { incrementCounter } from "../db/query";
|
|
17
|
-
import { defineFeature } from "../engine";
|
|
17
|
+
import { defineFeature, qn, type Registry, toKebab } from "../engine";
|
|
18
|
+
import {
|
|
19
|
+
type PendingGapEntry,
|
|
20
|
+
SHARED_INSTANCE_SENTINEL,
|
|
21
|
+
selectConsumerCursorForUpdate,
|
|
22
|
+
} from "../pipeline";
|
|
23
|
+
import { parseJsonOrThrow } from "../utils";
|
|
24
|
+
|
|
25
|
+
const FILE_REF_AGGREGATE_TYPE = "fileRef";
|
|
26
|
+
const STORAGE_TRACKING_FEATURE_NAME = "files-storage-tracking";
|
|
27
|
+
const TENANT_STORAGE_USAGE_MSP_NAME = "tenant-storage-usage";
|
|
18
28
|
|
|
19
29
|
// fileRef is a standard ES entity, so usage tracking subscribes to its
|
|
20
30
|
// auto-verb events. `fileRef.created` payload carries the entity fields
|
|
21
31
|
// (incl. size); `fileRef.deleted` / `fileRef.restored` carry `{ previous }`
|
|
22
32
|
// (the pre-event row), so the byte count to apply lives at previous.size.
|
|
23
|
-
const FILE_REF_CREATED = entityEventName(
|
|
24
|
-
const FILE_REF_DELETED = entityEventName(
|
|
25
|
-
const FILE_REF_RESTORED = entityEventName(
|
|
33
|
+
const FILE_REF_CREATED = entityEventName(FILE_REF_AGGREGATE_TYPE, "created");
|
|
34
|
+
const FILE_REF_DELETED = entityEventName(FILE_REF_AGGREGATE_TYPE, "deleted");
|
|
35
|
+
const FILE_REF_RESTORED = entityEventName(FILE_REF_AGGREGATE_TYPE, "restored");
|
|
26
36
|
|
|
27
37
|
function readNumber(value: unknown): number {
|
|
28
38
|
return typeof value === "number" ? value : 0;
|
|
@@ -36,6 +46,28 @@ function sizeFromPrevious(payload: Record<string, unknown>): number {
|
|
|
36
46
|
return 0;
|
|
37
47
|
}
|
|
38
48
|
|
|
49
|
+
// Single source of truth for "how much does this fileRef event move the
|
|
50
|
+
// tenant's storage counters" — shared by the MSP's own apply handlers and by
|
|
51
|
+
// transferTenantStorageUsage's replay of already-applied events during a
|
|
52
|
+
// tenant handover.
|
|
53
|
+
export function fileRefStorageDelta(event: {
|
|
54
|
+
readonly type: string;
|
|
55
|
+
readonly payload: Record<string, unknown>;
|
|
56
|
+
}): { readonly bytes: number; readonly files: number } | null {
|
|
57
|
+
switch (event.type) {
|
|
58
|
+
case FILE_REF_CREATED:
|
|
59
|
+
return { bytes: readNumber(event.payload["size"]), files: 1 };
|
|
60
|
+
case FILE_REF_DELETED:
|
|
61
|
+
// `0 - x`, not `-x`: a zero-byte previous size must stay `0`, not `-0`
|
|
62
|
+
// (toEqual/JSON round-trips treat -0 as distinct from 0).
|
|
63
|
+
return { bytes: 0 - sizeFromPrevious(event.payload), files: -1 };
|
|
64
|
+
case FILE_REF_RESTORED:
|
|
65
|
+
return { bytes: sizeFromPrevious(event.payload), files: 1 };
|
|
66
|
+
default:
|
|
67
|
+
return null;
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
|
|
39
71
|
// bigint in `mode: "number"` returns a JS number (safe up to 2^53 ≈ 9e15
|
|
40
72
|
// bytes ≈ 8 petabytes per tenant — large enough for any practical storage
|
|
41
73
|
// quota). Default "bigint" mode would hand back a bigint value, which
|
|
@@ -48,13 +80,15 @@ export const tenantStorageUsageTable = pgTable("read_tenant_storage_usage", {
|
|
|
48
80
|
lastUpdatedAt: instant("last_updated_at").default(sql`now()`).notNull(),
|
|
49
81
|
});
|
|
50
82
|
|
|
51
|
-
export const filesStorageTrackingFeature = defineFeature(
|
|
83
|
+
export const filesStorageTrackingFeature = defineFeature(STORAGE_TRACKING_FEATURE_NAME, (r) => {
|
|
52
84
|
r.multiStreamProjection({
|
|
53
|
-
name:
|
|
85
|
+
name: TENANT_STORAGE_USAGE_MSP_NAME,
|
|
54
86
|
table: tenantStorageUsageTable,
|
|
55
87
|
apply: {
|
|
56
88
|
[FILE_REF_CREATED]: async (event, tx) => {
|
|
57
|
-
const
|
|
89
|
+
const delta = fileRefStorageDelta(event);
|
|
90
|
+
// skip: not a tracked fileRef storage change
|
|
91
|
+
if (!delta) return;
|
|
58
92
|
|
|
59
93
|
// UPSERT: INSERT on first upload per tenant, otherwise atomic increment.
|
|
60
94
|
// The SQL increment guarantees correctness under concurrent dispatcher
|
|
@@ -63,13 +97,15 @@ export const filesStorageTrackingFeature = defineFeature("files-storage-tracking
|
|
|
63
97
|
await incrementCounter(
|
|
64
98
|
tx,
|
|
65
99
|
tenantStorageUsageTable,
|
|
66
|
-
{ tenantId: event.tenantId, totalBytes:
|
|
67
|
-
{ totalBytes:
|
|
100
|
+
{ tenantId: event.tenantId, totalBytes: delta.bytes, fileCount: delta.files },
|
|
101
|
+
{ totalBytes: delta.bytes, fileCount: delta.files },
|
|
68
102
|
{ set: { lastUpdatedAt: sql`now()` } },
|
|
69
103
|
);
|
|
70
104
|
},
|
|
71
105
|
[FILE_REF_DELETED]: async (event, tx) => {
|
|
72
|
-
const
|
|
106
|
+
const delta = fileRefStorageDelta(event);
|
|
107
|
+
// skip: not a tracked fileRef storage change
|
|
108
|
+
if (!delta) return;
|
|
73
109
|
// Decrement on delete. INSERT values are 0/0 so a delete that somehow
|
|
74
110
|
// precedes any upload can't create negative usage; the on-conflict
|
|
75
111
|
// path applies the real negative delta. Async (dispatcher) —
|
|
@@ -78,7 +114,7 @@ export const filesStorageTrackingFeature = defineFeature("files-storage-tracking
|
|
|
78
114
|
tx,
|
|
79
115
|
tenantStorageUsageTable,
|
|
80
116
|
{ tenantId: event.tenantId, totalBytes: 0, fileCount: 0 },
|
|
81
|
-
{ totalBytes:
|
|
117
|
+
{ totalBytes: delta.bytes, fileCount: delta.files },
|
|
82
118
|
{ set: { lastUpdatedAt: sql`now()` } },
|
|
83
119
|
);
|
|
84
120
|
},
|
|
@@ -86,15 +122,124 @@ export const filesStorageTrackingFeature = defineFeature("files-storage-tracking
|
|
|
86
122
|
// Restore re-increments by the soft-deleted row's size — symmetric
|
|
87
123
|
// to delete. Without this handler totalBytes/fileCount drift low
|
|
88
124
|
// after every delete→restore round-trip.
|
|
89
|
-
const
|
|
125
|
+
const delta = fileRefStorageDelta(event);
|
|
126
|
+
// skip: not a tracked fileRef storage change
|
|
127
|
+
if (!delta) return;
|
|
90
128
|
await incrementCounter(
|
|
91
129
|
tx,
|
|
92
130
|
tenantStorageUsageTable,
|
|
93
|
-
{ tenantId: event.tenantId, totalBytes:
|
|
94
|
-
{ totalBytes:
|
|
131
|
+
{ tenantId: event.tenantId, totalBytes: delta.bytes, fileCount: delta.files },
|
|
132
|
+
{ totalBytes: delta.bytes, fileCount: delta.files },
|
|
95
133
|
{ set: { lastUpdatedAt: sql`now()` } },
|
|
96
134
|
);
|
|
97
135
|
},
|
|
98
136
|
},
|
|
99
137
|
});
|
|
100
138
|
});
|
|
139
|
+
|
|
140
|
+
// Qualified name of the tenant-storage-usage MSP consumer row in
|
|
141
|
+
// kumiko_event_consumers — the same qualification registry-ingest.ts applies
|
|
142
|
+
// to every multiStreamProjection (qualify(featureName, "projection", name)).
|
|
143
|
+
const TENANT_STORAGE_USAGE_MSP_QN = qn(
|
|
144
|
+
toKebab(STORAGE_TRACKING_FEATURE_NAME),
|
|
145
|
+
"projection",
|
|
146
|
+
toKebab(TENANT_STORAGE_USAGE_MSP_NAME),
|
|
147
|
+
);
|
|
148
|
+
|
|
149
|
+
const TRACKED_FILE_REF_EVENT_TYPES = [FILE_REF_CREATED, FILE_REF_DELETED, FILE_REF_RESTORED];
|
|
150
|
+
|
|
151
|
+
function isWithinPendingGap(eventId: bigint, gaps: readonly PendingGapEntry[]): boolean {
|
|
152
|
+
return gaps.some((gap) => eventId >= BigInt(gap.from) && eventId <= BigInt(gap.to));
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
// Moves the already-applied share of the tenant-storage-usage counters from
|
|
156
|
+
// source to destination tenant when a tenant-handover claim moves fileRef
|
|
157
|
+
// rows. Must run BEFORE moveEventHistory rewrites the events' tenant_id —
|
|
158
|
+
// the query below finds "already applied" events by sourceTenantId, and once
|
|
159
|
+
// the rewrite runs there is no way to tell an already-applied event from a
|
|
160
|
+
// not-yet-applied one by tenant_id alone.
|
|
161
|
+
//
|
|
162
|
+
// Events with id above the cursor (or inside a pending gap) are NOT summed
|
|
163
|
+
// here: the dispatcher hasn't applied them yet, they will carry the
|
|
164
|
+
// destination tenant_id once moveEventHistory runs, and the dispatcher will
|
|
165
|
+
// apply them against the destination counter on its own next pass.
|
|
166
|
+
export async function transferTenantStorageUsage(args: {
|
|
167
|
+
readonly db: DbRunner;
|
|
168
|
+
readonly registry: Registry;
|
|
169
|
+
readonly fileRefIds: readonly string[];
|
|
170
|
+
readonly sourceTenantId: string;
|
|
171
|
+
readonly destinationTenantId: string;
|
|
172
|
+
}): Promise<void> {
|
|
173
|
+
const { db, registry, fileRefIds, sourceTenantId, destinationTenantId } = args;
|
|
174
|
+
// skip: no fileRefs moved, nothing to transfer
|
|
175
|
+
if (fileRefIds.length === 0) return;
|
|
176
|
+
// skip: storage tracking is not mounted in this app
|
|
177
|
+
if (!registry.getAllMultiStreamProjections().has(TENANT_STORAGE_USAGE_MSP_QN)) return;
|
|
178
|
+
|
|
179
|
+
// Locks the same consumer row the dispatcher's own FOR UPDATE SKIP LOCKED
|
|
180
|
+
// pass takes before applying a batch — serializing against it here means no
|
|
181
|
+
// fileRef event for these ids can be applied between reading the cursor
|
|
182
|
+
// below and the tenant move that follows in the caller.
|
|
183
|
+
const cursor = await selectConsumerCursorForUpdate(
|
|
184
|
+
db,
|
|
185
|
+
TENANT_STORAGE_USAGE_MSP_QN,
|
|
186
|
+
SHARED_INSTANCE_SENTINEL,
|
|
187
|
+
);
|
|
188
|
+
// skip: no consumer row means the MSP never applied anything yet
|
|
189
|
+
if (!cursor) return;
|
|
190
|
+
|
|
191
|
+
const rows = await executeRawQueryRead<{ id: string; type: string; payload: string }>(
|
|
192
|
+
db,
|
|
193
|
+
`SELECT "id"::text AS id, "type", "payload"::text AS payload FROM kumiko_events
|
|
194
|
+
WHERE "tenant_id" = $1 AND "aggregate_type" = $2 AND "aggregate_id" = ANY($3)
|
|
195
|
+
AND "type" = ANY($4) AND "id" <= $5`,
|
|
196
|
+
[
|
|
197
|
+
sourceTenantId,
|
|
198
|
+
FILE_REF_AGGREGATE_TYPE,
|
|
199
|
+
fileRefIds,
|
|
200
|
+
TRACKED_FILE_REF_EVENT_TYPES,
|
|
201
|
+
cursor.lastProcessedEventId,
|
|
202
|
+
],
|
|
203
|
+
);
|
|
204
|
+
|
|
205
|
+
let bytes = 0;
|
|
206
|
+
let files = 0;
|
|
207
|
+
for (const row of rows) {
|
|
208
|
+
if (isWithinPendingGap(BigInt(row.id), cursor.pendingGaps)) continue;
|
|
209
|
+
const delta = fileRefStorageDelta({
|
|
210
|
+
type: row.type,
|
|
211
|
+
payload: parseJsonOrThrow<Record<string, unknown>>(
|
|
212
|
+
row.payload,
|
|
213
|
+
"transferTenantStorageUsage: kumiko_events.payload",
|
|
214
|
+
),
|
|
215
|
+
});
|
|
216
|
+
if (!delta) continue;
|
|
217
|
+
bytes += delta.bytes;
|
|
218
|
+
files += delta.files;
|
|
219
|
+
}
|
|
220
|
+
// skip: the moved fileRefs carry no applied storage share
|
|
221
|
+
if (bytes === 0 && files === 0) return;
|
|
222
|
+
|
|
223
|
+
// Source: insert-values stay 0/0 (same guard as the DELETED handler above)
|
|
224
|
+
// — the row this subtracts from is expected to already exist (the MSP
|
|
225
|
+
// already applied the create against it), and if it somehow doesn't, this
|
|
226
|
+
// must not conjure a negative balance out of thin air.
|
|
227
|
+
await incrementCounter(
|
|
228
|
+
db,
|
|
229
|
+
tenantStorageUsageTable,
|
|
230
|
+
{ tenantId: sourceTenantId, totalBytes: 0, fileCount: 0 },
|
|
231
|
+
{ totalBytes: -bytes, fileCount: -files },
|
|
232
|
+
{ set: { lastUpdatedAt: sql`now()` } },
|
|
233
|
+
);
|
|
234
|
+
// Destination: insert-values ARE the transferred amount (same as CREATED/
|
|
235
|
+
// RESTORED above) — a tenant claiming its first-ever fileRef must start
|
|
236
|
+
// its counter row at the transferred total, not at 0 with an increment
|
|
237
|
+
// that only fires once a row already exists.
|
|
238
|
+
await incrementCounter(
|
|
239
|
+
db,
|
|
240
|
+
tenantStorageUsageTable,
|
|
241
|
+
{ tenantId: destinationTenantId, totalBytes: bytes, fileCount: files },
|
|
242
|
+
{ totalBytes: bytes, fileCount: files },
|
|
243
|
+
{ set: { lastUpdatedAt: sql`now()` } },
|
|
244
|
+
);
|
|
245
|
+
}
|
package/src/files/types.ts
CHANGED
|
@@ -50,6 +50,8 @@ const EXTENSION_MIME_WHITELIST: Record<string, readonly string[]> = {
|
|
|
50
50
|
csv: ["text/csv", "application/csv", "text/plain"],
|
|
51
51
|
json: ["application/json", "text/json"],
|
|
52
52
|
md: ["text/markdown", "text/plain"],
|
|
53
|
+
doc: ["application/msword"],
|
|
54
|
+
docx: ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"],
|
|
53
55
|
} satisfies Record<string, readonly string[]>;
|
|
54
56
|
|
|
55
57
|
// Magic-byte signatures for the subset of EXTENSION_MIME_WHITELIST that has
|
|
@@ -78,6 +80,19 @@ const MAGIC_BYTE_SIGNATURES: ReadonlyArray<{
|
|
|
78
80
|
matches: (bytes) => startsWithAscii(bytes, "RIFF") && startsWithAscii(bytes, "WEBP", 8),
|
|
79
81
|
},
|
|
80
82
|
{ mimeType: "application/pdf", matches: (bytes) => startsWithAscii(bytes, "%PDF-") },
|
|
83
|
+
// Full 8-byte OLE Compound File signature (legacy .doc/.xls/.ppt share it —
|
|
84
|
+
// sniffing can't tell them apart by bytes alone, only content-verification
|
|
85
|
+
// below narrows to msword specifically).
|
|
86
|
+
{
|
|
87
|
+
mimeType: "application/msword",
|
|
88
|
+
matches: (bytes) => startsWithBytes(bytes, [0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1]),
|
|
89
|
+
},
|
|
90
|
+
// ZIP local-file-header signature — .docx/.xlsx/.pptx/plain .zip all share
|
|
91
|
+
// it; same caveat as OLE above.
|
|
92
|
+
{
|
|
93
|
+
mimeType: "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
|
94
|
+
matches: (bytes) => startsWithBytes(bytes, [0x50, 0x4b, 0x03, 0x04]),
|
|
95
|
+
},
|
|
81
96
|
];
|
|
82
97
|
|
|
83
98
|
function startsWithBytes(bytes: Uint8Array, signature: readonly number[]): boolean {
|
|
@@ -100,6 +115,12 @@ export function sniffMimeType(bytes: Uint8Array): string | null {
|
|
|
100
115
|
return null;
|
|
101
116
|
}
|
|
102
117
|
|
|
118
|
+
// Strips a `; charset=…`-style suffix and trims/lowercases — the one
|
|
119
|
+
// normalization every mimeType comparison needs before comparing values.
|
|
120
|
+
export function normalizeMimeType(mimeType: string): string {
|
|
121
|
+
return mimeType.toLowerCase().split(";")[0]?.trim() ?? "";
|
|
122
|
+
}
|
|
123
|
+
|
|
103
124
|
// image/jpg is not a registered IANA type but EXTENSION_MIME_WHITELIST above
|
|
104
125
|
// accepts it as a jpg alias — sniffMimeType only ever returns the canonical
|
|
105
126
|
// image/jpeg, so without this a legitimately-declared "image/jpg" upload
|
|
@@ -119,11 +140,19 @@ const DECLARED_MIME_ALIASES: Readonly<Record<string, string>> = {
|
|
|
119
140
|
export function resolveServedContentType(bytes: Uint8Array, declaredMimeType: string): string {
|
|
120
141
|
const sniffed = sniffMimeType(bytes);
|
|
121
142
|
if (!sniffed) return "application/octet-stream";
|
|
122
|
-
const normalizedDeclared = declaredMimeType
|
|
143
|
+
const normalizedDeclared = normalizeMimeType(declaredMimeType);
|
|
123
144
|
const declared = DECLARED_MIME_ALIASES[normalizedDeclared] ?? normalizedDeclared;
|
|
124
145
|
return sniffed === declared ? sniffed : "application/octet-stream";
|
|
125
146
|
}
|
|
126
147
|
|
|
148
|
+
// Signature-checked against the declared mimeType regardless of
|
|
149
|
+
// `options.accept`. Deliberately narrow: most of the whitelist has no magic
|
|
150
|
+
// bytes at all and would reject honestly-mislabeled uploads if checked.
|
|
151
|
+
const CONTENT_VERIFIED_MIME_TYPES = new Set([
|
|
152
|
+
"application/msword",
|
|
153
|
+
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
|
154
|
+
]);
|
|
155
|
+
|
|
127
156
|
export function validateFile(
|
|
128
157
|
metadata: FileMetadata,
|
|
129
158
|
options: FileValidationOptions,
|
|
@@ -146,7 +175,7 @@ export function validateFile(
|
|
|
146
175
|
// and having the mimeType reflect that.
|
|
147
176
|
const allowedMimes = EXTENSION_MIME_WHITELIST[ext];
|
|
148
177
|
if (allowedMimes && metadata.mimeType) {
|
|
149
|
-
const normalized = metadata.mimeType
|
|
178
|
+
const normalized = normalizeMimeType(metadata.mimeType);
|
|
150
179
|
if (!allowedMimes.includes(normalized)) {
|
|
151
180
|
return `mime_mismatch: extension ".${ext}" does not match mimeType "${metadata.mimeType}"`;
|
|
152
181
|
}
|
|
@@ -156,6 +185,19 @@ export function validateFile(
|
|
|
156
185
|
return null;
|
|
157
186
|
}
|
|
158
187
|
|
|
188
|
+
// Separate from validateFile: metadata is known before the upload body is
|
|
189
|
+
// fully read, content-verification only after — callers reject early on
|
|
190
|
+
// metadata without buffering bytes they'll then have to reject anyway.
|
|
191
|
+
export function validateFileContent(mimeType: string, content: Uint8Array): string | null {
|
|
192
|
+
const normalizedDeclared = normalizeMimeType(mimeType);
|
|
193
|
+
if (!CONTENT_VERIFIED_MIME_TYPES.has(normalizedDeclared)) return null;
|
|
194
|
+
const signature = MAGIC_BYTE_SIGNATURES.find((s) => s.mimeType === normalizedDeclared);
|
|
195
|
+
if (signature && !signature.matches(content)) {
|
|
196
|
+
return `content_mismatch: bytes do not match declared mimeType "${mimeType}"`;
|
|
197
|
+
}
|
|
198
|
+
return null;
|
|
199
|
+
}
|
|
200
|
+
|
|
159
201
|
export function assertSafeStorageKey(key: string): void {
|
|
160
202
|
if (key.split("/").some((segment) => segment === "." || segment === "..")) {
|
|
161
203
|
throw new Error(`storage key contains a path-traversal segment: "${key}"`);
|
package/src/jobs/job-runner.ts
CHANGED
|
@@ -13,6 +13,7 @@ import {
|
|
|
13
13
|
type JobContext,
|
|
14
14
|
type JobDefinition,
|
|
15
15
|
type JobRunIn,
|
|
16
|
+
type JobTriggerEvent,
|
|
16
17
|
type MemberReader,
|
|
17
18
|
type Registry,
|
|
18
19
|
type SessionUser,
|
|
@@ -42,6 +43,22 @@ import {
|
|
|
42
43
|
} from "../pipeline/write-origin";
|
|
43
44
|
import { bridgeStub } from "../testing/handler-context";
|
|
44
45
|
|
|
46
|
+
// Shape-checked after the BullMQ/Redis JSON round-trip. Unlike `_writeOrigin`
|
|
47
|
+
// (a security-relevant escape hatch that must fail the run closed), a
|
|
48
|
+
// malformed `_triggerEvent` only costs an idempotency-key convenience — the
|
|
49
|
+
// run proceeds without it rather than failing over what would be a framework
|
|
50
|
+
// bug, not untrusted input.
|
|
51
|
+
function isJobTriggerEvent(value: unknown): value is JobTriggerEvent {
|
|
52
|
+
if (typeof value !== "object" || value === null) return false;
|
|
53
|
+
const candidate = value as Record<string, unknown>;
|
|
54
|
+
if (typeof candidate["id"] !== "string") return false;
|
|
55
|
+
const headers = candidate["headers"];
|
|
56
|
+
if (typeof headers !== "object" || headers === null) return false;
|
|
57
|
+
return Object.values(headers as Record<string, unknown>).every(
|
|
58
|
+
(v) => typeof v === "string" || typeof v === "number" || typeof v === "boolean",
|
|
59
|
+
);
|
|
60
|
+
}
|
|
61
|
+
|
|
45
62
|
// A payload's own `_writeOrigin` is never trusted; only the ambient gated origin is stamped.
|
|
46
63
|
function stampGatedWriteOrigin(data: Record<string, unknown>): void {
|
|
47
64
|
delete data["_writeOrigin"];
|
|
@@ -216,6 +233,7 @@ export type JobRunner = {
|
|
|
216
233
|
eventName: string,
|
|
217
234
|
payload: Record<string, unknown>,
|
|
218
235
|
user?: SessionUser,
|
|
236
|
+
triggerEvent?: JobTriggerEvent,
|
|
219
237
|
): Promise<void>;
|
|
220
238
|
// Wires JobContext.write/writeAs/queryAs to the real dispatcher — called
|
|
221
239
|
// once at boot, after the dispatcher exists (job-runner construction happens
|
|
@@ -367,6 +385,17 @@ function buildRetryBullOpts(jobDef: JobDefinition): Pick<JobsOptions, "attempts"
|
|
|
367
385
|
return opts;
|
|
368
386
|
}
|
|
369
387
|
|
|
388
|
+
// Equality filter on top-level payload fields for a job's trigger.where —
|
|
389
|
+
// lets N jobs share one broad event QN, partitioned by a payload
|
|
390
|
+
// discriminant instead of needing N distinct event types.
|
|
391
|
+
function payloadMatchesTriggerFilter(
|
|
392
|
+
where: Readonly<Record<string, string | number | boolean>> | undefined,
|
|
393
|
+
payload: Record<string, unknown>,
|
|
394
|
+
): boolean {
|
|
395
|
+
if (!where) return true;
|
|
396
|
+
return Object.entries(where).every(([key, value]) => payload[key] === value);
|
|
397
|
+
}
|
|
398
|
+
|
|
370
399
|
// BullMQ sweeps queue-wide: any job finishing with keepJobs set can evict
|
|
371
400
|
// OTHER jobs in the same completed/failed zset (moveToFinished lua,
|
|
372
401
|
// removeJobsByMaxAge/removeJobsByMaxCount), not just the finishing job, and
|
|
@@ -748,6 +777,12 @@ export function createJobRunner(options: JobRunnerOptions): JobRunner {
|
|
|
748
777
|
// don't dig through the raw payload themselves.
|
|
749
778
|
const triggerName = rawData["_triggerName"] as string | undefined; // @cast-boundary dynamic-key
|
|
750
779
|
|
|
780
|
+
// Same carry-through for `_triggerEvent` — handleEvent injects it when
|
|
781
|
+
// the job-trigger event consumer supplied the stored event that fired
|
|
782
|
+
// this run (see createJobTriggerEventConsumer).
|
|
783
|
+
const rawTriggerEvent = rawData["_triggerEvent"];
|
|
784
|
+
const triggerEvent = isJobTriggerEvent(rawTriggerEvent) ? rawTriggerEvent : undefined;
|
|
785
|
+
|
|
751
786
|
// Absent = legacy or ungated root; present but invalid fails the run closed.
|
|
752
787
|
const rawWriteOrigin = rawData["_writeOrigin"]; // @cast-boundary dynamic-key
|
|
753
788
|
let jobOrigin: WriteOrigin | undefined;
|
|
@@ -855,6 +890,7 @@ export function createJobRunner(options: JobRunnerOptions): JobRunner {
|
|
|
855
890
|
triggeredBy: triggeredById !== null ? { id: triggeredById, tenantId } : null,
|
|
856
891
|
log: createJobLogger(logs),
|
|
857
892
|
...(triggerName !== undefined && { triggerName }),
|
|
893
|
+
...(triggerEvent !== undefined && { triggerEvent }),
|
|
858
894
|
write: (qn: string, payload: unknown) => {
|
|
859
895
|
if (!dispatchWriteRef) {
|
|
860
896
|
throw new Error(
|
|
@@ -1247,6 +1283,7 @@ export function createJobRunner(options: JobRunnerOptions): JobRunner {
|
|
|
1247
1283
|
eventName: string,
|
|
1248
1284
|
payload: Record<string, unknown>,
|
|
1249
1285
|
user?: SessionUser,
|
|
1286
|
+
triggerEvent?: JobTriggerEvent,
|
|
1250
1287
|
): Promise<void> {
|
|
1251
1288
|
const traceCtx = captureTraceContext(tracer);
|
|
1252
1289
|
// Same correlation propagation as dispatch(): events triggered from
|
|
@@ -1263,6 +1300,7 @@ export function createJobRunner(options: JobRunnerOptions): JobRunner {
|
|
|
1263
1300
|
? triggerOn.includes(eventName)
|
|
1264
1301
|
: triggerOn === eventName;
|
|
1265
1302
|
if (!matches) continue;
|
|
1303
|
+
if (!payloadMatchesTriggerFilter(jobDef.trigger.where, payload)) continue;
|
|
1266
1304
|
const data: Record<string, unknown> = { ...payload };
|
|
1267
1305
|
if (user) {
|
|
1268
1306
|
data["_tenantId"] = user.tenantId;
|
|
@@ -1273,6 +1311,7 @@ export function createJobRunner(options: JobRunnerOptions): JobRunner {
|
|
|
1273
1311
|
// setzen wir es auch — kostet nichts und vereinfacht Handler-Code
|
|
1274
1312
|
// (kein "ist es Multi?"-Branch nötig).
|
|
1275
1313
|
data["_triggerName"] = eventName;
|
|
1314
|
+
if (triggerEvent) data["_triggerEvent"] = triggerEvent;
|
|
1276
1315
|
if (traceCtx) data[TRACE_CONTEXT_KEY] = traceCtx;
|
|
1277
1316
|
if (reqCtx?.correlationId) data["_correlationId"] = reqCtx.correlationId;
|
|
1278
1317
|
// Same maxPerTenant guard as dispatch — events that fan into many
|