cursedbelt-server 3.0.0 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/server/activity/index.d.ts +1 -1
- package/dist/server/activity/index.js +1 -1
- package/dist/server/auth/jwt.d.ts +1 -1
- package/dist/server/auth/jwt.js +2 -2
- package/dist/server/master-lock/accountsPage.d.ts +39 -0
- package/dist/server/master-lock/accountsPage.js +455 -0
- package/dist/server/master-lock/guard.js +57 -1
- package/dist/server/master-lock/index.d.ts +10 -2
- package/dist/server/master-lock/index.js +10 -2
- package/dist/server/master-lock/lockPage.d.ts +9 -0
- package/dist/server/master-lock/lockPage.js +44 -36
- package/dist/server/media-bun/streaming.d.ts +0 -1
- package/dist/server/media-bun/streaming.js +1 -1
- package/dist/server/storage/files/catalogue.d.ts +2 -2
- package/dist/server/storage/files/catalogue.js +18 -208
- package/dist/server/storage/files/fileRoutes.d.ts +2 -20
- package/dist/server/storage/files/fileRoutes.js +1 -74
- package/dist/server/storage/files/index.d.ts +1 -1
- package/dist/server/storage/files/index.js +1 -1
- package/dist/server/storage/files/migrations.js +9 -6
- package/dist/server/storage/files/stack.d.ts +6 -31
- package/dist/server/storage/files/stack.js +0 -27
- package/dist/server/storage/files/types.d.ts +10 -60
- package/dist/server/storage/files/types.js +9 -0
- package/dist/server/storage/index.d.ts +2 -2
- package/dist/server/storage/index.js +1 -1
- package/dist/server/storage/mountMediaRoutes.d.ts +1 -4
- package/dist/server/storage/types.d.ts +0 -9
- package/dist/server/storage/types.js +2 -9
- package/package.json +2 -2
- package/src/leafSubpathsImportNothing.spec.ts +6 -4
- package/src/server/activity/index.ts +1 -1
- package/src/server/auth/jwt.ts +2 -2
- package/src/server/master-lock/accountsPage.spec.ts +231 -0
- package/src/server/master-lock/accountsPage.ts +465 -0
- package/src/server/master-lock/guard.spec.ts +25 -0
- package/src/server/master-lock/guard.ts +65 -1
- package/src/server/master-lock/index.ts +22 -2
- package/src/server/master-lock/lockPage.spec.ts +20 -16
- package/src/server/master-lock/lockPage.ts +45 -36
- package/src/server/media-bun/streaming.ts +0 -6
- package/src/server/storage/fileUploadClient.integration.spec.ts +2 -21
- package/src/server/storage/files/adapter.photo.display.spec.ts +2 -2
- package/src/server/storage/files/adapter.photo.spec.ts +2 -2
- package/src/server/storage/files/adapter.spec.ts +0 -1
- package/src/server/storage/files/catalogue.media-host.spec.ts +6 -71
- package/src/server/storage/files/catalogue.recover.spec.ts +14 -23
- package/src/server/storage/files/catalogue.ts +19 -240
- package/src/server/storage/files/fileRoutes.replace.spec.ts +2 -2
- package/src/server/storage/files/fileRoutes.spec.ts +3 -97
- package/src/server/storage/files/fileRoutes.ts +2 -96
- package/src/server/storage/files/index.ts +1 -4
- package/src/server/storage/files/migrations.ts +9 -7
- package/src/server/storage/files/stack.spec.ts +11 -111
- package/src/server/storage/files/stack.ts +9 -61
- package/src/server/storage/files/types.ts +20 -68
- package/src/server/storage/index.ts +0 -4
- package/src/server/storage/mountMediaRoutes.ts +1 -7
- package/src/server/storage/types.ts +2 -11
- package/src/shippedFilesAreTracked.spec.ts +3 -2
- package/dist/server/private-media/crypto.d.ts +0 -37
- package/dist/server/private-media/crypto.js +0 -111
- package/dist/server/private-media/index-store.d.ts +0 -31
- package/dist/server/private-media/index-store.js +0 -36
- package/dist/server/private-media/mountPrivateRoutes.d.ts +0 -26
- package/dist/server/private-media/mountPrivateRoutes.js +0 -219
- package/dist/server/private-media/pipeline.d.ts +0 -37
- package/dist/server/private-media/pipeline.js +0 -97
- package/dist/server/private-media/unlock.d.ts +0 -52
- package/dist/server/private-media/unlock.js +0 -55
- package/src/server/private-media/blobRoute.spec.ts +0 -165
- package/src/server/private-media/crypto.spec.ts +0 -81
- package/src/server/private-media/crypto.ts +0 -155
- package/src/server/private-media/gatePolicy.spec.ts +0 -98
- package/src/server/private-media/index-store.ts +0 -83
- package/src/server/private-media/mountPrivateRoutes.ts +0 -265
- package/src/server/private-media/pipeline.spec.ts +0 -111
- package/src/server/private-media/pipeline.ts +0 -170
- package/src/server/private-media/routes.spec.ts +0 -265
- package/src/server/private-media/unlock.ts +0 -112
- package/src/server/storage/files/catalogue.private.spec.ts +0 -106
|
@@ -1,12 +1,9 @@
|
|
|
1
1
|
import type { Database } from 'bun:sqlite';
|
|
2
2
|
import type { Dirent } from 'node:fs';
|
|
3
|
-
import {
|
|
3
|
+
import { readdir, stat as fsStat } from 'node:fs/promises';
|
|
4
4
|
import path from 'node:path';
|
|
5
5
|
import type { ProcessingStatus } from 'cursedbelt-core/media';
|
|
6
6
|
import { renderImage } from '../../media-bun/image';
|
|
7
|
-
import { sealMeta } from '../../private-media/crypto';
|
|
8
|
-
import type { PrivateIndex, PrivateItem } from '../../private-media/index-store';
|
|
9
|
-
import { processPrivateBlob, processPrivateVideo } from '../../private-media/pipeline';
|
|
10
7
|
import { ffmpegImageThumbnail, isBrowserRenderableImage } from '../imageThumbnail';
|
|
11
8
|
import {
|
|
12
9
|
fsDisplayKey,
|
|
@@ -18,7 +15,6 @@ import {
|
|
|
18
15
|
fsVttKey,
|
|
19
16
|
type MediaCtx,
|
|
20
17
|
type MediaStore,
|
|
21
|
-
privateFsKey,
|
|
22
18
|
} from '../types';
|
|
23
19
|
import { type ProcessOptions, type ProcessResult, processVideo } from '../videoProcessor';
|
|
24
20
|
import { mimeByExt } from './mimeByExt';
|
|
@@ -31,7 +27,6 @@ import {
|
|
|
31
27
|
type FileRecord,
|
|
32
28
|
type FileType,
|
|
33
29
|
type ListFilesOptions,
|
|
34
|
-
type PrivateFileCatalogue,
|
|
35
30
|
type TagFacet,
|
|
36
31
|
} from './types';
|
|
37
32
|
|
|
@@ -63,30 +58,6 @@ export type ImageDerivativesOptions = {
|
|
|
63
58
|
* ffmpeg by default; remote binary-server job when the bytes live there). */
|
|
64
59
|
export type ImageDerivatives = (opts: ImageDerivativesOptions) => Promise<{ hasThumbnail: boolean }>;
|
|
65
60
|
|
|
66
|
-
/** Sealed descriptor payload (encrypted under the set key) — the private counterpart to the
|
|
67
|
-
* plaintext name/tags a public row stores in the clear. */
|
|
68
|
-
type SealedDescriptor = { name: string; mime: string; tags: string[] };
|
|
69
|
-
|
|
70
|
-
// Stream an upload body to a local plaintext file (staged for the private pipeline; wiped after).
|
|
71
|
-
// Bun.write(dest, new Response(stream)) hangs in Bun 1.3.x — pump via a FileSink instead.
|
|
72
|
-
async function streamToFile(body: ReadableStream<Uint8Array>, dest: string): Promise<number> {
|
|
73
|
-
await mkdir(path.dirname(dest), { recursive: true });
|
|
74
|
-
const sink = Bun.file(dest).writer();
|
|
75
|
-
const reader = body.getReader();
|
|
76
|
-
let size = 0;
|
|
77
|
-
try {
|
|
78
|
-
while (true) {
|
|
79
|
-
const { done, value } = await reader.read();
|
|
80
|
-
if (done) break;
|
|
81
|
-
size += value.byteLength;
|
|
82
|
-
sink.write(value);
|
|
83
|
-
}
|
|
84
|
-
} finally {
|
|
85
|
-
await sink.end();
|
|
86
|
-
}
|
|
87
|
-
return size;
|
|
88
|
-
}
|
|
89
|
-
|
|
90
61
|
// The general primary binary for a NON-video file (images/docs/…) lives here; video files store a
|
|
91
62
|
// raw source (fsSourceKey) + derived mp4/HLS/poster/sprite via the shared video keyspace.
|
|
92
63
|
export const fileBinaryKey = (owner: string, id: string) => `fs/${owner}/${id}`;
|
|
@@ -104,7 +75,6 @@ type Row = {
|
|
|
104
75
|
mime: string;
|
|
105
76
|
size: number;
|
|
106
77
|
file_type: string;
|
|
107
|
-
is_private: number;
|
|
108
78
|
has_thumbnail: number;
|
|
109
79
|
has_hls: number;
|
|
110
80
|
has_sprites: number;
|
|
@@ -120,9 +90,6 @@ type Row = {
|
|
|
120
90
|
view_key?: number;
|
|
121
91
|
};
|
|
122
92
|
|
|
123
|
-
/** A private row carries the sealed envelope + wrapped key + wrap mode alongside the base row. */
|
|
124
|
-
type PrivateRow = Row & { enc_name: string; enc_dek: string; enc_key_type: string };
|
|
125
|
-
|
|
126
93
|
type CollectionRow = {
|
|
127
94
|
id: string;
|
|
128
95
|
owner: string;
|
|
@@ -267,9 +234,7 @@ export type SqliteFileCatalogueOptions = {
|
|
|
267
234
|
imageDerivatives?: ImageDerivatives;
|
|
268
235
|
};
|
|
269
236
|
|
|
270
|
-
export function createSqliteFileCatalogue(
|
|
271
|
-
opts: SqliteFileCatalogueOptions,
|
|
272
|
-
): FileCatalogue & PrivateFileCatalogue {
|
|
237
|
+
export function createSqliteFileCatalogue(opts: SqliteFileCatalogueOptions): FileCatalogue {
|
|
273
238
|
const { db, store, workDir } = opts;
|
|
274
239
|
const ownerOf = opts.owner ?? ((c: MediaCtx) => c.tenantId);
|
|
275
240
|
const now = opts.now ?? (() => Date.now());
|
|
@@ -278,11 +243,11 @@ export function createSqliteFileCatalogue(
|
|
|
278
243
|
const randomSeed = opts.randomSeed ?? rand31;
|
|
279
244
|
|
|
280
245
|
const insertFile = db.query(
|
|
281
|
-
`INSERT INTO files (id, owner, name, mime, size, file_type,
|
|
282
|
-
VALUES ($id, $owner, $name, $mime, $size, $type,
|
|
246
|
+
`INSERT INTO files (id, owner, name, mime, size, file_type, processing_status, processing_progress, storage_backend, rand_key, created_at, updated_at)
|
|
247
|
+
VALUES ($id, $owner, $name, $mime, $size, $type, $status, $progress, 'disk', $rand, $now, $now)`,
|
|
283
248
|
);
|
|
284
249
|
const insertTag = db.query(
|
|
285
|
-
`INSERT OR IGNORE INTO file_tags (file_id, tag, owner
|
|
250
|
+
`INSERT OR IGNORE INTO file_tags (file_id, tag, owner) VALUES ($file, $tag, $owner)`,
|
|
286
251
|
);
|
|
287
252
|
const deleteTags = db.query(`DELETE FROM file_tags WHERE file_id = $file`);
|
|
288
253
|
const getRow = db.query<Row, { $id: string; $owner: string }>(
|
|
@@ -318,33 +283,7 @@ export function createSqliteFileCatalogue(
|
|
|
318
283
|
`UPDATE files SET name = $name, mime = $mime, size = $size, file_type = $type,
|
|
319
284
|
processing_status = $status, processing_progress = $progress,
|
|
320
285
|
has_hls = 0, has_thumbnail = 0, has_sprites = 0, duration = NULL, updated_at = $now
|
|
321
|
-
WHERE id = $id AND owner = $owner
|
|
322
|
-
);
|
|
323
|
-
|
|
324
|
-
// Private (encrypted) rows: name/tags are sealed in enc_name (sealedMeta); the wrapped AES-128
|
|
325
|
-
// content key is in enc_dek; enc_key_type records the wrap mode. is_private = 1 keeps them out of
|
|
326
|
-
// the public feed. These rows are surfaced to `mountPrivateRoutes` via `asPrivateIndex()`.
|
|
327
|
-
const insertPrivate = db.query(
|
|
328
|
-
`INSERT INTO files (id, owner, name, mime, size, file_type, is_private, has_hls, duration,
|
|
329
|
-
processing_status, processing_progress, storage_backend, enc_name, enc_dek, enc_key_type, rand_key, created_at, updated_at)
|
|
330
|
-
VALUES ($id, $owner, '', '', $size, $type, 1, 0, NULL, $status, $progress, 'disk', $sealed, $wrapped, $mode, $rand, $now, $now)`,
|
|
331
|
-
);
|
|
332
|
-
const finishPrivate = db.query(
|
|
333
|
-
`UPDATE files SET processing_status = $status, processing_progress = $progress, has_hls = $hls,
|
|
334
|
-
duration = $duration, size = $size, enc_name = $sealed, enc_dek = $wrapped, updated_at = $now WHERE id = $id`,
|
|
335
|
-
);
|
|
336
|
-
const getPrivateRow = db.query<PrivateRow, { $id: string; $owner: string }>(
|
|
337
|
-
`SELECT * FROM files WHERE id = $id AND owner = $owner AND is_private = 1`,
|
|
338
|
-
);
|
|
339
|
-
const listPrivateFullRows = db.query<PrivateRow, { $owner: string }>(
|
|
340
|
-
`SELECT * FROM files WHERE owner = $owner AND is_private = 1 ORDER BY created_at DESC, id DESC`,
|
|
341
|
-
);
|
|
342
|
-
const updatePrivateRow = db.query(
|
|
343
|
-
`UPDATE files SET size = $size, processing_status = $status, enc_name = $sealed, enc_dek = $wrapped,
|
|
344
|
-
enc_key_type = $mode, updated_at = $now WHERE id = $id AND owner = $owner AND is_private = 1`,
|
|
345
|
-
);
|
|
346
|
-
const deletePrivateRow = db.query(
|
|
347
|
-
`DELETE FROM files WHERE id = $id AND owner = $owner AND is_private = 1`,
|
|
286
|
+
WHERE id = $id AND owner = $owner`,
|
|
348
287
|
);
|
|
349
288
|
|
|
350
289
|
// ── Collection prepared statements ───────────────────────────────────────────────
|
|
@@ -405,17 +344,6 @@ export function createSqliteFileCatalogue(
|
|
|
405
344
|
);
|
|
406
345
|
const deleteFileViews = db.query(`DELETE FROM file_views WHERE file_id = $file`);
|
|
407
346
|
|
|
408
|
-
const rowToPrivateItem = (r: PrivateRow, owner: string): PrivateItem => ({
|
|
409
|
-
id: r.id,
|
|
410
|
-
owner,
|
|
411
|
-
sizeBytes: r.size,
|
|
412
|
-
createdAt: r.created_at,
|
|
413
|
-
status: r.processing_status as ProcessingStatus,
|
|
414
|
-
keyMode: (r.enc_key_type || 'deniable') as PrivateItem['keyMode'],
|
|
415
|
-
wrappedContentKey: r.enc_dek,
|
|
416
|
-
sealedMeta: r.enc_name,
|
|
417
|
-
});
|
|
418
|
-
|
|
419
347
|
const tagsFor = (ids: string[]): Map<string, string[]> => {
|
|
420
348
|
const map = new Map<string, string[]>();
|
|
421
349
|
if (ids.length === 0) return map;
|
|
@@ -472,7 +400,6 @@ export function createSqliteFileCatalogue(
|
|
|
472
400
|
size: row.size,
|
|
473
401
|
fileType: row.file_type as FileType,
|
|
474
402
|
tags,
|
|
475
|
-
isPrivate: row.is_private === 1,
|
|
476
403
|
hasThumbnail: row.has_thumbnail === 1,
|
|
477
404
|
hasHls: row.has_hls === 1,
|
|
478
405
|
hasSprites: row.has_sprites === 1,
|
|
@@ -665,10 +592,7 @@ export function createSqliteFileCatalogue(
|
|
|
665
592
|
|
|
666
593
|
const statsByType = db.query<{ file_type: string; n: number; bytes: number | null }, { $owner: string }>(
|
|
667
594
|
`SELECT file_type, COUNT(*) AS n, SUM(size) AS bytes FROM files
|
|
668
|
-
WHERE owner = $owner
|
|
669
|
-
);
|
|
670
|
-
const statsPrivate = db.query<{ n: number; bytes: number | null }, { $owner: string }>(
|
|
671
|
-
`SELECT COUNT(*) AS n, SUM(size) AS bytes FROM files WHERE owner = $owner AND is_private = 1`,
|
|
595
|
+
WHERE owner = $owner GROUP BY file_type`,
|
|
672
596
|
);
|
|
673
597
|
const statsStatus = db.query<{ processing_status: string; n: number }, { $owner: string }>(
|
|
674
598
|
`SELECT processing_status, COUNT(*) AS n FROM files
|
|
@@ -686,7 +610,6 @@ export function createSqliteFileCatalogue(
|
|
|
686
610
|
files += row.n;
|
|
687
611
|
bytes += row.bytes ?? 0;
|
|
688
612
|
}
|
|
689
|
-
const priv = statsPrivate.get({ $owner: owner });
|
|
690
613
|
let processing = 0;
|
|
691
614
|
let failed = 0;
|
|
692
615
|
for (const row of statsStatus.all({ $owner: owner })) {
|
|
@@ -698,8 +621,6 @@ export function createSqliteFileCatalogue(
|
|
|
698
621
|
files,
|
|
699
622
|
bytes,
|
|
700
623
|
byType,
|
|
701
|
-
privateItems: priv?.n ?? 0,
|
|
702
|
-
privateBytes: priv?.bytes ?? 0,
|
|
703
624
|
processing,
|
|
704
625
|
failed,
|
|
705
626
|
};
|
|
@@ -710,7 +631,6 @@ export function createSqliteFileCatalogue(
|
|
|
710
631
|
const params: Record<string, string | number> = { $owner: owner, $limit: limit + 1 };
|
|
711
632
|
const where: string[] = [
|
|
712
633
|
'f.owner = $owner',
|
|
713
|
-
'f.is_private = 0',
|
|
714
634
|
options.trashedOnly ? 'f.trashed_at IS NOT NULL' : 'f.trashed_at IS NULL',
|
|
715
635
|
];
|
|
716
636
|
let from = 'files f';
|
|
@@ -808,7 +728,7 @@ export function createSqliteFileCatalogue(
|
|
|
808
728
|
async tagFacets(ctx) {
|
|
809
729
|
const rows = db
|
|
810
730
|
.query<TagFacet, { $owner: string }>(
|
|
811
|
-
`SELECT tag, COUNT(*) as count FROM file_tags WHERE owner = $owner
|
|
731
|
+
`SELECT tag, COUNT(*) as count FROM file_tags WHERE owner = $owner
|
|
812
732
|
GROUP BY tag ORDER BY count DESC, tag ASC`,
|
|
813
733
|
)
|
|
814
734
|
.all({ $owner: ownerOf(ctx) });
|
|
@@ -859,9 +779,6 @@ export function createSqliteFileCatalogue(
|
|
|
859
779
|
const owner = ownerOf(ctx);
|
|
860
780
|
const row = getRow.get({ $id: id, $owner: owner });
|
|
861
781
|
if (!row) return null;
|
|
862
|
-
if (row.is_private === 1) {
|
|
863
|
-
throw new Error('replaceContent: private files cannot be replaced in place');
|
|
864
|
-
}
|
|
865
782
|
const isVideo = upload.mime.startsWith('video/');
|
|
866
783
|
// Store the new bytes FIRST — a failed write leaves the old row + bytes fully intact.
|
|
867
784
|
const key = isVideo ? fsSourceKey(owner, id) : fileBinaryKey(owner, id);
|
|
@@ -1001,26 +918,23 @@ export function createSqliteFileCatalogue(
|
|
|
1001
918
|
|
|
1002
919
|
recoverProcessing() {
|
|
1003
920
|
// A restart mid-transcode leaves a row stuck in `processing`/`pending` — the in-process
|
|
1004
|
-
// job is gone.
|
|
1005
|
-
//
|
|
1006
|
-
//
|
|
1007
|
-
// insert and never appear here. (Ported from ca's recoverOnBoot; our jobs are in-process,
|
|
1008
|
-
// so recovery is app-invoked at startup rather than a persistent queue.)
|
|
921
|
+
// job is gone. The source still sits at fsSourceKey, so re-kick the pipeline. Non-video rows
|
|
922
|
+
// are `ready` at insert and never appear here. (Ported from ca's recoverOnBoot; our jobs are
|
|
923
|
+
// in-process, so recovery is app-invoked at startup rather than a persistent queue.)
|
|
1009
924
|
// Re-kicks are CAPPED at maxProcessingAttempts total kicks: if the transcode itself is what
|
|
1010
925
|
// keeps killing the process (2026-07-04 prod outage: ffmpeg OOM on a 1GB host → systemd
|
|
1011
926
|
// restart → unconditional re-kick → OOM again, forever), the row must land on `failed`
|
|
1012
927
|
// instead of holding the whole service down.
|
|
1013
928
|
const stuck = db
|
|
1014
|
-
.query<{ id: string; owner: string;
|
|
1015
|
-
`SELECT id, owner,
|
|
1016
|
-
WHERE
|
|
1017
|
-
AND processing_status IN ('pending', 'processing')`,
|
|
929
|
+
.query<{ id: string; owner: string; processing_attempts: number }, []>(
|
|
930
|
+
`SELECT id, owner, processing_attempts FROM files
|
|
931
|
+
WHERE file_type = 'video' AND processing_status IN ('pending', 'processing')`,
|
|
1018
932
|
)
|
|
1019
933
|
.all();
|
|
1020
934
|
let requeued = 0;
|
|
1021
935
|
let failed = 0;
|
|
1022
936
|
for (const r of stuck) {
|
|
1023
|
-
if (r.
|
|
937
|
+
if (r.processing_attempts >= maxProcessingAttempts) {
|
|
1024
938
|
setProcessing.run({ $id: r.id, $status: 'failed', $progress: 0, $now: now() });
|
|
1025
939
|
failed++;
|
|
1026
940
|
} else {
|
|
@@ -1031,7 +945,7 @@ export function createSqliteFileCatalogue(
|
|
|
1031
945
|
return { requeued, failed };
|
|
1032
946
|
},
|
|
1033
947
|
|
|
1034
|
-
// Backfill grid thumbnails for
|
|
948
|
+
// Backfill grid thumbnails for video rows that finished (ready/failed) without a poster —
|
|
1035
949
|
// e.g. rows that landed while transcodeVideos was off entirely, or that failed a full transcode
|
|
1036
950
|
// before the poster step. Re-kicks the (now poster-capable) pipeline; the attempts counter is
|
|
1037
951
|
// reset first so the maxProcessingAttempts cap doesn't pre-empt a fresh, cheaper poster-only run.
|
|
@@ -1039,7 +953,7 @@ export function createSqliteFileCatalogue(
|
|
|
1039
953
|
const missing = db
|
|
1040
954
|
.query<{ id: string; owner: string }, []>(
|
|
1041
955
|
`SELECT id, owner FROM files
|
|
1042
|
-
WHERE file_type = 'video' AND
|
|
956
|
+
WHERE file_type = 'video' AND has_thumbnail = 0
|
|
1043
957
|
AND processing_status IN ('ready', 'failed')`,
|
|
1044
958
|
)
|
|
1045
959
|
.all();
|
|
@@ -1053,7 +967,7 @@ export function createSqliteFileCatalogue(
|
|
|
1053
967
|
return { requeued: missing.length };
|
|
1054
968
|
},
|
|
1055
969
|
|
|
1056
|
-
// Backfill grid thumbnails for
|
|
970
|
+
// Backfill grid thumbnails for image rows with no poster — e.g. HEIC/HEVC iPhone photos
|
|
1057
971
|
// whose sharp thumbnail silently failed before the ffmpeg fallback existed. Re-kicks the (now
|
|
1058
972
|
// HEIC-capable) image pipeline, serialized so at most one ffmpeg decode runs at a time.
|
|
1059
973
|
backfillImageThumbnails(): { requeued: number } {
|
|
@@ -1064,7 +978,7 @@ export function createSqliteFileCatalogue(
|
|
|
1064
978
|
const candidates = db
|
|
1065
979
|
.query<{ id: string; owner: string; mime: string; has_thumbnail: number }, []>(
|
|
1066
980
|
`SELECT id, owner, mime, has_thumbnail FROM files
|
|
1067
|
-
WHERE file_type = 'image'
|
|
981
|
+
WHERE file_type = 'image'
|
|
1068
982
|
AND (has_thumbnail = 0 OR mime IN ('image/heic', 'image/heif'))`,
|
|
1069
983
|
)
|
|
1070
984
|
.all();
|
|
@@ -1187,140 +1101,5 @@ export function createSqliteFileCatalogue(
|
|
|
1187
1101
|
// Upsert-gated to the caller's own files; unknown/other-owner ids record nothing.
|
|
1188
1102
|
recordViewStmt.run({ $id: fileId, $owner: ownerOf(ctx), $now: now() });
|
|
1189
1103
|
},
|
|
1190
|
-
|
|
1191
|
-
async ingestPrivate(ctx, upload, cfg) {
|
|
1192
|
-
const owner = ownerOf(ctx);
|
|
1193
|
-
const id = crypto.randomUUID();
|
|
1194
|
-
const ts = now();
|
|
1195
|
-
// Seal the descriptor NOW — sealing is independent of the transcode (only the wrapped content
|
|
1196
|
-
// KEY needs the pipeline). Storing enc_name at insert means a try-a-password reveal matches the
|
|
1197
|
-
// item immediately, before its HLS finishes encrypting (no seal race).
|
|
1198
|
-
const descriptor: SealedDescriptor = {
|
|
1199
|
-
name: upload.name,
|
|
1200
|
-
mime: upload.mime,
|
|
1201
|
-
tags: upload.tags ?? [],
|
|
1202
|
-
};
|
|
1203
|
-
const sealed = sealMeta(cfg.setKey, descriptor);
|
|
1204
|
-
const isVideo = upload.mime.startsWith('video/');
|
|
1205
|
-
insertPrivate.run({
|
|
1206
|
-
$id: id,
|
|
1207
|
-
$owner: owner,
|
|
1208
|
-
$size: 0,
|
|
1209
|
-
$type: classifyFileType(upload.mime),
|
|
1210
|
-
$mode: cfg.keyMode,
|
|
1211
|
-
$status: 'processing',
|
|
1212
|
-
$progress: 0,
|
|
1213
|
-
$sealed: sealed,
|
|
1214
|
-
$wrapped: '',
|
|
1215
|
-
$rand: randKey(),
|
|
1216
|
-
$now: ts,
|
|
1217
|
-
});
|
|
1218
|
-
|
|
1219
|
-
// Stage plaintext to a scratch file, then encrypt: video → AES-128 HLS ladder; anything
|
|
1220
|
-
// else (photo/doc/audio) → one AES-128-CTR blob at privateSourceKey. Plaintext is never
|
|
1221
|
-
// persisted either way. We MUST await: the wrapped key is produced by the pipeline and
|
|
1222
|
-
// there is nothing to serve without it.
|
|
1223
|
-
const srcPath = path.join(workDir, 'ingest', `${owner}-${id}.src`);
|
|
1224
|
-
let size = 0;
|
|
1225
|
-
try {
|
|
1226
|
-
size = await streamToFile(upload.body, srcPath);
|
|
1227
|
-
const res = isVideo
|
|
1228
|
-
? await processPrivateVideo({
|
|
1229
|
-
store,
|
|
1230
|
-
workDir,
|
|
1231
|
-
owner,
|
|
1232
|
-
id,
|
|
1233
|
-
sourcePath: srcPath,
|
|
1234
|
-
setKey: cfg.setKey,
|
|
1235
|
-
ladderHeights: opts.ladderHeights,
|
|
1236
|
-
onProgress: (status, pct) =>
|
|
1237
|
-
setProcessing.run({ $id: id, $status: status, $progress: pct, $now: now() }),
|
|
1238
|
-
})
|
|
1239
|
-
: {
|
|
1240
|
-
...(await processPrivateBlob({
|
|
1241
|
-
store,
|
|
1242
|
-
owner,
|
|
1243
|
-
id,
|
|
1244
|
-
sourcePath: srcPath,
|
|
1245
|
-
setKey: cfg.setKey,
|
|
1246
|
-
})),
|
|
1247
|
-
durationSeconds: 0,
|
|
1248
|
-
hasHls: false,
|
|
1249
|
-
};
|
|
1250
|
-
finishPrivate.run({
|
|
1251
|
-
$id: id,
|
|
1252
|
-
$status: res.status,
|
|
1253
|
-
$progress: res.status === 'ready' ? 100 : 0,
|
|
1254
|
-
$hls: res.hasHls ? 1 : 0,
|
|
1255
|
-
$duration: res.durationSeconds || null,
|
|
1256
|
-
$size: size,
|
|
1257
|
-
$sealed: sealed,
|
|
1258
|
-
$wrapped: res.wrappedContentKey ?? '',
|
|
1259
|
-
$now: now(),
|
|
1260
|
-
});
|
|
1261
|
-
} finally {
|
|
1262
|
-
await rm(srcPath, { force: true }).catch(() => {});
|
|
1263
|
-
}
|
|
1264
|
-
|
|
1265
|
-
const row = getRow.get({ $id: id, $owner: owner }) as Row;
|
|
1266
|
-
return {
|
|
1267
|
-
id,
|
|
1268
|
-
size,
|
|
1269
|
-
createdAt: ts,
|
|
1270
|
-
processingStatus: row.processing_status as ProcessingStatus,
|
|
1271
|
-
processingProgress: row.processing_progress,
|
|
1272
|
-
hasHls: row.has_hls === 1,
|
|
1273
|
-
};
|
|
1274
|
-
},
|
|
1275
|
-
|
|
1276
|
-
asPrivateIndex(): PrivateIndex {
|
|
1277
|
-
// A PrivateIndex view over the catalogue's private rows — the seam mountPrivateRoutes reads.
|
|
1278
|
-
// `owner` here is the private owner axis the routes pass (a userId), matching how ingestPrivate
|
|
1279
|
-
// stored the row (both go through the files.owner column).
|
|
1280
|
-
return {
|
|
1281
|
-
async add(item) {
|
|
1282
|
-
insertPrivate.run({
|
|
1283
|
-
$id: item.id,
|
|
1284
|
-
$owner: item.owner,
|
|
1285
|
-
$size: item.sizeBytes,
|
|
1286
|
-
// External index writers predate blob items and only ever added videos.
|
|
1287
|
-
$type: 'video',
|
|
1288
|
-
$mode: item.keyMode,
|
|
1289
|
-
$status: item.status,
|
|
1290
|
-
$progress: item.status === 'ready' ? 100 : 0,
|
|
1291
|
-
$sealed: item.sealedMeta,
|
|
1292
|
-
$wrapped: item.wrappedContentKey,
|
|
1293
|
-
$rand: randKey(),
|
|
1294
|
-
$now: now(),
|
|
1295
|
-
});
|
|
1296
|
-
},
|
|
1297
|
-
async get(owner, id) {
|
|
1298
|
-
const row = getPrivateRow.get({ $id: id, $owner: owner });
|
|
1299
|
-
return row ? rowToPrivateItem(row, owner) : null;
|
|
1300
|
-
},
|
|
1301
|
-
async list(owner) {
|
|
1302
|
-
return listPrivateFullRows.all({ $owner: owner }).map((r) => rowToPrivateItem(r, owner));
|
|
1303
|
-
},
|
|
1304
|
-
async update(owner, id, patch) {
|
|
1305
|
-
const cur = getPrivateRow.get({ $id: id, $owner: owner });
|
|
1306
|
-
if (!cur) return;
|
|
1307
|
-
const merged = { ...rowToPrivateItem(cur, owner), ...patch };
|
|
1308
|
-
updatePrivateRow.run({
|
|
1309
|
-
$id: id,
|
|
1310
|
-
$owner: owner,
|
|
1311
|
-
$size: merged.sizeBytes,
|
|
1312
|
-
$status: merged.status,
|
|
1313
|
-
$sealed: merged.sealedMeta,
|
|
1314
|
-
$wrapped: merged.wrappedContentKey,
|
|
1315
|
-
$mode: merged.keyMode,
|
|
1316
|
-
$now: now(),
|
|
1317
|
-
});
|
|
1318
|
-
},
|
|
1319
|
-
async remove(owner, id) {
|
|
1320
|
-
await store.removePrefix(privateFsKey(owner, id));
|
|
1321
|
-
deletePrivateRow.run({ $id: id, $owner: owner });
|
|
1322
|
-
},
|
|
1323
|
-
};
|
|
1324
|
-
},
|
|
1325
1104
|
};
|
|
1326
1105
|
}
|
|
@@ -11,7 +11,7 @@ import type { MediaCtx, MediaStore } from '../types';
|
|
|
11
11
|
import { createSqliteFileCatalogue, fileBinaryKey } from './catalogue';
|
|
12
12
|
import { mountFileRoutes } from './fileRoutes';
|
|
13
13
|
import { FILE_CATALOGUE_MIGRATIONS } from './migrations';
|
|
14
|
-
import type { FileCatalogue
|
|
14
|
+
import type { FileCatalogue } from './types';
|
|
15
15
|
|
|
16
16
|
// PUT /_files/replace/:id — the editor's "overwrite original" byte edge.
|
|
17
17
|
|
|
@@ -24,7 +24,7 @@ describe('fileRoutes — replace', () => {
|
|
|
24
24
|
let db: Database;
|
|
25
25
|
let root: string;
|
|
26
26
|
let store: MediaStore;
|
|
27
|
-
let cat: FileCatalogue
|
|
27
|
+
let cat: FileCatalogue;
|
|
28
28
|
let authed = true;
|
|
29
29
|
const app = new Hono();
|
|
30
30
|
let fileId: string;
|
|
@@ -6,115 +6,21 @@ import path from 'node:path';
|
|
|
6
6
|
import { Hono } from 'hono';
|
|
7
7
|
import { MAX_SINGLE_UPLOAD_BYTES } from 'cursedbelt-core/media/upload-limits';
|
|
8
8
|
import { runMigrations } from '../../migration/runner';
|
|
9
|
-
import { generateUserSalt } from '../../private-media/crypto';
|
|
10
|
-
import { createUnlockService } from '../../private-media/unlock';
|
|
11
9
|
import { createDiskStore } from '../diskStore';
|
|
12
10
|
import { createStreamTokenService } from '../streamToken';
|
|
13
11
|
import type { MediaCtx } from '../types';
|
|
14
12
|
import { createSqliteFileCatalogue } from './catalogue';
|
|
15
13
|
import { mountFileRoutes, SINGLE_UPLOAD_TOO_LARGE } from './fileRoutes';
|
|
16
14
|
import { FILE_CATALOGUE_MIGRATIONS } from './migrations';
|
|
17
|
-
import type { FileCatalogue, FilePage
|
|
15
|
+
import type { FileCatalogue, FilePage } from './types';
|
|
18
16
|
|
|
19
17
|
const CTX: MediaCtx = { userId: 'u1', tenantId: 't1' };
|
|
20
|
-
const unlock = createUnlockService({ serverSecret: 'srv', unlockSecret: 'unl' });
|
|
21
18
|
const streamToken = createStreamTokenService({ secret: 'stream' });
|
|
22
|
-
const salt = generateUserSalt();
|
|
23
|
-
|
|
24
|
-
describe('fileRoutes — private upload key derivation (password vs unlock token)', () => {
|
|
25
|
-
// ingestPrivate runs the ffmpeg pipeline — fake the catalogue and capture the key/mode the
|
|
26
|
-
// route derived. The derivation logic IS what this route owns.
|
|
27
|
-
let captured: { setKey: Buffer; keyMode: PrivateKeyMode } | null = null;
|
|
28
|
-
/*
|
|
29
|
-
* Read through a function, not the variable.
|
|
30
|
-
*
|
|
31
|
-
* Each test opens with `captured = null`, which narrows the binding to `null`
|
|
32
|
-
* for the rest of the block — TypeScript cannot see the assignment inside
|
|
33
|
-
* `fakeCatalogue.ingestPrivate`, so `captured?.keyMode` had type `never` and
|
|
34
|
-
* every assertion below was unchecked. A call boundary resets the narrowing,
|
|
35
|
-
* so the descriptor's fields are type-checked again.
|
|
36
|
-
*/
|
|
37
|
-
const derived = (): { setKey: Buffer; keyMode: PrivateKeyMode } | null => captured;
|
|
38
|
-
const fakeCatalogue = {
|
|
39
|
-
async ingestPrivate(_ctx: MediaCtx, _upload: unknown, cfg: unknown) {
|
|
40
|
-
captured = cfg as { setKey: Buffer; keyMode: PrivateKeyMode };
|
|
41
|
-
return {
|
|
42
|
-
id: 'p1',
|
|
43
|
-
size: 1,
|
|
44
|
-
createdAt: 1,
|
|
45
|
-
processingStatus: 'ready',
|
|
46
|
-
processingProgress: 100,
|
|
47
|
-
hasHls: false,
|
|
48
|
-
};
|
|
49
|
-
},
|
|
50
|
-
} as unknown as FileCatalogue & PrivateFileCatalogue;
|
|
51
|
-
|
|
52
|
-
const app = new Hono();
|
|
53
|
-
mountFileRoutes(app, {
|
|
54
|
-
store: createDiskStore('/tmp/unused-fr'),
|
|
55
|
-
token: streamToken,
|
|
56
|
-
catalogue: fakeCatalogue,
|
|
57
|
-
privateUpload: {
|
|
58
|
-
unlock,
|
|
59
|
-
authorize: async () => ({ userId: 'u1', userSalt: salt }),
|
|
60
|
-
},
|
|
61
|
-
});
|
|
62
|
-
|
|
63
|
-
const put = (headers: Record<string, string>) =>
|
|
64
|
-
app.request('/_files/upload-private', {
|
|
65
|
-
method: 'PUT',
|
|
66
|
-
headers: { 'content-type': 'video/mp4', 'x-filename': 'clip.mp4', ...headers },
|
|
67
|
-
body: 'bytes',
|
|
68
|
-
});
|
|
69
|
-
|
|
70
|
-
test('password header → deniable key derived from the password', async () => {
|
|
71
|
-
captured = null;
|
|
72
|
-
const res = await put({ 'x-private-password': 'pw1' });
|
|
73
|
-
expect(res.status).toBe(200);
|
|
74
|
-
expect(derived()?.keyMode).toBe('deniable');
|
|
75
|
-
const expected = unlock.deriveSetKey({ userId: 'u1', userSalt: salt, password: 'pw1' });
|
|
76
|
-
expect(derived()?.setKey.equals(expected)).toBe(true);
|
|
77
|
-
});
|
|
78
|
-
|
|
79
|
-
test('no password, no token → standard server key', async () => {
|
|
80
|
-
captured = null;
|
|
81
|
-
const res = await put({});
|
|
82
|
-
expect(res.status).toBe(200);
|
|
83
|
-
expect(derived()?.keyMode).toBe('standard');
|
|
84
|
-
const expected = unlock.deriveSetKey({ userId: 'u1', userSalt: salt });
|
|
85
|
-
expect(derived()?.setKey.equals(expected)).toBe(true);
|
|
86
|
-
});
|
|
87
|
-
|
|
88
|
-
test('x-unlock-token → setKey + keyMode recovered from the grant', async () => {
|
|
89
|
-
captured = null;
|
|
90
|
-
const { token } = await unlock.unlock({ userId: 'u1', userSalt: salt, password: 'pw2' });
|
|
91
|
-
const res = await put({ 'x-unlock-token': token });
|
|
92
|
-
expect(res.status).toBe(200);
|
|
93
|
-
expect(derived()?.keyMode).toBe('deniable');
|
|
94
|
-
const expected = unlock.deriveSetKey({ userId: 'u1', userSalt: salt, password: 'pw2' });
|
|
95
|
-
expect(derived()?.setKey.equals(expected)).toBe(true);
|
|
96
|
-
});
|
|
97
|
-
|
|
98
|
-
test('a standard-mode token carries keyMode standard', async () => {
|
|
99
|
-
captured = null;
|
|
100
|
-
const { token } = await unlock.unlock({ userId: 'u1', userSalt: salt });
|
|
101
|
-
await put({ 'x-unlock-token': token });
|
|
102
|
-
expect(derived()?.keyMode).toBe('standard');
|
|
103
|
-
});
|
|
104
|
-
|
|
105
|
-
test('invalid or other-user tokens are rejected (401), password path unaffected', async () => {
|
|
106
|
-
expect((await put({ 'x-unlock-token': 'garbage' })).status).toBe(401);
|
|
107
|
-
const { token: foreign } = await unlock.unlock({ userId: 'u2', userSalt: salt });
|
|
108
|
-
expect((await put({ 'x-unlock-token': foreign })).status).toBe(401);
|
|
109
|
-
// Password path still works after token rejections.
|
|
110
|
-
expect((await put({ 'x-private-password': 'pw1' })).status).toBe(200);
|
|
111
|
-
});
|
|
112
|
-
});
|
|
113
19
|
|
|
114
20
|
describe('fileRoutes — GET list route (filters + sort passthrough)', () => {
|
|
115
21
|
let db: Database;
|
|
116
22
|
let root: string;
|
|
117
|
-
let cat: FileCatalogue
|
|
23
|
+
let cat: FileCatalogue;
|
|
118
24
|
const app = new Hono();
|
|
119
25
|
let clock = 9000;
|
|
120
26
|
|
|
@@ -262,7 +168,7 @@ describe('fileRoutes — single-PUT upload cap (MAX_SINGLE_UPLOAD_BYTES → 413)
|
|
|
262
168
|
processingStatus: 'ready',
|
|
263
169
|
};
|
|
264
170
|
},
|
|
265
|
-
} as unknown as FileCatalogue
|
|
171
|
+
} as unknown as FileCatalogue;
|
|
266
172
|
|
|
267
173
|
const app = new Hono();
|
|
268
174
|
mountFileRoutes(app, {
|