cursedbelt-server 3.0.1 → 4.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/server/auth/jwt.d.ts +1 -1
- package/dist/server/auth/jwt.js +2 -2
- package/dist/server/master-lock/accountsPage.d.ts +39 -0
- package/dist/server/master-lock/accountsPage.js +455 -0
- package/dist/server/master-lock/guard.js +39 -1
- package/dist/server/master-lock/index.d.ts +10 -2
- package/dist/server/master-lock/index.js +10 -2
- package/dist/server/master-lock/lockPage.d.ts +9 -0
- package/dist/server/master-lock/lockPage.js +44 -36
- package/dist/server/media-bun/streaming.d.ts +0 -1
- package/dist/server/media-bun/streaming.js +1 -1
- package/dist/server/storage/binaryStore.d.ts +385 -0
- package/dist/server/storage/binaryStore.js +739 -0
- package/dist/server/storage/binaryStoreFake.d.ts +56 -0
- package/dist/server/storage/binaryStoreFake.js +63 -0
- package/dist/server/storage/files/catalogue.d.ts +2 -2
- package/dist/server/storage/files/catalogue.js +18 -208
- package/dist/server/storage/files/fileRoutes.d.ts +2 -20
- package/dist/server/storage/files/fileRoutes.js +1 -74
- package/dist/server/storage/files/index.d.ts +1 -1
- package/dist/server/storage/files/index.js +1 -1
- package/dist/server/storage/files/migrations.js +9 -6
- package/dist/server/storage/files/stack.d.ts +6 -31
- package/dist/server/storage/files/stack.js +0 -27
- package/dist/server/storage/files/types.d.ts +10 -60
- package/dist/server/storage/files/types.js +9 -0
- package/dist/server/storage/index.d.ts +2 -2
- package/dist/server/storage/index.js +1 -1
- package/dist/server/storage/mountMediaRoutes.d.ts +1 -4
- package/dist/server/storage/types.d.ts +0 -9
- package/dist/server/storage/types.js +2 -9
- package/package.json +14 -2
- package/src/leafSubpathsImportNothing.spec.ts +30 -4
- package/src/server/auth/jwt.ts +2 -2
- package/src/server/master-lock/accountsPage.spec.ts +231 -0
- package/src/server/master-lock/accountsPage.ts +465 -0
- package/src/server/master-lock/guard.ts +44 -1
- package/src/server/master-lock/index.ts +22 -2
- package/src/server/master-lock/lockPage.spec.ts +20 -16
- package/src/server/master-lock/lockPage.ts +45 -36
- package/src/server/media-bun/streaming.ts +0 -6
- package/src/server/storage/binaryStore.spec.ts +908 -0
- package/src/server/storage/binaryStore.ts +1049 -0
- package/src/server/storage/binaryStoreFake.ts +111 -0
- package/src/server/storage/fileUploadClient.integration.spec.ts +2 -21
- package/src/server/storage/files/adapter.photo.display.spec.ts +2 -2
- package/src/server/storage/files/adapter.photo.spec.ts +2 -2
- package/src/server/storage/files/adapter.spec.ts +0 -1
- package/src/server/storage/files/catalogue.media-host.spec.ts +6 -71
- package/src/server/storage/files/catalogue.recover.spec.ts +14 -23
- package/src/server/storage/files/catalogue.ts +19 -240
- package/src/server/storage/files/fileRoutes.replace.spec.ts +2 -2
- package/src/server/storage/files/fileRoutes.spec.ts +3 -97
- package/src/server/storage/files/fileRoutes.ts +2 -96
- package/src/server/storage/files/index.ts +1 -4
- package/src/server/storage/files/migrations.ts +9 -7
- package/src/server/storage/files/stack.spec.ts +11 -111
- package/src/server/storage/files/stack.ts +9 -61
- package/src/server/storage/files/types.ts +20 -68
- package/src/server/storage/index.ts +0 -4
- package/src/server/storage/mountMediaRoutes.ts +1 -7
- package/src/server/storage/types.ts +2 -11
- package/src/shippedFilesAreTracked.spec.ts +3 -2
- package/dist/server/private-media/crypto.d.ts +0 -37
- package/dist/server/private-media/crypto.js +0 -111
- package/dist/server/private-media/index-store.d.ts +0 -31
- package/dist/server/private-media/index-store.js +0 -36
- package/dist/server/private-media/mountPrivateRoutes.d.ts +0 -26
- package/dist/server/private-media/mountPrivateRoutes.js +0 -219
- package/dist/server/private-media/pipeline.d.ts +0 -37
- package/dist/server/private-media/pipeline.js +0 -97
- package/dist/server/private-media/unlock.d.ts +0 -52
- package/dist/server/private-media/unlock.js +0 -55
- package/src/server/private-media/blobRoute.spec.ts +0 -165
- package/src/server/private-media/crypto.spec.ts +0 -81
- package/src/server/private-media/crypto.ts +0 -155
- package/src/server/private-media/gatePolicy.spec.ts +0 -98
- package/src/server/private-media/index-store.ts +0 -83
- package/src/server/private-media/mountPrivateRoutes.ts +0 -265
- package/src/server/private-media/pipeline.spec.ts +0 -111
- package/src/server/private-media/pipeline.ts +0 -170
- package/src/server/private-media/routes.spec.ts +0 -265
- package/src/server/private-media/unlock.ts +0 -112
- package/src/server/storage/files/catalogue.private.spec.ts +0 -106
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A `BinaryStore` that does nothing, so adding a method to the interface stops
|
|
3
|
+
* breaking twenty-two test files in apps nobody touched.
|
|
4
|
+
*
|
|
5
|
+
* ── The measurement (2026-09-08) ────────────────────────────────────────────
|
|
6
|
+
* `BinaryStore.fetchMedia` was added to serve audio off this Mac while
|
|
7
|
+
* Cloudflare's daily Worker quota was spent — a good change, landed on `main`,
|
|
8
|
+
* with apps/music green. It also broke **apps/collections and apps/roms**, four
|
|
9
|
+
* test files each, with `Property 'fetchMedia' is missing in type`. Both apps
|
|
10
|
+
* were left unable to typecheck, and therefore unable to gate or deploy, by a
|
|
11
|
+
* commit neither of them appears in. Nobody noticed for hours: a worker's
|
|
12
|
+
* `verify:scoped` only typechecks the workspaces its own diff touches, which is
|
|
13
|
+
* exactly the property that makes it fast.
|
|
14
|
+
*
|
|
15
|
+
* The owner's rule, in his words: *"a change to roms for example cannot block
|
|
16
|
+
* family tree app from passing its tests."* This is that rule for the one
|
|
17
|
+
* interface twenty-two suites all fake by hand.
|
|
18
|
+
*
|
|
19
|
+
* 🔴 **This is why the fake travels WITH the interface, into this package.** An
|
|
20
|
+
* interface whose change can break two other apps from a third app's commit is a
|
|
21
|
+
* shared interface already; a fake living in a different repo from the interface it
|
|
22
|
+
* fakes cannot do the job it exists for, because the two can be changed
|
|
23
|
+
* independently. Published as `cursedbelt-server/binary-store/testing`, beside
|
|
24
|
+
* `cursedbelt-server/binary-store` — the same split as `./d1` and `./d1/testing`, and
|
|
25
|
+
* for the same reason: a consumer's PRODUCTION graph must not be able to reach a
|
|
26
|
+
* fake.
|
|
27
|
+
*
|
|
28
|
+
* ── 🔴 Why defaults and not optional members ────────────────────────────────
|
|
29
|
+
* The obvious alternative is to mark each new capability `fetchMedia?:` on the
|
|
30
|
+
* interface. That moves the cost to every REAL consumer — each one then has to
|
|
31
|
+
* narrow before calling, for a method the production store always has — and it
|
|
32
|
+
* makes "did we implement this?" unanswerable by the typechecker, which is the
|
|
33
|
+
* one question the interface exists to answer. The fakes are the side that
|
|
34
|
+
* should absorb the change, so the defaults live here.
|
|
35
|
+
*
|
|
36
|
+
* ── What a default may do ───────────────────────────────────────────────────
|
|
37
|
+
* As little as possible, and never something a test could mistake for a result.
|
|
38
|
+
* `get` and `meta` answer "not here"; `fetchMedia` answers 404; the mutators do
|
|
39
|
+
* nothing. A suite that cares about any of them overrides it — spread this first
|
|
40
|
+
* and the file's own members win:
|
|
41
|
+
*
|
|
42
|
+
* const store: BinaryStore = {
|
|
43
|
+
* ...binaryStoreFakeDefaults("collections"),
|
|
44
|
+
* async meta(key) { … what this suite is actually about … },
|
|
45
|
+
* };
|
|
46
|
+
*/
|
|
47
|
+
import type { BinaryStore } from './binaryStore';
|
|
48
|
+
/**
|
|
49
|
+
* Every member of {@link BinaryStore}, inert.
|
|
50
|
+
*
|
|
51
|
+
* @param appId the tenant the fake claims to be. Required, and deliberately so —
|
|
52
|
+
* a blank tenant is the mistake `binaryStoreTenant` exists to catch, and a
|
|
53
|
+
* fake that quietly defaults it would hide a test written against the
|
|
54
|
+
* wrong app.
|
|
55
|
+
*/
|
|
56
|
+
export declare function binaryStoreFakeDefaults(appId: string): BinaryStore;
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Every member of {@link BinaryStore}, inert.
|
|
3
|
+
*
|
|
4
|
+
* @param appId the tenant the fake claims to be. Required, and deliberately so —
|
|
5
|
+
* a blank tenant is the mistake `binaryStoreTenant` exists to catch, and a
|
|
6
|
+
* fake that quietly defaults it would hide a test written against the
|
|
7
|
+
* wrong app.
|
|
8
|
+
*/
|
|
9
|
+
export function binaryStoreFakeDefaults(appId) {
|
|
10
|
+
return {
|
|
11
|
+
appId,
|
|
12
|
+
async put() { },
|
|
13
|
+
async putLarge() { },
|
|
14
|
+
async get() {
|
|
15
|
+
return null;
|
|
16
|
+
},
|
|
17
|
+
async meta() {
|
|
18
|
+
return null;
|
|
19
|
+
},
|
|
20
|
+
async stat() {
|
|
21
|
+
return null;
|
|
22
|
+
},
|
|
23
|
+
async has() {
|
|
24
|
+
return false;
|
|
25
|
+
},
|
|
26
|
+
// 🔴 `"unknown"`, never `"absent"`. A fake that claimed to know a key is missing
|
|
27
|
+
// would let a listing route's "answer only from the memo" branch look exercised
|
|
28
|
+
// when nothing was ever memoized — the one thing a test of that branch is for.
|
|
29
|
+
known() {
|
|
30
|
+
return 'unknown';
|
|
31
|
+
},
|
|
32
|
+
// 🔴 `0`, and never a lie about work done. The real one returns how many
|
|
33
|
+
// negative memos it dropped, and an app's repair route reports that number to
|
|
34
|
+
// the owner — a fake that invented one would make a broken repair read as a
|
|
35
|
+
// working one in the only test that could catch it.
|
|
36
|
+
forgetMisses() {
|
|
37
|
+
return 0;
|
|
38
|
+
},
|
|
39
|
+
cacheStats() {
|
|
40
|
+
return { present: 0, absent: 0 };
|
|
41
|
+
},
|
|
42
|
+
async remove() { },
|
|
43
|
+
async removePrefix() { },
|
|
44
|
+
async mediaUrl(key) {
|
|
45
|
+
// Carries the key, so an assertion on "which object was linked" reads.
|
|
46
|
+
return `https://binary-server.test/media/${appId}/${key}`;
|
|
47
|
+
},
|
|
48
|
+
async mediaPrefixUrl(prefix, master) {
|
|
49
|
+
// Carries BOTH, because the bug this shape exists to prevent is signing the wrong
|
|
50
|
+
// one of them — a URL pointing at the master but scoped to the wrong prefix 403s
|
|
51
|
+
// every segment while the master itself loads fine.
|
|
52
|
+
return `https://binary-server.test/media/${appId}/${master}?p=${encodeURIComponent(prefix)}`;
|
|
53
|
+
},
|
|
54
|
+
async fetchMedia() {
|
|
55
|
+
// 🔴 404, not an empty 200. A fake that answers "here are zero bytes" makes a
|
|
56
|
+
// streaming route look like it worked on an object that does not exist.
|
|
57
|
+
return new Response(null, { status: 404 });
|
|
58
|
+
},
|
|
59
|
+
async signToken(_claims, _ttlSeconds) {
|
|
60
|
+
return 'fake-token';
|
|
61
|
+
},
|
|
62
|
+
};
|
|
63
|
+
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { Database } from 'bun:sqlite';
|
|
2
2
|
import { type MediaCtx, type MediaStore } from '../types';
|
|
3
3
|
import { type ProcessOptions, type ProcessResult } from '../videoProcessor';
|
|
4
|
-
import { type FileCatalogue
|
|
4
|
+
import { type FileCatalogue } from './types';
|
|
5
5
|
/** Swappable video ingest→derivatives implementation. The default runs ffmpeg
|
|
6
6
|
* locally ({@link processVideo}); a remote backend (binary-server) delegates
|
|
7
7
|
* the heavy work to the storage host instead — see binaryServerProcessing. */
|
|
@@ -69,4 +69,4 @@ export type SqliteFileCatalogueOptions = {
|
|
|
69
69
|
* with ffmpeg fallback). Inject a remote one when the byte store is remote. */
|
|
70
70
|
imageDerivatives?: ImageDerivatives;
|
|
71
71
|
};
|
|
72
|
-
export declare function createSqliteFileCatalogue(opts: SqliteFileCatalogueOptions): FileCatalogue
|
|
72
|
+
export declare function createSqliteFileCatalogue(opts: SqliteFileCatalogueOptions): FileCatalogue;
|
|
@@ -1,34 +1,11 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { readdir, stat as fsStat } from 'node:fs/promises';
|
|
2
2
|
import path from 'node:path';
|
|
3
3
|
import { renderImage } from '../../media-bun/image';
|
|
4
|
-
import { sealMeta } from '../../private-media/crypto';
|
|
5
|
-
import { processPrivateBlob, processPrivateVideo } from '../../private-media/pipeline';
|
|
6
4
|
import { ffmpegImageThumbnail, isBrowserRenderableImage } from '../imageThumbnail';
|
|
7
|
-
import { fsDisplayKey, fsHlsPrefix, fsMp4Key, fsPosterKey, fsSourceKey, fsSpriteKey, fsVttKey,
|
|
5
|
+
import { fsDisplayKey, fsHlsPrefix, fsMp4Key, fsPosterKey, fsSourceKey, fsSpriteKey, fsVttKey, } from '../types';
|
|
8
6
|
import { processVideo } from '../videoProcessor';
|
|
9
7
|
import { mimeByExt } from './mimeByExt';
|
|
10
8
|
import { classifyFileType, } from './types';
|
|
11
|
-
// Stream an upload body to a local plaintext file (staged for the private pipeline; wiped after).
|
|
12
|
-
// Bun.write(dest, new Response(stream)) hangs in Bun 1.3.x — pump via a FileSink instead.
|
|
13
|
-
async function streamToFile(body, dest) {
|
|
14
|
-
await mkdir(path.dirname(dest), { recursive: true });
|
|
15
|
-
const sink = Bun.file(dest).writer();
|
|
16
|
-
const reader = body.getReader();
|
|
17
|
-
let size = 0;
|
|
18
|
-
try {
|
|
19
|
-
while (true) {
|
|
20
|
-
const { done, value } = await reader.read();
|
|
21
|
-
if (done)
|
|
22
|
-
break;
|
|
23
|
-
size += value.byteLength;
|
|
24
|
-
sink.write(value);
|
|
25
|
-
}
|
|
26
|
-
}
|
|
27
|
-
finally {
|
|
28
|
-
await sink.end();
|
|
29
|
-
}
|
|
30
|
-
return size;
|
|
31
|
-
}
|
|
32
9
|
// The general primary binary for a NON-video file (images/docs/…) lives here; video files store a
|
|
33
10
|
// raw source (fsSourceKey) + derived mp4/HLS/poster/sprite via the shared video keyspace.
|
|
34
11
|
export const fileBinaryKey = (owner, id) => `fs/${owner}/${id}`;
|
|
@@ -120,9 +97,9 @@ export function createSqliteFileCatalogue(opts) {
|
|
|
120
97
|
const rand31 = () => Math.floor(Math.random() * RANDOM_MOD);
|
|
121
98
|
const randKey = opts.randKey ?? rand31;
|
|
122
99
|
const randomSeed = opts.randomSeed ?? rand31;
|
|
123
|
-
const insertFile = db.query(`INSERT INTO files (id, owner, name, mime, size, file_type,
|
|
124
|
-
VALUES ($id, $owner, $name, $mime, $size, $type,
|
|
125
|
-
const insertTag = db.query(`INSERT OR IGNORE INTO file_tags (file_id, tag, owner
|
|
100
|
+
const insertFile = db.query(`INSERT INTO files (id, owner, name, mime, size, file_type, processing_status, processing_progress, storage_backend, rand_key, created_at, updated_at)
|
|
101
|
+
VALUES ($id, $owner, $name, $mime, $size, $type, $status, $progress, 'disk', $rand, $now, $now)`);
|
|
102
|
+
const insertTag = db.query(`INSERT OR IGNORE INTO file_tags (file_id, tag, owner) VALUES ($file, $tag, $owner)`);
|
|
126
103
|
const deleteTags = db.query(`DELETE FROM file_tags WHERE file_id = $file`);
|
|
127
104
|
const getRow = db.query(`SELECT * FROM files WHERE id = $id AND owner = $owner`);
|
|
128
105
|
const deleteFile = db.query(`DELETE FROM files WHERE id = $id AND owner = $owner`);
|
|
@@ -140,20 +117,7 @@ export function createSqliteFileCatalogue(opts) {
|
|
|
140
117
|
const replaceMeta = db.query(`UPDATE files SET name = $name, mime = $mime, size = $size, file_type = $type,
|
|
141
118
|
processing_status = $status, processing_progress = $progress,
|
|
142
119
|
has_hls = 0, has_thumbnail = 0, has_sprites = 0, duration = NULL, updated_at = $now
|
|
143
|
-
WHERE id = $id AND owner = $owner
|
|
144
|
-
// Private (encrypted) rows: name/tags are sealed in enc_name (sealedMeta); the wrapped AES-128
|
|
145
|
-
// content key is in enc_dek; enc_key_type records the wrap mode. is_private = 1 keeps them out of
|
|
146
|
-
// the public feed. These rows are surfaced to `mountPrivateRoutes` via `asPrivateIndex()`.
|
|
147
|
-
const insertPrivate = db.query(`INSERT INTO files (id, owner, name, mime, size, file_type, is_private, has_hls, duration,
|
|
148
|
-
processing_status, processing_progress, storage_backend, enc_name, enc_dek, enc_key_type, rand_key, created_at, updated_at)
|
|
149
|
-
VALUES ($id, $owner, '', '', $size, $type, 1, 0, NULL, $status, $progress, 'disk', $sealed, $wrapped, $mode, $rand, $now, $now)`);
|
|
150
|
-
const finishPrivate = db.query(`UPDATE files SET processing_status = $status, processing_progress = $progress, has_hls = $hls,
|
|
151
|
-
duration = $duration, size = $size, enc_name = $sealed, enc_dek = $wrapped, updated_at = $now WHERE id = $id`);
|
|
152
|
-
const getPrivateRow = db.query(`SELECT * FROM files WHERE id = $id AND owner = $owner AND is_private = 1`);
|
|
153
|
-
const listPrivateFullRows = db.query(`SELECT * FROM files WHERE owner = $owner AND is_private = 1 ORDER BY created_at DESC, id DESC`);
|
|
154
|
-
const updatePrivateRow = db.query(`UPDATE files SET size = $size, processing_status = $status, enc_name = $sealed, enc_dek = $wrapped,
|
|
155
|
-
enc_key_type = $mode, updated_at = $now WHERE id = $id AND owner = $owner AND is_private = 1`);
|
|
156
|
-
const deletePrivateRow = db.query(`DELETE FROM files WHERE id = $id AND owner = $owner AND is_private = 1`);
|
|
120
|
+
WHERE id = $id AND owner = $owner`);
|
|
157
121
|
// ── Collection prepared statements ───────────────────────────────────────────────
|
|
158
122
|
const insertCollection = db.query(`INSERT INTO file_collections (id, owner, name, kind, visibility, created_at, updated_at)
|
|
159
123
|
VALUES ($id, $owner, $name, $kind, $visibility, $now, $now)
|
|
@@ -180,16 +144,6 @@ export function createSqliteFileCatalogue(opts) {
|
|
|
180
144
|
SELECT f.id, f.owner, $now, 1 FROM files f WHERE f.id = $id AND f.owner = $owner
|
|
181
145
|
ON CONFLICT(file_id) DO UPDATE SET viewed_at = excluded.viewed_at, view_count = view_count + 1`);
|
|
182
146
|
const deleteFileViews = db.query(`DELETE FROM file_views WHERE file_id = $file`);
|
|
183
|
-
const rowToPrivateItem = (r, owner) => ({
|
|
184
|
-
id: r.id,
|
|
185
|
-
owner,
|
|
186
|
-
sizeBytes: r.size,
|
|
187
|
-
createdAt: r.created_at,
|
|
188
|
-
status: r.processing_status,
|
|
189
|
-
keyMode: (r.enc_key_type || 'deniable'),
|
|
190
|
-
wrappedContentKey: r.enc_dek,
|
|
191
|
-
sealedMeta: r.enc_name,
|
|
192
|
-
});
|
|
193
147
|
const tagsFor = (ids) => {
|
|
194
148
|
const map = new Map();
|
|
195
149
|
if (ids.length === 0)
|
|
@@ -235,7 +189,6 @@ export function createSqliteFileCatalogue(opts) {
|
|
|
235
189
|
size: row.size,
|
|
236
190
|
fileType: row.file_type,
|
|
237
191
|
tags,
|
|
238
|
-
isPrivate: row.is_private === 1,
|
|
239
192
|
hasThumbnail: row.has_thumbnail === 1,
|
|
240
193
|
hasHls: row.has_hls === 1,
|
|
241
194
|
hasSprites: row.has_sprites === 1,
|
|
@@ -416,8 +369,7 @@ export function createSqliteFileCatalogue(opts) {
|
|
|
416
369
|
return rows[0];
|
|
417
370
|
};
|
|
418
371
|
const statsByType = db.query(`SELECT file_type, COUNT(*) AS n, SUM(size) AS bytes FROM files
|
|
419
|
-
WHERE owner = $owner
|
|
420
|
-
const statsPrivate = db.query(`SELECT COUNT(*) AS n, SUM(size) AS bytes FROM files WHERE owner = $owner AND is_private = 1`);
|
|
372
|
+
WHERE owner = $owner GROUP BY file_type`);
|
|
421
373
|
const statsStatus = db.query(`SELECT processing_status, COUNT(*) AS n FROM files
|
|
422
374
|
WHERE owner = $owner GROUP BY processing_status`);
|
|
423
375
|
return {
|
|
@@ -431,7 +383,6 @@ export function createSqliteFileCatalogue(opts) {
|
|
|
431
383
|
files += row.n;
|
|
432
384
|
bytes += row.bytes ?? 0;
|
|
433
385
|
}
|
|
434
|
-
const priv = statsPrivate.get({ $owner: owner });
|
|
435
386
|
let processing = 0;
|
|
436
387
|
let failed = 0;
|
|
437
388
|
for (const row of statsStatus.all({ $owner: owner })) {
|
|
@@ -444,8 +395,6 @@ export function createSqliteFileCatalogue(opts) {
|
|
|
444
395
|
files,
|
|
445
396
|
bytes,
|
|
446
397
|
byType,
|
|
447
|
-
privateItems: priv?.n ?? 0,
|
|
448
|
-
privateBytes: priv?.bytes ?? 0,
|
|
449
398
|
processing,
|
|
450
399
|
failed,
|
|
451
400
|
};
|
|
@@ -456,7 +405,6 @@ export function createSqliteFileCatalogue(opts) {
|
|
|
456
405
|
const params = { $owner: owner, $limit: limit + 1 };
|
|
457
406
|
const where = [
|
|
458
407
|
'f.owner = $owner',
|
|
459
|
-
'f.is_private = 0',
|
|
460
408
|
options.trashedOnly ? 'f.trashed_at IS NOT NULL' : 'f.trashed_at IS NULL',
|
|
461
409
|
];
|
|
462
410
|
let from = 'files f';
|
|
@@ -547,7 +495,7 @@ export function createSqliteFileCatalogue(opts) {
|
|
|
547
495
|
},
|
|
548
496
|
async tagFacets(ctx) {
|
|
549
497
|
const rows = db
|
|
550
|
-
.query(`SELECT tag, COUNT(*) as count FROM file_tags WHERE owner = $owner
|
|
498
|
+
.query(`SELECT tag, COUNT(*) as count FROM file_tags WHERE owner = $owner
|
|
551
499
|
GROUP BY tag ORDER BY count DESC, tag ASC`)
|
|
552
500
|
.all({ $owner: ownerOf(ctx) });
|
|
553
501
|
return rows;
|
|
@@ -591,9 +539,6 @@ export function createSqliteFileCatalogue(opts) {
|
|
|
591
539
|
const row = getRow.get({ $id: id, $owner: owner });
|
|
592
540
|
if (!row)
|
|
593
541
|
return null;
|
|
594
|
-
if (row.is_private === 1) {
|
|
595
|
-
throw new Error('replaceContent: private files cannot be replaced in place');
|
|
596
|
-
}
|
|
597
542
|
const isVideo = upload.mime.startsWith('video/');
|
|
598
543
|
// Store the new bytes FIRST — a failed write leaves the old row + bytes fully intact.
|
|
599
544
|
const key = isVideo ? fsSourceKey(owner, id) : fileBinaryKey(owner, id);
|
|
@@ -728,24 +673,21 @@ export function createSqliteFileCatalogue(opts) {
|
|
|
728
673
|
},
|
|
729
674
|
recoverProcessing() {
|
|
730
675
|
// A restart mid-transcode leaves a row stuck in `processing`/`pending` — the in-process
|
|
731
|
-
// job is gone.
|
|
732
|
-
//
|
|
733
|
-
//
|
|
734
|
-
// insert and never appear here. (Ported from ca's recoverOnBoot; our jobs are in-process,
|
|
735
|
-
// so recovery is app-invoked at startup rather than a persistent queue.)
|
|
676
|
+
// job is gone. The source still sits at fsSourceKey, so re-kick the pipeline. Non-video rows
|
|
677
|
+
// are `ready` at insert and never appear here. (Ported from ca's recoverOnBoot; our jobs are
|
|
678
|
+
// in-process, so recovery is app-invoked at startup rather than a persistent queue.)
|
|
736
679
|
// Re-kicks are CAPPED at maxProcessingAttempts total kicks: if the transcode itself is what
|
|
737
680
|
// keeps killing the process (2026-07-04 prod outage: ffmpeg OOM on a 1GB host → systemd
|
|
738
681
|
// restart → unconditional re-kick → OOM again, forever), the row must land on `failed`
|
|
739
682
|
// instead of holding the whole service down.
|
|
740
683
|
const stuck = db
|
|
741
|
-
.query(`SELECT id, owner,
|
|
742
|
-
WHERE
|
|
743
|
-
AND processing_status IN ('pending', 'processing')`)
|
|
684
|
+
.query(`SELECT id, owner, processing_attempts FROM files
|
|
685
|
+
WHERE file_type = 'video' AND processing_status IN ('pending', 'processing')`)
|
|
744
686
|
.all();
|
|
745
687
|
let requeued = 0;
|
|
746
688
|
let failed = 0;
|
|
747
689
|
for (const r of stuck) {
|
|
748
|
-
if (r.
|
|
690
|
+
if (r.processing_attempts >= maxProcessingAttempts) {
|
|
749
691
|
setProcessing.run({ $id: r.id, $status: 'failed', $progress: 0, $now: now() });
|
|
750
692
|
failed++;
|
|
751
693
|
}
|
|
@@ -756,14 +698,14 @@ export function createSqliteFileCatalogue(opts) {
|
|
|
756
698
|
}
|
|
757
699
|
return { requeued, failed };
|
|
758
700
|
},
|
|
759
|
-
// Backfill grid thumbnails for
|
|
701
|
+
// Backfill grid thumbnails for video rows that finished (ready/failed) without a poster —
|
|
760
702
|
// e.g. rows that landed while transcodeVideos was off entirely, or that failed a full transcode
|
|
761
703
|
// before the poster step. Re-kicks the (now poster-capable) pipeline; the attempts counter is
|
|
762
704
|
// reset first so the maxProcessingAttempts cap doesn't pre-empt a fresh, cheaper poster-only run.
|
|
763
705
|
backfillVideoPosters() {
|
|
764
706
|
const missing = db
|
|
765
707
|
.query(`SELECT id, owner FROM files
|
|
766
|
-
WHERE file_type = 'video' AND
|
|
708
|
+
WHERE file_type = 'video' AND has_thumbnail = 0
|
|
767
709
|
AND processing_status IN ('ready', 'failed')`)
|
|
768
710
|
.all();
|
|
769
711
|
// Serialize: on the memory-constrained hosts this exists for, running every poster extract
|
|
@@ -775,7 +717,7 @@ export function createSqliteFileCatalogue(opts) {
|
|
|
775
717
|
}, Promise.resolve());
|
|
776
718
|
return { requeued: missing.length };
|
|
777
719
|
},
|
|
778
|
-
// Backfill grid thumbnails for
|
|
720
|
+
// Backfill grid thumbnails for image rows with no poster — e.g. HEIC/HEVC iPhone photos
|
|
779
721
|
// whose sharp thumbnail silently failed before the ffmpeg fallback existed. Re-kicks the (now
|
|
780
722
|
// HEIC-capable) image pipeline, serialized so at most one ffmpeg decode runs at a time.
|
|
781
723
|
backfillImageThumbnails() {
|
|
@@ -785,7 +727,7 @@ export function createSqliteFileCatalogue(opts) {
|
|
|
785
727
|
// present, so a warm boot re-runs nothing.
|
|
786
728
|
const candidates = db
|
|
787
729
|
.query(`SELECT id, owner, mime, has_thumbnail FROM files
|
|
788
|
-
WHERE file_type = 'image'
|
|
730
|
+
WHERE file_type = 'image'
|
|
789
731
|
AND (has_thumbnail = 0 OR mime IN ('image/heic', 'image/heif'))`)
|
|
790
732
|
.all();
|
|
791
733
|
void candidates.reduce(async (prev, r) => {
|
|
@@ -897,137 +839,5 @@ export function createSqliteFileCatalogue(opts) {
|
|
|
897
839
|
// Upsert-gated to the caller's own files; unknown/other-owner ids record nothing.
|
|
898
840
|
recordViewStmt.run({ $id: fileId, $owner: ownerOf(ctx), $now: now() });
|
|
899
841
|
},
|
|
900
|
-
async ingestPrivate(ctx, upload, cfg) {
|
|
901
|
-
const owner = ownerOf(ctx);
|
|
902
|
-
const id = crypto.randomUUID();
|
|
903
|
-
const ts = now();
|
|
904
|
-
// Seal the descriptor NOW — sealing is independent of the transcode (only the wrapped content
|
|
905
|
-
// KEY needs the pipeline). Storing enc_name at insert means a try-a-password reveal matches the
|
|
906
|
-
// item immediately, before its HLS finishes encrypting (no seal race).
|
|
907
|
-
const descriptor = {
|
|
908
|
-
name: upload.name,
|
|
909
|
-
mime: upload.mime,
|
|
910
|
-
tags: upload.tags ?? [],
|
|
911
|
-
};
|
|
912
|
-
const sealed = sealMeta(cfg.setKey, descriptor);
|
|
913
|
-
const isVideo = upload.mime.startsWith('video/');
|
|
914
|
-
insertPrivate.run({
|
|
915
|
-
$id: id,
|
|
916
|
-
$owner: owner,
|
|
917
|
-
$size: 0,
|
|
918
|
-
$type: classifyFileType(upload.mime),
|
|
919
|
-
$mode: cfg.keyMode,
|
|
920
|
-
$status: 'processing',
|
|
921
|
-
$progress: 0,
|
|
922
|
-
$sealed: sealed,
|
|
923
|
-
$wrapped: '',
|
|
924
|
-
$rand: randKey(),
|
|
925
|
-
$now: ts,
|
|
926
|
-
});
|
|
927
|
-
// Stage plaintext to a scratch file, then encrypt: video → AES-128 HLS ladder; anything
|
|
928
|
-
// else (photo/doc/audio) → one AES-128-CTR blob at privateSourceKey. Plaintext is never
|
|
929
|
-
// persisted either way. We MUST await: the wrapped key is produced by the pipeline and
|
|
930
|
-
// there is nothing to serve without it.
|
|
931
|
-
const srcPath = path.join(workDir, 'ingest', `${owner}-${id}.src`);
|
|
932
|
-
let size = 0;
|
|
933
|
-
try {
|
|
934
|
-
size = await streamToFile(upload.body, srcPath);
|
|
935
|
-
const res = isVideo
|
|
936
|
-
? await processPrivateVideo({
|
|
937
|
-
store,
|
|
938
|
-
workDir,
|
|
939
|
-
owner,
|
|
940
|
-
id,
|
|
941
|
-
sourcePath: srcPath,
|
|
942
|
-
setKey: cfg.setKey,
|
|
943
|
-
ladderHeights: opts.ladderHeights,
|
|
944
|
-
onProgress: (status, pct) => setProcessing.run({ $id: id, $status: status, $progress: pct, $now: now() }),
|
|
945
|
-
})
|
|
946
|
-
: {
|
|
947
|
-
...(await processPrivateBlob({
|
|
948
|
-
store,
|
|
949
|
-
owner,
|
|
950
|
-
id,
|
|
951
|
-
sourcePath: srcPath,
|
|
952
|
-
setKey: cfg.setKey,
|
|
953
|
-
})),
|
|
954
|
-
durationSeconds: 0,
|
|
955
|
-
hasHls: false,
|
|
956
|
-
};
|
|
957
|
-
finishPrivate.run({
|
|
958
|
-
$id: id,
|
|
959
|
-
$status: res.status,
|
|
960
|
-
$progress: res.status === 'ready' ? 100 : 0,
|
|
961
|
-
$hls: res.hasHls ? 1 : 0,
|
|
962
|
-
$duration: res.durationSeconds || null,
|
|
963
|
-
$size: size,
|
|
964
|
-
$sealed: sealed,
|
|
965
|
-
$wrapped: res.wrappedContentKey ?? '',
|
|
966
|
-
$now: now(),
|
|
967
|
-
});
|
|
968
|
-
}
|
|
969
|
-
finally {
|
|
970
|
-
await rm(srcPath, { force: true }).catch(() => { });
|
|
971
|
-
}
|
|
972
|
-
const row = getRow.get({ $id: id, $owner: owner });
|
|
973
|
-
return {
|
|
974
|
-
id,
|
|
975
|
-
size,
|
|
976
|
-
createdAt: ts,
|
|
977
|
-
processingStatus: row.processing_status,
|
|
978
|
-
processingProgress: row.processing_progress,
|
|
979
|
-
hasHls: row.has_hls === 1,
|
|
980
|
-
};
|
|
981
|
-
},
|
|
982
|
-
asPrivateIndex() {
|
|
983
|
-
// A PrivateIndex view over the catalogue's private rows — the seam mountPrivateRoutes reads.
|
|
984
|
-
// `owner` here is the private owner axis the routes pass (a userId), matching how ingestPrivate
|
|
985
|
-
// stored the row (both go through the files.owner column).
|
|
986
|
-
return {
|
|
987
|
-
async add(item) {
|
|
988
|
-
insertPrivate.run({
|
|
989
|
-
$id: item.id,
|
|
990
|
-
$owner: item.owner,
|
|
991
|
-
$size: item.sizeBytes,
|
|
992
|
-
// External index writers predate blob items and only ever added videos.
|
|
993
|
-
$type: 'video',
|
|
994
|
-
$mode: item.keyMode,
|
|
995
|
-
$status: item.status,
|
|
996
|
-
$progress: item.status === 'ready' ? 100 : 0,
|
|
997
|
-
$sealed: item.sealedMeta,
|
|
998
|
-
$wrapped: item.wrappedContentKey,
|
|
999
|
-
$rand: randKey(),
|
|
1000
|
-
$now: now(),
|
|
1001
|
-
});
|
|
1002
|
-
},
|
|
1003
|
-
async get(owner, id) {
|
|
1004
|
-
const row = getPrivateRow.get({ $id: id, $owner: owner });
|
|
1005
|
-
return row ? rowToPrivateItem(row, owner) : null;
|
|
1006
|
-
},
|
|
1007
|
-
async list(owner) {
|
|
1008
|
-
return listPrivateFullRows.all({ $owner: owner }).map((r) => rowToPrivateItem(r, owner));
|
|
1009
|
-
},
|
|
1010
|
-
async update(owner, id, patch) {
|
|
1011
|
-
const cur = getPrivateRow.get({ $id: id, $owner: owner });
|
|
1012
|
-
if (!cur)
|
|
1013
|
-
return;
|
|
1014
|
-
const merged = { ...rowToPrivateItem(cur, owner), ...patch };
|
|
1015
|
-
updatePrivateRow.run({
|
|
1016
|
-
$id: id,
|
|
1017
|
-
$owner: owner,
|
|
1018
|
-
$size: merged.sizeBytes,
|
|
1019
|
-
$status: merged.status,
|
|
1020
|
-
$sealed: merged.sealedMeta,
|
|
1021
|
-
$wrapped: merged.wrappedContentKey,
|
|
1022
|
-
$mode: merged.keyMode,
|
|
1023
|
-
$now: now(),
|
|
1024
|
-
});
|
|
1025
|
-
},
|
|
1026
|
-
async remove(owner, id) {
|
|
1027
|
-
await store.removePrefix(privateFsKey(owner, id));
|
|
1028
|
-
deletePrivateRow.run({ $id: id, $owner: owner });
|
|
1029
|
-
},
|
|
1030
|
-
};
|
|
1031
|
-
},
|
|
1032
842
|
};
|
|
1033
843
|
}
|
|
@@ -1,9 +1,8 @@
|
|
|
1
1
|
import type { Context, Hono } from 'hono';
|
|
2
|
-
import type { UnlockPasswordPolicy, UnlockService } from '../../private-media/unlock';
|
|
3
2
|
import type { StreamTokenService } from '../streamToken';
|
|
4
3
|
import type { MediaCtx, MediaStore } from '../types';
|
|
5
4
|
import type { ChunkedUploadService } from './chunkedUpload';
|
|
6
|
-
import { type FileCatalogue
|
|
5
|
+
import { type FileCatalogue } from './types';
|
|
7
6
|
/** The 413 message a single-PUT over the cap returns — the client treats it as a hard error. */
|
|
8
7
|
export declare const SINGLE_UPLOAD_TOO_LARGE = "file too large for a single request \u2014 use chunked upload";
|
|
9
8
|
export interface FileUrls {
|
|
@@ -26,24 +25,7 @@ export type MountFileRoutesOptions = {
|
|
|
26
25
|
/** Session auth for the upload PUTs (a fetch CAN send the Bearer). Return a ctx or null. */
|
|
27
26
|
authorizeUpload?: (c: Context) => Promise<MediaCtx | null> | MediaCtx | null;
|
|
28
27
|
/** The catalogue uploads land in. Required for the upload routes to be mounted. */
|
|
29
|
-
catalogue?: FileCatalogue
|
|
30
|
-
/** Wire private uploads: derive a per-user set key from the request (password header ⇒ deniable,
|
|
31
|
-
* absent ⇒ standard server key). Omit ⇒ the private-upload route is disabled. */
|
|
32
|
-
privateUpload?: {
|
|
33
|
-
unlock: UnlockService;
|
|
34
|
-
authorize: (c: Context) => Promise<{
|
|
35
|
-
userId: string;
|
|
36
|
-
userSalt: string;
|
|
37
|
-
} | null>;
|
|
38
|
-
/** Header carrying the plaintext password for a deniable set (default 'x-private-password'). */
|
|
39
|
-
passwordHeader?: string;
|
|
40
|
-
/** Header carrying an UNLOCK TOKEN (an already-unlocked session) — takes precedence over the
|
|
41
|
-
* password header; setKey + keyMode come from the verified grant (default 'x-unlock-token'). */
|
|
42
|
-
tokenHeader?: string;
|
|
43
|
-
/** What a supplied password means (default derive — deniable set). `gate` verifies it
|
|
44
|
-
* against the app's accounts and wraps under the recoverable server key instead. */
|
|
45
|
-
passwordPolicy?: UnlockPasswordPolicy;
|
|
46
|
-
};
|
|
28
|
+
catalogue?: FileCatalogue;
|
|
47
29
|
/** Wire large-file chunked uploads (direct-to-backend). Omit ⇒ init/complete routes are absent
|
|
48
30
|
* and clients use the streamed `PUT /upload`. Uses the same `authorizeUpload` guard. */
|
|
49
31
|
chunked?: ChunkedUploadService;
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { MAX_SINGLE_UPLOAD_BYTES } from 'cursedbelt-core/media/upload-limits';
|
|
2
2
|
import { serveRange } from '../streamRange';
|
|
3
3
|
import { fileBinaryKey } from './catalogue';
|
|
4
|
-
import { FILE_SORTS
|
|
4
|
+
import { FILE_SORTS } from './types';
|
|
5
5
|
// ── File binary + upload routes ─────────────────────────────────────────────────
|
|
6
6
|
// The FileCatalogue's byte edge for GENERAL files (images/docs/audio/any) — the piece
|
|
7
7
|
// mountMediaRoutes (video-only: mp4/hls/poster/sprite) doesn't cover. A file's primary binary
|
|
@@ -144,79 +144,6 @@ export function mountFileRoutes(app, opts) {
|
|
|
144
144
|
}
|
|
145
145
|
});
|
|
146
146
|
}
|
|
147
|
-
// PUT a PRIVATE upload → catalogue.ingestPrivate under a set key derived from the request:
|
|
148
|
-
// either an UNLOCK TOKEN (an already-unlocked session — same mechanism the content-key route
|
|
149
|
-
// uses) or a password header (deniable) / neither (standard server key).
|
|
150
|
-
const pv = opts.privateUpload;
|
|
151
|
-
if (pv && catalogue) {
|
|
152
|
-
const passwordHeader = pv.passwordHeader ?? 'x-private-password';
|
|
153
|
-
const tokenHeader = pv.tokenHeader ?? 'x-unlock-token';
|
|
154
|
-
app.put(`${base}/upload-private`, async (c) => {
|
|
155
|
-
const who = await pv.authorize(c);
|
|
156
|
-
if (!who)
|
|
157
|
-
return c.json({ error: 'unauthorized' }, 401);
|
|
158
|
-
const raw = c.req.raw.body;
|
|
159
|
-
if (!raw)
|
|
160
|
-
return c.json({ error: 'empty body' }, 400);
|
|
161
|
-
const body = guardSingleUpload(c, raw);
|
|
162
|
-
if (body instanceof Response)
|
|
163
|
-
return body;
|
|
164
|
-
let setKey;
|
|
165
|
-
let keyMode;
|
|
166
|
-
const unlockToken = c.req.header(tokenHeader) || undefined;
|
|
167
|
-
if (unlockToken) {
|
|
168
|
-
// Token path: recover the sealed set key + mode from the grant. The token must belong
|
|
169
|
-
// to the session caller.
|
|
170
|
-
const grant = await pv.unlock.verify(unlockToken);
|
|
171
|
-
if (!grant || grant.userId !== who.userId) {
|
|
172
|
-
return c.json({ error: 'invalid unlock token' }, 401);
|
|
173
|
-
}
|
|
174
|
-
setKey = grant.setKey;
|
|
175
|
-
keyMode = grant.mode;
|
|
176
|
-
}
|
|
177
|
-
else {
|
|
178
|
-
// Password path: derive fresh from the header (deniable) or the server key. Under the
|
|
179
|
-
// gate policy the password is an access check only — verify, then wrap standard.
|
|
180
|
-
const password = c.req.header(passwordHeader) || undefined;
|
|
181
|
-
const policy = pv.passwordPolicy ?? { mode: 'derive' };
|
|
182
|
-
let derivePassword = password;
|
|
183
|
-
if (policy.mode === 'gate' && password) {
|
|
184
|
-
if (!(await policy.verifyPassword(who, password))) {
|
|
185
|
-
return c.json({ error: 'invalid password' }, 401);
|
|
186
|
-
}
|
|
187
|
-
derivePassword = undefined;
|
|
188
|
-
}
|
|
189
|
-
setKey = pv.unlock.deriveSetKey({
|
|
190
|
-
userId: who.userId,
|
|
191
|
-
userSalt: who.userSalt,
|
|
192
|
-
password: derivePassword,
|
|
193
|
-
});
|
|
194
|
-
keyMode = derivePassword ? 'deniable' : 'standard';
|
|
195
|
-
}
|
|
196
|
-
// The private owner axis is the userId (deniability is personal), matching mountPrivateRoutes.
|
|
197
|
-
const ctx = { userId: who.userId, tenantId: who.userId };
|
|
198
|
-
const tags = c.req.header('x-tags');
|
|
199
|
-
try {
|
|
200
|
-
const item = await catalogue.ingestPrivate(ctx, {
|
|
201
|
-
name: c.req.header('x-filename') ?? 'upload',
|
|
202
|
-
mime: c.req.header('content-type') ?? 'application/octet-stream',
|
|
203
|
-
body,
|
|
204
|
-
tags: tags
|
|
205
|
-
? tags
|
|
206
|
-
.split(',')
|
|
207
|
-
.map((t) => t.trim())
|
|
208
|
-
.filter(Boolean)
|
|
209
|
-
: undefined,
|
|
210
|
-
}, { setKey, keyMode });
|
|
211
|
-
return c.json({ item });
|
|
212
|
-
}
|
|
213
|
-
catch (e) {
|
|
214
|
-
if (isUploadTooLarge(e))
|
|
215
|
-
return c.json({ error: SINGLE_UPLOAD_TOO_LARGE }, 413);
|
|
216
|
-
throw e;
|
|
217
|
-
}
|
|
218
|
-
});
|
|
219
|
-
}
|
|
220
147
|
// Chunked upload (large files, direct-to-backend). init → the client PUTs each chunk straight to
|
|
221
148
|
// the backend (URLs the app minted) → complete registers the row over the assembled bytes. The
|
|
222
149
|
// bytes never traverse this origin.
|
|
@@ -6,4 +6,4 @@ export { createFileUrls, type FileUrls, type MountFileRoutesOptions, mountFileRo
|
|
|
6
6
|
export { FILE_CATALOGUE_MIGRATIONS } from './migrations';
|
|
7
7
|
export { createFilesStack, type FilesStack, type FilesStackMountAuth, type FilesStackOptions, } from './stack';
|
|
8
8
|
export { createLedgeredDeletes, createStorageOpsLedger, createStorageReconciler, type StorageAction, type StorageOpRow, type StorageOpsLedger, type StorageReconciler, } from './storageOps';
|
|
9
|
-
export { type CollectionKind, type CollectionVisibility, classifyFileType, FILE_SORTS, type FileCatalogue, type FileCollection, type FilePage, type FileRecord, type FileSort, type FileToPlayable, type FileType, type ListFilesOptions, type
|
|
9
|
+
export { type CollectionKind, type CollectionVisibility, classifyFileType, FILE_SORTS, type FileCatalogue, type FileCollection, type FilePage, type FileRecord, type FileSort, type FileToPlayable, type FileType, type ListFilesOptions, type TagFacet, } from './types';
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// cursedbelt-server/storage FileCatalogue — a SQLite-backed tagged flat file system over the
|
|
2
|
-
// MediaStore + media pipeline
|
|
2
|
+
// MediaStore + media pipeline. One source of truth for a user's binaries; the
|
|
3
3
|
// video gallery is a video-filtered view over it.
|
|
4
4
|
export { createCatalogueMediaSource, createCataloguePhotoSource, createCatalogueVideoSource, createFileToPlayable, } from './adapter';
|
|
5
5
|
export { catalogueUploadKey, createSqliteFileCatalogue, fileBinaryKey, } from './catalogue';
|