cursedbelt-server 1.0.0

package/src/server/storage/files/migrations.ts

@@ -0,0 +1,187 @@
+import type { Database } from 'bun:sqlite';
+import type { Migration } from '../../migration/types';
+
+// Schema for the FileCatalogue. Runs through cursedbelt's migration runner (idempotent, recorded
+// in `_migrations`). Improves on ca's fs_nodes: the smart-view `file_type` is stored + indexed
+// (not re-derived per list), tags live in an indexed junction (fast facets, not a JSON scan),
+// processing state is typed columns (not an opaque blob), and a storage-operations ledger backs
+// durable cross-boundary (binary-server/S3) ops. Flat — no parent_id tree.
+
+export const FILE_CATALOGUE_MIGRATIONS: readonly Migration[] = [
+  {
+    name: '0001_file_catalogue',
+    up(db: Database): void {
+      db.run(`
+        CREATE TABLE IF NOT EXISTS files (
+          id                  TEXT PRIMARY KEY,
+          owner               TEXT NOT NULL,
+          name                TEXT NOT NULL DEFAULT '',
+          mime                TEXT NOT NULL DEFAULT '',
+          size                INTEGER NOT NULL DEFAULT 0,
+          file_type           TEXT NOT NULL DEFAULT 'other',
+          is_private          INTEGER NOT NULL DEFAULT 0,
+          has_thumbnail       INTEGER NOT NULL DEFAULT 0,
+          has_hls             INTEGER NOT NULL DEFAULT 0,
+          has_sprites         INTEGER NOT NULL DEFAULT 0,
+          duration            REAL,
+          processing_status   TEXT NOT NULL DEFAULT 'ready',
+          processing_progress INTEGER NOT NULL DEFAULT 100,
+          storage_backend     TEXT NOT NULL DEFAULT 'disk',
+          enc_name            TEXT NOT NULL DEFAULT '',
+          enc_dek             TEXT NOT NULL DEFAULT '',
+          enc_key_type        TEXT NOT NULL DEFAULT '',
+          created_at          INTEGER NOT NULL,
+          updated_at          INTEGER NOT NULL
+        )
+      `);
+      // Listing is cursor-keyed on (created_at, id); these cover the flat feed + smart-view filter.
+      db.run(
+        `CREATE INDEX IF NOT EXISTS idx_files_owner_created ON files(owner, created_at DESC, id DESC)`,
+      );
+      db.run(
+        `CREATE INDEX IF NOT EXISTS idx_files_owner_type_created ON files(owner, file_type, created_at DESC, id DESC)`,
+      );
+      db.run(`CREATE INDEX IF NOT EXISTS idx_files_owner_private ON files(owner, is_private)`);
+
+      // Tags as a junction (indexed) — single-tag filter + facet counts are index lookups, not a
+      // full-table JSON parse (ca's weakness). `owner` denormalized so facet queries stay covering.
+      db.run(`
+        CREATE TABLE IF NOT EXISTS file_tags (
+          file_id TEXT NOT NULL REFERENCES files(id) ON DELETE CASCADE,
+          tag     TEXT NOT NULL,
+          owner   TEXT NOT NULL,
+          is_private INTEGER NOT NULL DEFAULT 0,
+          PRIMARY KEY (file_id, tag)
+        )
+      `);
+      db.run(`CREATE INDEX IF NOT EXISTS idx_file_tags_owner_tag ON file_tags(owner, tag)`);
+
+      // Durable ledger for cross-boundary storage ops (binary-server/S3 put/delete) — the
+      // reconciler heals `pending` rows the server never got an ack for.
+      db.run(`
+        CREATE TABLE IF NOT EXISTS storage_operations (
+          op_id      TEXT PRIMARY KEY,
+          owner      TEXT NOT NULL,
+          file_id    TEXT,
+          action     TEXT NOT NULL,
+          key        TEXT NOT NULL,
+          backend    TEXT NOT NULL,
+          status     TEXT NOT NULL DEFAULT 'pending',
+          attempts   INTEGER NOT NULL DEFAULT 0,
+          error      TEXT,
+          created_at INTEGER NOT NULL,
+          updated_at INTEGER NOT NULL
+        )
+      `);
+      db.run(
+        `CREATE INDEX IF NOT EXISTS idx_storage_ops_status ON storage_operations(status, created_at)`,
+      );
+    },
+  },
+  {
+    name: '0002_file_collections',
+    up(db: Database): void {
+      // Collections (Apple-Photos-style groupings). kind='favorites' is a system singleton per owner;
+      // kind='custom' are user-defined. visibility='unlock' hides the collection unless explicitly
+      // requested (like a private shelf). Membership is in file_collection_files; files are never
+      // deleted by removing a collection.
+      db.run(`
+        CREATE TABLE IF NOT EXISTS file_collections (
+          id         TEXT PRIMARY KEY,
+          owner      TEXT NOT NULL,
+          name       TEXT NOT NULL,
+          kind       TEXT NOT NULL DEFAULT 'custom',
+          visibility TEXT NOT NULL DEFAULT 'normal',
+          created_at INTEGER NOT NULL,
+          updated_at INTEGER NOT NULL
+        )
+      `);
+      db.run(
+        `CREATE INDEX IF NOT EXISTS idx_file_collections_owner ON file_collections(owner, name)`,
+      );
+      // Unique partial index: at most one 'favorites' collection per owner.
+      db.run(
+        `CREATE UNIQUE INDEX IF NOT EXISTS idx_file_collections_owner_favorites
+         ON file_collections(owner) WHERE kind = 'favorites'`,
+      );
+
+      // Junction: which files belong to which collection. owner is denormalized so membership
+      // queries can stay covering without a join back to files. Foreign-key cascades are declared
+      // but SQLite's FK pragma may be off at runtime, so catalogue.remove() explicitly cleans
+      // file_collection_files rows (matching the file_tags pattern).
+      db.run(`
+        CREATE TABLE IF NOT EXISTS file_collection_files (
+          collection_id TEXT NOT NULL REFERENCES file_collections(id) ON DELETE CASCADE,
+          file_id       TEXT NOT NULL REFERENCES files(id) ON DELETE CASCADE,
+          owner         TEXT NOT NULL,
+          added_at      INTEGER NOT NULL,
+          PRIMARY KEY (collection_id, file_id)
+        )
+      `);
+      db.run(
+        `CREATE INDEX IF NOT EXISTS idx_fcf_owner_file ON file_collection_files(owner, file_id)`,
+      );
+      db.run(
+        `CREATE INDEX IF NOT EXISTS idx_fcf_collection ON file_collection_files(collection_id, added_at DESC)`,
+      );
+    },
+  },
+  {
+    name: '0003_file_sort_indexes',
+    up(db: Database): void {
+      // Keyset pagination under the `name` / `size` sorts stays an index walk, mirroring
+      // idx_files_owner_created for the default `recent` sort.
+      db.run(`CREATE INDEX IF NOT EXISTS idx_files_owner_name ON files(owner, name ASC, id ASC)`);
+      db.run(`CREATE INDEX IF NOT EXISTS idx_files_owner_size ON files(owner, size DESC, id DESC)`);
+    },
+  },
+  {
+    name: '0004_file_random_and_views',
+    up(db: Database): void {
+      // Per-row stable random key backing the seeded `random` gallery order. A session mints a
+      // seed (folded into the cursor); the sort key is `rand_key` remapped by that seed, so the
+      // order is a stable total order for keyset pagination while each visit reshuffles. ADD COLUMN
+      // can't take a non-constant DEFAULT, so seed existing rows here; new rows set it at insert.
+      db.run(`ALTER TABLE files ADD COLUMN rand_key INTEGER NOT NULL DEFAULT 0`);
+      // random() is a 64-bit int; mask to a non-negative 31-bit value so the seeded remap
+      // (rand_key * mul + add) stays well inside SQLite's signed-64-bit integer range.
+      db.run(`UPDATE files SET rand_key = (random() & 2147483647)`);
+
+      // View-event ledger: last-viewed timestamp + count per file, powering the `lastViewed` sort.
+      // One row per file (upserted on each open). owner is denormalized so the sort's join stays
+      // owner-scoped. FK cascade is declared but the pragma may be off, so catalogue.remove() also
+      // cleans these rows explicitly (matching the file_tags / file_collection_files pattern).
+      db.run(`
+        CREATE TABLE IF NOT EXISTS file_views (
+          file_id    TEXT PRIMARY KEY REFERENCES files(id) ON DELETE CASCADE,
+          owner      TEXT NOT NULL,
+          viewed_at  INTEGER NOT NULL,
+          view_count INTEGER NOT NULL DEFAULT 0
+        )
+      `);
+      db.run(
+        `CREATE INDEX IF NOT EXISTS idx_file_views_owner_viewed ON file_views(owner, viewed_at DESC)`,
+      );
+    },
+  },
+  {
+    name: '0005_processing_attempts',
+    up(db: Database): void {
+      // Counts transcode kicks per file so recoverProcessing can stop re-kicking a job that
+      // keeps dying (e.g. ffmpeg OOM-killed on a small host → service restart → re-kick loop).
+      db.run(`ALTER TABLE files ADD COLUMN processing_attempts INTEGER NOT NULL DEFAULT 0`);
+    },
+  },
+  {
+    name: '0006_file_trash',
+    up(db: Database): void {
+      // Trash-first delete: `trash()` sets trashed_at instead of dropping the row, so a mistaken
+      // delete is recoverable via `restore()`. `remove()` stays the PERMANENT hard-delete — now a
+      // deliberate second action (typically only invoked on an already-trashed row) rather than the
+      // primary delete path. `list()` excludes trashed rows by default (NULL check stays an index
+      // hit); pass `trashedOnly` to browse the trash bin itself.
+      db.run(`ALTER TABLE files ADD COLUMN trashed_at INTEGER`);
+      db.run(`CREATE INDEX IF NOT EXISTS idx_files_owner_trashed ON files(owner, trashed_at)`);
+    },
+  },
+];

package/src/server/storage/files/mimeByExt.ts

@@ -0,0 +1,14 @@
+import { mimeForFilename } from 'cwip/string';
+
+// Ext → mime for local-disk import (catalogue.importFromPath) — the catalogue needs SOME mime
+// to classify + serve a file that never passed through a browser's upload Content-Type.
+//
+// The table itself moved DOWN to `cwip/string`'s `fileKind` (2026-07-31): the viewer UI and the
+// notes ingest pipeline need the same knowledge, and cwip is the only layer all three can read
+// from. This wrapper stays so existing server call sites keep working; it is now one line, and
+// the lookup it delegates to is a superset of what used to live here (code, data, archive and
+// markup types the old table lacked). Unknown still falls back to 'application/octet-stream',
+// which `classifyFileType` maps to 'other' — importable, just without image/video processing.
+export function mimeByExt(name: string): string {
+  return mimeForFilename(name);
+}

package/src/server/storage/files/stack.spec.ts

@@ -0,0 +1,258 @@
+import { Database } from 'bun:sqlite';
+import { afterAll, beforeAll, describe, expect, test } from 'bun:test';
+import { mkdtemp, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import path from 'node:path';
+import { Hono } from 'hono';
+import { runMigrations } from '../../migration/runner';
+import { generateUserSalt } from '../../private-media/crypto';
+import { createDiskStore } from '../diskStore';
+import type { MediaCtx } from '../types';
+import { FILE_CATALOGUE_MIGRATIONS } from './migrations';
+import { createFilesStack, type FilesStack } from './stack';
+
+const CTX: MediaCtx = { userId: 'u1', tenantId: 't1' };
+const salt = generateUserSalt();
+
+// Seed a row recoverProcessing acts on WITHOUT ffmpeg: a private video stuck in `processing`
+// is marked failed synchronously (its set key isn't available at boot).
+function seedStuckPrivateRow(db: Database, id: string): void {
+  runMigrations(db, FILE_CATALOGUE_MIGRATIONS); // idempotent — the stack re-runs it harmlessly
+  db.run(
+    `INSERT INTO files (id, owner, name, mime, size, file_type, is_private, processing_status,
+       processing_progress, storage_backend, enc_name, enc_dek, enc_key_type, created_at, updated_at)
+     VALUES (?, 'u1', '', '', 1, 'video', 1, 'processing', 40, 'disk', 'x', 'x', 'deniable', 1, 1)`,
+    [id],
+  );
+}
+
+const statusOf = (db: Database, id: string): string | undefined =>
+  db
+    .query<{ processing_status: string }, [string]>(
+      'SELECT processing_status FROM files WHERE id = ?',
+    )
+    .get(id)?.processing_status;
+
+describe('createFilesStack — public stack', () => {
+  let db: Database;
+  let root: string;
+  let stack: FilesStack;
+  const app = new Hono();
+
+  beforeAll(async () => {
+    db = new Database(':memory:');
+    root = await mkdtemp(path.join(tmpdir(), 'stack-'));
+    stack = createFilesStack({
+      db,
+      store: createDiskStore(root),
+      workDir: path.join(root, 'work'),
+      streamSecret: 'stream-secret',
+    });
+    stack.mount(app, { authorizeUpload: () => CTX });
+  });
+  afterAll(async () => {
+    db.close();
+    await rm(root, { recursive: true, force: true });
+  });
+
+  test('migrations ran — catalogue.list works on a fresh db', async () => {
+    const page = await stack.catalogue.list(CTX);
+    expect(page.items).toEqual([]);
+  });
+
+  test('no private layer configured ⇒ unlock is null', () => {
+    expect(stack.unlock).toBeNull();
+  });
+
+  test('URL builders are wired to the default bases', async () => {
+    const f = await stack.fileUrls.build(CTX, 'fid', 'image/png');
+    expect(f.url).toStartWith('/_files/s/');
+    expect(f.url).toContain('ct=image%2Fpng');
+    const m = await stack.mediaUrls.build(CTX, 'files', 'vid');
+    expect(m.streamUrl).toStartWith('/_media/s/');
+    expect(m.hlsUrl).toEndWith('/hls/files/vid/master.m3u8');
+  });
+
+  test('videoSource is registered in the registry under the default sourceId', () => {
+    expect(stack.videoSource.id).toBe('files');
+    expect(stack.videoSource.title).toBe('Files');
+    expect(stack.registry.get('files')).toBe(stack.videoSource);
+    expect(stack.registry.list()).toEqual([stack.videoSource]);
+  });
+
+  test('mounted routes: upload → list shows it → binary route serves the bytes', async () => {
+    const up = await app.request('/_files/upload', {
+      method: 'PUT',
+      headers: { 'content-type': 'image/png', 'x-filename': 'pic.png', 'x-tags': 'demo' },
+      body: 'png-bytes',
+    });
+    expect(up.status).toBe(200);
+    const { file } = (await up.json()) as { file: { id: string; name: string } };
+    expect(file.name).toBe('pic.png');
+
+    const page = await stack.catalogue.list(CTX);
+    expect(page.items.map((f) => f.id)).toEqual([file.id]);
+
+    const { url } = await stack.fileUrls.build(CTX, file.id, 'image/png');
+    const res = await app.request(url);
+    expect(res.status).toBe(200);
+    expect(res.headers.get('content-type')).toBe('image/png');
+    expect(await res.text()).toBe('png-bytes');
+  });
+
+  test('upload is rejected when the host auth says no', async () => {
+    const denied = new Hono();
+    stack.mount(denied, { authorizeUpload: () => null });
+    const res = await denied.request('/_files/upload', { method: 'PUT', body: 'x' });
+    expect(res.status).toBe(401);
+  });
+
+  test('custom sourceId/basePaths flow through', async () => {
+    const db2 = new Database(':memory:');
+    const s = createFilesStack({
+      db: db2,
+      store: createDiskStore(root),
+      workDir: path.join(root, 'work2'),
+      streamSecret: 's',
+      sourceId: 'vault',
+      sourceTitle: 'Vault',
+      mediaBasePath: '/media2',
+      filesBasePath: '/files2',
+    });
+    expect(s.registry.get('vault')?.title).toBe('Vault');
+    expect((await s.fileUrls.build(CTX, 'f', 'a/b')).url).toStartWith('/files2/s/');
+    expect((await s.mediaUrls.build(CTX, 'vault', 'v')).streamUrl).toStartWith('/media2/s/');
+    db2.close();
+  });
+});
+
+describe('createFilesStack — recoverProcessing', () => {
+  let root: string;
+  beforeAll(async () => {
+    root = await mkdtemp(path.join(tmpdir(), 'stack-recover-'));
+  });
+  afterAll(async () => {
+    await rm(root, { recursive: true, force: true });
+  });
+
+  const build = (db: Database, recoverProcessing?: boolean) =>
+    createFilesStack({
+      db,
+      store: createDiskStore(root),
+      workDir: path.join(root, 'work'),
+      streamSecret: 's',
+      recoverProcessing,
+    });
+
+  test('runs by default — a stuck private row is marked failed at build', () => {
+    const db = new Database(':memory:');
+    seedStuckPrivateRow(db, 'stuck1');
+    build(db);
+    expect(statusOf(db, 'stuck1')).toBe('failed');
+    db.close();
+  });
+
+  test('recoverProcessing: false skips it', () => {
+    const db = new Database(':memory:');
+    seedStuckPrivateRow(db, 'stuck2');
+    build(db, false);
+    expect(statusOf(db, 'stuck2')).toBe('processing');
+    db.close();
+  });
+});
+
+describe('createFilesStack — private stack', () => {
+  let db: Database;
+  let root: string;
+  let stack: FilesStack;
+  const app = new Hono();
+
+  beforeAll(async () => {
+    db = new Database(':memory:');
+    root = await mkdtemp(path.join(tmpdir(), 'stack-priv-'));
+    stack = createFilesStack({
+      db,
+      store: createDiskStore(root),
+      workDir: path.join(root, 'work'),
+      streamSecret: 'stream-secret',
+      privateMedia: { serverSecret: 'srv', unlockSecret: 'unl' },
+    });
+    stack.mount(app, {
+      authorizeUpload: () => CTX,
+      authorizePrivate: async () => ({ userId: CTX.userId, userSalt: salt }),
+    });
+  });
+  afterAll(async () => {
+    db.close();
+    await rm(root, { recursive: true, force: true });
+  });
+
+  test('unlock service exists and its tokens verify', async () => {
+    expect(stack.unlock).not.toBeNull();
+    const { token, mode } = await stack.unlock!.unlock({
+      userId: CTX.userId,
+      userSalt: salt,
+      password: 'pw',
+    });
+    expect(mode).toBe('deniable');
+    expect((await stack.unlock!.verify(token))?.userId).toBe(CTX.userId);
+  });
+
+  test('POST /_private/unlock with a password mints a deniable token', async () => {
+    const res = await app.request('/_private/unlock', {
+      method: 'POST',
+      headers: { 'content-type': 'application/json' },
+      body: JSON.stringify({ password: 'hunter2' }),
+    });
+    expect(res.status).toBe(200);
+    const body = (await res.json()) as { token: string; mode: string };
+    expect(body.mode).toBe('deniable');
+    expect((await stack.unlock!.verify(body.token))?.mode).toBe('deniable');
+  });
+
+  test('reveal route is mounted (password reveal over an empty set)', async () => {
+    const res = await app.request('/_private/reveal', {
+      method: 'POST',
+      headers: { 'content-type': 'application/json' },
+      body: JSON.stringify({ password: 'hunter2' }),
+    });
+    expect(res.status).toBe(200);
+    expect((await res.json()) as { items: unknown[] }).toEqual({ items: [] });
+  });
+
+  test('private upload route is mounted and session-guarded', async () => {
+    const denied = new Hono();
+    stack.mount(denied, {
+      authorizeUpload: () => CTX,
+      authorizePrivate: async () => null,
+    });
+    const res = await denied.request('/_files/upload-private', { method: 'PUT', body: 'x' });
+    expect(res.status).toBe(401); // mounted (not 404) but rejected by the host auth
+  });
+
+  test('mount() throws without authorizePrivate when privateMedia is configured', () => {
+    expect(() => stack.mount(new Hono(), { authorizeUpload: () => CTX })).toThrow(
+      /authorizePrivate/,
+    );
+  });
+
+  test('privateBasePath flows through to the private mount', async () => {
+    const db2 = new Database(':memory:');
+    const s = createFilesStack({
+      db: db2,
+      store: createDiskStore(root),
+      workDir: path.join(root, 'work2'),
+      streamSecret: 's',
+      privateBasePath: '/vault',
+      privateMedia: { serverSecret: 'srv', unlockSecret: 'unl' },
+    });
+    const a = new Hono();
+    s.mount(a, {
+      authorizeUpload: () => CTX,
+      authorizePrivate: async () => ({ userId: CTX.userId, userSalt: salt }),
+    });
+    const res = await a.request('/vault/unlock', { method: 'POST', body: '{}' });
+    expect(res.status).toBe(200);
+    db2.close();
+  });
+});