@db3.ai/app 0.1.0-beta.1

package/src/media/README.md

@@ -0,0 +1,358 @@
+# @db3.ai/app/media
+
+`@db3.ai/app/media` is the framework-level file and media-browser layer that
+sits above `@db3.ai/app/storage`.
+
+Storage owns disks, paths, streams, and bytes. Media owns durable file ULIDs,
+library scopes, metadata, and optional browser tree rows.
+
+## Tables
+
+The media package uses three tables:
+
+```text
+media_libraries
+media_files
+media_items
+```
+
+`media_libraries` is the ownership and configuration boundary. It stores an
+app-owned `scope_type`, `scope_id`, and `library_key`, plus optional storage
+defaults such as `default_disk` and `path_prefix`.
+
+`media_files` stores managed file records. A file row can exist without being
+visible in a browser.
+
+`media_items` stores browser-visible rows: root folders, directories, and file
+placements. A `file` item points to a `media_files` row.
+
+## Scoped Libraries
+
+The framework does not know about application concepts such as websites,
+organizations, teams, or users. Apps map those concepts into a scoped library:
+
+```ts
+const library = await app.media.libraryFor({
+	scopeType: 'app.website',
+	scopeId: website.id,
+	key: 'default',
+	name: 'Website media',
+	pathPrefix: `websites/${website.id}/media`,
+});
+```
+
+The unique library identity is:
+
+```text
+scope_type + scope_id + library_key
+```
+
+This lets one app start with one default library per scope, while still allowing
+future libraries such as `generated-images`, `reference-material`, or
+`brand-assets` when they need separate configuration or workflows.
+
+## Managed Files Without Browser Items
+
+Use `storeFile()` when a file should be managed, loaded by ULID, and scoped to a
+library, but should not appear in the media browser.
+
+```ts
+const file = await app.media.storeFile({
+	library,
+	contents: Buffer.from('private export'),
+	name: 'export.txt',
+	mimeType: 'text/plain',
+	source: 'export',
+	meta: {
+		jobId: job.id,
+	},
+});
+
+const bytes = await app.media.readFile(file.id);
+```
+
+This creates a `media_files` row only.
+
+## Browser-Visible Files
+
+Use `storeVisibleFile()` when a file should also appear in the browser tree.
+
+```ts
+const stored = await app.media.storeVisibleFile({
+	library,
+	contents: imageBytes,
+	name: 'hero-preview.png',
+	mimeType: 'image/png',
+	source: 'generated-image',
+	folderPath: '/Generated images',
+	meta: {
+		model: 'gpt-image-2',
+		prompt,
+	},
+});
+
+stored.file.id;
+stored.item.path;
+```
+
+This creates:
+
+- one `media_files` row for the bytes and metadata
+- one `media_items` file row for browser placement
+- any missing folder rows in the requested folder path
+
+Browser paths are unique within a library. When the requested filename already
+exists in the destination folder, media placement keeps the managed file and
+its ULID unchanged while numbering the browser item: `moon.png`, `moon-2.png`,
+`moon-3.png`, and so on.
+
+## Streaming Completed Uploads
+
+Trusted HTTP adapters can stream an assembled file into managed media without
+buffering the complete payload in application memory:
+
+```ts
+import { createReadStream } from 'node:fs';
+
+const stored = await app.media.storeVisibleFileStream({
+	library,
+	stream: createReadStream(temporaryPath),
+	size: upload.size,
+	name: upload.name,
+	mimeType: upload.mimeType,
+	source: 'upload',
+	folderPath: '/Campaigns',
+});
+```
+
+Use `storeFileStream()` for a managed file that should not appear in the
+browser. Supply a trusted byte size when it is known. When size is omitted,
+storage measures the completed object before the `media_files` row is saved.
+
+Untrusted image uploads should instead use canonical reconstruction:
+
+```ts
+const stored = await app.media.storeReconstructedVisibleImageStream({
+	library,
+	stream: createReadStream(temporaryPath),
+	name: upload.name,
+	mimeType: detectedMimeType,
+	source: 'upload',
+	folderPath: '/Campaigns',
+});
+```
+
+The upload adapter should first apply a cheap signature allowlist and require
+the detected type to match its declared type. The framework then passes the
+complete stream through `ImageProcessor.reconstruct()`. Sharp fully decodes the
+input within its pixel and timeout limits, applies orientation, strips metadata
+and trailing bytes, and emits a new image in the same canonical format.
+
+The reconstructed stream writes directly to a generated durable path without
+buffering the complete source or output in JavaScript memory. The framework
+does not save a media row or browser item until the stream completes. Processor
+failures remove the partial durable object and raise `ImageReconstructionError`,
+allowing an upload adapter to return a validation response rather than a
+storage error.
+
+The media package deliberately does not implement an HTTP upload protocol.
+Authentication, resumable offsets, temporary-resource expiry, and protocol
+responses belong to an app or server adapter above this storage-neutral seam.
+
+## Loading Files
+
+The file ULID is the durable reference. Folder paths can change without
+changing the file id.
+
+```ts
+const file = await app.media.file(fileId);
+const bytes = file ? await app.media.readFile(file) : null;
+const stream = file ? await app.media.readFileStream(file) : null;
+```
+
+Routes remain responsible for authentication, authorization, headers, and
+response shaping. The media package only knows the library scope, not whether a
+given user can access that scope.
+
+## Deleting Files
+
+Use `deleteFile()` after the application has applied its own ownership and usage
+policy:
+
+```ts
+await app.media.deleteFile(file);
+```
+
+The manager removes disposable responsive-image variants, deletes the durable
+source bytes, and then deletes the `media_files` row. Database foreign keys
+remove every browser placement that points at the file. Application references
+such as article content remain app-owned and must be checked before calling this
+framework method.
+
+## Visibility And Public Delivery
+
+`media_files.visibility` records the storage visibility used when the file was
+written. It does not make a file public by itself. Apps should expose public
+media through deliberate routes that load the file by ULID, check the source and
+visibility they are willing to publish, and then write the response headers.
+
+Generated article images can therefore be stored as browser-visible public
+files. Media browsers can render those deliberate public routes directly so
+normal browser image caching applies, while private files continue to use an
+authenticated app route and browser object URL.
+
+## Responsive Image Variants
+
+The `media/image` module translates an allowlisted URL query into a framework
+image command, renders cache misses through an `ImageProcessor`, and stores the
+result on a configured disposable storage disk.
+
+```ts
+import { imageVariantOptionsFromSearchParams } from '@db3.ai/app/media';
+
+const widthOptions = imageVariantOptionsFromSearchParams(requestUrl.searchParams);
+const options = requestUrl.pathname.endsWith('.webp')
+	? widthOptions ?? {}
+	: widthOptions;
+
+if (options) {
+	const image = await app.media.renderImage(file, options);
+
+	reply
+		.header('content-type', image.mimeType)
+		.send(image.stream);
+}
+```
+
+The request contract supports width while preserving aspect ratio:
+
+```text
+/api/media/images/<media-file-ulid>.webp?w=640
+```
+
+To compress an image at its original dimensions, request WebP without a width:
+
+```text
+/api/media/images/<media-file-ulid>.webp
+```
+
+An article can use that full-size compressed image as its fallback while
+letting the browser select a smaller cached width:
+
+```html
+<img
+	src="/api/media/images/<media-file-ulid>.webp"
+	srcset="
+		/api/media/images/<media-file-ulid>.webp?w=480 480w,
+		/api/media/images/<media-file-ulid>.webp?w=960 960w,
+		/api/media/images/<media-file-ulid>.webp?w=1440 1440w
+	"
+	sizes="(max-width: 48rem) 100vw, 48rem"
+	width="1536"
+	height="1024"
+	loading="lazy"
+	decoding="async"
+	alt="Useful description of the image"
+/>
+```
+
+JPEG, PNG, WebP, and AVIF sources produce quality-controlled WebP variants,
+apply EXIF orientation, and never enlarge the original. The source URL without
+any transform query remains untouched. SVG and GIF files pass through unchanged
+because SVG is already responsive and GIF animation must be preserved.
+
+Generated variants are streamed from durable source storage through the
+processor and into disposable cache storage without buffering the complete
+source or result in JavaScript memory. Cache paths are versioned and keyed by
+the immutable managed-file ULID:
+
+```text
+image-cache/v1/<file-ulid>/w-640.webp
+image-cache/v1/<file-ulid>/original.webp
+```
+
+Configure the cache disk and prefix under `config.media`. Omitting `cacheDisk`
+keeps the cache on the source disk, while applications with a disposable disk
+can isolate every generated variant:
+
+```ts
+const media = {
+	images: {
+		cacheDisk: 'tmp',
+		cachePrefix: 'image-cache',
+	},
+};
+```
+
+Deleting the cache directory never removes originals. A later request
+regenerates the missing variant using the managed source file.
+Failed renders stop both streams and finish pending storage writes before
+removing partial cache files and returning the error.
+
+The default `SharpImageProcessor` uses Sharp/libvips. It is isolated behind the
+`ImageProcessor` contract so another engine, such as an ImageMagick adapter, can
+replace it without changing request parsing, cache identity, media ownership, or
+storage.
+
+Safety bounds currently include:
+
+- widths from 1 through 4096 pixels
+- a 40-megapixel decoded input limit
+- a 20-second processing timeout
+- at most two active cache-generating renders per application process
+- coalescing concurrent requests for the same variant into one render
+
+Canonical upload reconstruction uses the same 40-megapixel and 20-second
+processor bounds. Animated GIF and WebP inputs decode and reconstruct every
+frame within the total pixel limit.
+
+Unknown query parameters never become processor commands. Apps remain
+responsible for resolving the media ULID and applying authorization before
+calling the framework renderer.
+
+## Installation
+
+For an application, add the three models through committed migrations. The
+`install` call below is appropriate only for a disposable lab or explicit local
+schema setup, not request handling or production server startup.
+
+Install the models alongside your app models:
+
+```ts
+import { MediaFile, MediaItem, MediaLibrary } from '@db3.ai/app/media';
+
+await app.db.install(
+	MediaLibrary,
+	MediaFile,
+	MediaItem,
+);
+```
+
+When using `App`, the media manager is available as:
+
+```ts
+await app.media.storeFile(...);
+```
+
+## Runnable example and testing
+
+For an authenticated route, copy the Media examples and install `fastify@5`.
+Run `npx tsx examples/runPrivateFiles.ts` with the same disposable SQL settings.
+It exercises a 64 KiB bounded text upload, owner-scoped streamed download,
+private response headers, cross-user rejection, invalid input without writes,
+deletion and retry. The exact consumer test is
+`tests/examples/runPrivateFiles.test.ts`. Bearer tokens stay inside the lab;
+deployment needs HTTPS and the application's session/issuance policy. This is
+not a resumable upload, arbitrary-file validator or public publishing endpoint.
+
+The [project media lab](./examples/runProjectMedia.ts) streams a private project
+file, creates browser placements, handles duplicate names, scopes a read to the
+authorized library and checks byte/placement cleanup on deletion. Copy it into
+`examples` and run `npx tsx examples/runProjectMedia.ts` with a dedicated SQL
+test account allowed to create/drop `db3_app_test_*` databases.
+
+The lab removes its database and temporary local disk in `finally`. The website
+includes the exact consumer-copyable test. Image reconstruction and variant
+caching have separate service tests; this text-file workflow does not exercise
+them. Framework contributors run
+`npm run test:service --workspace packages/app -- media`.

package/src/media/constants.ts

@@ -0,0 +1,31 @@
+/**
+ * Default media library key used when an app needs one library per scope.
+ */
+export const MEDIA_LIBRARY_DEFAULT_KEY = 'default';
+
+/**
+ * Media item types used by the folder browser tree.
+ */
+export const MEDIA_ITEM_TYPE = {
+	root: 'root',
+	directory: 'dir',
+	file: 'file',
+} as const;
+
+/**
+ * Storage visibility values recorded for managed files.
+ */
+export const MEDIA_FILE_VISIBILITY = {
+	private: 'private',
+	public: 'public',
+} as const;
+
+/**
+ * Browser tree item type.
+ */
+export type MediaItemType = typeof MEDIA_ITEM_TYPE[keyof typeof MEDIA_ITEM_TYPE];
+
+/**
+ * Visibility applied when writing a managed file to storage.
+ */
+export type MediaFileVisibility = typeof MEDIA_FILE_VISIBILITY[keyof typeof MEDIA_FILE_VISIBILITY];

package/src/media/contracts/MediaOptions.ts

@@ -0,0 +1,9 @@
+import type { ImageVariantCacheOptions } from '../image/contracts';
+
+/**
+ * Framework media-service configuration.
+ */
+export interface MediaOptions {
+	/** Responsive-image rendering and disposable cache configuration. */
+	images?: ImageVariantCacheOptions;
+}

package/src/media/contracts/index.ts

@@ -0,0 +1 @@
+export type { MediaOptions } from './MediaOptions';

package/src/media/examples/createPrivateFileServer.ts

@@ -0,0 +1,55 @@
+import Fastify, { type FastifyInstance, type FastifyReply, type FastifyRequest } from 'fastify';
+import { MediaFile } from '@db3.ai/app/media';
+import type { App } from '@db3.ai/app/server';
+
+/**
+ * Builds a deliberately small private-text-file API without opening a socket.
+ *
+ * The caller owns App and server cleanup. Uploads are bounded to 64 KiB and
+ * buffered by the HTTP parser; downloads stream only after owner authorization.
+ * Bearer tokens are lab inputs. A deployed endpoint requires HTTPS and an
+ * application-owned token issuance and storage policy.
+ *
+ * @param application - Active application with Auth, Media and local storage.
+ * @returns Injectable HTTP server exposing POST, GET and DELETE /files.
+ */
+export function createPrivateFileServer(application: App): FastifyInstance {
+	const server = Fastify({ bodyLimit: 64 * 1024 });
+	server.addHook('onRequest', (_request, _reply, done) => application.requestContext.run(done));
+
+	/** Resolves a bearer token without accepting a caller-supplied owner id. */
+	async function owner(request: FastifyRequest, reply: FastifyReply): Promise<string | null> {
+		const authorization = request.headers.authorization;
+		const token = authorization?.startsWith('Bearer ') ? authorization.slice(7) : '';
+		const user = token ? await application.auth.authenticateToken(token) : null;
+		if (!user?.id) { reply.code(401).send({ error: 'Sign in first.' }); return null; }
+		return user.id;
+	}
+
+	server.post<{ Body: string }>('/files', async (request, reply) => {
+		const ownerId = await owner(request, reply);
+		if (!ownerId) return;
+		if (request.headers['content-type']?.split(';')[0]?.trim() !== 'text/plain' || typeof request.body !== 'string' || !request.body.trim()) return reply.code(400).send({ error: 'Supply non-empty text/plain.' });
+		const library = await application.media.libraryFor({ scopeType: 'user', scopeId: ownerId, pathPrefix: `users/${ownerId}` });
+		const file = await application.media.storeFile({ library, name: 'brief.txt', mimeType: 'text/plain', contents: request.body, visibility: 'private', source: 'upload' });
+		return reply.code(201).send({ id: file.id, name: file.name, size: file.size });
+	});
+	server.get<{ Params: { id: string } }>('/files/:id', async (request, reply) => {
+		const ownerId = await owner(request, reply);
+		if (!ownerId) return;
+		const library = await application.media.libraryFor({ scopeType: 'user', scopeId: ownerId });
+		const file = await MediaFile.where({ id: request.params.id, library }).first();
+		if (!file) return reply.code(404).send({ error: 'File not found.' });
+		return reply.header('Cache-Control', 'private, no-store').header('Content-Type', 'text/plain; charset=utf-8').header('Content-Disposition', 'attachment; filename="brief.txt"').header('X-Content-Type-Options', 'nosniff').send(await application.media.readFileStream(file));
+	});
+	server.delete<{ Params: { id: string } }>('/files/:id', async (request, reply) => {
+		const ownerId = await owner(request, reply);
+		if (!ownerId) return;
+		const library = await application.media.libraryFor({ scopeType: 'user', scopeId: ownerId });
+		const file = await MediaFile.where({ id: request.params.id, library }).first();
+		if (!file) return reply.code(404).send({ error: 'File not found.' });
+		await application.media.deleteFile(file);
+		return reply.code(204).send();
+	});
+	return server;
+}

package/src/media/examples/runPrivateFiles.ts

@@ -0,0 +1,45 @@
+import { mkdtemp, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { pathToFileURL } from 'node:url';
+import { AuthProvider, AuthToken, PasswordResetToken, UserIdentity } from '@db3.ai/app/auth';
+import { createGeneratedTestDatabase } from '@db3.ai/app/db/test/db';
+import { MediaFile, MediaItem, MediaLibrary } from '@db3.ai/app/media';
+import { App } from '@db3.ai/app/server';
+import { createPrivateFileServer } from './createPrivateFileServer';
+
+/** Runs authorized upload/download/deletion against real Auth, SQL and storage. */
+export async function runPrivateFiles() {
+	const root = await mkdtemp(join(tmpdir(), 'db3-private-files-'));
+	try {
+		const database = await createGeneratedTestDatabase('private_files');
+		const application = new App({ db: database.db, config: { auth: { providers: { password: true } } }, storage: { disks: { local: { driver: 'local', root } } } });
+		const server = createPrivateFileServer(application);
+		try {
+			await application.db.install(UserIdentity, AuthProvider, AuthToken, PasswordResetToken, MediaLibrary, MediaFile, MediaItem);
+			const ada = await application.auth.registerWithPassword({ name: 'Ada', email: 'ada@example.test', password: 'example-password-123' });
+			const grace = await application.auth.registerWithPassword({ name: 'Grace', email: 'grace@example.test', password: 'example-password-456' });
+			const headers = { authorization: `Bearer ${ada.token}`, 'content-type': 'text/plain' };
+			const other = { authorization: `Bearer ${grace.token}` };
+			const unsigned = await server.inject({ method: 'POST', url: '/files', headers: { 'content-type': 'text/plain' }, payload: 'Hidden' });
+			const upload = await server.inject({ method: 'POST', url: '/files', headers, payload: 'Private client brief' });
+			if (upload.statusCode !== 201) throw new Error(`Upload failed: ${upload.statusCode}`);
+			const id = upload.json<{ id: string }>().id;
+			const file = await MediaFile.where('id', id).firstOrFail();
+			const path = file.path!;
+			const download = await server.inject({ url: `/files/${id}`, headers });
+			const stranger = await server.inject({ url: `/files/${id}`, headers: other });
+			const strangerDelete = await server.inject({ method: 'DELETE', url: `/files/${id}`, headers: other });
+			const empty = await server.inject({ method: 'POST', url: '/files', headers, payload: '  ' });
+			const wrongType = await server.inject({ method: 'POST', url: '/files', headers: { ...headers, 'content-type': 'application/json' }, payload: { owner: grace.user.id } });
+			const oversized = await server.inject({ method: 'POST', url: '/files', headers, payload: 'x'.repeat(64 * 1024 + 1) });
+			const failedWritesAbsent = await MediaFile.query().count() === 1;
+			const deleted = await server.inject({ method: 'DELETE', url: `/files/${id}`, headers });
+			const missing = await server.inject({ url: `/files/${id}`, headers });
+			const retry = await server.inject({ method: 'POST', url: '/files', headers, payload: 'Revised brief' });
+			return { uploaded: upload.statusCode, text: download.body, unsigned: unsigned.statusCode, stranger: stranger.statusCode, strangerDelete: strangerDelete.statusCode, empty: empty.statusCode, wrongType: wrongType.statusCode, oversized: oversized.statusCode, failedWritesAbsent, privateHeaders: download.headers['cache-control'] === 'private, no-store' && download.headers['x-content-type-options'] === 'nosniff', deleted: deleted.statusCode, bytesRemoved: await application.storage.missing(path), missing: missing.statusCode, retry: retry.statusCode };
+		} finally { try { await server.close(); } finally { try { await application.close(); } finally { await database.destroy(); } } }
+	} finally { await rm(root, { recursive: true, force: true }); }
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) console.log(JSON.stringify(await runPrivateFiles(), null, 2));

package/src/media/examples/runProjectMedia.ts

@@ -0,0 +1,46 @@
+import { mkdtemp, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { Readable } from 'node:stream';
+import { pathToFileURL } from 'node:url';
+import { createGeneratedTestDatabase } from '@db3.ai/app/db/test/db';
+import { MediaFile, MediaItem, MediaLibrary } from '@db3.ai/app/media';
+import { App } from '@db3.ai/app/server';
+
+/**
+ * Stores a project brief, makes a browser placement and proves library isolation.
+ *
+ * The project identity is trusted lab input. Real endpoints must authorize the
+ * project before resolving its library, and scope file reads to that library.
+ *
+ * @returns Stable results after removing the lab's database and temporary files.
+ */
+export async function runProjectMedia() {
+	const root = await mkdtemp(join(tmpdir(), 'db3-media-guide-'));
+	try {
+		const database = await createGeneratedTestDatabase('media_guide');
+		const application = new App({ db: database.db, storage: { disks: { local: { driver: 'local', root } } } });
+		try {
+			await application.db.install(MediaLibrary, MediaFile, MediaItem);
+			const library = await application.media.libraryFor({ scopeType: 'project', scopeId: 'client-a', name: 'Client files', pathPrefix: 'projects/client-a' });
+			const otherLibrary = await application.media.libraryFor({ scopeType: 'project', scopeId: 'client-b' });
+			const stored = await application.media.storeVisibleFileStream({ library, name: 'brief.txt', stream: Readable.from(['Client brief']), mimeType: 'text/plain', folderPath: '/Briefs', source: 'upload' });
+			const duplicate = await application.media.storeVisibleFile({ library, name: 'brief.txt', contents: 'Revised brief', mimeType: 'text/plain', folderPath: '/Briefs' });
+			const file = await MediaFile.where({ id: stored.file.id, library }).firstOrFail();
+			const text = (await application.media.readFile(file)).toString('utf8');
+			const crossProjectRejected = await MediaFile.where({ id: file.id, library: otherLibrary }).first() === null;
+			const result = { text, path: stored.item.path, duplicatePath: duplicate.item.path, private: file.visibility === 'private', crossProjectRejected };
+			const originalPath = file.path!;
+			await application.media.deleteFile(file);
+			return { ...result, deleted: await application.storage.missing(originalPath), placementRemoved: await MediaItem.where('id', stored.item.id).first() === null };
+		} finally {
+			try { await application.close(); } finally { await database.destroy(); }
+		}
+	} finally {
+		await rm(root, { recursive: true, force: true });
+	}
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+	console.log(JSON.stringify(await runProjectMedia(), null, 2));
+}

package/src/media/image/ImageReconstructionError.ts

@@ -0,0 +1,18 @@
+/**
+ * Indicates that untrusted source bytes could not be decoded and reconstructed.
+ *
+ * Storage and database failures remain their original error types so callers
+ * can distinguish invalid image input from application infrastructure errors.
+ */
+export class ImageReconstructionError extends Error {
+	/**
+	 * Creates an image reconstruction error with its processor failure attached.
+	 *
+	 * @param message - Stable application-facing failure summary.
+	 * @param options - Standard error options containing the processor cause.
+	 */
+	constructor(message: string, options?: ErrorOptions) {
+		super(message, options);
+		this.name = 'ImageReconstructionError';
+	}
+}