@db3.ai/app 0.1.0-beta.1

package/src/storage/drivers/FlystorageDisk.ts

@@ -0,0 +1,295 @@
+import type { FileStorage } from '@flystorage/file-storage';
+import type { Readable } from 'node:stream';
+
+import { storageContentsToBuffer } from '../contents';
+import { mimeTypeFromPath } from '../mime';
+import { normalizeStorageDirectoryPath, normalizeStoragePath } from '../path';
+import type { StorageListing, StorageListOptions } from '../contracts';
+import type { StorageContents, StorageDisk, StoragePutOptions } from '../types';
+import { FlystorageListing } from './FlystorageListing';
+
+/**
+ * Base storage disk for drivers backed by Flystorage.
+ */
+export class FlystorageDisk implements StorageDisk {
+	/**
+	 * Creates a Flystorage-backed disk wrapper.
+	 *
+	 * @param name - Configured disk name.
+	 * @param storage - Flystorage instance used for disk operations.
+	 */
+	constructor(
+		readonly name: string,
+		protected readonly storage: FileStorage,
+	) {}
+
+	/**
+	 * Writes file contents to this disk.
+	 *
+	 * Alias for put(), useful when testing storage from a shell or REPL.
+	 *
+	 * @param path - Relative storage path to write.
+	 * @param contents - Bytes or text content to persist.
+	 * @param options - Optional write metadata reserved for compatible drivers.
+	 * @returns Promise that resolves once the file is written.
+	 */
+	write(path: string, contents: StorageContents, options: StoragePutOptions = {}): Promise<void> {
+		return this.put(path, contents, options);
+	}
+
+	/**
+	 * Writes file contents to this disk.
+	 *
+	 * @param path - Relative storage path to write.
+	 * @param contents - Bytes or text content to persist.
+	 * @param options - Optional write metadata reserved for compatible drivers.
+	 * @returns Promise that resolves once the file is written.
+	 */
+	async put(path: string, contents: StorageContents, options: StoragePutOptions = {}): Promise<void> {
+		await this.storage.write(
+			normalizeStoragePath(path),
+			storageContentsToBuffer(contents),
+			{
+				mimeType: options.mimeType,
+				visibility: options.visibility,
+			},
+		);
+	}
+
+	/**
+	 * Writes streamed file contents to this disk.
+	 *
+	 * @param path - Relative storage path to write.
+	 * @param stream - Readable stream containing file bytes.
+	 * @param options - Optional write metadata reserved for compatible drivers.
+	 * @returns Promise that resolves once the stream is written.
+	 */
+	async writeStream(path: string, stream: Readable, options: StoragePutOptions = {}): Promise<void> {
+		await this.storage.write(
+			normalizeStoragePath(path),
+			stream,
+			{
+				mimeType: options.mimeType,
+				visibility: options.visibility,
+			},
+		);
+	}
+
+	/**
+	 * Lists files and directories beneath a path on this disk.
+	 *
+	 * @param path - Relative directory path, or an empty string for the disk root.
+	 * @param options - Directory traversal options.
+	 * @returns Lazy provider-neutral directory listing.
+	 */
+	list(path = '', options: StorageListOptions = {}): StorageListing {
+		return new FlystorageListing(this.storage.list(
+			normalizeStorageDirectoryPath(path),
+			{
+				deep: options.deep ?? false,
+			},
+		));
+	}
+
+	/**
+	 * Reads a file from this disk as bytes.
+	 *
+	 * @param path - Relative storage path to read.
+	 * @returns Stored file bytes.
+	 */
+	async get(path: string): Promise<Buffer> {
+		return this.readToBuffer(path);
+	}
+
+	/**
+	 * Reads a file from this disk as text.
+	 *
+	 * Matches Flystorage's readToString() API.
+	 *
+	 * @param path - Relative storage path to read.
+	 * @returns Stored text content.
+	 */
+	async readToString(path: string): Promise<string> {
+		return this.storage.readToString(normalizeStoragePath(path));
+	}
+
+	/**
+	 * Reads a file from this disk as a Buffer.
+	 *
+	 * Matches Flystorage's readToBuffer() API.
+	 *
+	 * @param path - Relative storage path to read.
+	 * @returns Stored file bytes.
+	 */
+	async readToBuffer(path: string): Promise<Buffer> {
+		return this.storage.readToBuffer(normalizeStoragePath(path));
+	}
+
+	/**
+	 * Reads a file from this disk as a Uint8Array.
+	 *
+	 * Matches Flystorage's readToUint8Array() API.
+	 *
+	 * @param path - Relative storage path to read.
+	 * @returns Stored file bytes.
+	 */
+	async readToUint8Array(path: string): Promise<Uint8Array> {
+		return this.storage.readToUint8Array(normalizeStoragePath(path));
+	}
+
+	/**
+	 * Reads a file from this disk as a stream.
+	 *
+	 * Matches Flystorage's read() API.
+	 *
+	 * @param path - Relative storage path to read.
+	 * @returns Readable stream for the stored file.
+	 */
+	async read(path: string): Promise<Readable> {
+		return this.storage.read(normalizeStoragePath(path));
+	}
+
+	/**
+	 * Reads a file from this disk as a stream.
+	 *
+	 * @param path - Relative storage path to read.
+	 * @returns Readable stream for the stored file.
+	 */
+	async readStream(path: string): Promise<Readable> {
+		return this.read(path);
+	}
+
+	/**
+	 * Reads a file from this disk as text.
+	 *
+	 * @param path - Relative storage path to read.
+	 * @param encoding - Text encoding used to decode the bytes.
+	 * @returns Stored text content.
+	 */
+	async getText(path: string, encoding: BufferEncoding = 'utf8'): Promise<string> {
+		return (await this.readToBuffer(path)).toString(encoding);
+	}
+
+	/**
+	 * Checks whether a file exists on this disk.
+	 *
+	 * @param path - Relative storage path to inspect.
+	 * @returns True when the file exists.
+	 */
+	async exists(path: string): Promise<boolean> {
+		return this.storage.fileExists(normalizeStoragePath(path));
+	}
+
+	/**
+	 * Checks whether a file is missing from this disk.
+	 *
+	 * @param path - Relative storage path to inspect.
+	 * @returns True when the file does not exist.
+	 */
+	async missing(path: string): Promise<boolean> {
+		return !(await this.exists(path));
+	}
+
+	/**
+	 * Deletes a file from this disk.
+	 *
+	 * @param path - Relative storage path to delete.
+	 * @returns True when a file was removed.
+	 */
+	async delete(path: string): Promise<boolean> {
+		const normalized = normalizeStoragePath(path);
+		const exists = await this.storage.fileExists(normalized);
+
+		await this.storage.deleteFile(normalized);
+
+		return exists;
+	}
+
+	/**
+	 * Copies a file to another path on this disk.
+	 *
+	 * @param from - Existing relative storage path.
+	 * @param to - New relative storage path.
+	 * @returns Promise that resolves once the file is copied.
+	 */
+	async copy(from: string, to: string): Promise<void> {
+		await this.storage.copyFile(
+			normalizeStoragePath(from),
+			normalizeStoragePath(to),
+		);
+	}
+
+	/**
+	 * Moves a file to another path on this disk.
+	 *
+	 * @param from - Existing relative storage path.
+	 * @param to - New relative storage path.
+	 * @returns Promise that resolves once the file is moved.
+	 */
+	async move(from: string, to: string): Promise<void> {
+		await this.storage.moveFile(
+			normalizeStoragePath(from),
+			normalizeStoragePath(to),
+		);
+	}
+
+	/**
+	 * Returns file size in bytes.
+	 *
+	 * @param path - Relative storage path to inspect.
+	 * @returns File size in bytes.
+	 */
+	async size(path: string): Promise<number> {
+		return this.storage.fileSize(normalizeStoragePath(path));
+	}
+
+	/**
+	 * Returns the last modification date for a file.
+	 *
+	 * @param path - Relative storage path to inspect.
+	 * @returns Last modified date.
+	 */
+	async lastModified(path: string): Promise<Date> {
+		return new Date(await this.storage.lastModified(normalizeStoragePath(path)));
+	}
+
+	/**
+	 * Returns a MIME type for a storage path.
+	 *
+	 * @param path - Relative storage path to inspect.
+	 * @returns MIME type inferred from the file extension or storage metadata.
+	 */
+	async mimeType(path: string): Promise<string> {
+		const normalized = normalizeStoragePath(path);
+
+		try {
+			return await this.storage.mimeType(normalized, {
+				fallbackMethod: 'path',
+			});
+		} catch {
+			return mimeTypeFromPath(normalized);
+		}
+	}
+
+	/**
+	 * Builds a public URL for this disk when supported by the driver.
+	 *
+	 * @param path - Relative storage path to expose.
+	 * @returns Public URL for the storage path.
+	 */
+	async url(path: string): Promise<string> {
+		return this.storage.publicUrl(normalizeStoragePath(path));
+	}
+
+	/**
+	 * Reports that this disk is not backed by a local filesystem path.
+	 *
+	 * @param path - Relative storage path requested by the caller.
+	 * @returns This method never returns for non-local disks.
+	 */
+	path(path: string): string {
+		normalizeStoragePath(path);
+
+		throw new Error(`Storage disk "${this.name}" does not expose local filesystem paths.`);
+	}
+}

package/src/storage/drivers/FlystorageListing.ts

@@ -0,0 +1,78 @@
+import type { StatEntry } from '@flystorage/file-storage';
+
+import type { StorageEntry, StorageListing } from '../contracts';
+
+/**
+ * Normalizes a Flystorage directory listing behind Platform's storage contract.
+ */
+export class FlystorageListing implements StorageListing {
+	/**
+	 * Creates a Platform listing over a Flystorage stat-entry source.
+	 *
+	 * @param listing - Lazy Flystorage directory entries.
+	 */
+	constructor(private readonly listing: AsyncIterable<StatEntry>) {}
+
+	/**
+	 * Collects all remaining entries from the underlying listing.
+	 *
+	 * @param sorted - Whether entries should be sorted by path.
+	 * @returns Normalized storage entries.
+	 */
+	async toArray(sorted = true): Promise<StorageEntry[]> {
+		const entries: StorageEntry[] = [];
+
+		for await (const entry of this) {
+			entries.push(entry);
+		}
+
+		if (sorted) {
+			entries.sort((left, right) => left.path.localeCompare(right.path));
+		}
+
+		return entries;
+	}
+
+	/**
+	 * Streams normalized storage entries from the underlying listing.
+	 *
+	 * @returns Async generator of normalized storage entries.
+	 */
+	async *[Symbol.asyncIterator](): AsyncGenerator<StorageEntry> {
+		for await (const entry of this.listing) {
+			yield storageEntry(entry);
+		}
+	}
+}
+
+/**
+ * Converts one Flystorage stat entry into Platform's provider-neutral shape.
+ *
+ * @param entry - Flystorage file or directory metadata.
+ * @returns Normalized Platform storage entry.
+ */
+function storageEntry(entry: StatEntry): StorageEntry {
+	const lastModified = entry.lastModifiedMs === undefined
+		? undefined
+		: new Date(entry.lastModifiedMs);
+
+	if (entry.type === 'file') {
+		return {
+			type: 'file',
+			isFile: true,
+			isDirectory: false,
+			path: entry.path,
+			lastModified,
+			size: entry.size,
+			mimeType: entry.mimeType,
+		};
+	}
+
+	return {
+		type: 'directory',
+		isFile: false,
+		isDirectory: true,
+		path: entry.path,
+		lastModified,
+	};
+}

package/src/storage/drivers/LocalStorageDisk.ts

@@ -0,0 +1,45 @@
+import { FileStorage } from '@flystorage/file-storage';
+import { LocalStorageAdapter } from '@flystorage/local-fs';
+import { resolve } from 'node:path';
+
+import { resolveLocalStoragePath } from '../path';
+import type { LocalStorageDiskConfig } from '../types';
+import { FlystorageDisk } from './FlystorageDisk';
+
+/**
+ * Local filesystem storage disk.
+ *
+ * Paths are always relative to the configured disk root. The disk rejects empty,
+ * absolute, and parent-directory paths before touching the filesystem.
+ */
+export class LocalStorageDisk extends FlystorageDisk {
+	private readonly root: string;
+
+	/**
+	 * Creates a local disk backed by Flystorage's local filesystem adapter.
+	 *
+	 * @param name - Configured disk name.
+	 * @param config - Local disk configuration.
+	 */
+	constructor(name: string, config: LocalStorageDiskConfig = { driver: 'local' }) {
+		const root = resolve(config.root ?? 'storage/app');
+
+		super(name, new FileStorage(new LocalStorageAdapter(root, {
+			publicUrlOptions: {
+				baseUrl: config.url?.replace(/\/+$/, ''),
+			},
+		})));
+
+		this.root = root;
+	}
+
+	/**
+	 * Returns the absolute local filesystem path for a storage path.
+	 *
+	 * @param path - Relative storage path to resolve.
+	 * @returns Absolute filesystem path inside the disk root.
+	 */
+	path(path: string): string {
+		return resolveLocalStoragePath(this.root, path);
+	}
+}

package/src/storage/drivers/S3StorageDisk.ts

@@ -0,0 +1,114 @@
+import { S3Client, type S3ClientConfig } from '@aws-sdk/client-s3';
+import { AwsS3StorageAdapter } from '@flystorage/aws-s3';
+import { FileStorage } from '@flystorage/file-storage';
+
+import type { S3StorageDiskConfig } from '../types';
+import { FlystorageDisk } from './FlystorageDisk';
+
+/**
+ * S3-compatible object storage disk.
+ *
+ * This driver supports AWS S3 plus providers that implement the S3 protocol,
+ * such as DigitalOcean Spaces, by using a custom endpoint.
+ */
+export class S3StorageDisk extends FlystorageDisk {
+	/**
+	 * Creates an S3-compatible disk backed by Flystorage's AWS S3 adapter.
+	 *
+	 * @param name - Configured disk name.
+	 * @param config - S3 disk configuration.
+	 */
+	constructor(name: string, config: S3StorageDiskConfig) {
+		const bucket = requiredS3Option(config.bucket, 'bucket');
+		const client = new S3Client(s3ClientConfig(config));
+
+		super(name, new FileStorage(new AwsS3StorageAdapter(client, {
+			bucket,
+			prefix: trimSlashes(config.prefix),
+			region: config.region,
+			publicUrlOptions: {
+				baseUrl: publicUrlBase(config.url),
+				forcePathStyle: config.forcePathStyle,
+				region: config.region,
+			} as Record<string, unknown>,
+		})));
+	}
+}
+
+/**
+ * Builds AWS SDK S3 client config from a storage disk config.
+ *
+ * @param config - S3 disk configuration.
+ * @returns AWS SDK S3 client config.
+ */
+function s3ClientConfig(config: S3StorageDiskConfig): S3ClientConfig {
+	return {
+		region: config.region ?? 'us-east-1',
+		endpoint: config.endpoint,
+		forcePathStyle: config.forcePathStyle,
+		credentials: credentials(config),
+	};
+}
+
+/**
+ * Resolves optional static S3 credentials from disk config.
+ *
+ * @param config - S3 disk configuration.
+ * @returns AWS SDK credentials or undefined for default provider resolution.
+ */
+function credentials(config: S3StorageDiskConfig): S3ClientConfig['credentials'] {
+	const accessKeyId = config.accessKeyId ?? config.key;
+	const secretAccessKey = config.secretAccessKey ?? config.secret;
+
+	if (!accessKeyId && !secretAccessKey) return undefined;
+
+	if (!accessKeyId || !secretAccessKey) {
+		throw new Error('S3 storage disk requires both accessKeyId and secretAccessKey when either credential is configured.');
+	}
+
+	return {
+		accessKeyId,
+		secretAccessKey,
+		sessionToken: config.sessionToken,
+	};
+}
+
+/**
+ * Returns a required S3 string option.
+ *
+ * @param value - Option value.
+ * @param name - Option name used in error messages.
+ * @returns Required non-empty string.
+ */
+function requiredS3Option(value: string | undefined, name: string): string {
+	if (!value?.trim()) {
+		throw new Error(`S3 storage disk requires a ${name}.`);
+	}
+
+	return value;
+}
+
+/**
+ * Removes leading and trailing slashes from a configured prefix.
+ *
+ * @param value - Prefix value to normalize.
+ * @returns Normalized prefix or undefined.
+ */
+function trimSlashes(value: string | undefined): string | undefined {
+	const normalized = value?.replace(/^\/+|\/+$/g, '');
+
+	return normalized ? normalized : undefined;
+}
+
+/**
+ * Converts a configured public base URL into Flystorage's URL template shape.
+ *
+ * @param value - Configured public URL or template.
+ * @returns URL template with a path placeholder.
+ */
+function publicUrlBase(value: string | undefined): string | undefined {
+	if (!value) return undefined;
+	if (value.includes('{uri}')) return value;
+
+	return `${value.replace(/\/+$/, '')}/{uri}`;
+}

package/src/storage/drivers/index.ts

@@ -0,0 +1,3 @@
+export * from './FlystorageDisk';
+export * from './LocalStorageDisk';
+export * from './S3StorageDisk';

package/src/storage/examples/runStorage.ts

@@ -0,0 +1,37 @@
+import { mkdtemp, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { Readable } from 'node:stream';
+import { pathToFileURL } from 'node:url';
+import { App } from '@db3.ai/app/server';
+
+/**
+ * Writes, reads, lists and removes an export on an isolated real local disk.
+ *
+ * This runnable lab owns its temporary directory, not application storage.
+ * The same storage calls can use a configured app().storage in feature code.
+ *
+ * @returns Stable observations; no temporary absolute path is exposed.
+ */
+export async function runStorage() {
+	const root = await mkdtemp(join(tmpdir(), 'db3-storage-guide-'));
+	const application = new App({ storage: { default: 'exports', disks: { exports: { driver: 'local', root } } } });
+	try {
+		const disk = application.storage.disk('exports');
+		await disk.write('reports/latest.json', JSON.stringify({ status: 'ready' }), { mimeType: 'application/json' });
+		const report = JSON.parse(await disk.readToString('reports/latest.json'));
+		await disk.writeStream('reports/export.csv', Readable.from(['name\n', 'Ada\n']), { mimeType: 'text/csv' });
+		const csv = await disk.readToString('reports/export.csv');
+		const files = (await disk.list('reports').toArray()).filter(entry => entry.isFile).map(entry => entry.path);
+		let traversalRejected = false;
+		try { await disk.write('../outside.txt', 'not allowed'); } catch { traversalRejected = true; }
+		await disk.delete('reports/latest.json');
+		return { report, csv, files, traversalRejected, deleted: await disk.missing('reports/latest.json') };
+	} finally {
+		try { await application.close(); } finally { await rm(root, { recursive: true, force: true }); }
+	}
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+	console.log(JSON.stringify(await runStorage(), null, 2));
+}

package/src/storage/examples/runStreamExport.ts

@@ -0,0 +1,41 @@
+import { mkdtemp, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { Writable } from 'node:stream';
+import { pipeline } from 'node:stream/promises';
+import { pathToFileURL } from 'node:url';
+import { Storage } from '@db3.ai/app/storage';
+import { writeCsvExport } from './writeCsvExport';
+
+/**
+ * Exercises streamed file publication, interrupted-write cleanup and retry.
+ * @returns Counts and cleanup outcomes without buffering the complete export.
+ */
+export async function runStreamExport() {
+	const root = await mkdtemp(join(tmpdir(), 'db3-stream-guide-'));
+	const disk = new Storage({ disks: { local: { driver: 'local', root } } }).disk('local');
+	try {
+		await writeCsvExport(disk, 'exports/latest.csv', { rows: 50_000 });
+		const sizeBefore = await disk.size('exports/latest.csv');
+		let interruptionRejected = false;
+		try { await writeCsvExport(disk, 'exports/latest.csv', { rows: 10, failAt: 2 }); } catch (error) {
+			// Drivers may wrap failures while preserving the original stream error.
+			const cause = error instanceof Error && error.cause instanceof Error ? error.cause : error;
+			if (!(cause instanceof Error) || cause.message !== 'Controlled export interruption') throw error;
+			interruptionRejected = true;
+		}
+		const previousPreserved = await disk.size('exports/latest.csv') === sizeBefore;
+		const pendingFiles = (await disk.list('pending').toArray()).filter(entry => entry.isFile).length;
+		await writeCsvExport(disk, 'exports/repaired.csv', { rows: 3 });
+		let lines = 0;
+		let bytes = 0;
+		const sink = new Writable({
+			/** Counts each chunk without retaining the complete file. */
+			write(chunk: Buffer, _encoding, callback) { bytes += chunk.length; for (const byte of chunk) if (byte === 10) lines++; callback(); },
+		});
+		await pipeline(await disk.readStream('exports/latest.csv'), sink);
+		return { lines, complete: bytes === sizeBefore, interruptionRejected, previousPreserved, pendingFiles, repaired: await disk.exists('exports/repaired.csv') };
+	} finally { await rm(root, { recursive: true, force: true }); }
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) console.log(JSON.stringify(await runStreamExport(), null, 2));

package/src/storage/examples/writeCsvExport.ts

@@ -0,0 +1,45 @@
+import { randomUUID } from 'node:crypto';
+import { Readable } from 'node:stream';
+import type { StorageDisk } from '@db3.ai/app/storage';
+
+/** Options for a bounded-memory CSV export and deliberate failure in the lab. */
+export interface CsvExportOptions {
+	/** Number of synthetic rows to generate without retaining all rows in memory. */
+	rows: number;
+	/** Lab-only failure point; omit for a normal export. */
+	failAt?: number;
+}
+
+/**
+ * Streams a generated CSV to a staging path, publishing only after success.
+ *
+ * The staging path is owned by this operation. A failed stream removes that
+ * object and leaves the previous destination untouched. Cross-provider moves
+ * are not assumed atomic; concurrent publication needs an application policy.
+ *
+ * @param disk - Application-owned disk selected before the operation.
+ * @param destination - Authorized relative output path, not raw request input.
+ * @param options - Row count and optional controlled failure point.
+ */
+export async function writeCsvExport(disk: StorageDisk, destination: string, options: CsvExportOptions): Promise<void> {
+	if (!Number.isInteger(options.rows) || options.rows < 0 || options.rows > 1_000_000) throw new Error('rows must be an integer between 0 and 1000000');
+	const staging = `pending/${randomUUID()}.csv`;
+	const stream = Readable.from(csvRows(options));
+	try {
+		await disk.writeStream(staging, stream, { mimeType: 'text/csv', visibility: 'private' });
+		await disk.move(staging, destination);
+	} catch (error) {
+		stream.destroy();
+		if (await disk.exists(staging)) await disk.delete(staging);
+		throw error;
+	}
+}
+
+/** Produces one safe synthetic CSV row at a time; real exports must escape data. */
+function* csvRows(options: CsvExportOptions): Generator<string> {
+	yield 'id,value\n';
+	for (let index = 0; index < options.rows; index++) {
+		if (index === options.failAt) throw new Error('Controlled export interruption');
+		yield `${index},note-${index}\n`;
+	}
+}

package/src/storage/index.ts

@@ -0,0 +1,7 @@
+export * from './Storage';
+export * from './contents';
+export * from './contracts';
+export * from './drivers';
+export * from './mime';
+export * from './path';
+export * from './types';