@rebasepro/server-core 0.0.1-canary.09e5ec5

package/src/storage/LocalStorageController.ts

@@ -0,0 +1,365 @@
+/**
+ * Local filesystem storage controller
+ */
+
+import * as fs from "fs";
+import * as path from "path";
+import { promisify } from "util";
+import {
+    StorageController,
+    LocalStorageConfig,
+    DEFAULT_MAX_FILE_SIZE
+} from "./types";
+import {
+    UploadFileProps,
+    UploadFileResult,
+    DownloadConfig,
+    DownloadMetadata,
+    StorageListResult,
+    StorageReference
+} from "@rebasepro/types";
+
+const mkdir = promisify(fs.mkdir);
+const writeFile = promisify(fs.writeFile);
+const readFile = promisify(fs.readFile);
+const unlink = promisify(fs.unlink);
+const readdir = promisify(fs.readdir);
+const stat = promisify(fs.stat);
+const access = promisify(fs.access);
+
+/**
+ * Remove initial and trailing slashes from a path.
+ * Handles paths like "/images/", "images/", "/images" → "images"
+ */
+function normalizeStoragePath(s: string): string {
+    let result = s;
+    while (result.startsWith("/")) {
+        result = result.slice(1);
+    }
+    while (result.endsWith("/")) {
+        result = result.slice(0, -1);
+    }
+    return result;
+}
+
+/**
+ * Local filesystem storage implementation
+ * Stores files in a directory structure: {basePath}/{bucket}/{path}
+ */
+export class LocalStorageController implements StorageController {
+    private config: LocalStorageConfig;
+    private basePath: string;
+
+    constructor(config: LocalStorageConfig) {
+        this.config = config;
+        this.basePath = path.resolve(config.basePath);
+    }
+
+    getType(): "local" {
+        return "local";
+    }
+
+    /**
+     * Ensure directory exists, creating it if necessary
+     */
+    private async ensureDir(dirPath: string): Promise<void> {
+        try {
+            await mkdir(dirPath, { recursive: true });
+        } catch (error: unknown) {
+            if (error instanceof Error && (error as NodeJS.ErrnoException).code !== "EEXIST") {
+                throw error;
+            }
+        }
+    }
+
+    /**
+     * Get the full filesystem path for a storage path.
+     * Includes a path traversal guard to prevent escaping the base directory.
+     */
+    private getFullPath(storagePath: string, bucket?: string): string {
+        const parts = bucket ? [this.basePath, bucket, storagePath] : [this.basePath, storagePath];
+        const resolved = path.resolve(path.join(...parts));
+        if (!resolved.startsWith(this.basePath + path.sep) && resolved !== this.basePath) {
+            throw new Error("Path traversal detected: resolved storage path is outside the base directory.");
+        }
+        return resolved;
+    }
+
+    /**
+     * Validate file before upload
+     */
+    private validateFile(file: File): void {
+        const maxSize = this.config.maxFileSize ?? DEFAULT_MAX_FILE_SIZE;
+        if (file.size > maxSize) {
+            throw new Error(`File size ${file.size} exceeds maximum allowed size ${maxSize}`);
+        }
+
+        if (this.config.allowedMimeTypes && this.config.allowedMimeTypes.length > 0) {
+            if (!this.config.allowedMimeTypes.includes(file.type)) {
+                throw new Error(`File type ${file.type} is not allowed. Allowed types: ${this.config.allowedMimeTypes.join(", ")}`);
+            }
+        }
+    }
+
+    async putObject({
+        file,
+        key,
+        metadata,
+        bucket
+    }: UploadFileProps): Promise<UploadFileResult> {
+        this.validateFile(file);
+
+        // Always use a bucket (default to 'default')
+        const usedBucket = bucket ?? "default";
+        const fullStoragePath = key;
+        const fullPath = this.getFullPath(fullStoragePath, usedBucket);
+
+        // Ensure parent directory exists
+        await this.ensureDir(path.dirname(fullPath));
+
+        // Convert File to Buffer and write
+        const arrayBuffer = await file.arrayBuffer();
+        const buffer = Buffer.from(arrayBuffer);
+        await writeFile(fullPath, buffer);
+
+        // Always save metadata file with at least contentType (required for preview)
+        const metadataPath = `${fullPath}.metadata.json`;
+        await writeFile(metadataPath, JSON.stringify({
+            ...(metadata || {}),
+            contentType: file.type,
+            size: file.size,
+            uploadedAt: new Date().toISOString()
+        }, null, 2));
+
+        return {
+            key: fullStoragePath,
+            bucket: usedBucket,
+            storageUrl: `local://${usedBucket}/${fullStoragePath}`
+        };
+    }
+
+    async getSignedUrl(key: string, bucket?: string): Promise<DownloadConfig> {
+        // Handle local:// URLs
+        let resolvedPath = key;
+        let resolvedBucket = bucket;
+
+        if (key.startsWith("local://")) {
+            const withoutProtocol = key.substring("local://".length);
+            const firstSlash = withoutProtocol.indexOf("/");
+            if (firstSlash > 0) {
+                resolvedBucket = withoutProtocol.substring(0, firstSlash);
+                resolvedPath = withoutProtocol.substring(firstSlash + 1);
+            }
+        }
+
+        // Normalize path to handle leading/trailing slashes
+        resolvedPath = normalizeStoragePath(resolvedPath);
+        const fullPath = this.getFullPath(resolvedPath, resolvedBucket);
+
+        try {
+            await access(fullPath, fs.constants.R_OK);
+        } catch {
+            return {
+                url: null,
+                fileNotFound: true
+            };
+        }
+
+        // Read metadata if available
+        let metadata: DownloadMetadata | undefined;
+        const metadataPath = `${fullPath}.metadata.json`;
+        try {
+            const metadataContent = await readFile(metadataPath, "utf-8");
+            const savedMetadata = JSON.parse(metadataContent);
+            const fileStat = await stat(fullPath);
+
+            metadata = {
+                bucket: resolvedBucket ?? "default",
+                fullPath: resolvedPath,
+                name: path.basename(resolvedPath),
+                size: fileStat.size,
+                contentType: savedMetadata.contentType || "application/octet-stream",
+                customMetadata: savedMetadata
+            };
+        } catch {
+            // No metadata file, create basic metadata from stat
+            try {
+                const fileStat = await stat(fullPath);
+                metadata = {
+                    bucket: resolvedBucket ?? "default",
+                    fullPath: resolvedPath,
+                    name: path.basename(resolvedPath),
+                    size: fileStat.size,
+                    contentType: "application/octet-stream",
+                    customMetadata: {}
+                };
+            } catch {
+                // Stat failed
+            }
+        }
+
+        // Return a relative URL that will be served by the storage routes
+        const bucketPath = resolvedBucket ? `${resolvedBucket}/` : "";
+        const url = `/api/storage/file/${bucketPath}${resolvedPath}`;
+
+        return {
+            url,
+            metadata
+        };
+    }
+
+    async getObject(key: string, bucket?: string): Promise<File | null> {
+        // Handle local:// URLs
+        let resolvedPath = key;
+        let resolvedBucket = bucket;
+
+        if (key.startsWith("local://")) {
+            const withoutProtocol = key.substring("local://".length);
+            const firstSlash = withoutProtocol.indexOf("/");
+            if (firstSlash > 0) {
+                resolvedBucket = withoutProtocol.substring(0, firstSlash);
+                resolvedPath = withoutProtocol.substring(firstSlash + 1);
+            }
+        }
+
+        // Normalize path to handle leading/trailing slashes
+        resolvedPath = normalizeStoragePath(resolvedPath);
+        const fullPath = this.getFullPath(resolvedPath, resolvedBucket);
+
+        try {
+            await access(fullPath, fs.constants.R_OK);
+            const buffer = await readFile(fullPath);
+
+            // Try to get content type from metadata
+            let contentType = "application/octet-stream";
+            try {
+                const metadataPath = `${fullPath}.metadata.json`;
+                const metadataContent = await readFile(metadataPath, "utf-8");
+                const metadata = JSON.parse(metadataContent);
+                contentType = metadata.contentType || contentType;
+            } catch {
+                // No metadata, use default content type
+            }
+
+            const blob = new Blob([buffer], { type: contentType });
+            return new File([blob], path.basename(resolvedPath), { type: contentType });
+        } catch {
+            return null;
+        }
+    }
+
+    async deleteObject(key: string, bucket?: string): Promise<void> {
+        // Handle local:// URLs
+        let resolvedPath = key;
+        let resolvedBucket = bucket;
+
+        if (key.startsWith("local://")) {
+            const withoutProtocol = key.substring("local://".length);
+            const firstSlash = withoutProtocol.indexOf("/");
+            if (firstSlash > 0) {
+                resolvedBucket = withoutProtocol.substring(0, firstSlash);
+                resolvedPath = withoutProtocol.substring(firstSlash + 1);
+            }
+        }
+
+        // Normalize path to handle leading/trailing slashes
+        resolvedPath = normalizeStoragePath(resolvedPath);
+        const fullPath = this.getFullPath(resolvedPath, resolvedBucket);
+
+        try {
+            await unlink(fullPath);
+            // Also delete metadata file if exists
+            try {
+                await unlink(`${fullPath}.metadata.json`);
+            } catch {
+                // Metadata file might not exist
+            }
+        } catch (error: unknown) {
+            if (error instanceof Error && (error as NodeJS.ErrnoException).code !== "ENOENT") {
+                throw error;
+            }
+            // File doesn't exist, nothing to delete
+        }
+    }
+
+    async listObjects(prefix: string, options?: {
+        bucket?: string;
+        maxResults?: number;
+        pageToken?: string;
+    }): Promise<StorageListResult> {
+        // Normalize path to handle leading/trailing slashes
+        const normalizedPath = normalizeStoragePath(prefix);
+        const fullPath = this.getFullPath(normalizedPath, options?.bucket);
+        const items: StorageReference[] = [];
+        const prefixes: StorageReference[] = [];
+
+        try {
+            await access(fullPath, fs.constants.R_OK);
+            const entries = await readdir(fullPath, { withFileTypes: true });
+
+            let count = 0;
+            const maxResults = options?.maxResults ?? 1000;
+            const startIndex = options?.pageToken ? parseInt(options.pageToken, 10) : 0;
+
+            for (let i = startIndex; i < entries.length && count < maxResults; i++) {
+                const entry = entries[i];
+
+                // Skip metadata files
+                if (entry.name.endsWith(".metadata.json")) {
+                    continue;
+                }
+
+                const entryPath = prefix ? `${prefix}/${entry.name}` : entry.name;
+                const bucket = options?.bucket ?? "default";
+
+                const ref: StorageReference = {
+                    bucket,
+                    fullPath: entryPath,
+                    name: entry.name,
+                    parent: null as never, // Simplified - not fully implementing parent chain
+                    root: null as never,
+                    toString: () => `local://${bucket}/${entryPath}`
+                };
+
+                if (entry.isDirectory()) {
+                    prefixes.push(ref);
+                } else {
+                    items.push(ref);
+                }
+                count++;
+            }
+
+            const nextPageToken = startIndex + count < entries.length
+                ? String(startIndex + count)
+                : undefined;
+
+            return {
+                items,
+                prefixes,
+                nextPageToken
+            };
+        } catch (error: unknown) {
+            const code = (error as NodeJS.ErrnoException)?.code;
+            if (code === "ENOENT" || code === "ENOTDIR") {
+                return { items: [],
+prefixes: [] };
+            }
+            throw error;
+        }
+    }
+
+    /**
+     * Get the absolute filesystem path for serving files
+     * Used by the storage routes to serve files directly
+     */
+    getAbsolutePath(key: string, bucket?: string): string {
+        return this.getFullPath(key, bucket);
+    }
+
+    /**
+     * Get the base path for the storage
+     */
+    getBasePath(): string {
+        return this.basePath;
+    }
+}

package/src/storage/S3StorageController.ts

@@ -0,0 +1,298 @@
+/**
+ * S3-compatible storage controller (works with AWS S3 and MinIO)
+ */
+
+import {
+    S3Client,
+    PutObjectCommand,
+    GetObjectCommand,
+    DeleteObjectCommand,
+    ListObjectsV2Command,
+    HeadObjectCommand,
+    _Object,
+    CommonPrefix
+} from "@aws-sdk/client-s3";
+import { getSignedUrl } from "@aws-sdk/s3-request-presigner";
+import {
+    StorageController,
+    S3StorageConfig,
+    DEFAULT_MAX_FILE_SIZE
+} from "./types";
+import {
+    UploadFileProps,
+    UploadFileResult,
+    DownloadConfig,
+    DownloadMetadata,
+    StorageListResult,
+    StorageReference
+} from "@rebasepro/types";
+
+/**
+ * S3-compatible storage implementation
+ * Works with AWS S3 and MinIO (with forcePathStyle option)
+ */
+export class S3StorageController implements StorageController {
+    private config: S3StorageConfig;
+    private client: S3Client;
+
+    constructor(config: S3StorageConfig) {
+        this.config = config;
+        this.client = new S3Client({
+            region: config.region || "us-east-1",
+            endpoint: config.endpoint,
+            forcePathStyle: config.forcePathStyle ?? !!config.endpoint, // Auto-enable for custom endpoints (MinIO)
+            credentials: {
+                accessKeyId: config.accessKeyId,
+                secretAccessKey: config.secretAccessKey
+            }
+        });
+    }
+
+    getType(): "s3" {
+        return "s3";
+    }
+
+    /**
+     * Validate file before upload
+     */
+    private validateFile(file: File): void {
+        const maxSize = this.config.maxFileSize ?? DEFAULT_MAX_FILE_SIZE;
+        if (file.size > maxSize) {
+            throw new Error(`File size ${file.size} exceeds maximum allowed size ${maxSize}`);
+        }
+
+        if (this.config.allowedMimeTypes && this.config.allowedMimeTypes.length > 0) {
+            if (!this.config.allowedMimeTypes.includes(file.type)) {
+                throw new Error(`File type ${file.type} is not allowed. Allowed types: ${this.config.allowedMimeTypes.join(", ")}`);
+            }
+        }
+    }
+
+    /**
+     * Get the bucket name - either from parameter or config
+     */
+    private getBucket(bucket?: string): string {
+        return bucket ?? this.config.bucket;
+    }
+
+    async putObject({
+        file,
+        key,
+        metadata,
+        bucket
+    }: UploadFileProps): Promise<UploadFileResult> {
+        this.validateFile(file);
+
+        const usedBucket = this.getBucket(bucket);
+
+        // Convert File to Buffer
+        const arrayBuffer = await file.arrayBuffer();
+        const buffer = Buffer.from(arrayBuffer);
+
+        const command = new PutObjectCommand({
+            Bucket: usedBucket,
+            Key: key,
+            Body: buffer,
+            ContentType: file.type,
+            Metadata: metadata ? this.flattenMetadata(metadata) : undefined
+        });
+
+        await this.client.send(command);
+
+        return {
+            key,
+            bucket: usedBucket,
+            storageUrl: `s3://${usedBucket}/${key}`
+        };
+    }
+
+    /**
+     * Flatten nested metadata to string values (S3 requirement)
+     */
+    private flattenMetadata(metadata: Record<string, unknown>): Record<string, string> {
+        const flattened: Record<string, string> = {};
+        for (const [key, value] of Object.entries(metadata)) {
+            if (typeof value === "string") {
+                flattened[key] = value;
+            } else if (value !== undefined && value !== null) {
+                flattened[key] = JSON.stringify(value);
+            }
+        }
+        return flattened;
+    }
+
+    async getSignedUrl(key: string, bucket?: string): Promise<DownloadConfig> {
+        // Handle s3:// and gs:// URLs for backward compatibility
+        let resolvedPath = key;
+        let resolvedBucket = this.getBucket(bucket);
+
+        const match = key.match(/^(s3|gs):\/\//);
+        if (match) {
+            const protocolLength = match[0].length;
+            const withoutProtocol = key.substring(protocolLength);
+            const firstSlash = withoutProtocol.indexOf("/");
+            if (firstSlash > 0) {
+                resolvedBucket = withoutProtocol.substring(0, firstSlash);
+                resolvedPath = withoutProtocol.substring(firstSlash + 1);
+            }
+        }
+
+        try {
+            // First check if the object exists and get metadata
+            const headCommand = new HeadObjectCommand({
+                Bucket: resolvedBucket,
+                Key: resolvedPath
+            });
+
+            const headResult = await this.client.send(headCommand);
+
+            // Generate a signed URL
+            const getCommand = new GetObjectCommand({
+                Bucket: resolvedBucket,
+                Key: resolvedPath
+            });
+
+            const expiresIn = this.config.signedUrlExpiration ?? 3600;
+            const url = await getSignedUrl(this.client, getCommand, { expiresIn });
+
+            const metadata: DownloadMetadata = {
+                bucket: resolvedBucket,
+                fullPath: resolvedPath,
+                name: resolvedPath.split("/").pop() || resolvedPath,
+                size: headResult.ContentLength || 0,
+                contentType: headResult.ContentType || "application/octet-stream",
+                customMetadata: headResult.Metadata || {}
+            };
+
+            return {
+                url,
+                metadata
+            };
+        } catch (error: unknown) {
+            const s3Error = error as { name?: string; $metadata?: { httpStatusCode?: number } };
+            if (s3Error.name === "NotFound" || s3Error.$metadata?.httpStatusCode === 404) {
+                return {
+                    url: null,
+                    fileNotFound: true
+                };
+            }
+            throw error;
+        }
+    }
+
+    async getObject(key: string, bucket?: string): Promise<File | null> {
+        // Handle s3:// and gs:// URLs
+        let resolvedPath = key;
+        let resolvedBucket = this.getBucket(bucket);
+
+        const match = key.match(/^(s3|gs):\/\//);
+        if (match) {
+            const protocolLength = match[0].length;
+            const withoutProtocol = key.substring(protocolLength);
+            const firstSlash = withoutProtocol.indexOf("/");
+            if (firstSlash > 0) {
+                resolvedBucket = withoutProtocol.substring(0, firstSlash);
+                resolvedPath = withoutProtocol.substring(firstSlash + 1);
+            }
+        }
+
+        try {
+            const command = new GetObjectCommand({
+                Bucket: resolvedBucket,
+                Key: resolvedPath
+            });
+
+            const response = await this.client.send(command);
+
+            if (!response.Body) {
+                return null;
+            }
+
+            // Convert stream to buffer
+            const chunks: Uint8Array[] = [];
+            // @ts-ignore - Body is a ReadableStream in Node.js
+            for await (const chunk of response.Body) {
+                chunks.push(chunk);
+            }
+            const buffer = Buffer.concat(chunks);
+
+            const contentType = response.ContentType || "application/octet-stream";
+            const fileName = resolvedPath.split("/").pop() || resolvedPath;
+
+            const blob = new Blob([buffer], { type: contentType });
+            return new File([blob], fileName, { type: contentType });
+        } catch (error: unknown) {
+            const s3Error = error as { name?: string; $metadata?: { httpStatusCode?: number } };
+            if (s3Error.name === "NoSuchKey" || s3Error.$metadata?.httpStatusCode === 404) {
+                return null;
+            }
+            throw error;
+        }
+    }
+
+    async deleteObject(key: string, bucket?: string): Promise<void> {
+        // Handle s3:// and gs:// URLs
+        let resolvedPath = key;
+        let resolvedBucket = this.getBucket(bucket);
+
+        const match = key.match(/^(s3|gs):\/\//);
+        if (match) {
+            const protocolLength = match[0].length;
+            const withoutProtocol = key.substring(protocolLength);
+            const firstSlash = withoutProtocol.indexOf("/");
+            if (firstSlash > 0) {
+                resolvedBucket = withoutProtocol.substring(0, firstSlash);
+                resolvedPath = withoutProtocol.substring(firstSlash + 1);
+            }
+        }
+
+        const command = new DeleteObjectCommand({
+            Bucket: resolvedBucket,
+            Key: resolvedPath
+        });
+
+        await this.client.send(command);
+    }
+
+    async listObjects(prefix: string, options?: {
+        bucket?: string;
+        maxResults?: number;
+        pageToken?: string;
+    }): Promise<StorageListResult> {
+        const resolvedBucket = this.getBucket(options?.bucket);
+
+        const command = new ListObjectsV2Command({
+            Bucket: resolvedBucket,
+            Prefix: prefix || undefined,
+            MaxKeys: options?.maxResults ?? 1000,
+            ContinuationToken: options?.pageToken,
+            Delimiter: "/" // This gives us folder-like behavior
+        });
+
+        const response = await this.client.send(command);
+
+        const items: StorageReference[] = (response.Contents || []).map(obj => ({
+            bucket: resolvedBucket,
+            fullPath: obj.Key || "",
+            name: (obj.Key || "").split("/").pop() || "",
+            parent: null as never,
+            root: null as never,
+            toString: () => `s3://${resolvedBucket}/${obj.Key}`
+        }));
+
+        const prefixes: StorageReference[] = (response.CommonPrefixes || []).map(prefix => ({
+            bucket: resolvedBucket,
+            fullPath: prefix.Prefix || "",
+            name: (prefix.Prefix || "").replace(/\/$/, "").split("/").pop() || "",
+            parent: null as never,
+            root: null as never,
+            toString: () => `s3://${resolvedBucket}/${prefix.Prefix}`
+        }));
+
+        return {
+            items,
+            prefixes,
+            nextPageToken: response.NextContinuationToken
+        };
+    }
+}

package/src/storage/index.ts

@@ -0,0 +1,43 @@
+/**
+ * Storage module for Rebase backend
+ *
+ * Provides pluggable file storage with two built-in providers:
+ * - **Local filesystem** — zero config, great for dev and single-server deployments.
+ * - **S3-compatible** — works with AWS S3, Cloudflare R2, MinIO, Hetzner Object Storage,
+ *   Backblaze B2, DigitalOcean Spaces, and GCS (via S3 interop).
+ *
+ * For other providers (native GCS SDK, Azure Blob, etc.), implement the
+ * `StorageController` interface and pass the instance directly to the `storage` config.
+ */
+
+export * from "./types";
+export { LocalStorageController } from "./LocalStorageController";
+export { S3StorageController } from "./S3StorageController";
+export { createStorageRoutes } from "./routes";
+export type { StorageRoutesConfig } from "./routes";
+export * from "./storage-registry";
+
+import { BackendStorageConfig, StorageController } from "./types";
+import { LocalStorageController } from "./LocalStorageController";
+import { S3StorageController } from "./S3StorageController";
+
+/**
+ * Create a storage controller from a config object.
+ *
+ * For custom providers, implement `StorageController` directly instead
+ * of going through this factory.
+ */
+export function createStorageController(config: BackendStorageConfig): StorageController {
+    switch (config.type) {
+        case "local":
+            return new LocalStorageController(config);
+        case "s3":
+            return new S3StorageController(config);
+        default:
+            throw new Error(
+                `Unknown storage type: ${(config as Record<string, unknown>).type}. ` +
+                "Built-in types: local, s3. " +
+                "For other providers, implement the StorageController interface directly."
+            );
+    }
+}