@rebasepro/server-core 0.0.1-canary.4d4fb3e

package/src/storage/S3StorageController.ts

@@ -0,0 +1,295 @@
+/**
+ * S3-compatible storage controller (works with AWS S3 and MinIO)
+ */
+
+import {
+    S3Client,
+    PutObjectCommand,
+    GetObjectCommand,
+    DeleteObjectCommand,
+    ListObjectsV2Command,
+    HeadObjectCommand,
+    _Object,
+    CommonPrefix
+} from '@aws-sdk/client-s3';
+import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
+import {
+    StorageController,
+    S3StorageConfig,
+    DEFAULT_MAX_FILE_SIZE
+} from './types';
+import {
+    UploadFileProps,
+    UploadFileResult,
+    DownloadConfig,
+    DownloadMetadata,
+    StorageListResult,
+    StorageReference
+} from '@rebasepro/types';
+
+/**
+ * S3-compatible storage implementation
+ * Works with AWS S3 and MinIO (with forcePathStyle option)
+ */
+export class S3StorageController implements StorageController {
+    private config: S3StorageConfig;
+    private client: S3Client;
+
+    constructor(config: S3StorageConfig) {
+        this.config = config;
+        this.client = new S3Client({
+            region: config.region || 'us-east-1',
+            endpoint: config.endpoint,
+            forcePathStyle: config.forcePathStyle ?? !!config.endpoint, // Auto-enable for custom endpoints (MinIO)
+            credentials: {
+                accessKeyId: config.accessKeyId,
+                secretAccessKey: config.secretAccessKey
+            }
+        });
+    }
+
+    getType(): 's3' {
+        return 's3';
+    }
+
+    /**
+     * Validate file before upload
+     */
+    private validateFile(file: File): void {
+        const maxSize = this.config.maxFileSize ?? DEFAULT_MAX_FILE_SIZE;
+        if (file.size > maxSize) {
+            throw new Error(`File size ${file.size} exceeds maximum allowed size ${maxSize}`);
+        }
+
+        if (this.config.allowedMimeTypes && this.config.allowedMimeTypes.length > 0) {
+            if (!this.config.allowedMimeTypes.includes(file.type)) {
+                throw new Error(`File type ${file.type} is not allowed. Allowed types: ${this.config.allowedMimeTypes.join(', ')}`);
+            }
+        }
+    }
+
+    /**
+     * Get the bucket name - either from parameter or config
+     */
+    private getBucket(bucket?: string): string {
+        return bucket ?? this.config.bucket;
+    }
+
+    async uploadFile({
+        file,
+        fileName,
+        path: storagePath,
+        metadata,
+        bucket
+    }: UploadFileProps): Promise<UploadFileResult> {
+        this.validateFile(file);
+
+        const usedFileName = fileName ?? file.name;
+        const key = storagePath ? `${storagePath}/${usedFileName}` : usedFileName;
+        const usedBucket = this.getBucket(bucket);
+
+        // Convert File to Buffer
+        const arrayBuffer = await file.arrayBuffer();
+        const buffer = Buffer.from(arrayBuffer);
+
+        const command = new PutObjectCommand({
+            Bucket: usedBucket,
+            Key: key,
+            Body: buffer,
+            ContentType: file.type,
+            Metadata: metadata ? this.flattenMetadata(metadata) : undefined
+        });
+
+        await this.client.send(command);
+
+        return {
+            path: key,
+            bucket: usedBucket,
+            storageUrl: `s3://${usedBucket}/${key}`
+        };
+    }
+
+    /**
+     * Flatten nested metadata to string values (S3 requirement)
+     */
+    private flattenMetadata(metadata: Record<string, unknown>): Record<string, string> {
+        const flattened: Record<string, string> = {};
+        for (const [key, value] of Object.entries(metadata)) {
+            if (typeof value === 'string') {
+                flattened[key] = value;
+            } else if (value !== undefined && value !== null) {
+                flattened[key] = JSON.stringify(value);
+            }
+        }
+        return flattened;
+    }
+
+    async getDownloadURL(storagePath: string, bucket?: string): Promise<DownloadConfig> {
+        // Handle s3:// URLs
+        let resolvedPath = storagePath;
+        let resolvedBucket = this.getBucket(bucket);
+
+        if (storagePath.startsWith('s3://')) {
+            const withoutProtocol = storagePath.substring('s3://'.length);
+            const firstSlash = withoutProtocol.indexOf('/');
+            if (firstSlash > 0) {
+                resolvedBucket = withoutProtocol.substring(0, firstSlash);
+                resolvedPath = withoutProtocol.substring(firstSlash + 1);
+            }
+        }
+
+        try {
+            // First check if the object exists and get metadata
+            const headCommand = new HeadObjectCommand({
+                Bucket: resolvedBucket,
+                Key: resolvedPath
+            });
+
+            const headResult = await this.client.send(headCommand);
+
+            // Generate a signed URL
+            const getCommand = new GetObjectCommand({
+                Bucket: resolvedBucket,
+                Key: resolvedPath
+            });
+
+            const expiresIn = this.config.signedUrlExpiration ?? 3600;
+            const url = await getSignedUrl(this.client, getCommand, { expiresIn });
+
+            const metadata: DownloadMetadata = {
+                bucket: resolvedBucket,
+                fullPath: resolvedPath,
+                name: resolvedPath.split('/').pop() || resolvedPath,
+                size: headResult.ContentLength || 0,
+                contentType: headResult.ContentType || 'application/octet-stream',
+                customMetadata: headResult.Metadata || {}
+            };
+
+            return {
+                url,
+                metadata
+            };
+        } catch (error: unknown) {
+            const s3Error = error as { name?: string; $metadata?: { httpStatusCode?: number } };
+            if (s3Error.name === 'NotFound' || s3Error.$metadata?.httpStatusCode === 404) {
+                return {
+                    url: null,
+                    fileNotFound: true
+                };
+            }
+            throw error;
+        }
+    }
+
+    async getFile(storagePath: string, bucket?: string): Promise<File | null> {
+        // Handle s3:// URLs
+        let resolvedPath = storagePath;
+        let resolvedBucket = this.getBucket(bucket);
+
+        if (storagePath.startsWith('s3://')) {
+            const withoutProtocol = storagePath.substring('s3://'.length);
+            const firstSlash = withoutProtocol.indexOf('/');
+            if (firstSlash > 0) {
+                resolvedBucket = withoutProtocol.substring(0, firstSlash);
+                resolvedPath = withoutProtocol.substring(firstSlash + 1);
+            }
+        }
+
+        try {
+            const command = new GetObjectCommand({
+                Bucket: resolvedBucket,
+                Key: resolvedPath
+            });
+
+            const response = await this.client.send(command);
+
+            if (!response.Body) {
+                return null;
+            }
+
+            // Convert stream to buffer
+            const chunks: Uint8Array[] = [];
+            // @ts-ignore - Body is a ReadableStream in Node.js
+            for await (const chunk of response.Body) {
+                chunks.push(chunk);
+            }
+            const buffer = Buffer.concat(chunks);
+
+            const contentType = response.ContentType || 'application/octet-stream';
+            const fileName = resolvedPath.split('/').pop() || resolvedPath;
+
+            const blob = new Blob([buffer], { type: contentType });
+            return new File([blob], fileName, { type: contentType });
+        } catch (error: unknown) {
+            const s3Error = error as { name?: string; $metadata?: { httpStatusCode?: number } };
+            if (s3Error.name === 'NoSuchKey' || s3Error.$metadata?.httpStatusCode === 404) {
+                return null;
+            }
+            throw error;
+        }
+    }
+
+    async deleteFile(storagePath: string, bucket?: string): Promise<void> {
+        // Handle s3:// URLs
+        let resolvedPath = storagePath;
+        let resolvedBucket = this.getBucket(bucket);
+
+        if (storagePath.startsWith('s3://')) {
+            const withoutProtocol = storagePath.substring('s3://'.length);
+            const firstSlash = withoutProtocol.indexOf('/');
+            if (firstSlash > 0) {
+                resolvedBucket = withoutProtocol.substring(0, firstSlash);
+                resolvedPath = withoutProtocol.substring(firstSlash + 1);
+            }
+        }
+
+        const command = new DeleteObjectCommand({
+            Bucket: resolvedBucket,
+            Key: resolvedPath
+        });
+
+        await this.client.send(command);
+    }
+
+    async list(storagePath: string, options?: {
+        bucket?: string;
+        maxResults?: number;
+        pageToken?: string;
+    }): Promise<StorageListResult> {
+        const resolvedBucket = this.getBucket(options?.bucket);
+
+        const command = new ListObjectsV2Command({
+            Bucket: resolvedBucket,
+            Prefix: storagePath || undefined,
+            MaxKeys: options?.maxResults ?? 1000,
+            ContinuationToken: options?.pageToken,
+            Delimiter: '/' // This gives us folder-like behavior
+        });
+
+        const response = await this.client.send(command);
+
+        const items: StorageReference[] = (response.Contents || []).map(obj => ({
+            bucket: resolvedBucket,
+            fullPath: obj.Key || '',
+            name: (obj.Key || '').split('/').pop() || '',
+            parent: null as never,
+            root: null as never,
+            toString: () => `s3://${resolvedBucket}/${obj.Key}`
+        }));
+
+        const prefixes: StorageReference[] = (response.CommonPrefixes || []).map(prefix => ({
+            bucket: resolvedBucket,
+            fullPath: prefix.Prefix || '',
+            name: (prefix.Prefix || '').replace(/\/$/, '').split('/').pop() || '',
+            parent: null as never,
+            root: null as never,
+            toString: () => `s3://${resolvedBucket}/${prefix.Prefix}`
+        }));
+
+        return {
+            items,
+            prefixes,
+            nextPageToken: response.NextContinuationToken
+        };
+    }
+}

package/src/storage/index.ts

@@ -0,0 +1,32 @@
+/**
+ * Storage module for Rebase backend
+ *
+ * Provides file storage functionality with support for:
+ * - Local filesystem storage (default, zero config)
+ * - S3-compatible storage (AWS S3, MinIO)
+ */
+
+export * from './types';
+export { LocalStorageController } from './LocalStorageController';
+export { S3StorageController } from './S3StorageController';
+export { createStorageRoutes } from './routes';
+export type { StorageRoutesConfig } from './routes';
+export * from './storage-registry';
+
+import { BackendStorageConfig, StorageController } from './types';
+import { LocalStorageController } from './LocalStorageController';
+import { S3StorageController } from './S3StorageController';
+
+/**
+ * Factory function to create a storage controller based on configuration
+ */
+export function createStorageController(config: BackendStorageConfig): StorageController {
+    switch (config.type) {
+        case 'local':
+            return new LocalStorageController(config);
+        case 's3':
+            return new S3StorageController(config);
+        default:
+            throw new Error(`Unknown storage type: ${(config as Record<string, unknown>).type}`);
+    }
+}

package/src/storage/routes.ts

@@ -0,0 +1,247 @@
+/**
+ * Storage REST API routes using Hono
+ */
+
+import { Hono } from 'hono';
+import * as fs from 'fs';
+import { StorageController } from './types';
+import { LocalStorageController } from './LocalStorageController';
+import { requireAuth as jwtRequireAuth, optionalAuth } from '../auth/middleware';
+import { ApiError } from '../api/errors';
+import { HonoEnv } from '../api/types';
+
+export interface StorageRoutesConfig {
+    controller: StorageController;
+    /** Base path for storage routes (default: '/api/storage') */
+    basePath?: string;
+    /** Require authentication for write operations (default: true) */
+    requireAuth?: boolean;
+    /** Allow unauthenticated read access to stored files (default: false).
+     *  When false and requireAuth is true, reads also require authentication. */
+    publicRead?: boolean;
+}
+
+/**
+ * Extract the wildcard portion of a route path from the full request path.
+ *
+ * Hono's `c.req.param('*')` does not work reliably in sub-routers mounted
+ * via `app.route(prefix, subRouter)`. Instead we derive the wildcard value
+ * from the fully-resolved `c.req.path` and `c.req.routePath`.
+ *
+ * For a route `/metadata/*` mounted at `/api/storage`, a request to
+ * `/api/storage/metadata/default/file.jpg` yields routePath
+ * `/api/storage/metadata/*`.  We strip the prefix (everything before `/*`)
+ * plus one character for the trailing `/` to obtain `default/file.jpg`.
+ */
+export function extractWildcardPath(c: { req: { path: string; routePath: string } }): string {
+    const routePath = c.req.routePath;           // e.g. "/api/storage/metadata/*"
+    const prefix = routePath.replace('/*', '');   // e.g. "/api/storage/metadata"
+    const fullPath = c.req.path;                  // e.g. "/api/storage/metadata/default/file.jpg"
+    const idx = fullPath.indexOf(prefix);
+    if (idx < 0) return '';
+    // +1 to skip the '/' after the prefix
+    return fullPath.substring(idx + prefix.length + 1);
+}
+
+/**
+ * Create storage REST API routes
+ */
+export function createStorageRoutes(config: StorageRoutesConfig): Hono<HonoEnv> {
+    const router = new Hono<HonoEnv>();
+    const { controller, requireAuth = true, publicRead = false } = config;
+
+    // Use actual JWT auth middleware from auth module
+    const writeAuthMiddleware = requireAuth ? jwtRequireAuth : optionalAuth;
+
+    // For read operations: respect publicRead config.
+    const readAuthMiddleware = (publicRead || !requireAuth) ? optionalAuth : jwtRequireAuth;
+
+    /**
+     * Parse bucket and path from a combined file path.
+     */
+    const parseBucketAndPath = (filePath: string): { bucket: string; resolvedPath: string } => {
+        const parts = filePath.split('/');
+
+        // Only recognize 'default' as an explicit bucket prefix
+        if (parts.length > 1 && parts[0].toLowerCase() === 'default') {
+            return {
+                bucket: 'default',
+                resolvedPath: parts.slice(1).join('/')
+            };
+        }
+
+        // All other paths use 'default' bucket with the full path
+        return {
+            bucket: 'default',
+            resolvedPath: filePath
+        };
+    };
+
+    /**
+     * POST /upload - Upload a file
+     * Body: multipart/form-data with 'file' field
+     * Request body can also contain metadata keys 'metadata_*'
+     */
+    router.post('/upload', writeAuthMiddleware, async (c) => {
+        const body = await c.req.parseBody();
+        const uploadedFile = body['file'];
+
+        if (!uploadedFile || typeof uploadedFile === 'string') {
+            throw ApiError.badRequest('No file provided');
+        }
+
+        const storagePath = typeof body['path'] === 'string' ? body['path'] : '';
+        const bucket = typeof body['bucket'] === 'string' ? body['bucket'] : undefined;
+        let fileName = typeof body['fileName'] === 'string' ? body['fileName'] : undefined;
+        
+        if (!fileName) {
+            fileName = uploadedFile.name;
+        }
+
+        // Extract custom metadata from request body
+        const metadata: Record<string, unknown> = {};
+        for (const [key, value] of Object.entries(body)) {
+            if (key.startsWith('metadata_')) {
+                metadata[key.replace('metadata_', '')] = value;
+            }
+        }
+
+        const result = await controller.uploadFile({
+            file: uploadedFile,
+            fileName: fileName || 'unnamed',
+            path: storagePath,
+            metadata: Object.keys(metadata).length > 0 ? metadata : undefined,
+            bucket
+        });
+
+        return c.json({
+            success: true,
+            data: result
+        }, 201);
+    });
+
+    /**
+     * GET /file/* - Download/serve a file
+     * Path: /file/{bucket}/{path} or /file/{path}
+     */
+    router.get('/file/*', readAuthMiddleware, async (c) => {
+        const rawPath = extractWildcardPath(c);
+        if (!rawPath) {
+            throw ApiError.notFound('File not found');
+        }
+
+        const filePath = decodeURIComponent(rawPath);
+
+        // For local storage, serve the file directly
+        if (controller.getType() === 'local') {
+            const localController = controller as LocalStorageController;
+            const { bucket, resolvedPath } = parseBucketAndPath(filePath);
+
+            const absolutePath = localController.getAbsolutePath(resolvedPath, bucket);
+
+            // Check if file exists
+            if (!fs.existsSync(absolutePath)) {
+                throw ApiError.notFound('File not found');
+            }
+
+            // Get content type from metadata or infer from extension
+            let contentType = 'application/octet-stream';
+            const metadataPath = `${absolutePath}.metadata.json`;
+            if (fs.existsSync(metadataPath)) {
+                try {
+                    const metadata = JSON.parse(fs.readFileSync(metadataPath, 'utf-8'));
+                    contentType = metadata.contentType || contentType;
+                } catch {
+                    // Ignore metadata errors
+                }
+            }
+
+            c.header('Content-Type', contentType);
+            // In a better scenario, we should pipe the stream instead of reading whole file
+            const fileContent = fs.readFileSync(absolutePath);
+            return c.body(new Uint8Array(fileContent)); 
+        }
+
+        // For S3 storage, redirect to signed URL
+        const downloadConfig = await controller.getDownloadURL(filePath);
+        if (downloadConfig.fileNotFound || !downloadConfig.url) {
+            throw ApiError.notFound('File not found');
+        }
+
+        return c.redirect(downloadConfig.url);
+    });
+
+    /**
+     * GET /metadata/* - Get file metadata
+     */
+    router.get('/metadata/*', readAuthMiddleware, async (c) => {
+        const rawPath = extractWildcardPath(c);
+        if (!rawPath) {
+            return c.json({
+                success: true,
+                data: null,
+                fileNotFound: true
+            }, 404);
+        }
+
+        const filePath = decodeURIComponent(rawPath);
+        const { bucket, resolvedPath } = parseBucketAndPath(filePath);
+
+        const downloadConfig = await controller.getDownloadURL(resolvedPath, bucket);
+
+        if (downloadConfig.fileNotFound) {
+            throw ApiError.notFound('File not found');
+        }
+
+        return c.json({
+            success: true,
+            data: downloadConfig.metadata
+        });
+    });
+
+    /**
+     * DELETE /file/* - Delete a file
+     */
+    router.delete('/file/*', writeAuthMiddleware, async (c) => {
+        const rawPath = extractWildcardPath(c);
+        if (!rawPath) {
+            return c.json({ success: true, message: 'No file to delete' });
+        }
+
+        const filePath = decodeURIComponent(rawPath);
+        const { bucket, resolvedPath } = parseBucketAndPath(filePath);
+
+        await controller.deleteFile(resolvedPath, bucket);
+
+        return c.json({
+            success: true,
+            message: 'File deleted'
+        });
+    });
+
+    /**
+     * GET /list - List files in a path
+     */
+    router.get('/list', writeAuthMiddleware, async (c) => {
+        const storagePath = c.req.query('path') || '';
+        const bucket = c.req.query('bucket');
+        const maxResults = c.req.query('maxResults');
+        const pageToken = c.req.query('pageToken');
+
+        const result = await controller.list(
+            storagePath,
+            {
+                bucket,
+                maxResults: maxResults ? parseInt(maxResults, 10) : undefined,
+                pageToken
+            }
+        );
+
+        return c.json({
+            success: true,
+            data: result
+        });
+    });
+
+    return router;
+}