json-rest-api 0.1.1 → 1.0.1

package/plugins/core/file-handling-plugin.js

@@ -0,0 +1,338 @@
+/**
+ * File Handling Plugin for JSON REST API
+ * 
+ * This plugin provides automatic file upload handling based on schema definitions.
+ * It works with any protocol connector (HTTP, Express, WebSocket, etc.) by using
+ * a detector registry pattern.
+ * 
+ * Features:
+ * - Schema-driven: Detects file fields from type: 'file' in schemas
+ * - Protocol-agnostic: Works with any connector that registers a detector
+ * - Storage pluggable: Different fields can use different storage backends
+ * - Zero configuration: Just define file fields in your schema
+ * 
+ * Usage:
+ * ```javascript
+ * // 1. Define schema with file fields
+ * const imageSchema = {
+ *   title: { type: 'string' },
+ *   uploadedImage: { 
+ *     type: 'file',
+ *     storage: S3Storage,
+ *     accepts: ['image/jpeg', 'image/png']
+ *   }
+ * };
+ * 
+ * // 2. Use plugins (order matters - file-handling depends on rest-api)
+ * api.use(RestApiPlugin);
+ * api.use(FileHandlingPlugin);
+ * api.use(ExpressPlugin); // Or any other connector
+ * 
+ * // 3. Files are automatically handled!
+ * ```
+ */
+
+import { RestApiValidationError } from '../../lib/rest-api-errors.js';
+
+export const FileHandlingPlugin = {
+  name: 'file-handling',
+  dependencies: ['rest-api'],
+  
+  install({ addHook, helpers, scopes, log, vars, on, api }) {
+    // Track which scopes have file fields
+    const fileScopes = new Map(); // scopeName -> fileField[]
+    
+    // Registry of file detectors from various protocols
+    const detectorRegistry = [];
+    
+    /**
+     * Register a file detector from a protocol plugin
+     * 
+     * @param {Object} detector - The detector object
+     * @param {string} detector.name - Name of the detector (e.g., 'http-multipart')
+     * @param {Function} detector.detect - Function to check if this detector applies
+     * @param {Function} detector.parse - Function to parse files from the request
+     */
+    api.rest.registerFileDetector = (detector) => {
+      if (!detector || !detector.name || !detector.detect || !detector.parse) {
+        throw new Error('File detector must have name, detect(), and parse() properties');
+      }
+      
+      detectorRegistry.push(detector);
+      log.debug(`Registered file detector: ${detector.name}`);
+    };
+    
+    // Store detectors array for inspection
+    api.rest.fileDetectors = detectorRegistry;
+    
+    /**
+     * Analyze a scope's schema to find file fields
+     */
+    const analyzeScopeSchema = (scopeName, scopeOptions) => {
+      const schema = scopeOptions?.schema;
+      if (!schema) return;
+      
+      const fileFields = [];
+      
+      // Look for fields with type: 'file'
+      for (const [fieldName, fieldConfig] of Object.entries(schema)) {
+        if (fieldConfig && fieldConfig.type === 'file') {
+          fileFields.push({
+            field: fieldName,
+            storage: fieldConfig.storage,
+            accepts: fieldConfig.accepts || ['*'],
+            maxSize: fieldConfig.maxSize,
+            required: fieldConfig.required || false
+          });
+        }
+      }
+      
+      if (fileFields.length > 0) {
+        fileScopes.set(scopeName, fileFields);
+        log.info(`Scope '${scopeName}' has ${fileFields.length} file field(s): ${fileFields.map(f => f.field).join(', ')}`);
+      }
+    };
+    
+    // Analyze existing scopes
+    for (const [scopeName, scope] of Object.entries(scopes)) {
+      if (scope._scopeOptions) {
+        analyzeScopeSchema(scopeName, scope._scopeOptions);
+      }
+    }
+    
+    // Listen for new scopes being added
+    on('scope:added', 'analyzeFileFields', ({ eventData }) => {
+      const { scopeName, scopeOptions } = eventData;
+      analyzeScopeSchema(scopeName, scopeOptions);
+    });
+    
+    /**
+     * Process files for a scope if it has file fields
+     */
+    const processFiles = async (scopeName, params) => {
+      const fileFields = fileScopes.get(scopeName);
+      if (!fileFields || fileFields.length === 0) {
+        return; // This scope doesn't have file fields
+      }
+      
+      // Try each detector to see if we have files
+      let parsed = null;
+      let detectorUsed = null;
+      
+      for (const detector of detectorRegistry) {
+        try {
+          if (await detector.detect(params)) {
+            log.debug(`Detector '${detector.name}' matched for scope '${scopeName}'`);
+            parsed = await detector.parse(params);
+            detectorUsed = detector.name;
+            break;
+          }
+        } catch (error) {
+          log.warn(`Detector '${detector.name}' failed:`, error);
+        }
+      }
+      
+      if (!parsed) {
+        // No files detected - check if any were required
+        for (const fieldConfig of fileFields) {
+          if (fieldConfig.required && !params.inputRecord?.data?.attributes?.[fieldConfig.field]) {
+            throw new RestApiValidationError(
+              `Required file field '${fieldConfig.field}' is missing`,
+              {
+                fields: [fieldConfig.field],
+                violations: [{
+                  field: fieldConfig.field,
+                  message: 'This field is required'
+                }]
+              }
+            );
+          }
+        }
+        return;
+      }
+      
+      log.debug(`Processing files with detector '${detectorUsed}'`);
+      const { fields, files } = parsed;
+      
+      // Process each file field defined in schema
+      for (const fieldConfig of fileFields) {
+        const file = files[fieldConfig.field];
+        
+        if (!file) {
+          if (fieldConfig.required) {
+            throw new RestApiValidationError(
+              `Required file field '${fieldConfig.field}' is missing`,
+              {
+                fields: [fieldConfig.field],
+                violations: [{
+                  field: fieldConfig.field,
+                  message: 'This field is required'
+                }]
+              }
+            );
+          }
+          continue;
+        }
+        
+        // Validate mime type
+        if (fieldConfig.accepts[0] !== '*') {
+          const acceptable = fieldConfig.accepts.some(pattern => {
+            if (pattern.endsWith('/*')) {
+              // e.g., 'image/*'
+              const prefix = pattern.slice(0, -2);
+              return file.mimetype.startsWith(prefix + '/');
+            }
+            return file.mimetype === pattern;
+          });
+          
+          if (!acceptable) {
+            throw new RestApiValidationError(
+              `Invalid file type for field '${fieldConfig.field}'`,
+              {
+                fields: [fieldConfig.field],
+                violations: [{
+                  field: fieldConfig.field,
+                  message: `Expected ${fieldConfig.accepts.join(' or ')}, got ${file.mimetype}`
+                }]
+              }
+            );
+          }
+        }
+        
+        // Validate file size
+        if (fieldConfig.maxSize) {
+          const maxBytes = parseSize(fieldConfig.maxSize);
+          if (file.size > maxBytes) {
+            throw new RestApiValidationError(
+              `File too large for field '${fieldConfig.field}'`,
+              {
+                fields: [fieldConfig.field],
+                violations: [{
+                  field: fieldConfig.field,
+                  message: `Maximum size is ${fieldConfig.maxSize}, got ${formatSize(file.size)}`
+                }]
+              }
+            );
+          }
+        }
+        
+        // Upload to storage
+        if (!fieldConfig.storage) {
+          throw new Error(`No storage configured for file field '${fieldConfig.field}'`);
+        }
+        
+        try {
+          const storedUrl = await fieldConfig.storage.upload(file);
+          fields[fieldConfig.field] = storedUrl;
+          log.debug(`Uploaded file for field '${fieldConfig.field}' to: ${storedUrl}`);
+        } catch (error) {
+          // Cleanup if file has cleanup function
+          if (file.cleanup) {
+            try {
+              await file.cleanup();
+            } catch (cleanupError) {
+              log.warn(`Failed to cleanup file after upload error:`, cleanupError);
+            }
+          }
+          
+          throw new RestApiValidationError(
+            `Failed to upload file for field '${fieldConfig.field}': ${error.message}`,
+            {
+              fields: [fieldConfig.field],
+              violations: [{
+                field: fieldConfig.field,
+                message: error.message
+              }]
+            }
+          );
+        }
+      }
+      
+      // Replace inputRecord with processed data
+      if (!params.inputRecord) {
+        params.inputRecord = { data: { attributes: {} } };
+      }
+      if (!params.inputRecord.data) {
+        params.inputRecord.data = { attributes: {} };
+      }
+      if (!params.inputRecord.data.attributes) {
+        params.inputRecord.data.attributes = {};
+      }
+      
+      // Merge fields into attributes
+      Object.assign(params.inputRecord.data.attributes, fields);
+      
+      // Cleanup any remaining temp files
+      for (const file of Object.values(files)) {
+        if (file.cleanup) {
+          try {
+            await file.cleanup();
+          } catch (error) {
+            log.warn(`Failed to cleanup temp file:`, error);
+          }
+        }
+      }
+    };
+    
+    /**
+     * Hook into REST API methods to process files
+     */
+    addHook('beforeProcessing', 'processFiles', {}, async (context) => {
+      // Only process for mutation methods
+      if (!['post', 'put', 'patch'].includes(context.method)) {
+        return;
+      }
+      
+      // Process files if this scope has file fields
+      await processFiles(context.scopeName, context.params);
+    });
+    
+    log.info('File handling plugin initialized successfully');
+  }
+};
+
+/**
+ * Parse size string to bytes
+ * @param {string} size - Size string like '10mb', '1.5GB'
+ * @returns {number} Size in bytes
+ */
+function parseSize(size) {
+  const units = {
+    b: 1,
+    kb: 1024,
+    mb: 1024 * 1024,
+    gb: 1024 * 1024 * 1024
+  };
+  
+  const match = size.toLowerCase().match(/^(\d+(?:\.\d+)?)\s*([a-z]+)$/);
+  if (!match) {
+    throw new Error(`Invalid size format: ${size}`);
+  }
+  
+  const [, num, unit] = match;
+  const multiplier = units[unit];
+  
+  if (!multiplier) {
+    throw new Error(`Unknown size unit: ${unit}`);
+  }
+  
+  return parseFloat(num) * multiplier;
+}
+
+/**
+ * Format bytes to human readable size
+ * @param {number} bytes - Size in bytes
+ * @returns {string} Human readable size
+ */
+function formatSize(bytes) {
+  const units = ['B', 'KB', 'MB', 'GB'];
+  let size = bytes;
+  let unitIndex = 0;
+  
+  while (size >= 1024 && unitIndex < units.length - 1) {
+    size /= 1024;
+    unitIndex++;
+  }
+  
+  return `${size.toFixed(1)}${units[unitIndex]}`;
+}

package/plugins/core/lib/busboy-detector.js

@@ -0,0 +1,167 @@
+/**
+ * Busboy File Detector for HTTP Multipart Uploads
+ * 
+ * This detector handles multipart/form-data uploads using the busboy library.
+ * It's suitable for both HTTP and Express connectors.
+ * 
+ * Features:
+ * - Streaming parser (memory efficient)
+ * - Configurable file size limits
+ * - Automatic field parsing
+ * - No temporary files by default (keeps in memory)
+ * 
+ * Usage:
+ * ```javascript
+ * import { createBusboyDetector } from 'jsonrestapi/plugins/core/lib/busboy-detector.js';
+ * 
+ * api.use(HttpPlugin, {
+ *   fileParser: 'busboy',
+ *   fileParserOptions: {
+ *     limits: { fileSize: 10 * 1024 * 1024 } // 10MB
+ *   }
+ * });
+ * ```
+ */
+
+import Busboy from 'busboy';
+import { Readable } from 'stream';
+
+/**
+ * Creates a busboy-based file detector
+ * 
+ * @param {Object} options - Busboy configuration options
+ * @param {Object} options.limits - Size limits
+ * @param {number} options.limits.fileSize - Max file size in bytes
+ * @param {number} options.limits.files - Max number of files
+ * @param {number} options.limits.fields - Max number of fields
+ * @returns {Object} Detector object with detect() and parse() methods
+ */
+export function createBusboyDetector(options = {}) {
+  return {
+    name: 'busboy-multipart',
+    
+    /**
+     * Check if this detector can handle the request
+     * @param {Object} params - Request parameters
+     * @returns {boolean} True if this is a multipart request
+     */
+    detect: (params) => {
+      const req = params._httpReq || params._expressReq;
+      if (!req || !req.headers) return false;
+      
+      const contentType = req.headers['content-type'] || '';
+      return contentType.includes('multipart/form-data');
+    },
+    
+    /**
+     * Parse multipart data from the request
+     * @param {Object} params - Request parameters
+     * @returns {Promise<{fields: Object, files: Object}>} Parsed data
+     */
+    parse: async (params) => {
+      const req = params._httpReq || params._expressReq;
+      
+      return new Promise((resolve, reject) => {
+        const busboy = new Busboy({
+          headers: req.headers,
+          ...options
+        });
+        
+        const fields = {};
+        const files = {};
+        const filePromises = [];
+        
+        // Handle fields
+        busboy.on('field', (fieldname, val) => {
+          // Handle array notation (field[] or field[0])
+          const arrayMatch = fieldname.match(/^(.+)\[\d*\]$/);
+          if (arrayMatch) {
+            const baseName = arrayMatch[1];
+            if (!fields[baseName]) {
+              fields[baseName] = [];
+            }
+            if (Array.isArray(fields[baseName])) {
+              fields[baseName].push(val);
+            }
+          } else {
+            fields[fieldname] = val;
+          }
+        });
+        
+        // Handle files
+        busboy.on('file', (fieldname, file, filename, encoding, mimetype) => {
+          const chunks = [];
+          let size = 0;
+          
+          const filePromise = new Promise((fileResolve, fileReject) => {
+            file.on('data', (chunk) => {
+              chunks.push(chunk);
+              size += chunk.length;
+            });
+            
+            file.on('limit', () => {
+              fileReject(new Error(`File size limit exceeded for field '${fieldname}'`));
+            });
+            
+            file.on('end', () => {
+              files[fieldname] = {
+                filename: filename,
+                mimetype: mimetype,
+                encoding: encoding,
+                size: size,
+                data: Buffer.concat(chunks)
+              };
+              fileResolve();
+            });
+            
+            file.on('error', fileReject);
+          });
+          
+          filePromises.push(filePromise);
+        });
+        
+        // Handle completion
+        busboy.on('finish', async () => {
+          try {
+            // Wait for all files to be fully read
+            await Promise.all(filePromises);
+            resolve({ fields, files });
+          } catch (error) {
+            reject(error);
+          }
+        });
+        
+        // Handle errors
+        busboy.on('error', (error) => {
+          reject(error);
+        });
+        
+        // Handle limit errors
+        busboy.on('partsLimit', () => {
+          reject(new Error('Parts limit exceeded'));
+        });
+        
+        busboy.on('filesLimit', () => {
+          reject(new Error('Files limit exceeded'));
+        });
+        
+        busboy.on('fieldsLimit', () => {
+          reject(new Error('Fields limit exceeded'));
+        });
+        
+        // Pipe request to busboy
+        if (req.pipe) {
+          req.pipe(busboy);
+        } else if (req.on) {
+          // For already buffered requests
+          const stream = new Readable();
+          stream.push(req.body || req);
+          stream.push(null);
+          stream.pipe(busboy);
+        } else {
+          reject(new Error('Request object is not a stream'));
+        }
+      });
+    }
+  };
+}

package/plugins/core/lib/formidable-detector.js

@@ -0,0 +1,130 @@
+/**
+ * Formidable File Detector for HTTP Multipart Uploads
+ * 
+ * This detector handles multipart/form-data uploads using the formidable library.
+ * Unlike busboy, formidable saves files to disk temporarily, which is better
+ * for large files.
+ * 
+ * Features:
+ * - Disk-based storage (better for large files)
+ * - Automatic temp file cleanup
+ * - Progress tracking support
+ * - Built-in file type detection
+ * 
+ * Usage:
+ * ```javascript
+ * import { createFormidableDetector } from 'jsonrestapi/plugins/core/lib/formidable-detector.js';
+ * 
+ * api.use(ExpressPlugin, {
+ *   fileParser: 'formidable',
+ *   fileParserOptions: {
+ *     uploadDir: './uploads/temp',
+ *     keepExtensions: true,
+ *     maxFileSize: 200 * 1024 * 1024 // 200MB
+ *   }
+ * });
+ * ```
+ */
+
+import formidable from 'formidable';
+import { promises as fs } from 'fs';
+import path from 'path';
+
+/**
+ * Creates a formidable-based file detector
+ * 
+ * @param {Object} options - Formidable configuration options
+ * @param {string} options.uploadDir - Directory for temporary files
+ * @param {boolean} options.keepExtensions - Keep file extensions
+ * @param {number} options.maxFileSize - Max file size in bytes
+ * @param {boolean} options.multiples - Allow multiple files per field
+ * @returns {Object} Detector object with detect() and parse() methods
+ */
+export function createFormidableDetector(options = {}) {
+  // Set default upload directory
+  const uploadDir = options.uploadDir || path.join(process.cwd(), 'uploads', 'temp');
+  
+  return {
+    name: 'formidable-multipart',
+    
+    /**
+     * Check if this detector can handle the request
+     * @param {Object} params - Request parameters
+     * @returns {boolean} True if this is a multipart request
+     */
+    detect: (params) => {
+      const req = params._httpReq || params._expressReq;
+      if (!req || !req.headers) return false;
+      
+      const contentType = req.headers['content-type'] || '';
+      return contentType.includes('multipart/form-data');
+    },
+    
+    /**
+     * Parse multipart data from the request
+     * @param {Object} params - Request parameters
+     * @returns {Promise<{fields: Object, files: Object}>} Parsed data
+     */
+    parse: async (params) => {
+      const req = params._httpReq || params._expressReq;
+      
+      // Ensure upload directory exists
+      await fs.mkdir(uploadDir, { recursive: true });
+      
+      // Create form parser
+      const form = formidable({
+        uploadDir,
+        keepExtensions: true,
+        ...options
+      });
+      
+      // Parse the request
+      const [fields, fileUploads] = await form.parse(req);
+      
+      // Normalize formidable's file format to our standard format
+      const files = {};
+      
+      for (const [fieldname, uploadedFiles] of Object.entries(fileUploads)) {
+        // Formidable returns arrays for multiple files
+        const fileArray = Array.isArray(uploadedFiles) ? uploadedFiles : [uploadedFiles];
+        
+        // For now, just take the first file (TODO: handle multiple files per field)
+        const file = fileArray[0];
+        
+        if (file) {
+          // Read file data into memory (for small files)
+          // For large files, you might want to keep them on disk
+          const data = await fs.readFile(file.filepath);
+          
+          files[fieldname] = {
+            filename: file.originalFilename || 'unknown',
+            mimetype: file.mimetype || 'application/octet-stream',
+            size: file.size,
+            data: data,
+            filepath: file.filepath, // Keep for reference
+            
+            // Cleanup function to remove temp file
+            cleanup: async () => {
+              try {
+                await fs.unlink(file.filepath);
+              } catch (error) {
+                // File might already be deleted, ignore
+                if (error.code !== 'ENOENT') {
+                  throw error;
+                }
+              }
+            }
+          };
+        }
+      }
+      
+      // Normalize fields (formidable returns arrays for repeated fields)
+      const normalizedFields = {};
+      for (const [key, value] of Object.entries(fields)) {
+        normalizedFields[key] = Array.isArray(value) && value.length === 1 ? value[0] : value;
+      }
+      
+      return { fields: normalizedFields, files };
+    }
+  };
+}