@xenterprises/fastify-ximagepipeline 1.1.0 → 1.2.0

package/src/workers/processor.js

@@ -1,33 +1,41 @@
 // src/workers/processor.js
 import { downloadFromS3, deleteFromS3, uploadToS3, getPublicUrl, batchDeleteFromS3 } from "../services/s3.js";
-import { stripExif, generateVariants, generateBlurhash, getImageMetadata, compressToJpeg } from "../utils/image.js";
-import { getVariantPresets } from "../xImagePipeline.js";
+import { stripExif, generateVariants, generateBlurhash, getImageMetadata, getAspectRatio, compressToJpeg } from "../utils/image.js";
 
 /**
  * Setup media processor worker
  * Polls job queue and processes media files
  */
 export function setupWorker(fastify, context, config) {
-  // Worker state
   const workerId = `worker-${Date.now()}-${Math.random().toString(36).substring(7)}`;
   let isRunning = true;
 
-  console.info(`   📊 Media Worker initialized (${workerId})`);
+  fastify.log.info(`[xImagePipeline] Media Worker initialized (${workerId})`);
 
   // Start polling
   const pollInterval = setInterval(() => {
     if (isRunning) {
       processNextJob(fastify, context, config, workerId).catch((err) => {
-        console.error("Worker error:", err.message);
+        fastify.log.error({ err }, "[xImagePipeline] Worker error");
       });
     }
   }, config.pollInterval);
 
+  // Periodically recover stale locks
+  const staleLockInterval = setInterval(() => {
+    if (isRunning) {
+      recoverStaleLocks(context.db, config.lockTimeout).catch((err) => {
+        fastify.log.error({ err }, "[xImagePipeline] Stale lock recovery error");
+      });
+    }
+  }, config.lockTimeout);
+
   // Cleanup on fastify close
   fastify.addHook("onClose", async () => {
     isRunning = false;
     clearInterval(pollInterval);
-    console.info("   📊 Media Worker stopped");
+    clearInterval(staleLockInterval);
+    fastify.log.info("[xImagePipeline] Media Worker stopped");
   });
 
   return { workerId, stop: () => { isRunning = false; } };
@@ -57,7 +65,6 @@ async function processNextJob(fastify, context, config, workerId) {
     }
 
     // Lock the job
-    const lockExpiry = new Date(Date.now() + config.lockTimeout);
     const locked = await context.db.mediaQueue.updateMany({
       where: {
         id: job.id,
@@ -74,7 +81,7 @@ async function processNextJob(fastify, context, config, workerId) {
       return; // Job was locked by another worker
     }
 
-    console.info(`Processing job ${job.id}...`);
+    fastify.log.info({ jobId: job.id }, "[xImagePipeline] Processing job");
 
     try {
       // Download file from staging
@@ -96,7 +103,6 @@ async function processNextJob(fastify, context, config, workerId) {
           const moderationResult = await moderateImage(cleanBuffer, context.moderation);
 
           if (!moderationResult.passed) {
-            // Content rejected
             await context.db.mediaQueue.update({
               where: { id: job.id },
               data: {
@@ -108,28 +114,28 @@ async function processNextJob(fastify, context, config, workerId) {
               },
             });
 
-            // Clean up staging
             await deleteFromS3(context.s3Client, context.r2Config.bucket, job.stagingKey);
-            console.info(`Job ${job.id} rejected by moderation`);
+            fastify.log.info({ jobId: job.id }, "[xImagePipeline] Job rejected by moderation");
             return;
           }
         } catch (err) {
-          console.error(`Moderation failed: ${err.message}`);
+          fastify.log.error({ err, jobId: job.id }, "[xImagePipeline] Moderation failed");
           throw err;
         }
       }
 
-      // Step 4: Generate variants
-      const variantPresets = getVariantPresets();
-      const preset = variantPresets[job.sourceType] || [];
+      // Step 4: Generate variants using sourceType config
+      const sourceTypeConfig = context.sourceTypes[job.sourceType];
+      const variantNames = sourceTypeConfig?.variants || [];
       const variantSpecs = {};
-      for (const variantName of preset) {
+      for (const variantName of variantNames) {
         if (context.variants[variantName]) {
           variantSpecs[variantName] = context.variants[variantName];
         }
       }
 
-      const variants = await generateVariants(cleanBuffer, variantSpecs, job.sourceType);
+      const quality = sourceTypeConfig?.quality || 85;
+      const variants = await generateVariants(cleanBuffer, variantSpecs, job.sourceType, quality);
 
       // Step 5: Generate blurhash
       const blurhash = await generateBlurhash(cleanBuffer);
@@ -150,10 +156,11 @@ async function processNextJob(fastify, context, config, workerId) {
         urls[variantName] = getPublicUrl(context.r2Config, variantKey);
       }
 
-      // Upload compressed original (JPEG for space efficiency)
+      // Upload compressed original if sourceType config says to
       let originalUrl = null;
-      if (context.storeOriginal) {
-        const compressedOriginal = await compressToJpeg(cleanBuffer, context.originalQuality);
+      const storeOriginal = sourceTypeConfig?.storeOriginal ?? false;
+      if (storeOriginal) {
+        const compressedOriginal = await compressToJpeg(cleanBuffer, quality);
         const originalKey = `${originalPath}/original.jpg`;
         await uploadToS3(context.s3Client, context.r2Config.bucket, originalKey, compressedOriginal, {
           contentType: "image/jpeg",
@@ -171,7 +178,7 @@ async function processNextJob(fastify, context, config, workerId) {
           width: metadata.width,
           height: metadata.height,
           format: metadata.format,
-          aspectRatio: `${metadata.width}:${metadata.height}`,
+          aspectRatio: getAspectRatio(metadata.width, metadata.height),
           blurhash,
           originalFilename: job.originalFilename,
           mimeType: job.mimeType,
@@ -193,9 +200,9 @@ async function processNextJob(fastify, context, config, workerId) {
       // Step 9: Clean up staging
       await deleteFromS3(context.s3Client, context.r2Config.bucket, job.stagingKey);
 
-      console.info(`Job ${job.id} completed successfully`);
+      fastify.log.info({ jobId: job.id, mediaId: media.id }, "[xImagePipeline] Job completed");
     } catch (error) {
-      console.error(`Job ${job.id} processing error: ${error.message}`);
+      fastify.log.error({ err: error, jobId: job.id }, "[xImagePipeline] Job processing error");
 
       // Update job with error
       const nextAttempt = job.attempts + 1;
@@ -213,26 +220,28 @@ async function processNextJob(fastify, context, config, workerId) {
       });
 
       if (!shouldRetry) {
-        // Clean up staging after max retries
         try {
           await deleteFromS3(context.s3Client, context.r2Config.bucket, job.stagingKey);
         } catch (cleanupErr) {
-          console.error(`Failed to cleanup staging: ${cleanupErr.message}`);
+          fastify.log.error({ err: cleanupErr }, "[xImagePipeline] Failed to cleanup staging");
         }
       }
     }
   } catch (error) {
-    console.error("Worker process error:", error.message);
+    fastify.log.error({ err: error }, "[xImagePipeline] Worker process error");
   }
 }
 
 /**
  * Moderate image content
- * Currently a stub - implement with actual API (Rekognition, Vision, etc.)
+ * Stub implementation — override via options.moderation.handler to provide real moderation.
+ * Expected signature: async (buffer, config) => { passed: boolean, flags: string[], confidence: object }
  */
 async function moderateImage(buffer, moderationConfig) {
-  // TODO: Implement actual moderation API call
-  // For now, always approve
+  if (typeof moderationConfig.handler === "function") {
+    return moderationConfig.handler(buffer, moderationConfig);
+  }
+  // Default: always approve
   return {
     passed: true,
     flags: [],
@@ -241,7 +250,7 @@ async function moderateImage(buffer, moderationConfig) {
 }
 
 /**
- * Recover stale locks (jobs locked > lockTimeout)
+ * Recover stale locks (jobs locked longer than lockTimeout)
  */
 export async function recoverStaleLocks(db, lockTimeout) {
   const staleThreshold = new Date(Date.now() - lockTimeout);

package/src/xImagePipeline.js

@@ -1,6 +1,6 @@
 // src/xImagePipeline.js
 import fp from "fastify-plugin";
-import { initializeS3Client } from "./services/s3.js";
+import { initializeS3Client, deleteFromS3, batchDeleteFromS3 } from "./services/s3.js";
 import { setupUploadRoute } from "./routes/upload.js";
 import { setupStatusRoute } from "./routes/status.js";
 import { setupWorker } from "./workers/processor.js";
@@ -9,26 +9,34 @@ import { setupWorker } from "./workers/processor.js";
  * xImagePipeline Plugin for Fastify
  * Handles image uploads with EXIF stripping, moderation, variant generation, and R2 storage
  *
- * @param {Object} fastify - Fastify instance
+ * @param {import('fastify').FastifyInstance} fastify - Fastify instance
  * @param {Object} options - Plugin options
  * @param {Object} options.r2 - R2 configuration (endpoint, accessKeyId, secretAccessKey, bucket)
- * @param {Object} options.db - Database instance (Prisma client or similar)
- * @param {Object} options.moderation - Moderation config (provider, apiKey, etc.)
- * @param {Object} options.variants - Variant size definitions (optional, uses defaults)
- * @param {Object} options.sourceTypes - Source type configurations (optional, uses defaults)
- *   Each source type defines: variants[], formats[], quality, storeOriginal
- * @param {Object} options.worker - Worker configuration (enabled, pollInterval, maxAttempts)
+ * @param {Object} options.db - Database instance (Prisma client)
+ * @param {Object} [options.moderation] - Moderation config ({ handler: async (buffer, config) => result })
+ * @param {Object} [options.variants] - Variant size definitions (default: xs/sm/md/lg/xl/2xl)
+ * @param {Object} [options.sourceTypes] - Source type configurations (default: avatar/member_photo/gallery/hero/content)
+ * @param {Object} [options.worker] - Worker config ({ enabled, pollInterval, maxAttempts, lockTimeout })
+ * @param {string} [options.stagingPath='staging'] - R2 prefix for staging uploads
+ * @param {string} [options.mediaPath='media'] - R2 prefix for processed media
+ * @param {string} [options.originalsPath='originals'] - R2 prefix for originals
+ * @param {number} [options.maxFileSize=52428800] - Max upload size in bytes (default 50MB)
+ * @param {string[]} [options.allowedMimeTypes] - Allowed MIME types (default: jpeg/png/webp/gif)
  */
 async function xImagePipeline(fastify, options) {
   // Validate required configuration
   if (!options.r2) {
-    throw new Error("R2 configuration is required");
+    throw new Error("[xImagePipeline] R2 configuration is required");
   }
   if (!options.db) {
-    throw new Error("Database instance (Prisma client) is required");
+    throw new Error("[xImagePipeline] Database instance (Prisma client) is required");
   }
 
-  console.info("\n 🎬 Starting xImagePipeline...\n");
+  if (typeof options.r2 !== "object" || !options.r2.endpoint || !options.r2.accessKeyId || !options.r2.secretAccessKey || !options.r2.bucket) {
+    throw new Error("[xImagePipeline] R2 configuration must include: endpoint, accessKeyId, secretAccessKey, bucket");
+  }
+
+  fastify.log.info("[xImagePipeline] Starting...");
 
   // Initialize R2 S3 client
   const s3Client = initializeS3Client(options.r2);
@@ -41,7 +49,7 @@ async function xImagePipeline(fastify, options) {
     variants: options.variants || getDefaultVariants(),
     sourceTypes: options.sourceTypes || getDefaultSourceTypes(),
     r2Config: options.r2,
-    maxFileSize: options.maxFileSize || 50 * 1024 * 1024, // 50MB default
+    maxFileSize: options.maxFileSize || 50 * 1024 * 1024,
     allowedMimeTypes: options.allowedMimeTypes || [
       "image/jpeg",
       "image/png",
@@ -55,15 +63,77 @@ async function xImagePipeline(fastify, options) {
 
   // Decorate fastify instance with image pipeline utilities
   fastify.decorate("xImagePipeline", {
-    upload: async (file, metadata) => {
-      // Placeholder - implemented in upload route
-    },
+    /**
+     * Get the status of a processing job
+     * @param {string} jobId
+     * @returns {Promise<Object|null>} Job with media relation, or null
+     */
     getStatus: async (jobId) => {
-      // Placeholder - implemented in status route
+      return context.db.mediaQueue.findUnique({
+        where: { id: jobId },
+        include: { media: true },
+      });
     },
+
+    /**
+     * Delete a media record and all associated R2 objects
+     * @param {string} mediaId
+     * @returns {Promise<{ deleted: boolean, r2Deleted: number }>}
+     */
     deleteMedia: async (mediaId) => {
-      // Placeholder - implemented as utility
+      const media = await context.db.media.findUnique({ where: { id: mediaId } });
+      if (!media) {
+        throw new Error(`[xImagePipeline] Media not found: ${mediaId}`);
+      }
+
+      // Delete variant files from R2
+      const mediaPrefix = `${context.mediaPath}/${media.sourceType}/${media.sourceId}/${mediaId}`;
+      const r2Result = await batchDeleteFromS3(s3Client, context.r2Config.bucket, mediaPrefix);
+
+      // Delete original if stored
+      if (media.originalUrl) {
+        const originalPrefix = `${context.originalsPath}/${media.sourceType}/${media.sourceId}/${mediaId}`;
+        const origResult = await batchDeleteFromS3(s3Client, context.r2Config.bucket, originalPrefix);
+        r2Result.deleted += origResult.deleted;
+      }
+
+      // Delete database records
+      await context.db.mediaQueue.deleteMany({ where: { mediaId } });
+      await context.db.media.delete({ where: { id: mediaId } });
+
+      return { deleted: true, r2Deleted: r2Result.deleted };
+    },
+
+    /**
+     * List media by sourceType and sourceId
+     * @param {string} sourceType
+     * @param {string} sourceId
+     * @returns {Promise<Object[]>}
+     */
+    listMedia: async (sourceType, sourceId) => {
+      return context.db.media.findMany({
+        where: { sourceType, sourceId },
+        orderBy: { createdAt: "desc" },
+      });
     },
+
+    /**
+     * Get variant presets for all source types
+     * @returns {Object}
+     */
+    getVariantPresets,
+
+    /**
+     * Get the source type configurations
+     * @returns {Object}
+     */
+    getSourceTypes: () => context.sourceTypes,
+
+    /**
+     * Get variant definitions
+     * @returns {Object}
+     */
+    getVariants: () => context.variants,
   });
 
   // Register routes
@@ -80,16 +150,16 @@ async function xImagePipeline(fastify, options) {
 
     try {
       setupWorker(fastify, context, workerConfig);
-      console.info("   ✅ Image Pipeline Worker Started");
+      fastify.log.info("[xImagePipeline] Worker started");
     } catch (err) {
-      console.error("   ❌ Failed to start image pipeline worker:", err.message);
+      fastify.log.error({ err }, "[xImagePipeline] Failed to start worker");
       if (options.worker?.failOnError !== false) {
         throw err;
       }
     }
   }
 
-  console.info("\n 🎬 xImagePipeline Ready!\n");
+  fastify.log.info("[xImagePipeline] Ready");
 }
 
 /**
@@ -108,7 +178,6 @@ function getDefaultVariants() {
 
 /**
  * Get default source type configurations
- * Each source type can have different processing settings
  */
 function getDefaultSourceTypes() {
   return {
@@ -146,7 +215,8 @@ function getDefaultSourceTypes() {
 }
 
 /**
- * Get variant presets for different source types
+ * Get variant presets for all source types
+ * @returns {Object<string, string[]>}
  */
 export function getVariantPresets() {
   return {
@@ -161,4 +231,5 @@ export function getVariantPresets() {
 export default fp(xImagePipeline, {
   name: "xImagePipeline",
   fastify: "5.x",
+  dependencies: [],
 });

package/FILES.md

@@ -1,212 +0,0 @@
-# xMedia Plugin - File Listing
-
-## Core Files
-
-### Main Plugin
-- **src/xMedia.js** (145 lines)
-  - Plugin initialization and configuration
-  - Route registration
-  - Worker setup
-  - Variant presets
-
-### Services
-- **src/services/s3.js** (168 lines)
-  - R2/S3 client initialization
-  - Upload, download, delete operations
-  - Public URL generation
-  - Batch operations
-
-### Utilities
-- **src/utils/image.js** (230 lines)
-  - EXIF stripping
-  - Image metadata extraction
-  - Variant generation
-  - Blurhash generation
-  - Image validation
-
-### Routes
-- **src/routes/upload.js** (97 lines)
-  - POST /media/upload endpoint
-  - File validation
-  - Job creation
-  - 202 response
-
-- **src/routes/status.js** (77 lines)
-  - GET /media/status/:jobId endpoint
-  - Job status retrieval
-  - Media data response
-  - HTTP status codes
-
-### Workers
-- **src/workers/processor.js** (368 lines)
-  - Job queue polling
-  - Processing pipeline
-  - Pessimistic locking
-  - Error handling and retry
-  - Stale lock recovery
-
-## Configuration Files
-
-- **package.json** (40 lines)
-  - Dependencies and dev dependencies
-  - Scripts
-  - Package metadata
-
-- **SCHEMA.prisma** (76 lines)
-  - MediaStatus enum
-  - MediaQueue model
-  - Media model
-  - Database indexes
-  - Example relations
-
-## Documentation
-
-- **README.md** (500+ lines)
-  - Feature overview
-  - Installation guide
-  - Setup instructions
-  - API documentation
-  - Frontend examples
-  - Configuration guide
-  - Troubleshooting
-
-- **IMPLEMENTATION_SUMMARY.md** (400+ lines)
-  - Completed components
-  - Directory structure
-  - Processing pipeline
-  - Configuration options
-  - Quick start guide
-  - Design decisions
-  - Future enhancements
-
-- **INTEGRATION_GUIDE.md** (350+ lines)
-  - Step-by-step integration
-  - Environment setup
-  - Prisma schema addition
-  - Plugin registration
-  - Frontend examples
-  - R2 setup
-  - Troubleshooting
-  - Usage examples
-
-- **FILES.md** (this file)
-  - File listing and line counts
-
-## Testing
-
-- **test/xMedia.test.js** (280 lines)
-  - Plugin registration tests
-  - Configuration tests
-  - Route tests
-  - Variant preset tests
-
-## Statistics
-
-### Code Files
-- Total lines: ~1,200+
-- Files: 7 source files
-- Services: 1
-- Utilities: 1
-- Routes: 2
-- Workers: 1
-
-### Documentation
-- README: 500+ lines
-- Implementation Summary: 400+ lines
-- Integration Guide: 350+ lines
-- Total docs: 1,250+ lines
-
-### Total Project Size
-- Source code: 1,200+ lines
-- Tests: 280 lines
-- Documentation: 1,250+ lines
-- **Grand Total: 2,730+ lines**
-
-## Directory Tree
-
-```
-xMedia/
-├── src/
-│   ├── xMedia.js                    (145 lines) - Main plugin
-│   ├── services/
-│   │   └── s3.js                    (168 lines) - R2/S3 integration
-│   ├── utils/
-│   │   └── image.js                 (230 lines) - Image processing
-│   ├── routes/
-│   │   ├── upload.js                (97 lines) - Upload endpoint
-│   │   └── status.js                (77 lines) - Status endpoint
-│   └── workers/
-│       └── processor.js             (368 lines) - Job processor
-├── test/
-│   └── xMedia.test.js               (280 lines) - Test suite
-├── package.json                     (40 lines)
-├── SCHEMA.prisma                    (76 lines)
-├── README.md                        (500+ lines)
-├── IMPLEMENTATION_SUMMARY.md        (400+ lines)
-├── INTEGRATION_GUIDE.md             (350+ lines)
-└── FILES.md                         (this file)
-```
-
-## Feature Matrix
-
-| Feature | File | Status |
-|---------|------|--------|
-| Plugin initialization | xMedia.js | ✅ |
-| R2/S3 integration | services/s3.js | ✅ |
-| File uploads | routes/upload.js | ✅ |
-| Status checking | routes/status.js | ✅ |
-| EXIF stripping | utils/image.js | ✅ |
-| Variant generation | utils/image.js | ✅ |
-| Blurhash generation | utils/image.js | ✅ |
-| Image validation | utils/image.js | ✅ |
-| Job queueing | routes/upload.js | ✅ |
-| Worker processing | workers/processor.js | ✅ |
-| Moderation hooks | workers/processor.js | ✅ |
-| Retry logic | workers/processor.js | ✅ |
-| Lock management | workers/processor.js | ✅ |
-| Database models | SCHEMA.prisma | ✅ |
-| API documentation | README.md | ✅ |
-| Setup guide | INTEGRATION_GUIDE.md | ✅ |
-| Tests | test/xMedia.test.js | ✅ |
-
-## Implementation Status
-
-- ✅ Core plugin complete
-- ✅ S3/R2 integration complete
-- ✅ Image processing complete
-- ✅ Upload endpoint complete
-- ✅ Status endpoint complete
-- ✅ Worker processor complete
-- ✅ Database schema defined
-- ✅ Comprehensive documentation
-- ✅ Integration guide
-- ✅ Basic test suite
-- ⏳ Moderation API integration (hook exists, stub ready)
-- ⏳ Production monitoring setup
-- ⏳ CDN cache configuration
-
-## Ready for Production
-
-This plugin is production-ready with:
-- ✅ Complete error handling
-- ✅ Comprehensive logging
-- ✅ Database transactions
-- ✅ Job retry logic
-- ✅ Stale lock recovery
-- ✅ Rate limiting hooks
-- ✅ Security validation
-- ✅ CORS configuration
-- ✅ Performance optimization
-
-## Next Steps for User
-
-1. Copy files to your project
-2. Install dependencies
-3. Add Prisma schema
-4. Configure R2 bucket
-5. Register plugin
-6. Build frontend integration
-7. Deploy to production
-8. Monitor job queue
-
-**Total implementation time: ~2,730 lines of code and documentation**