@xenterprises/fastify-ximagepipeline 1.1.1 → 1.2.0

package/src/workers/processor.js

@@ -1,33 +1,41 @@
 // src/workers/processor.js
 import { downloadFromS3, deleteFromS3, uploadToS3, getPublicUrl, batchDeleteFromS3 } from "../services/s3.js";
-import { stripExif, generateVariants, generateBlurhash, getImageMetadata, compressToJpeg } from "../utils/image.js";
-import { getVariantPresets } from "../xImagePipeline.js";
+import { stripExif, generateVariants, generateBlurhash, getImageMetadata, getAspectRatio, compressToJpeg } from "../utils/image.js";
 
 /**
  * Setup media processor worker
  * Polls job queue and processes media files
  */
 export function setupWorker(fastify, context, config) {
-  // Worker state
   const workerId = `worker-${Date.now()}-${Math.random().toString(36).substring(7)}`;
   let isRunning = true;
 
-  console.info(`   📊 Media Worker initialized (${workerId})`);
+  fastify.log.info(`[xImagePipeline] Media Worker initialized (${workerId})`);
 
   // Start polling
   const pollInterval = setInterval(() => {
     if (isRunning) {
       processNextJob(fastify, context, config, workerId).catch((err) => {
-        console.error("Worker error:", err.message);
+        fastify.log.error({ err }, "[xImagePipeline] Worker error");
       });
     }
   }, config.pollInterval);
 
+  // Periodically recover stale locks
+  const staleLockInterval = setInterval(() => {
+    if (isRunning) {
+      recoverStaleLocks(context.db, config.lockTimeout).catch((err) => {
+        fastify.log.error({ err }, "[xImagePipeline] Stale lock recovery error");
+      });
+    }
+  }, config.lockTimeout);
+
   // Cleanup on fastify close
   fastify.addHook("onClose", async () => {
     isRunning = false;
     clearInterval(pollInterval);
-    console.info("   📊 Media Worker stopped");
+    clearInterval(staleLockInterval);
+    fastify.log.info("[xImagePipeline] Media Worker stopped");
   });
 
   return { workerId, stop: () => { isRunning = false; } };
@@ -57,7 +65,6 @@ async function processNextJob(fastify, context, config, workerId) {
     }
 
     // Lock the job
-    const lockExpiry = new Date(Date.now() + config.lockTimeout);
     const locked = await context.db.mediaQueue.updateMany({
       where: {
         id: job.id,
@@ -74,7 +81,7 @@ async function processNextJob(fastify, context, config, workerId) {
       return; // Job was locked by another worker
     }
 
-    console.info(`Processing job ${job.id}...`);
+    fastify.log.info({ jobId: job.id }, "[xImagePipeline] Processing job");
 
     try {
       // Download file from staging
@@ -96,7 +103,6 @@ async function processNextJob(fastify, context, config, workerId) {
           const moderationResult = await moderateImage(cleanBuffer, context.moderation);
 
           if (!moderationResult.passed) {
-            // Content rejected
             await context.db.mediaQueue.update({
               where: { id: job.id },
               data: {
@@ -108,28 +114,28 @@ async function processNextJob(fastify, context, config, workerId) {
               },
             });
 
-            // Clean up staging
             await deleteFromS3(context.s3Client, context.r2Config.bucket, job.stagingKey);
-            console.info(`Job ${job.id} rejected by moderation`);
+            fastify.log.info({ jobId: job.id }, "[xImagePipeline] Job rejected by moderation");
             return;
           }
         } catch (err) {
-          console.error(`Moderation failed: ${err.message}`);
+          fastify.log.error({ err, jobId: job.id }, "[xImagePipeline] Moderation failed");
           throw err;
         }
       }
 
-      // Step 4: Generate variants
-      const variantPresets = getVariantPresets();
-      const preset = variantPresets[job.sourceType] || [];
+      // Step 4: Generate variants using sourceType config
+      const sourceTypeConfig = context.sourceTypes[job.sourceType];
+      const variantNames = sourceTypeConfig?.variants || [];
       const variantSpecs = {};
-      for (const variantName of preset) {
+      for (const variantName of variantNames) {
         if (context.variants[variantName]) {
           variantSpecs[variantName] = context.variants[variantName];
         }
       }
 
-      const variants = await generateVariants(cleanBuffer, variantSpecs, job.sourceType);
+      const quality = sourceTypeConfig?.quality || 85;
+      const variants = await generateVariants(cleanBuffer, variantSpecs, job.sourceType, quality);
 
       // Step 5: Generate blurhash
       const blurhash = await generateBlurhash(cleanBuffer);
@@ -150,10 +156,11 @@ async function processNextJob(fastify, context, config, workerId) {
         urls[variantName] = getPublicUrl(context.r2Config, variantKey);
       }
 
-      // Upload compressed original (JPEG for space efficiency)
+      // Upload compressed original if sourceType config says to
       let originalUrl = null;
-      if (context.storeOriginal) {
-        const compressedOriginal = await compressToJpeg(cleanBuffer, context.originalQuality);
+      const storeOriginal = sourceTypeConfig?.storeOriginal ?? false;
+      if (storeOriginal) {
+        const compressedOriginal = await compressToJpeg(cleanBuffer, quality);
         const originalKey = `${originalPath}/original.jpg`;
         await uploadToS3(context.s3Client, context.r2Config.bucket, originalKey, compressedOriginal, {
           contentType: "image/jpeg",
@@ -171,7 +178,7 @@ async function processNextJob(fastify, context, config, workerId) {
           width: metadata.width,
           height: metadata.height,
           format: metadata.format,
-          aspectRatio: `${metadata.width}:${metadata.height}`,
+          aspectRatio: getAspectRatio(metadata.width, metadata.height),
           blurhash,
           originalFilename: job.originalFilename,
           mimeType: job.mimeType,
@@ -193,9 +200,9 @@ async function processNextJob(fastify, context, config, workerId) {
       // Step 9: Clean up staging
       await deleteFromS3(context.s3Client, context.r2Config.bucket, job.stagingKey);
 
-      console.info(`Job ${job.id} completed successfully`);
+      fastify.log.info({ jobId: job.id, mediaId: media.id }, "[xImagePipeline] Job completed");
     } catch (error) {
-      console.error(`Job ${job.id} processing error: ${error.message}`);
+      fastify.log.error({ err: error, jobId: job.id }, "[xImagePipeline] Job processing error");
 
       // Update job with error
       const nextAttempt = job.attempts + 1;
@@ -213,26 +220,28 @@ async function processNextJob(fastify, context, config, workerId) {
       });
 
       if (!shouldRetry) {
-        // Clean up staging after max retries
         try {
           await deleteFromS3(context.s3Client, context.r2Config.bucket, job.stagingKey);
         } catch (cleanupErr) {
-          console.error(`Failed to cleanup staging: ${cleanupErr.message}`);
+          fastify.log.error({ err: cleanupErr }, "[xImagePipeline] Failed to cleanup staging");
         }
       }
     }
   } catch (error) {
-    console.error("Worker process error:", error.message);
+    fastify.log.error({ err: error }, "[xImagePipeline] Worker process error");
   }
 }
 
 /**
  * Moderate image content
- * Currently a stub - implement with actual API (Rekognition, Vision, etc.)
+ * Stub implementation — override via options.moderation.handler to provide real moderation.
+ * Expected signature: async (buffer, config) => { passed: boolean, flags: string[], confidence: object }
  */
 async function moderateImage(buffer, moderationConfig) {
-  // TODO: Implement actual moderation API call
-  // For now, always approve
+  if (typeof moderationConfig.handler === "function") {
+    return moderationConfig.handler(buffer, moderationConfig);
+  }
+  // Default: always approve
   return {
     passed: true,
     flags: [],
@@ -241,7 +250,7 @@ async function moderateImage(buffer, moderationConfig) {
 }
 
 /**
- * Recover stale locks (jobs locked > lockTimeout)
+ * Recover stale locks (jobs locked longer than lockTimeout)
  */
 export async function recoverStaleLocks(db, lockTimeout) {
   const staleThreshold = new Date(Date.now() - lockTimeout);

package/src/xImagePipeline.js

@@ -1,6 +1,6 @@
 // src/xImagePipeline.js
 import fp from "fastify-plugin";
-import { initializeS3Client } from "./services/s3.js";
+import { initializeS3Client, deleteFromS3, batchDeleteFromS3 } from "./services/s3.js";
 import { setupUploadRoute } from "./routes/upload.js";
 import { setupStatusRoute } from "./routes/status.js";
 import { setupWorker } from "./workers/processor.js";
@@ -9,26 +9,34 @@ import { setupWorker } from "./workers/processor.js";
  * xImagePipeline Plugin for Fastify
  * Handles image uploads with EXIF stripping, moderation, variant generation, and R2 storage
  *
- * @param {Object} fastify - Fastify instance
+ * @param {import('fastify').FastifyInstance} fastify - Fastify instance
  * @param {Object} options - Plugin options
  * @param {Object} options.r2 - R2 configuration (endpoint, accessKeyId, secretAccessKey, bucket)
- * @param {Object} options.db - Database instance (Prisma client or similar)
- * @param {Object} options.moderation - Moderation config (provider, apiKey, etc.)
- * @param {Object} options.variants - Variant size definitions (optional, uses defaults)
- * @param {Object} options.sourceTypes - Source type configurations (optional, uses defaults)
- *   Each source type defines: variants[], formats[], quality, storeOriginal
- * @param {Object} options.worker - Worker configuration (enabled, pollInterval, maxAttempts)
+ * @param {Object} options.db - Database instance (Prisma client)
+ * @param {Object} [options.moderation] - Moderation config ({ handler: async (buffer, config) => result })
+ * @param {Object} [options.variants] - Variant size definitions (default: xs/sm/md/lg/xl/2xl)
+ * @param {Object} [options.sourceTypes] - Source type configurations (default: avatar/member_photo/gallery/hero/content)
+ * @param {Object} [options.worker] - Worker config ({ enabled, pollInterval, maxAttempts, lockTimeout })
+ * @param {string} [options.stagingPath='staging'] - R2 prefix for staging uploads
+ * @param {string} [options.mediaPath='media'] - R2 prefix for processed media
+ * @param {string} [options.originalsPath='originals'] - R2 prefix for originals
+ * @param {number} [options.maxFileSize=52428800] - Max upload size in bytes (default 50MB)
+ * @param {string[]} [options.allowedMimeTypes] - Allowed MIME types (default: jpeg/png/webp/gif)
  */
 async function xImagePipeline(fastify, options) {
   // Validate required configuration
   if (!options.r2) {
-    throw new Error("R2 configuration is required");
+    throw new Error("[xImagePipeline] R2 configuration is required");
   }
   if (!options.db) {
-    throw new Error("Database instance (Prisma client) is required");
+    throw new Error("[xImagePipeline] Database instance (Prisma client) is required");
   }
 
-  console.info("\n 🎬 Starting xImagePipeline...\n");
+  if (typeof options.r2 !== "object" || !options.r2.endpoint || !options.r2.accessKeyId || !options.r2.secretAccessKey || !options.r2.bucket) {
+    throw new Error("[xImagePipeline] R2 configuration must include: endpoint, accessKeyId, secretAccessKey, bucket");
+  }
+
+  fastify.log.info("[xImagePipeline] Starting...");
 
   // Initialize R2 S3 client
   const s3Client = initializeS3Client(options.r2);
@@ -41,7 +49,7 @@ async function xImagePipeline(fastify, options) {
     variants: options.variants || getDefaultVariants(),
     sourceTypes: options.sourceTypes || getDefaultSourceTypes(),
     r2Config: options.r2,
-    maxFileSize: options.maxFileSize || 50 * 1024 * 1024, // 50MB default
+    maxFileSize: options.maxFileSize || 50 * 1024 * 1024,
     allowedMimeTypes: options.allowedMimeTypes || [
       "image/jpeg",
       "image/png",
@@ -55,15 +63,77 @@ async function xImagePipeline(fastify, options) {
 
   // Decorate fastify instance with image pipeline utilities
   fastify.decorate("xImagePipeline", {
-    upload: async (file, metadata) => {
-      // Placeholder - implemented in upload route
-    },
+    /**
+     * Get the status of a processing job
+     * @param {string} jobId
+     * @returns {Promise<Object|null>} Job with media relation, or null
+     */
     getStatus: async (jobId) => {
-      // Placeholder - implemented in status route
+      return context.db.mediaQueue.findUnique({
+        where: { id: jobId },
+        include: { media: true },
+      });
     },
+
+    /**
+     * Delete a media record and all associated R2 objects
+     * @param {string} mediaId
+     * @returns {Promise<{ deleted: boolean, r2Deleted: number }>}
+     */
     deleteMedia: async (mediaId) => {
-      // Placeholder - implemented as utility
+      const media = await context.db.media.findUnique({ where: { id: mediaId } });
+      if (!media) {
+        throw new Error(`[xImagePipeline] Media not found: ${mediaId}`);
+      }
+
+      // Delete variant files from R2
+      const mediaPrefix = `${context.mediaPath}/${media.sourceType}/${media.sourceId}/${mediaId}`;
+      const r2Result = await batchDeleteFromS3(s3Client, context.r2Config.bucket, mediaPrefix);
+
+      // Delete original if stored
+      if (media.originalUrl) {
+        const originalPrefix = `${context.originalsPath}/${media.sourceType}/${media.sourceId}/${mediaId}`;
+        const origResult = await batchDeleteFromS3(s3Client, context.r2Config.bucket, originalPrefix);
+        r2Result.deleted += origResult.deleted;
+      }
+
+      // Delete database records
+      await context.db.mediaQueue.deleteMany({ where: { mediaId } });
+      await context.db.media.delete({ where: { id: mediaId } });
+
+      return { deleted: true, r2Deleted: r2Result.deleted };
+    },
+
+    /**
+     * List media by sourceType and sourceId
+     * @param {string} sourceType
+     * @param {string} sourceId
+     * @returns {Promise<Object[]>}
+     */
+    listMedia: async (sourceType, sourceId) => {
+      return context.db.media.findMany({
+        where: { sourceType, sourceId },
+        orderBy: { createdAt: "desc" },
+      });
     },
+
+    /**
+     * Get variant presets for all source types
+     * @returns {Object}
+     */
+    getVariantPresets,
+
+    /**
+     * Get the source type configurations
+     * @returns {Object}
+     */
+    getSourceTypes: () => context.sourceTypes,
+
+    /**
+     * Get variant definitions
+     * @returns {Object}
+     */
+    getVariants: () => context.variants,
   });
 
   // Register routes
@@ -80,16 +150,16 @@ async function xImagePipeline(fastify, options) {
 
     try {
       setupWorker(fastify, context, workerConfig);
-      console.info("   ✅ Image Pipeline Worker Started");
+      fastify.log.info("[xImagePipeline] Worker started");
     } catch (err) {
-      console.error("   ❌ Failed to start image pipeline worker:", err.message);
+      fastify.log.error({ err }, "[xImagePipeline] Failed to start worker");
       if (options.worker?.failOnError !== false) {
         throw err;
       }
     }
   }
 
-  console.info("\n 🎬 xImagePipeline Ready!\n");
+  fastify.log.info("[xImagePipeline] Ready");
 }
 
 /**
@@ -108,7 +178,6 @@ function getDefaultVariants() {
 
 /**
  * Get default source type configurations
- * Each source type can have different processing settings
  */
 function getDefaultSourceTypes() {
   return {
@@ -146,7 +215,8 @@ function getDefaultSourceTypes() {
 }
 
 /**
- * Get variant presets for different source types
+ * Get variant presets for all source types
+ * @returns {Object<string, string[]>}
  */
 export function getVariantPresets() {
   return {
@@ -161,4 +231,5 @@ export function getVariantPresets() {
 export default fp(xImagePipeline, {
   name: "xImagePipeline",
   fastify: "5.x",
+  dependencies: [],
 });

package/SCHEMA.prisma

@@ -1,113 +0,0 @@
-// This schema should be added to your main Prisma schema.prisma file
-// It defines the models needed for the xMedia pipeline
-
-enum MediaStatus {
-  PENDING
-  PROCESSING
-  COMPLETE
-  REJECTED
-  FAILED
-}
-
-enum ModerationResult {
-  APPROVED
-  REJECTED
-  FLAGGED
-}
-
-model MediaQueue {
-  id String @id @default(cuid())
-
-  // Job status
-  status MediaStatus @default(PENDING)
-
-  // Source information
-  sourceType String // avatar, gallery, hero, member_photo, content, etc.
-  sourceId String // userId, bandId, etc.
-
-  // File information
-  stagingKey String // Key in R2 staging bucket
-  originalFilename String
-  mimeType String
-  fileSize Int
-
-  // Processing results
-  mediaId String? // FK to Media after processing
-  media Media? @relation(fields: [mediaId], references: [id], onDelete: SetNull)
-
-  // Error tracking
-  attempts Int @default(0)
-  maxAttempts Int @default(3)
-  errorMsg String?
-
-  // Moderation results
-  moderationResult ModerationResult?
-  moderationDetails Json? // Full moderation API response
-
-  // Locking for worker process
-  lockedAt DateTime?
-  lockedBy String? // Worker ID that locked this job
-
-  // Timestamps
-  createdAt DateTime @default(now())
-  updatedAt DateTime @updatedAt
-
-  @@index([status, createdAt])
-  @@index([sourceType, sourceId])
-  @@index([lockedAt])
-}
-
-model Media {
-  id String @id @default(cuid())
-
-  // Variant URLs (object mapping variant name to URL)
-  // e.g., { "xs": "https://...", "sm": "https://...", "md": "https://..." }
-  urls Json @default("{}")
-  originalUrl String // URL to full-resolution original
-
-  // Image properties
-  width Int
-  height Int
-  format String // jpeg, png, webp, gif
-  aspectRatio String // e.g., "16:9", "4:3", "1:1"
-
-  // Loading placeholder
-  blurhash String // For instant UI placeholder
-
-  // For smart cropping
-  focalPoint Json @default("{\"x\": 0.5, \"y\": 0.5}") // { x: 0-1, y: 0-1 }
-
-  // Source information (denormalized for queries)
-  sourceType String
-  sourceId String
-
-  // File information
-  originalFilename String
-  mimeType String
-  fileSize Int
-
-  // Metadata
-  exifStripped Boolean @default(true)
-  createdAt DateTime @default(now())
-  updatedAt DateTime @updatedAt
-
-  // Relations (optional - add as needed)
-  // User.avatar => Media
-  // Band.avatar => Media
-  // Gallery items, etc.
-
-  queue MediaQueue[]
-
-  @@index([sourceType, sourceId])
-  @@index([createdAt])
-}
-
-// Example: Add to User model
-// avatar Media? @relation(fields: [avatarId], references: [id], onDelete: SetNull)
-// avatarId String?
-
-// Example: Add to Band model
-// avatar Media? @relation(fields: [avatarId], references: [id], onDelete: SetNull)
-// avatarId String?
-// hero Media? @relation(fields: [heroId], references: [id], onDelete: SetNull)
-// heroId String?