@xenterprises/fastify-ximagepipeline 1.0.0

package/src/workers/processor.js

@@ -0,0 +1,264 @@
+// src/workers/processor.js
+import { downloadFromS3, deleteFromS3, uploadToS3, getPublicUrl, batchDeleteFromS3 } from "../services/s3.js";
+import { stripExif, generateVariants, generateBlurhash, getImageMetadata, compressToJpeg } from "../utils/image.js";
+import { getVariantPresets } from "../xImagePipeline.js";
+
+/**
+ * Setup media processor worker
+ * Polls job queue and processes media files
+ */
+export function setupWorker(fastify, context, config) {
+  // Worker state
+  const workerId = `worker-${Date.now()}-${Math.random().toString(36).substring(7)}`;
+  let isRunning = true;
+
+  console.info(`   📊 Media Worker initialized (${workerId})`);
+
+  // Start polling
+  const pollInterval = setInterval(() => {
+    if (isRunning) {
+      processNextJob(fastify, context, config, workerId).catch((err) => {
+        console.error("Worker error:", err.message);
+      });
+    }
+  }, config.pollInterval);
+
+  // Cleanup on fastify close
+  fastify.addHook("onClose", async () => {
+    isRunning = false;
+    clearInterval(pollInterval);
+    console.info("   📊 Media Worker stopped");
+  });
+
+  return { workerId, stop: () => { isRunning = false; } };
+}
+
+/**
+ * Process next job in queue
+ */
+async function processNextJob(fastify, context, config, workerId) {
+  try {
+    // Find next available job with locking
+    const job = await context.db.mediaQueue.findFirst({
+      where: {
+        status: "PENDING",
+        lockedAt: null,
+        attempts: {
+          lt: config.maxAttempts,
+        },
+      },
+      orderBy: {
+        createdAt: "asc",
+      },
+    });
+
+    if (!job) {
+      return; // No jobs available
+    }
+
+    // Lock the job
+    const lockExpiry = new Date(Date.now() + config.lockTimeout);
+    const locked = await context.db.mediaQueue.updateMany({
+      where: {
+        id: job.id,
+        lockedAt: null, // Ensure no one else locked it
+      },
+      data: {
+        lockedAt: new Date(),
+        lockedBy: workerId,
+        status: "PROCESSING",
+      },
+    });
+
+    if (locked.count === 0) {
+      return; // Job was locked by another worker
+    }
+
+    console.info(`Processing job ${job.id}...`);
+
+    try {
+      // Download file from staging
+      const buffer = await downloadFromS3(
+        context.s3Client,
+        context.r2Config.bucket,
+        job.stagingKey
+      );
+
+      // Step 1: Strip EXIF
+      const cleanBuffer = await stripExif(buffer);
+
+      // Step 2: Get image metadata
+      const metadata = await getImageMetadata(cleanBuffer);
+
+      // Step 3: Content moderation (if configured)
+      if (context.moderation) {
+        try {
+          const moderationResult = await moderateImage(cleanBuffer, context.moderation);
+
+          if (!moderationResult.passed) {
+            // Content rejected
+            await context.db.mediaQueue.update({
+              where: { id: job.id },
+              data: {
+                status: "REJECTED",
+                moderationResult: "REJECTED",
+                moderationDetails: moderationResult,
+                lockedAt: null,
+                lockedBy: null,
+              },
+            });
+
+            // Clean up staging
+            await deleteFromS3(context.s3Client, context.r2Config.bucket, job.stagingKey);
+            console.info(`Job ${job.id} rejected by moderation`);
+            return;
+          }
+        } catch (err) {
+          console.error(`Moderation failed: ${err.message}`);
+          throw err;
+        }
+      }
+
+      // Step 4: Generate variants
+      const variantPresets = getVariantPresets();
+      const preset = variantPresets[job.sourceType] || [];
+      const variantSpecs = {};
+      for (const variantName of preset) {
+        if (context.variants[variantName]) {
+          variantSpecs[variantName] = context.variants[variantName];
+        }
+      }
+
+      const variants = await generateVariants(cleanBuffer, variantSpecs, job.sourceType);
+
+      // Step 5: Generate blurhash
+      const blurhash = await generateBlurhash(cleanBuffer);
+
+      // Step 6: Upload to R2
+      const mediaId = `media-${Date.now()}-${Math.random().toString(36).substring(7)}`;
+      const mediaPath = `${context.mediaPath}/${job.sourceType}/${job.sourceId}/${mediaId}`;
+      const originalPath = `${context.originalsPath}/${job.sourceType}/${job.sourceId}/${mediaId}`;
+
+      const urls = {};
+
+      // Upload variants
+      for (const [variantName, variantBuffer] of Object.entries(variants)) {
+        const variantKey = `${mediaPath}/${variantName}.webp`;
+        await uploadToS3(context.s3Client, context.r2Config.bucket, variantKey, variantBuffer, {
+          contentType: "image/webp",
+        });
+        urls[variantName] = getPublicUrl(context.r2Config, variantKey);
+      }
+
+      // Upload compressed original (JPEG for space efficiency)
+      let originalUrl = null;
+      if (context.storeOriginal) {
+        const compressedOriginal = await compressToJpeg(cleanBuffer, context.originalQuality);
+        const originalKey = `${originalPath}/original.jpg`;
+        await uploadToS3(context.s3Client, context.r2Config.bucket, originalKey, compressedOriginal, {
+          contentType: "image/jpeg",
+        });
+        originalUrl = getPublicUrl(context.r2Config, originalKey);
+      }
+
+      // Step 7: Create Media record
+      const media = await context.db.media.create({
+        data: {
+          sourceType: job.sourceType,
+          sourceId: job.sourceId,
+          urls,
+          originalUrl,
+          width: metadata.width,
+          height: metadata.height,
+          format: metadata.format,
+          aspectRatio: `${metadata.width}:${metadata.height}`,
+          blurhash,
+          originalFilename: job.originalFilename,
+          mimeType: job.mimeType,
+          fileSize: job.fileSize,
+        },
+      });
+
+      // Step 8: Update job
+      await context.db.mediaQueue.update({
+        where: { id: job.id },
+        data: {
+          status: "COMPLETE",
+          mediaId: media.id,
+          lockedAt: null,
+          lockedBy: null,
+        },
+      });
+
+      // Step 9: Clean up staging
+      await deleteFromS3(context.s3Client, context.r2Config.bucket, job.stagingKey);
+
+      console.info(`Job ${job.id} completed successfully`);
+    } catch (error) {
+      console.error(`Job ${job.id} processing error: ${error.message}`);
+
+      // Update job with error
+      const nextAttempt = job.attempts + 1;
+      const shouldRetry = nextAttempt < config.maxAttempts;
+
+      await context.db.mediaQueue.update({
+        where: { id: job.id },
+        data: {
+          status: shouldRetry ? "PENDING" : "FAILED",
+          errorMsg: error.message,
+          attempts: nextAttempt,
+          lockedAt: null,
+          lockedBy: null,
+        },
+      });
+
+      if (!shouldRetry) {
+        // Clean up staging after max retries
+        try {
+          await deleteFromS3(context.s3Client, context.r2Config.bucket, job.stagingKey);
+        } catch (cleanupErr) {
+          console.error(`Failed to cleanup staging: ${cleanupErr.message}`);
+        }
+      }
+    }
+  } catch (error) {
+    console.error("Worker process error:", error.message);
+  }
+}
+
+/**
+ * Moderate image content
+ * Currently a stub - implement with actual API (Rekognition, Vision, etc.)
+ */
+async function moderateImage(buffer, moderationConfig) {
+  // TODO: Implement actual moderation API call
+  // For now, always approve
+  return {
+    passed: true,
+    flags: [],
+    confidence: {},
+  };
+}
+
+/**
+ * Recover stale locks (jobs locked > lockTimeout)
+ */
+export async function recoverStaleLocks(db, lockTimeout) {
+  const staleThreshold = new Date(Date.now() - lockTimeout);
+
+  const recovered = await db.mediaQueue.updateMany({
+    where: {
+      lockedAt: {
+        lt: staleThreshold,
+      },
+      status: "PROCESSING",
+    },
+    data: {
+      lockedAt: null,
+      lockedBy: null,
+      status: "PENDING",
+    },
+  });
+
+  return recovered.count;
+}

package/src/xImagePipeline.js

@@ -0,0 +1,164 @@
+// src/xImagePipeline.js
+import fp from "fastify-plugin";
+import { initializeS3Client } from "./services/s3.js";
+import { setupUploadRoute } from "./routes/upload.js";
+import { setupStatusRoute } from "./routes/status.js";
+import { setupWorker } from "./workers/processor.js";
+
+/**
+ * xImagePipeline Plugin for Fastify
+ * Handles image uploads with EXIF stripping, moderation, variant generation, and R2 storage
+ *
+ * @param {Object} fastify - Fastify instance
+ * @param {Object} options - Plugin options
+ * @param {Object} options.r2 - R2 configuration (endpoint, accessKeyId, secretAccessKey, bucket)
+ * @param {Object} options.db - Database instance (Prisma client or similar)
+ * @param {Object} options.moderation - Moderation config (provider, apiKey, etc.)
+ * @param {Object} options.variants - Variant size definitions (optional, uses defaults)
+ * @param {Object} options.sourceTypes - Source type configurations (optional, uses defaults)
+ *   Each source type defines: variants[], formats[], quality, storeOriginal
+ * @param {Object} options.worker - Worker configuration (enabled, pollInterval, maxAttempts)
+ */
+async function xImagePipeline(fastify, options) {
+  // Validate required configuration
+  if (!options.r2) {
+    throw new Error("R2 configuration is required");
+  }
+  if (!options.db) {
+    throw new Error("Database instance (Prisma client) is required");
+  }
+
+  console.info("\n 🎬 Starting xImagePipeline...\n");
+
+  // Initialize R2 S3 client
+  const s3Client = initializeS3Client(options.r2);
+
+  // Store configuration in plugin context
+  const context = {
+    s3Client,
+    db: options.db,
+    moderation: options.moderation || null,
+    variants: options.variants || getDefaultVariants(),
+    sourceTypes: options.sourceTypes || getDefaultSourceTypes(),
+    r2Config: options.r2,
+    maxFileSize: options.maxFileSize || 50 * 1024 * 1024, // 50MB default
+    allowedMimeTypes: options.allowedMimeTypes || [
+      "image/jpeg",
+      "image/png",
+      "image/webp",
+      "image/gif",
+    ],
+    stagingPath: options.stagingPath || "staging",
+    mediaPath: options.mediaPath || "media",
+    originalsPath: options.originalsPath || "originals",
+  };
+
+  // Decorate fastify instance with image pipeline utilities
+  fastify.decorate("xImagePipeline", {
+    upload: async (file, metadata) => {
+      // Placeholder - implemented in upload route
+    },
+    getStatus: async (jobId) => {
+      // Placeholder - implemented in status route
+    },
+    deleteMedia: async (mediaId) => {
+      // Placeholder - implemented as utility
+    },
+  });
+
+  // Register routes
+  await setupUploadRoute(fastify, context);
+  await setupStatusRoute(fastify, context);
+
+  // Start worker if enabled
+  if (options.worker?.enabled !== false) {
+    const workerConfig = {
+      pollInterval: options.worker?.pollInterval || 5000,
+      maxAttempts: options.worker?.maxAttempts || 3,
+      lockTimeout: options.worker?.lockTimeout || 300000, // 5 minutes
+    };
+
+    try {
+      setupWorker(fastify, context, workerConfig);
+      console.info("   ✅ Image Pipeline Worker Started");
+    } catch (err) {
+      console.error("   ❌ Failed to start image pipeline worker:", err.message);
+      if (options.worker?.failOnError !== false) {
+        throw err;
+      }
+    }
+  }
+
+  console.info("\n 🎬 xImagePipeline Ready!\n");
+}
+
+/**
+ * Get default variant specifications
+ */
+function getDefaultVariants() {
+  return {
+    xs: { width: 80, height: 80, fit: "cover" },
+    sm: { width: 200, height: 200, fit: "cover" },
+    md: { width: 600, height: null, fit: "inside" },
+    lg: { width: 1200, height: null, fit: "inside" },
+    xl: { width: 1920, height: null, fit: "inside" },
+    "2xl": { width: 2560, height: null, fit: "inside" },
+  };
+}
+
+/**
+ * Get default source type configurations
+ * Each source type can have different processing settings
+ */
+function getDefaultSourceTypes() {
+  return {
+    avatar: {
+      variants: ["xs", "sm"],
+      formats: ["webp"],
+      quality: 85,
+      storeOriginal: true,
+    },
+    member_photo: {
+      variants: ["xs", "sm", "md"],
+      formats: ["webp"],
+      quality: 85,
+      storeOriginal: true,
+    },
+    gallery: {
+      variants: ["md", "lg", "xl"],
+      formats: ["webp"],
+      quality: 85,
+      storeOriginal: false,
+    },
+    hero: {
+      variants: ["lg", "xl", "2xl"],
+      formats: ["webp"],
+      quality: 80,
+      storeOriginal: false,
+    },
+    content: {
+      variants: ["md", "lg"],
+      formats: ["webp"],
+      quality: 85,
+      storeOriginal: true,
+    },
+  };
+}
+
+/**
+ * Get variant presets for different source types
+ */
+export function getVariantPresets() {
+  return {
+    avatar: ["xs", "sm"],
+    member_photo: ["xs", "sm", "md"],
+    gallery: ["md", "lg", "xl"],
+    hero: ["lg", "xl", "2xl"],
+    content: ["md", "lg"],
+  };
+}
+
+export default fp(xImagePipeline, {
+  name: "xImagePipeline",
+  fastify: "5.x",
+});

package/test/xImagePipeline.test.js

@@ -0,0 +1,196 @@
+// test/xImagePipeline.test.js
+import { test } from "node:test";
+import assert from "node:assert";
+import Fastify from "fastify";
+import multipart from "@fastify/multipart";
+import xImagePipeline from "../src/xImagePipeline.js";
+
+// Mock database
+const mockDb = {
+  mediaQueue: {
+    create: async (data) => ({
+      id: "test-job-" + Date.now(),
+      ...data.data,
+    }),
+    findUnique: async (query) => ({
+      id: query.where.id,
+      status: "COMPLETE",
+      sourceType: "avatar",
+      sourceId: "user123",
+    }),
+    findFirst: async () => null,
+    update: async (query) => ({ count: 1 }),
+    updateMany: async () => ({ count: 0 }),
+  },
+  media: {
+    create: async (data) => ({
+      id: "media-123",
+      ...data.data,
+    }),
+  },
+};
+
+// Mock R2 config
+const mockR2Config = {
+  endpoint: "https://example.r2.cloudflarestorage.com",
+  region: "auto",
+  accessKeyId: "mock-key",
+  secretAccessKey: "mock-secret",
+  bucket: "test-bucket",
+};
+
+test("xImagePipeline Plugin - registers successfully with required config", async () => {
+  const fastify = Fastify({ logger: false });
+  try {
+    await fastify.register(multipart);
+    await fastify.register(xImagePipeline, {
+      r2: mockR2Config,
+      db: mockDb,
+    });
+    assert.ok(true, "Plugin registered successfully");
+  } finally {
+    try {
+      await fastify.close();
+    } catch {
+      // Ignore
+    }
+  }
+});
+
+test("xImagePipeline Plugin - throws error without R2 config", async () => {
+  const fastify = Fastify({ logger: false });
+  try {
+    await assert.rejects(
+      async () => {
+        await fastify.register(multipart);
+        await fastify.register(xImagePipeline, {
+          db: mockDb,
+        });
+      },
+      /R2 configuration is required/
+    );
+  } finally {
+    try {
+      await fastify.close();
+    } catch {
+      // Ignore
+    }
+  }
+});
+
+test("xImagePipeline Plugin - throws error without database", async () => {
+  const fastify = Fastify({ logger: false });
+  try {
+    await assert.rejects(
+      async () => {
+        await fastify.register(multipart);
+        await fastify.register(xImagePipeline, {
+          r2: mockR2Config,
+        });
+      },
+      /Database instance/
+    );
+  } finally {
+    try {
+      await fastify.close();
+    } catch {
+      // Ignore
+    }
+  }
+});
+
+test("xImagePipeline Routes - GET /media/status/:jobId returns job status", async () => {
+  const fastify = Fastify({ logger: false });
+  try {
+    await fastify.register(multipart);
+    await fastify.register(xImagePipeline, {
+      r2: mockR2Config,
+      db: mockDb,
+    });
+
+    const response = await fastify.inject({
+      method: "GET",
+      url: "/image-pipeline/status/test-job-123",
+    });
+
+    assert.equal(response.statusCode, 200);
+    const body = JSON.parse(response.payload);
+    assert.ok(body.jobId, "Response should have jobId");
+    assert.ok(body.status, "Response should have status");
+  } finally {
+    try {
+      await fastify.close();
+    } catch {
+      // Ignore
+    }
+  }
+});
+
+test("xImagePipeline Configuration - accepts custom variants config", async () => {
+  const fastify = Fastify({ logger: false });
+  const customConfig = {
+    r2: mockR2Config,
+    db: mockDb,
+    variants: {
+      custom: {
+        xs: { width: 100, height: 100, fit: "cover" },
+        lg: { width: 800, height: 600, fit: "inside" },
+      },
+    },
+  };
+  try {
+    await fastify.register(multipart);
+    await fastify.register(xImagePipeline, customConfig);
+    assert.ok(true, "Plugin registered with custom variants");
+  } finally {
+    try {
+      await fastify.close();
+    } catch {
+      // Ignore
+    }
+  }
+});
+
+test("xImagePipeline Configuration - accepts worker configuration", async () => {
+  const fastify = Fastify({ logger: false });
+  const configWithWorker = {
+    r2: mockR2Config,
+    db: mockDb,
+    worker: {
+      enabled: true,
+      pollInterval: 10000,
+      maxAttempts: 5,
+    },
+  };
+  try {
+    await fastify.register(multipart);
+    await fastify.register(xImagePipeline, configWithWorker);
+    assert.ok(true, "Plugin registered with worker config");
+  } finally {
+    try {
+      await fastify.close();
+    } catch {
+      // Ignore
+    }
+  }
+});
+
+test("xImagePipeline Variant Presets - has predefined variant presets", async () => {
+  const fastify = Fastify({ logger: false });
+  try {
+    await fastify.register(multipart);
+    await fastify.register(xImagePipeline, {
+      r2: mockR2Config,
+      db: mockDb,
+    });
+
+    // Just verify plugin registered - variants are available via plugin configuration
+    assert.ok(true, "Plugin has variant presets configured");
+  } finally {
+    try {
+      await fastify.close();
+    } catch {
+      // Ignore
+    }
+  }
+});