@gallop.software/studio 1.5.10 → 2.0.1

package/src/handlers/scan.ts

@@ -0,0 +1,311 @@
+import { NextRequest, NextResponse } from 'next/server'
+import { promises as fs } from 'fs'
+import path from 'path'
+import sharp from 'sharp'
+import { encode } from 'blurhash'
+import { loadMeta, saveMeta, isMediaFile, isImageFile, getFileEntries } from './utils'
+import { getAllThumbnailPaths, isProcessed } from '../types'
+import { getPublicPath } from '../config'
+
+/**
+ * Streaming scan handler - scans filesystem for new files not in meta
+ * For images, reads dimensions (w/h)
+ * Handles collisions by renaming files with -1, -2, etc.
+ * Also detects orphaned files in the images folder
+ */
+export async function handleScanStream() {
+  const encoder = new TextEncoder()
+  
+  const stream = new ReadableStream({
+    async start(controller) {
+      const sendEvent = (data: object) => {
+        controller.enqueue(encoder.encode(`data: ${JSON.stringify(data)}\n\n`))
+      }
+
+      try {
+        const meta = await loadMeta()
+        const existingCount = Object.keys(meta).filter(k => !k.startsWith('_')).length
+        const existingKeys = new Set(Object.keys(meta))
+        const added: string[] = []
+        const renamed: Array<{ from: string; to: string }> = []
+        const errors: string[] = []
+        const orphanedFiles: string[] = []
+
+        // Collect all files first
+        const allFiles: Array<{ relativePath: string; fullPath: string }> = []
+
+        async function scanDir(dir: string, relativePath: string = ''): Promise<void> {
+          try {
+            const entries = await fs.readdir(dir, { withFileTypes: true })
+            
+            for (const entry of entries) {
+              if (entry.name.startsWith('.')) continue
+              
+              const fullPath = path.join(dir, entry.name)
+              const relPath = relativePath ? `${relativePath}/${entry.name}` : entry.name
+
+              // Skip the images folder (generated thumbnails)
+              if (relPath === 'images' || relPath.startsWith('images/')) continue
+
+              if (entry.isDirectory()) {
+                await scanDir(fullPath, relPath)
+              } else if (isMediaFile(entry.name)) {
+                allFiles.push({ relativePath: relPath, fullPath })
+              }
+            }
+          } catch {
+            // Directory might not exist
+          }
+        }
+
+        const publicDir = getPublicPath()
+        await scanDir(publicDir)
+
+        const total = allFiles.length
+        sendEvent({ type: 'start', total })
+
+        for (let i = 0; i < allFiles.length; i++) {
+          let { relativePath, fullPath } = allFiles[i]
+          let imageKey = '/' + relativePath
+          
+          sendEvent({ 
+            type: 'progress', 
+            current: i + 1, 
+            total, 
+            percent: Math.round(((i + 1) / total) * 100),
+            currentFile: relativePath 
+          })
+
+          // Check if already in meta
+          if (existingKeys.has(imageKey)) {
+            // File already tracked - skip
+            continue
+          }
+
+          // Check for collision (path exists in meta but file is new)
+          if (meta[imageKey]) {
+            // Need to rename this file to avoid collision
+            const ext = path.extname(relativePath)
+            const baseName = relativePath.slice(0, -ext.length)
+            let counter = 1
+            let newKey = `/${baseName}-${counter}${ext}`
+            
+            while (meta[newKey]) {
+              counter++
+              newKey = `/${baseName}-${counter}${ext}`
+            }
+            
+            // Rename the physical file
+            const newRelativePath = `${baseName}-${counter}${ext}`
+            const newFullPath = getPublicPath(newRelativePath)
+            
+            try {
+              await fs.rename(fullPath, newFullPath)
+              renamed.push({ from: relativePath, to: newRelativePath })
+              relativePath = newRelativePath
+              fullPath = newFullPath
+              imageKey = newKey
+            } catch (err) {
+              console.error(`Failed to rename ${relativePath}:`, err)
+              errors.push(`Failed to rename ${relativePath}`)
+              continue
+            }
+          }
+
+          try {
+            const isImage = isImageFile(relativePath)
+            
+            if (isImage) {
+              // Read dimensions and generate blurhash for images
+              const ext = path.extname(relativePath).toLowerCase()
+              
+              if (ext === '.svg') {
+                // SVGs don't have pixel dimensions in the same way
+                meta[imageKey] = { o: { w: 0, h: 0 }, b: '' }
+              } else {
+                try {
+                  const buffer = await fs.readFile(fullPath)
+                  const metadata = await sharp(buffer).metadata()
+                  
+                  // Generate blurhash
+                  const { data, info } = await sharp(buffer)
+                    .resize(32, 32, { fit: 'inside' })
+                    .ensureAlpha()
+                    .raw()
+                    .toBuffer({ resolveWithObject: true })
+                  
+                  const blurhash = encode(new Uint8ClampedArray(data), info.width, info.height, 4, 4)
+                  
+                  meta[imageKey] = {
+                    o: { w: metadata.width || 0, h: metadata.height || 0 },
+                    b: blurhash,
+                  }
+                } catch {
+                  // Couldn't read dimensions
+                  meta[imageKey] = { o: { w: 0, h: 0 } }
+                }
+              }
+            } else {
+              // Non-image files - just add empty entry
+              meta[imageKey] = {}
+            }
+            
+            existingKeys.add(imageKey)
+            added.push(imageKey)
+          } catch (error) {
+            console.error(`Failed to process ${relativePath}:`, error)
+            errors.push(relativePath)
+          }
+        }
+
+        // Check for orphaned files in the images folder
+        sendEvent({ type: 'cleanup', message: 'Checking for orphaned thumbnails...' })
+        
+        // Build set of expected thumbnail paths from meta entries
+        const expectedThumbnails = new Set<string>()
+        const fileEntries = getFileEntries(meta)
+        for (const [imageKey, entry] of fileEntries) {
+          // Only track local thumbnails (not pushed to CDN)
+          if (entry.c === undefined && isProcessed(entry)) {
+            for (const thumbPath of getAllThumbnailPaths(imageKey)) {
+              expectedThumbnails.add(thumbPath)
+            }
+          }
+        }
+
+        // Scan the images folder for orphaned files
+        async function findOrphans(dir: string, relativePath: string = ''): Promise<void> {
+          try {
+            const entries = await fs.readdir(dir, { withFileTypes: true })
+            
+            for (const entry of entries) {
+              if (entry.name.startsWith('.')) continue
+
+              const fullPath = path.join(dir, entry.name)
+              const relPath = relativePath ? `${relativePath}/${entry.name}` : entry.name
+
+              if (entry.isDirectory()) {
+                await findOrphans(fullPath, relPath)
+              } else if (isImageFile(entry.name)) {
+                const publicPath = `/images/${relPath}`
+                if (!expectedThumbnails.has(publicPath)) {
+                  orphanedFiles.push(publicPath)
+                }
+              }
+            }
+          } catch {
+            // Directory might not exist
+          }
+        }
+
+        const imagesDir = getPublicPath('images')
+        try {
+          await findOrphans(imagesDir)
+        } catch {
+          // images dir might not exist
+        }
+
+        await saveMeta(meta)
+
+        sendEvent({ 
+          type: 'complete', 
+          existingCount,
+          added: added.length, 
+          renamed: renamed.length,
+          errors: errors.length,
+          renamedFiles: renamed,
+          orphanedFiles: orphanedFiles.length > 0 ? orphanedFiles : undefined,
+        })
+      } catch (error) {
+        console.error('Scan failed:', error)
+        sendEvent({ type: 'error', message: 'Scan failed' })
+      } finally {
+        controller.close()
+      }
+    }
+  })
+
+  return new Response(stream, {
+    headers: {
+      'Content-Type': 'text/event-stream',
+      'Cache-Control': 'no-cache',
+      'Connection': 'keep-alive',
+    },
+  })
+}
+
+/**
+ * Delete orphaned files from the images folder
+ */
+export async function handleDeleteOrphans(request: NextRequest) {
+  try {
+    const { paths } = await request.json() as { paths: string[] }
+    
+    if (!paths || !Array.isArray(paths) || paths.length === 0) {
+      return NextResponse.json({ error: 'No paths provided' }, { status: 400 })
+    }
+    
+    const deleted: string[] = []
+    const errors: string[] = []
+    
+    for (const orphanPath of paths) {
+      // Ensure the path is within the images folder for safety
+      if (!orphanPath.startsWith('/images/')) {
+        errors.push(`Invalid path: ${orphanPath}`)
+        continue
+      }
+      
+      const fullPath = getPublicPath(orphanPath)
+      
+      try {
+        await fs.unlink(fullPath)
+        deleted.push(orphanPath)
+      } catch (err) {
+        console.error(`Failed to delete ${orphanPath}:`, err)
+        errors.push(orphanPath)
+      }
+    }
+    
+    // Clean up empty directories
+    const imagesDir = getPublicPath('images')
+    
+    async function removeEmptyDirs(dir: string): Promise<boolean> {
+      try {
+        const entries = await fs.readdir(dir, { withFileTypes: true })
+        let isEmpty = true
+        
+        for (const entry of entries) {
+          if (entry.isDirectory()) {
+            const subDirEmpty = await removeEmptyDirs(path.join(dir, entry.name))
+            if (!subDirEmpty) isEmpty = false
+          } else {
+            isEmpty = false
+          }
+        }
+        
+        if (isEmpty && dir !== imagesDir) {
+          await fs.rmdir(dir)
+        }
+        
+        return isEmpty
+      } catch {
+        return true
+      }
+    }
+    
+    try {
+      await removeEmptyDirs(imagesDir)
+    } catch {
+      // images dir might not exist
+    }
+    
+    return NextResponse.json({
+      success: true,
+      deleted: deleted.length,
+      errors: errors.length,
+    })
+  } catch (error) {
+    console.error('Failed to delete orphans:', error)
+    return NextResponse.json({ error: 'Failed to delete orphaned files' }, { status: 500 })
+  }
+}

package/src/handlers/utils/cdn.ts

@@ -0,0 +1,234 @@
+import { promises as fs } from 'fs'
+import { S3Client, GetObjectCommand, PutObjectCommand, DeleteObjectCommand } from '@aws-sdk/client-s3'
+import { getAllThumbnailPaths } from '../../types'
+import { getContentType } from './files'
+import { getPublicPath } from '../../config'
+
+/**
+ * Purge URLs from Cloudflare cache
+ * Requires CLOUDFLARE_ZONE_ID and CLOUDFLARE_API_TOKEN environment variables
+ */
+export async function purgeCloudflareCache(urls: string[]): Promise<void> {
+  const zoneId = process.env.CLOUDFLARE_ZONE_ID
+  const apiToken = process.env.CLOUDFLARE_API_TOKEN
+  
+  if (!zoneId || !apiToken || urls.length === 0) {
+    return // Cache purge not configured or no URLs to purge
+  }
+  
+  try {
+    const response = await fetch(
+      `https://api.cloudflare.com/client/v4/zones/${zoneId}/purge_cache`,
+      {
+        method: 'POST',
+        headers: {
+          'Authorization': `Bearer ${apiToken}`,
+          'Content-Type': 'application/json',
+        },
+        body: JSON.stringify({ files: urls }),
+      }
+    )
+    
+    if (!response.ok) {
+      console.error('Cache purge failed:', await response.text())
+    }
+  } catch (error) {
+    console.error('Cache purge error:', error)
+  }
+}
+
+function getR2Client() {
+  const accountId = process.env.CLOUDFLARE_R2_ACCOUNT_ID
+  const accessKeyId = process.env.CLOUDFLARE_R2_ACCESS_KEY_ID
+  const secretAccessKey = process.env.CLOUDFLARE_R2_SECRET_ACCESS_KEY
+
+  if (!accountId || !accessKeyId || !secretAccessKey) {
+    throw new Error('R2 not configured')
+  }
+
+  return new S3Client({
+    region: 'auto',
+    endpoint: `https://${accountId}.r2.cloudflarestorage.com`,
+    credentials: { accessKeyId, secretAccessKey },
+  })
+}
+
+export async function downloadFromCdn(originalPath: string): Promise<Buffer> {
+  const bucketName = process.env.CLOUDFLARE_R2_BUCKET_NAME
+  if (!bucketName) throw new Error('R2 bucket not configured')
+
+  const r2 = getR2Client()
+  const maxRetries = 3
+  let lastError: Error | undefined
+
+  for (let attempt = 0; attempt < maxRetries; attempt++) {
+    try {
+      const response = await r2.send(
+        new GetObjectCommand({
+          Bucket: bucketName,
+          Key: originalPath.replace(/^\//, ''),
+        })
+      )
+
+      const stream = response.Body as NodeJS.ReadableStream
+      const chunks: Buffer[] = []
+      for await (const chunk of stream) {
+        chunks.push(Buffer.from(chunk))
+      }
+      return Buffer.concat(chunks)
+    } catch (error) {
+      lastError = error as Error
+      // Wait before retry (exponential backoff: 500ms, 1s)
+      if (attempt < maxRetries - 1) {
+        await new Promise(resolve => setTimeout(resolve, 500 * (attempt + 1)))
+      }
+    }
+  }
+
+  throw lastError || new Error(`Failed to download ${originalPath} after ${maxRetries} attempts`)
+}
+
+export async function uploadToCdn(imageKey: string): Promise<void> {
+  const bucketName = process.env.CLOUDFLARE_R2_BUCKET_NAME
+  if (!bucketName) throw new Error('R2 bucket not configured')
+
+  const r2 = getR2Client()
+
+  // Upload all thumbnail sizes derived from imageKey
+  for (const thumbPath of getAllThumbnailPaths(imageKey)) {
+    const localPath = getPublicPath(thumbPath)
+    try {
+      const fileBuffer = await fs.readFile(localPath)
+      await r2.send(
+        new PutObjectCommand({
+          Bucket: bucketName,
+          Key: thumbPath.replace(/^\//, ''),
+          Body: fileBuffer,
+          ContentType: getContentType(thumbPath),
+        })
+      )
+    } catch {
+      // File might not exist (e.g., if image is smaller than thumbnail size)
+    }
+  }
+}
+
+export async function deleteLocalThumbnails(imageKey: string): Promise<void> {
+  for (const thumbPath of getAllThumbnailPaths(imageKey)) {
+    const localPath = getPublicPath(thumbPath)
+    try {
+      await fs.unlink(localPath)
+    } catch {
+      // File might not exist
+    }
+  }
+}
+
+/**
+ * Download image from a remote URL (not R2)
+ */
+export async function downloadFromRemoteUrl(url: string): Promise<Buffer> {
+  const maxRetries = 3
+  let lastError: Error | undefined
+
+  for (let attempt = 0; attempt < maxRetries; attempt++) {
+    try {
+      const response = await fetch(url)
+      if (!response.ok) {
+        throw new Error(`Failed to download from ${url}: ${response.status}`)
+      }
+      const arrayBuffer = await response.arrayBuffer()
+      return Buffer.from(arrayBuffer)
+    } catch (error) {
+      lastError = error as Error
+      // Wait before retry (exponential backoff: 500ms, 1s)
+      if (attempt < maxRetries - 1) {
+        await new Promise(resolve => setTimeout(resolve, 500 * (attempt + 1)))
+      }
+    }
+  }
+
+  throw lastError || new Error(`Failed to download from ${url} after ${maxRetries} attempts`)
+}
+
+/**
+ * Upload original image to R2 CDN
+ */
+export async function uploadOriginalToCdn(imageKey: string): Promise<void> {
+  const bucketName = process.env.CLOUDFLARE_R2_BUCKET_NAME
+  if (!bucketName) throw new Error('R2 bucket not configured')
+
+  const r2 = getR2Client()
+  const localPath = getPublicPath(imageKey)
+  const fileBuffer = await fs.readFile(localPath)
+  
+  await r2.send(
+    new PutObjectCommand({
+      Bucket: bucketName,
+      Key: imageKey.replace(/^\//, ''),
+      Body: fileBuffer,
+      ContentType: getContentType(imageKey),
+    })
+  )
+}
+
+/**
+ * Delete original and thumbnails from R2 CDN
+ */
+export async function deleteFromCdn(imageKey: string, hasThumbnails: boolean): Promise<void> {
+  const bucketName = process.env.CLOUDFLARE_R2_BUCKET_NAME
+  if (!bucketName) throw new Error('R2 bucket not configured')
+
+  const r2 = getR2Client()
+
+  // Delete original
+  try {
+    await r2.send(
+      new DeleteObjectCommand({
+        Bucket: bucketName,
+        Key: imageKey.replace(/^\//, ''),
+      })
+    )
+  } catch {
+    // May not exist
+  }
+
+  // Delete thumbnails if they exist
+  if (hasThumbnails) {
+    for (const thumbPath of getAllThumbnailPaths(imageKey)) {
+      try {
+        await r2.send(
+          new DeleteObjectCommand({
+            Bucket: bucketName,
+            Key: thumbPath.replace(/^\//, ''),
+          })
+        )
+      } catch {
+        // May not exist
+      }
+    }
+  }
+}
+
+/**
+ * Delete only thumbnails from R2 CDN (keeps original)
+ */
+export async function deleteThumbnailsFromCdn(imageKey: string): Promise<void> {
+  const bucketName = process.env.CLOUDFLARE_R2_BUCKET_NAME
+  if (!bucketName) throw new Error('R2 bucket not configured')
+
+  const r2 = getR2Client()
+
+  for (const thumbPath of getAllThumbnailPaths(imageKey)) {
+    try {
+      await r2.send(
+        new DeleteObjectCommand({
+          Bucket: bucketName,
+          Key: thumbPath.replace(/^\//, ''),
+        })
+      )
+    } catch {
+      // May not exist
+    }
+  }
+}

package/src/handlers/utils/files.ts

@@ -0,0 +1,64 @@
+import { promises as fs } from 'fs'
+import path from 'path'
+
+export function isImageFile(filename: string): boolean {
+  const ext = path.extname(filename).toLowerCase()
+  return ['.jpg', '.jpeg', '.png', '.gif', '.webp', '.svg', '.ico', '.bmp', '.tiff', '.tif'].includes(ext)
+}
+
+export function isMediaFile(filename: string): boolean {
+  const ext = path.extname(filename).toLowerCase()
+  // Images
+  if (['.jpg', '.jpeg', '.png', '.gif', '.webp', '.svg', '.ico', '.bmp', '.tiff', '.tif'].includes(ext)) return true
+  // Videos
+  if (['.mp4', '.webm', '.mov', '.avi', '.mkv', '.m4v'].includes(ext)) return true
+  // Audio
+  if (['.mp3', '.wav', '.ogg', '.m4a', '.flac', '.aac'].includes(ext)) return true
+  // Documents/PDFs
+  if (['.pdf'].includes(ext)) return true
+  return false
+}
+
+export function getContentType(filePath: string): string {
+  const ext = path.extname(filePath).toLowerCase()
+  switch (ext) {
+    case '.jpg':
+    case '.jpeg':
+      return 'image/jpeg'
+    case '.png':
+      return 'image/png'
+    case '.gif':
+      return 'image/gif'
+    case '.webp':
+      return 'image/webp'
+    case '.svg':
+      return 'image/svg+xml'
+    default:
+      return 'application/octet-stream'
+  }
+}
+
+export async function getFolderStats(folderPath: string): Promise<{ fileCount: number; totalSize: number }> {
+  let fileCount = 0
+  let totalSize = 0
+
+  async function scanFolder(dir: string): Promise<void> {
+    try {
+      const entries = await fs.readdir(dir, { withFileTypes: true })
+      for (const entry of entries) {
+        if (entry.name.startsWith('.')) continue
+        const fullPath = path.join(dir, entry.name)
+        if (entry.isDirectory()) {
+          await scanFolder(fullPath)
+        } else if (isMediaFile(entry.name)) {
+          fileCount++
+          const stats = await fs.stat(fullPath)
+          totalSize += stats.size
+        }
+      }
+    } catch { /* ignore errors */ }
+  }
+
+  await scanFolder(folderPath)
+  return { fileCount, totalSize }
+}

package/src/handlers/utils/index.ts

@@ -0,0 +1,4 @@
+export { loadMeta, saveMeta, getCdnUrls, setCdnUrls, getOrAddCdnIndex, getMetaEntry, setMetaEntry, deleteMetaEntry, getFileEntries } from './meta'
+export { isImageFile, isMediaFile, getContentType, getFolderStats } from './files'
+export { processImage, DEFAULT_SIZES } from './thumbnails'
+export { downloadFromCdn, uploadToCdn, deleteLocalThumbnails, downloadFromRemoteUrl, uploadOriginalToCdn, deleteFromCdn, deleteThumbnailsFromCdn, purgeCloudflareCache } from './cdn'