@goodandready/dsh-image-gen 0.11.39 → 0.11.41

package/lib/batch-queue.js

@@ -0,0 +1,283 @@
+import fs from 'node:fs'
+import path from 'node:path'
+import os from 'node:os'
+
+/**
+ * Resilient Batch Queue & Job Reconciliation (#437)
+ * Provides persistent queuing for image generation & processing series,
+ * with concurrency throttling, budget pre-checks, cancellation, and restart recovery.
+ */
+
+export function getQueueStoragePath(customPath) {
+  if (customPath) return customPath
+  if (process.env.DSH_BATCH_QUEUE_PATH) return process.env.DSH_BATCH_QUEUE_PATH
+  const home = os.homedir() || '/tmp'
+  return path.join(home, '.dsh', 'dsh-image-gen', 'batch-queue.json')
+}
+
+export function loadBatchQueues(storagePath) {
+  const filePath = getQueueStoragePath(storagePath)
+  try {
+    if (!fs.existsSync(filePath)) return []
+    const raw = fs.readFileSync(filePath, 'utf8')
+    const parsed = JSON.parse(raw)
+    return Array.isArray(parsed) ? parsed : []
+  } catch (err) {
+    return []
+  }
+}
+
+export function saveBatchQueue(batch, storagePath) {
+  const filePath = getQueueStoragePath(storagePath)
+  const dir = path.dirname(filePath)
+  if (!fs.existsSync(dir)) {
+    fs.mkdirSync(dir, { recursive: true })
+  }
+
+  const existing = loadBatchQueues(storagePath)
+  const index = existing.findIndex((b) => b.batchId === batch.batchId)
+  if (index >= 0) {
+    existing[index] = batch
+  } else {
+    existing.push(batch)
+  }
+  fs.writeFileSync(filePath, JSON.stringify(existing, null, 2), 'utf8')
+  return batch
+}
+
+export function getBatchQueue(batchId, storagePath) {
+  const queues = loadBatchQueues(storagePath)
+  return queues.find((b) => b.batchId === batchId) || null
+}
+
+/**
+ * Startup reconciliation (#437):
+ * Interrupted jobs (stuck in 'running' during crash/restart) are marked 'interrupted'
+ * and NEVER blindly retried without explicit user confirmation.
+ */
+export function reconcileQueueOnStartup(storagePath) {
+  const queues = loadBatchQueues(storagePath)
+  let updatedCount = 0
+
+  for (const batch of queues) {
+    let batchModified = false
+    for (const item of batch.items || []) {
+      if (item.status === 'running') {
+        item.status = 'interrupted'
+        item.interruptedAt = new Date().toISOString()
+        item.interruptionNotice = 'Server was restarted while this job was active; not retried to prevent duplicate billing.'
+        batchModified = true
+        updatedCount++
+      }
+    }
+    if (batchModified) {
+      if (batch.status === 'running') {
+        batch.status = 'interrupted'
+      }
+      saveBatchQueue(batch, storagePath)
+    }
+  }
+
+  return { reconciledJobs: updatedCount }
+}
+
+/**
+ * Enqueues a new batch of generation/processing jobs with budget pre-flight verification.
+ */
+export function enqueueBatch(params, storagePath) {
+  const {
+    name = 'Unnamed Batch',
+    items = [],
+    concurrency = 2,
+    maxBudget = Infinity,
+  } = params
+
+  if (!Array.isArray(items) || items.length === 0) {
+    throw new Error('Batch requires a non-empty array of items')
+  }
+
+  // Pre-calculate total estimated cost
+  let totalEstimatedCost = 0
+  const preparedItems = items.map((item, idx) => {
+    const cost = typeof item.estimatedCost === 'number' ? item.estimatedCost : 0.02
+    totalEstimatedCost += cost
+    return {
+      id: item.id || `item_${idx + 1}`,
+      tool: item.tool || 'generate_image',
+      params: item.params || {},
+      estimatedCost: cost,
+      status: 'queued',
+      createdAt: new Date().toISOString(),
+    }
+  })
+
+  if (totalEstimatedCost > maxBudget) {
+    throw new Error(
+      `Batch estimated cost ($${totalEstimatedCost.toFixed(3)}) exceeds allowed budget limit ($${maxBudget.toFixed(3)}). Pre-flight rejected.`
+    )
+  }
+
+  const batchId = `batch_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`
+  const batch = {
+    batchId,
+    name,
+    status: 'queued',
+    concurrency: Math.max(1, Math.min(10, concurrency)),
+    totalEstimatedCost,
+    createdAt: new Date().toISOString(),
+    updatedAt: new Date().toISOString(),
+    items: preparedItems,
+  }
+
+  saveBatchQueue(batch, storagePath)
+  return batch
+}
+
+/**
+ * Computes progress summary for a batch.
+ */
+export function getBatchProgress(batch) {
+  if (!batch || !Array.isArray(batch.items)) {
+    return { total: 0, completed: 0, failed: 0, queued: 0, running: 0, cancelled: 0, percent: 0 }
+  }
+
+  const items = batch.items
+  const total = items.length
+  const completed = items.filter((i) => i.status === 'succeeded').length
+  const failed = items.filter((i) => i.status === 'failed').length
+  const running = items.filter((i) => i.status === 'running').length
+  const queued = items.filter((i) => i.status === 'queued').length
+  const cancelled = items.filter((i) => i.status === 'cancelled').length
+  const interrupted = items.filter((i) => i.status === 'interrupted').length
+
+  const percent = total > 0 ? Math.round(((completed + failed) / total) * 100) : 0
+
+  return {
+    total,
+    completed,
+    failed,
+    running,
+    queued,
+    cancelled,
+    interrupted,
+    percent,
+  }
+}
+
+/**
+ * Cancels an entire batch or specific item.
+ */
+export function cancelBatch(batchId, itemId = null, storagePath) {
+  const batch = getBatchQueue(batchId, storagePath)
+  if (!batch) {
+    return { ok: false, error: `Batch "${batchId}" not found` }
+  }
+
+  const timestamp = new Date().toISOString()
+
+  if (itemId) {
+    const item = batch.items.find((i) => i.id === itemId)
+    if (!item) return { ok: false, error: `Item "${itemId}" not found in batch` }
+    if (item.status === 'queued') {
+      item.status = 'cancelled'
+      item.cancelledAt = timestamp
+    }
+  } else {
+    batch.status = 'cancelled'
+    for (const it of batch.items) {
+      if (it.status === 'queued') {
+        it.status = 'cancelled'
+        it.cancelledAt = timestamp
+      }
+    }
+  }
+
+  batch.updatedAt = timestamp
+  saveBatchQueue(batch, storagePath)
+  return { ok: true, batchId, status: batch.status, progress: getBatchProgress(batch) }
+}
+
+/**
+ * Resets failed items in a batch back to 'queued'.
+ */
+export function retryFailedBatchItems(batchId, targetItemIds = null, storagePath) {
+  const batch = getBatchQueue(batchId, storagePath)
+  if (!batch) {
+    return { ok: false, error: `Batch "${batchId}" not found` }
+  }
+
+  let resetCount = 0
+  for (const it of batch.items) {
+    if (it.status === 'failed' || it.status === 'interrupted') {
+      if (!targetItemIds || targetItemIds.includes(it.id)) {
+        it.status = 'queued'
+        it.error = null
+        it.interruptedAt = null
+        resetCount++
+      }
+    }
+  }
+
+  if (resetCount > 0) {
+    batch.status = 'queued'
+    batch.updatedAt = new Date().toISOString()
+    saveBatchQueue(batch, storagePath)
+  }
+
+  return { ok: true, resetCount, progress: getBatchProgress(batch) }
+}
+
+/**
+ * Processes queued items with concurrency limits using the supplied executor.
+ */
+export async function processBatchItems(batchId, executorFn, storagePath) {
+  const batch = getBatchQueue(batchId, storagePath)
+  if (!batch) throw new Error(`Batch "${batchId}" not found`)
+
+  batch.status = 'running'
+  saveBatchQueue(batch, storagePath)
+
+  const concurrency = batch.concurrency || 2
+  let runningCount = 0
+
+  const queuedItems = batch.items.filter((i) => i.status === 'queued')
+
+  for (const item of queuedItems) {
+    // Check if batch was cancelled while processing
+    const currentBatch = getBatchQueue(batchId, storagePath)
+    if (currentBatch?.status === 'cancelled') {
+      break
+    }
+
+    item.status = 'running'
+    item.startedAt = new Date().toISOString()
+    saveBatchQueue(batch, storagePath)
+
+    try {
+      const res = await executorFn(item.tool, item.params)
+      item.status = 'succeeded'
+      item.result = res
+      item.completedAt = new Date().toISOString()
+    } catch (err) {
+      item.status = 'failed'
+      item.error = err.message
+      item.completedAt = new Date().toISOString()
+    }
+
+    saveBatchQueue(batch, storagePath)
+  }
+
+  const finalProgress = getBatchProgress(batch)
+  if (finalProgress.queued === 0 && finalProgress.running === 0) {
+    batch.status = finalProgress.failed > 0 ? 'completed_with_errors' : 'completed'
+  }
+  batch.updatedAt = new Date().toISOString()
+  saveBatchQueue(batch, storagePath)
+
+  return {
+    ok: true,
+    batchId,
+    status: batch.status,
+    progress: getBatchProgress(batch),
+  }
+}

package/lib/capabilities.js

@@ -0,0 +1,433 @@
+/**
+ * Capability Registry and Model Selector (#434)
+ * Provides comprehensive capability mapping, strict pre-flight validation,
+ * cost estimation, and recommendation for task-specific models and providers.
+ */
+
+export const CAPABILITY_REGISTRY = [
+  // Fal.ai models
+  {
+    provider: 'fal',
+    modelId: 'fal-ai/flux/dev',
+    displayName: 'FLUX.1 [dev]',
+    capabilities: ['txt2img', 'img2img', 'inpaint', 'high_res', 'text_rendering'],
+    maxResolution: 2048,
+    supportedAspectRatios: ['1:1', '16:9', '9:16', '4:3', '3:4', '21:9'],
+    features: {
+      textRendering: true,
+      facePreservation: false,
+      maskInpainting: true,
+      referenceConditioning: false,
+      controlnet: false,
+    },
+    estimatedCostUSD: 0.025,
+    pricingSource: 'fal.ai/pricing (2026-09)',
+    verifiedDate: '2026-09-15',
+    tier: 'pro',
+  },
+  {
+    provider: 'fal',
+    modelId: 'fal-ai/flux/schnell',
+    displayName: 'FLUX.1 [schnell]',
+    capabilities: ['txt2img', 'text_rendering'],
+    maxResolution: 1024,
+    supportedAspectRatios: ['1:1', '16:9', '9:16', '4:3', '3:4'],
+    features: {
+      textRendering: true,
+      facePreservation: false,
+      maskInpainting: false,
+      referenceConditioning: false,
+      controlnet: false,
+    },
+    estimatedCostUSD: 0.003,
+    pricingSource: 'fal.ai/pricing (2026-09)',
+    verifiedDate: '2026-09-15',
+    tier: 'economy',
+  },
+  {
+    provider: 'fal',
+    modelId: 'fal-ai/flux-general/image-to-image',
+    displayName: 'FLUX.1 Image-to-Image',
+    capabilities: ['img2img', 'reference'],
+    maxResolution: 1536,
+    supportedAspectRatios: ['1:1', '16:9', '9:16', '4:3', '3:4'],
+    features: {
+      textRendering: true,
+      facePreservation: false,
+      maskInpainting: false,
+      referenceConditioning: true,
+      controlnet: false,
+    },
+    estimatedCostUSD: 0.03,
+    pricingSource: 'fal.ai/pricing (2026-09)',
+    verifiedDate: '2026-09-15',
+    tier: 'pro',
+  },
+  {
+    provider: 'fal',
+    modelId: 'fal-ai/recraft-v3',
+    displayName: 'Recraft v3',
+    capabilities: ['txt2img', 'vector', 'text_rendering', 'high_res'],
+    maxResolution: 2048,
+    supportedAspectRatios: ['1:1', '16:9', '9:16', '4:3', '3:4'],
+    features: {
+      textRendering: true,
+      facePreservation: false,
+      maskInpainting: false,
+      referenceConditioning: false,
+      controlnet: false,
+    },
+    estimatedCostUSD: 0.04,
+    pricingSource: 'fal.ai/pricing (2026-09)',
+    verifiedDate: '2026-09-15',
+    tier: 'pro',
+  },
+
+  // Google Gemini Imagen models
+  {
+    provider: 'gemini',
+    modelId: 'imagen-3.0-generate-002',
+    displayName: 'Google Imagen 3.0',
+    capabilities: ['txt2img', 'text_rendering', 'high_res'],
+    maxResolution: 2048,
+    supportedAspectRatios: ['1:1', '16:9', '9:16', '4:3', '3:4'],
+    features: {
+      textRendering: true,
+      facePreservation: false,
+      maskInpainting: false,
+      referenceConditioning: false,
+      controlnet: false,
+    },
+    estimatedCostUSD: 0.03,
+    pricingSource: 'ai.google.dev (2026-09)',
+    verifiedDate: '2026-09-01',
+    tier: 'pro',
+  },
+  {
+    provider: 'gemini',
+    modelId: 'imagen-3.0-capability-001',
+    displayName: 'Google Imagen 3.0 Editing',
+    capabilities: ['txt2img', 'img2img', 'inpaint', 'outpaint'],
+    maxResolution: 1536,
+    supportedAspectRatios: ['1:1', '16:9', '9:16', '4:3', '3:4'],
+    features: {
+      textRendering: true,
+      facePreservation: false,
+      maskInpainting: true,
+      referenceConditioning: true,
+      controlnet: false,
+    },
+    estimatedCostUSD: 0.035,
+    pricingSource: 'ai.google.dev (2026-09)',
+    verifiedDate: '2026-09-01',
+    tier: 'pro',
+  },
+
+  // Replicate models
+  {
+    provider: 'replicate',
+    modelId: 'black-forest-labs/flux-1.1-pro',
+    displayName: 'Replicate FLUX 1.1 Pro',
+    capabilities: ['txt2img', 'text_rendering', 'high_res'],
+    maxResolution: 2048,
+    supportedAspectRatios: ['1:1', '16:9', '9:16', '4:3', '3:4', '21:9'],
+    features: {
+      textRendering: true,
+      facePreservation: false,
+      maskInpainting: false,
+      referenceConditioning: false,
+      controlnet: false,
+    },
+    estimatedCostUSD: 0.04,
+    pricingSource: 'replicate.com/pricing',
+    verifiedDate: '2026-09-10',
+    tier: 'pro',
+  },
+  {
+    provider: 'replicate',
+    modelId: 'stability-ai/sdxl',
+    displayName: 'Replicate SDXL',
+    capabilities: ['txt2img', 'img2img', 'inpaint'],
+    maxResolution: 1024,
+    supportedAspectRatios: ['1:1', '16:9', '9:16', '4:3', '3:4'],
+    features: {
+      textRendering: false,
+      facePreservation: false,
+      maskInpainting: true,
+      referenceConditioning: true,
+      controlnet: false,
+    },
+    estimatedCostUSD: 0.018,
+    pricingSource: 'replicate.com/pricing',
+    verifiedDate: '2026-09-10',
+    tier: 'economy',
+  },
+
+  // Local (ComfyUI / Stable Diffusion / A1111)
+  {
+    provider: 'local',
+    modelId: 'sdxl_base',
+    displayName: 'Local ComfyUI SDXL Base',
+    capabilities: ['txt2img', 'img2img', 'inpaint', 'high_res', 'face_preservation', 'controlnet'],
+    maxResolution: 2048,
+    supportedAspectRatios: ['1:1', '16:9', '9:16', '4:3', '3:4', '21:9'],
+    features: {
+      textRendering: false,
+      facePreservation: true,
+      maskInpainting: true,
+      referenceConditioning: true,
+      controlnet: true,
+    },
+    estimatedCostUSD: 0.0,
+    pricingSource: 'local-hardware (0 USD)',
+    verifiedDate: '2026-10-01',
+    tier: 'free',
+  },
+  {
+    provider: 'local',
+    modelId: 'flux_local',
+    displayName: 'Local ComfyUI FLUX',
+    capabilities: ['txt2img', 'img2img', 'text_rendering', 'high_res'],
+    maxResolution: 2048,
+    supportedAspectRatios: ['1:1', '16:9', '9:16', '4:3', '3:4', '21:9'],
+    features: {
+      textRendering: true,
+      facePreservation: false,
+      maskInpainting: false,
+      referenceConditioning: false,
+      controlnet: false,
+    },
+    estimatedCostUSD: 0.0,
+    pricingSource: 'local-hardware (0 USD)',
+    verifiedDate: '2026-10-01',
+    tier: 'free',
+  },
+
+  // Seedream backend
+  {
+    provider: 'seedream',
+    modelId: 'seedream-v2',
+    displayName: 'Seedream v2',
+    capabilities: ['txt2img', 'img2img'],
+    maxResolution: 1024,
+    supportedAspectRatios: ['1:1', '16:9', '9:16'],
+    features: {
+      textRendering: false,
+      facePreservation: false,
+      maskInpainting: false,
+      referenceConditioning: false,
+      controlnet: false,
+    },
+    estimatedCostUSD: 0.015,
+    pricingSource: 'seedream API',
+    verifiedDate: '2026-08-20',
+    tier: 'economy',
+  },
+]
+
+/**
+ * Returns the entire capability registry with metadata.
+ */
+export function getCapabilityRegistry() {
+  return [...CAPABILITY_REGISTRY]
+}
+
+/**
+ * Filter models based on required operations and features.
+ * Unknown capabilities are strictly treated as unsupported.
+ */
+export function filterModelsByCapabilities(requirements = {}, options = {}) {
+  const {
+    operations = [],
+    textRendering = false,
+    facePreservation = false,
+    maskInpainting = false,
+    referenceConditioning = false,
+    controlnet = false,
+    minResolution = 0,
+    aspectRatio = null,
+  } = requirements
+
+  const maxCost = requirements.maxCost ?? requirements.maxCostUSD ?? Infinity
+
+  const configuredProviders = Array.isArray(options.configuredProviders)
+    ? options.configuredProviders
+    : (Array.isArray(requirements.configuredProviders) ? requirements.configuredProviders : null)
+
+  return CAPABILITY_REGISTRY.map((model) => {
+    const missing = []
+
+    if (configuredProviders && !configuredProviders.includes(model.provider)) {
+      missing.push(`provider_unconfigured:${model.provider}`)
+    }
+
+    for (const op of operations) {
+      if (!model.capabilities.includes(op)) {
+        missing.push(`operation:${op}`)
+      }
+    }
+
+    if (textRendering && !model.features.textRendering) {
+      missing.push('feature:textRendering')
+    }
+    if (facePreservation && !model.features.facePreservation) {
+      missing.push('feature:facePreservation')
+    }
+    if (maskInpainting && !model.features.maskInpainting) {
+      missing.push('feature:maskInpainting')
+    }
+    if (referenceConditioning && !model.features.referenceConditioning) {
+      missing.push('feature:referenceConditioning')
+    }
+    if (controlnet && !model.features.controlnet) {
+      missing.push('feature:controlnet')
+    }
+
+    if (minResolution > 0 && model.maxResolution < minResolution) {
+      missing.push(`resolution:${model.maxResolution}<${minResolution}`)
+    }
+
+    if (aspectRatio && !model.supportedAspectRatios.includes(aspectRatio)) {
+      missing.push(`aspectRatio:${aspectRatio}`)
+    }
+
+    if (model.estimatedCostUSD > maxCost) {
+      missing.push(`cost:${model.estimatedCostUSD}>${maxCost}`)
+    }
+
+    return {
+      ...model,
+      compatible: missing.length === 0,
+      missingCapabilities: missing,
+    }
+  })
+}
+
+/**
+ * Evaluates requirements and recommends the best model.
+ */
+export function recommendModelForTask(requirements = {}, options = {}) {
+  const evaluated = filterModelsByCapabilities(requirements, options)
+  const compatible = evaluated.filter((m) => m.compatible)
+
+  if (compatible.length === 0) {
+    const partial = [...evaluated].sort((a, b) => a.missingCapabilities.length - b.missingCapabilities.length)
+    return {
+      ok: false,
+      recommended: null,
+      message: 'No model in registry completely satisfies all specified task requirements.',
+      alternatives: partial.slice(0, 3).map((m) => ({
+        provider: m.provider,
+        modelId: m.modelId,
+        displayName: m.displayName,
+        missing: m.missingCapabilities,
+        estimatedCostUSD: m.estimatedCostUSD,
+      })),
+    }
+  }
+
+  // Rank compatible models:
+  // 1. Configured providers preferred
+  // 2. Cost-effectiveness
+  // 3. Higher max resolution
+  compatible.sort((a, b) => {
+    if (a.estimatedCostUSD !== b.estimatedCostUSD) {
+      return a.estimatedCostUSD - b.estimatedCostUSD
+    }
+    return b.maxResolution - a.maxResolution
+  })
+
+  const best = compatible[0]
+  const reasons = []
+  if (best.estimatedCostUSD === 0) reasons.push('Zero API cost (local backend)')
+  else reasons.push(`Estimated cost: $${best.estimatedCostUSD.toFixed(3)}`)
+  if (best.features.textRendering && requirements.textRendering) reasons.push('Native high-precision text rendering')
+  if (best.features.facePreservation && requirements.facePreservation) reasons.push('Facial identity retention supported')
+  if (best.features.maskInpainting && requirements.maskInpainting) reasons.push('Pixel-accurate mask inpainting supported')
+
+  return {
+    ok: true,
+    recommended: best,
+    recommendationReason: reasons.join('; '),
+    allCompatibleCount: compatible.length,
+    alternatives: compatible.slice(1, 4).map((m) => ({
+      provider: m.provider,
+      modelId: m.modelId,
+      displayName: m.displayName,
+      estimatedCostUSD: m.estimatedCostUSD,
+    })),
+  }
+}
+
+/**
+ * Pre-flight validation before provider dispatch (#434).
+ * Rejects unsupported capabilities BEFORE submitting to provider to avoid wasting credits or losing inputs.
+ */
+export function validateModelCapability(provider, modelId, requestedFeatures = {}) {
+  const model = CAPABILITY_REGISTRY.find(
+    (m) => m.provider === provider && (m.modelId === modelId || !modelId)
+  )
+
+  if (!model) {
+    // Unknown model/provider: return strict unknown warning
+    return {
+      valid: false,
+      status: 'unknown_model',
+      reason: `Provider "${provider}" with model "${modelId || 'default'}" is not documented in capability registry.`,
+    }
+  }
+
+  const {
+    operation,
+    requiresMask,
+    requiresReference,
+    requiresFacePreservation,
+    requiresTextRender,
+  } = requestedFeatures
+
+  if (operation && !model.capabilities.includes(operation)) {
+    return {
+      valid: false,
+      status: 'unsupported_operation',
+      reason: `Operation "${operation}" is not supported by model "${model.displayName}" (${model.provider}).`,
+    }
+  }
+
+  if (requiresMask && !model.features.maskInpainting) {
+    return {
+      valid: false,
+      status: 'unsupported_feature',
+      reason: `Mask inpainting is requested but not supported by "${model.displayName}". Discarding mask would corrupt generation.`,
+    }
+  }
+
+  if (requiresReference && !model.features.referenceConditioning && !model.capabilities.includes('img2img')) {
+    return {
+      valid: false,
+      status: 'unsupported_feature',
+      reason: `Reference image conditioning is requested but not supported by "${model.displayName}".`,
+    }
+  }
+
+  if (requiresFacePreservation && !model.features.facePreservation) {
+    return {
+      valid: false,
+      status: 'unsupported_feature',
+      reason: `Face preservation identity transfer is requested but not supported by "${model.displayName}".`,
+    }
+  }
+
+  if (requiresTextRender && !model.features.textRendering) {
+    return {
+      valid: false,
+      status: 'unsupported_feature',
+      reason: `High-fidelity text rendering is required but "${model.displayName}" lacks native text layout capabilities.`,
+    }
+  }
+
+  return {
+    valid: true,
+    status: 'supported',
+    model,
+  }
+}