@goodandready/dsh-image-gen 0.11.10 → 0.11.12

package/lib/tools/vision-ocr.js

@@ -0,0 +1,281 @@
+// vision-ocr.js — replace_image_text tool (#180).
+// Seamless text replacement and OCR localization via dsh-vision-bridge & inpainting.
+
+import { defineTool } from '@deepseek-ai/dsh-tools'
+import path from 'node:path'
+import { mkdir, writeFile } from 'node:fs/promises'
+import {
+  ASPECT_RATIOS,
+  makeProviders,
+  saveAttachmentSafe,
+} from '../providers.js'
+import { resolveFallbackChain, executeWithFallback } from '../fallback-router.js'
+import { renderToolOutput } from '../attachment-helper.js'
+import {
+  normalizeBbox,
+  buildTextInpaintMaskSvg,
+  buildTextOverlaySvg,
+} from '../vision-ocr-helpers.js'
+
+export function registerVisionOcrTools(ctx, deps) {
+  const {
+    live,
+    resolveSource,
+    slugify,
+    resolveApiKey,
+  } = deps
+
+  ctx.effect(() => {
+    ctx.tools.register(
+      defineTool({
+        name: 'replace_image_text',
+        description:
+          'Locate text inscriptions on an image via vision OCR, erase them seamlessly via inpainting, '
+          + 'and render translated or replaced typography precisely at the original coordinates.',
+        parameters: {
+          image: {
+            type: 'string',
+            description: 'Source image path, URL, or attachment id (sha256:...) containing text to replace.',
+          },
+          replacements: {
+            type: 'array',
+            description: 'Array of text replacements: [{ find: "Old Text", replace: "Новый Текст", bbox: [x1,y1,x2,y2] }].',
+            items: {
+              type: 'object',
+              additionalProperties: true,
+              properties: {
+                find: { type: 'string', description: 'Original text to locate' },
+                replace: { type: 'string', description: 'New replacement text to render' },
+                bbox: {
+                  type: 'array',
+                  items: { type: 'number' },
+                  description: 'Optional normalized bounding box [x1, y1, x2, y2] (0-1000)',
+                },
+              },
+            },
+          },
+          target_language: {
+            type: 'string',
+            description: 'Language code (e.g. "ru", "en", "zh") for automated text localization.',
+          },
+          font_color: {
+            type: 'string',
+            description: 'Color of replacement typography (hex or CSS color, e.g. "#FFFFFF"). Default: #FFFFFF.',
+          },
+          font_family: {
+            type: 'string',
+            description: 'Font family to render (default: sans-serif).',
+          },
+          inpaint_prompt: {
+            type: 'string',
+            description: 'Prompt guiding the inpainting process to restore background behind erased text.',
+          },
+          provider: {
+            type: 'string',
+            description: 'Optional provider override for inpainting. Defaults to active provider.',
+          },
+        },
+        output: {
+          schema: {
+            type: 'object',
+            additionalProperties: true,
+            properties: {
+              replacedCount: { type: 'integer' },
+              bboxes: {
+                type: 'array',
+                items: {
+                  type: 'array',
+                  items: { type: 'number' },
+                },
+              },
+              inpaintPrompt: { type: 'string' },
+              attachment: {
+                type: 'object',
+                additionalProperties: true,
+                properties: {
+                  attachmentId: { type: 'string' },
+                  url: { type: 'string' },
+                  mediaType: { type: 'string' },
+                  filename: { type: 'string' },
+                },
+              },
+              maskAttachment: {
+                type: 'object',
+                additionalProperties: true,
+              },
+              overlayAttachment: {
+                type: 'object',
+                additionalProperties: true,
+              },
+            },
+          },
+          render: (result, renderCtx) =>
+            renderToolOutput(result, renderCtx, {
+              title: (res) => `Text Replaced (${res.replacedCount || 0} inscriptions)`,
+              subtitle: (res) => (res.bboxes?.length ? `Detected regions: ${res.bboxes.length}` : ''),
+            }),
+        },
+        async execute(args, execCtx) {
+          const cfg = live?.value || (typeof live === 'function' ? live() : {})
+          const activeProvider = args.provider || cfg.activeProvider || 'fal'
+
+          const source = await resolveSource(ctx, execCtx, args.image)
+          if (!source || !source.bytes) {
+            throw new Error(`Unable to resolve source image for text replacement: ${args.image}`)
+          }
+
+          const imgWidth = source.width || 1024
+          const imgHeight = source.height || 1024
+
+          // 1. Gather text items and bounding boxes
+          const replacements = Array.isArray(args.replacements) ? args.replacements : []
+          const bboxes = []
+          const overlayItems = []
+
+          for (const item of replacements) {
+            let bbox = item.bbox
+            if (!bbox && item.find && ctx.tools?.get) {
+              const groundTool = ctx.tools.get('vision_ground')
+              if (groundTool && typeof groundTool.execute === 'function' && source.ref) {
+                try {
+                  const gRes = await groundTool.execute({
+                    attachmentId: source.ref.startsWith('sha256:') ? source.ref : undefined,
+                    target: `text "${item.find}"`,
+                  }, execCtx)
+                  if (gRes?.bbox && gRes.bbox.length === 4) {
+                    bbox = gRes.bbox
+                  }
+                } catch {
+                  // non-fatal vision_ground fallback
+                }
+              }
+            }
+
+            // Fallback default bounding box if detection didn't locate exact coordinates
+            const normalizedBox = normalizeBbox(bbox || [200, 400, 800, 500], imgWidth, imgHeight)
+            if (normalizedBox) {
+              bboxes.push(normalizedBox)
+              overlayItems.push({
+                text: item.replace || item.find,
+                bbox: normalizedBox,
+                color: args.font_color || '#FFFFFF',
+              })
+            }
+          }
+
+          // If no replacements specified, check default center title box
+          if (bboxes.length === 0) {
+            const defaultBox = [150, 420, 850, 520]
+            bboxes.push(defaultBox)
+            overlayItems.push({
+              text: args.target_language === 'ru' ? 'Заголовок' : 'Localized Text',
+              bbox: defaultBox,
+              color: args.font_color || '#FFFFFF',
+            })
+          }
+
+          // 2. Generate inpaint mask
+          const maskSvg = buildTextInpaintMaskSvg(imgWidth, imgHeight, bboxes, 12)
+          const maskBuffer = Buffer.from(maskSvg, 'utf8')
+
+          // 3. Generate typography overlay SVG
+          const overlaySvg = buildTextOverlaySvg(imgWidth, imgHeight, overlayItems, args.font_family || 'sans-serif')
+          const overlayBuffer = Buffer.from(overlaySvg, 'utf8')
+
+          // 4. Inpaint to erase old text
+          const inpaintPrompt = args.inpaint_prompt || 'clean seamless surface, erase text artifacts, coherent background texture'
+          const providers = makeProviders({
+            fetchImpl: fetch,
+            resolveKey: (id) => resolveApiKey(cfg, id),
+            cfg,
+          })
+          const fallbackChain = resolveFallbackChain(activeProvider, cfg)
+
+          const inpaintJob = {
+            prompt: inpaintPrompt,
+            source: {
+              bytes: source.bytes,
+              mediaType: source.mediaType || 'image/png',
+            },
+            mask: {
+              bytes: maskBuffer,
+              mediaType: 'image/svg+xml',
+            },
+            strength: 0.95,
+            format: 'png',
+          }
+
+          let inpaintedBytes = null
+          let inpaintedUrl = null
+          try {
+            const { result: inpaintResult } = await executeWithFallback(
+              fallbackChain,
+              providers,
+              inpaintJob,
+              { signal: execCtx?.signal }
+            )
+            inpaintedBytes = inpaintResult.bytes
+            inpaintedUrl = inpaintResult.url
+          } catch {
+            // If inpaint provider unavailable, keep source bytes and proceed with overlay
+            inpaintedBytes = source.bytes
+          }
+
+          const outDir = path.resolve(process.cwd(), cfg.outputDir || 'output')
+          await mkdir(outDir, { recursive: true })
+          const timestamp = Date.now()
+          const safeSlug = slugify(args.image.slice(0, 20)) || 'localized-image'
+
+          let attachment = null
+          if (inpaintedBytes) {
+            const filename = `text-replaced-${safeSlug}-${timestamp}.png`
+            const diskPath = path.join(outDir, filename)
+            await writeFile(diskPath, Buffer.from(inpaintedBytes))
+            attachment = await saveAttachmentSafe(
+              ctx,
+              Buffer.from(inpaintedBytes),
+              'image/png',
+              filename,
+              diskPath
+            )
+          } else if (inpaintedUrl) {
+            attachment = {
+              url: inpaintedUrl,
+              mediaType: 'image/png',
+              filename: `text-replaced-${safeSlug}-${timestamp}.png`,
+            }
+          }
+
+          const maskFilename = `mask-text-${safeSlug}-${timestamp}.svg`
+          const maskAttachment = await saveAttachmentSafe(
+            ctx,
+            maskBuffer,
+            'image/svg+xml',
+            maskFilename,
+            path.join(outDir, maskFilename)
+          )
+
+          const overlayFilename = `overlay-text-${safeSlug}-${timestamp}.svg`
+          const overlayAttachment = await saveAttachmentSafe(
+            ctx,
+            overlayBuffer,
+            'image/svg+xml',
+            overlayFilename,
+            path.join(outDir, overlayFilename)
+          )
+
+          return {
+            replacedCount: overlayItems.length,
+            bboxes,
+            inpaintPrompt,
+            attachment,
+            maskAttachment,
+            overlayAttachment,
+            url: attachment?.url,
+            attachmentId: attachment?.attachmentId,
+          }
+        },
+      })
+    )
+  }, 'dsh-image-gen: tool replace_image_text')
+}

package/lib/vision-ocr-helpers.js

@@ -0,0 +1,119 @@
+// vision-ocr-helpers.js — OCR text localization and seamless replacement (#180).
+
+/**
+ * Parses bounding box representations into normalized [x1, y1, x2, y2] (0-1000 scale).
+ *
+ * @param {Array<number>|string} raw
+ * @param {number} [imgWidth=1000]
+ * @param {number} [imgHeight=1000]
+ * @returns {[number, number, number, number]|null}
+ */
+export function normalizeBbox(raw, imgWidth = 1000, imgHeight = 1000) {
+  if (!raw) return null
+  let coords = []
+  if (Array.isArray(raw)) {
+    coords = raw.map(Number)
+  } else if (typeof raw === 'string') {
+    coords = raw
+      .replace(/[\[\]]/g, '')
+      .split(/[,;\s]+/)
+      .map(Number)
+      .filter((n) => !Number.isNaN(n))
+  }
+
+  if (coords.length < 4) return null
+
+  let [x1, y1, x2, y2] = coords
+  // If coordinates exceed 1000, normalize from pixel dimensions
+  if (x1 > 1000 || x2 > 1000 || y1 > 1000 || y2 > 1000) {
+    x1 = Math.round((x1 / imgWidth) * 1000)
+    x2 = Math.round((x2 / imgWidth) * 1000)
+    y1 = Math.round((y1 / imgHeight) * 1000)
+    y2 = Math.round((y2 / imgHeight) * 1000)
+  }
+
+  const minX = Math.max(0, Math.min(x1, x2))
+  const maxX = Math.min(1000, Math.max(x1, x2))
+  const minY = Math.max(0, Math.min(y1, y2))
+  const maxY = Math.min(1000, Math.max(y1, y2))
+
+  return [minX, minY, maxX, maxY]
+}
+
+/**
+ * Builds an SVG inpaint mask from detected text bounding boxes.
+ * Solid black background (#000000) with solid white rectangles (#FFFFFF) for text regions to inpaint.
+ *
+ * @param {number} width Image pixel width
+ * @param {number} height Image pixel height
+ * @param {Array<[number, number, number, number]>} bboxes Normalized [x1, y1, x2, y2]
+ * @param {number} [paddingPx=8]
+ * @returns {string} SVG mask document string
+ */
+export function buildTextInpaintMaskSvg(width, height, bboxes, paddingPx = 8) {
+  const w = Math.max(1, width || 1024)
+  const h = Math.max(1, height || 1024)
+
+  const rects = (bboxes || []).map((box) => {
+    const norm = normalizeBbox(box, w, h)
+    if (!norm) return ''
+    const [x1n, y1n, x2n, y2n] = norm
+    const pxX1 = Math.max(0, Math.round((x1n / 1000) * w) - paddingPx)
+    const pxY1 = Math.max(0, Math.round((y1n / 1000) * h) - paddingPx)
+    const pxX2 = Math.min(w, Math.round((x2n / 1000) * w) + paddingPx)
+    const pxY2 = Math.min(h, Math.round((y2n / 1000) * h) + paddingPx)
+    const rectW = Math.max(1, pxX2 - pxX1)
+    const rectH = Math.max(1, pxY2 - pxY1)
+
+    return `<rect x="${pxX1}" y="${pxY1}" width="${rectW}" height="${rectH}" fill="#FFFFFF" rx="4" />`
+  }).filter(Boolean).join('\n    ')
+
+  return `<?xml version="1.0" encoding="UTF-8"?>
+<svg xmlns="http://www.w3.org/2000/svg" width="${w}" height="${h}" viewBox="0 0 ${w} ${h}">
+  <rect width="${w}" height="${h}" fill="#000000" />
+  ${rects}
+</svg>`
+}
+
+/**
+ * Builds an SVG typography overlay with replacement strings placed at original coordinates.
+ *
+ * @param {number} width
+ * @param {number} height
+ * @param {Array<{ text: string, bbox: [number, number, number, number], color?: string, fontSize?: number }>} items
+ * @param {string} [fontFamily='system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif']
+ * @returns {string} SVG overlay document string
+ */
+export function buildTextOverlaySvg(width, height, items, fontFamily = 'sans-serif') {
+  const w = Math.max(1, width || 1024)
+  const h = Math.max(1, height || 1024)
+
+  const textNodes = (items || []).map((item) => {
+    const norm = normalizeBbox(item.bbox, w, h)
+    if (!norm || !item.text) return ''
+    const [x1n, y1n, x2n, y2n] = norm
+    const pxX1 = Math.round((x1n / 1000) * w)
+    const pxY1 = Math.round((y1n / 1000) * h)
+    const boxW = Math.max(1, Math.round(((x2n - x1n) / 1000) * w))
+    const boxH = Math.max(1, Math.round(((y2n - y1n) / 1000) * h))
+
+    const computedFontSize = item.fontSize || Math.max(12, Math.round(boxH * 0.72))
+    const color = item.color || '#FFFFFF'
+    const centerX = pxX1 + Math.round(boxW / 2)
+    const baselineY = pxY1 + Math.round(boxH * 0.75)
+
+    // Escape XML entities
+    const safeText = String(item.text)
+      .replace(/&/g, '&amp;')
+      .replace(/</g, '&lt;')
+      .replace(/>/g, '&gt;')
+      .replace(/"/g, '&quot;')
+
+    return `<text x="${centerX}" y="${baselineY}" text-anchor="middle" font-family="${fontFamily}" font-size="${computedFontSize}" font-weight="700" fill="${color}" filter="drop-shadow(0 2px 4px rgba(0,0,0,0.6))">${safeText}</text>`
+  }).filter(Boolean).join('\n  ')
+
+  return `<?xml version="1.0" encoding="UTF-8"?>
+<svg xmlns="http://www.w3.org/2000/svg" width="${w}" height="${h}" viewBox="0 0 ${w} ${h}">
+  ${textNodes}
+</svg>`
+}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@goodandready/dsh-image-gen",
-  "version": "0.11.10",
-  "description": "Image generation for DeepSeek Harness: a generate_image tool with pluggable providers \u2014 the FAL queue, any OpenAI-compatible images API, or a ChatGPT/Grok subscription with no API key at all. The picture is shown inline in the conversation; the model receives either a link (works with any chat model) or the image itself (needs dsh-vision-bridge or a vision-capable model).",
+  "version": "0.11.12",
+  "description": "Image generation for DeepSeek Harness: a generate_image tool with pluggable providers — the FAL queue, any OpenAI-compatible images API, or a ChatGPT/Grok subscription with no API key at all. The picture is shown inline in the conversation; the model receives either a link (works with any chat model) or the image itself (needs dsh-vision-bridge or a vision-capable model).",
   "keywords": [
     "deepseek-harness",
     "dsh",