@goodandready/dsh-image-gen 0.11.11 → 0.11.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -0
- package/lib/anchor-helpers.js +38 -9
- package/lib/character-sheet-helpers.js +115 -0
- package/lib/client.js +80 -13
- package/lib/diagram-helpers.js +105 -0
- package/lib/register-tools.js +7 -1
- package/lib/tools/anchor.js +2 -2
- package/lib/tools/character-sheet.js +187 -0
- package/lib/tools/diagram.js +172 -0
- package/lib/tools/generation.js +28 -3
- package/lib/tools/vision-ocr.js +281 -0
- package/lib/vision-ocr-helpers.js +119 -0
- package/package.json +2 -2
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
// character-sheet.js — generate_character_sheet tool (#181).
|
|
2
|
+
// Generates multi-angle character model sheets and turnarounds with visual consistency.
|
|
3
|
+
|
|
4
|
+
import { defineTool } from '@deepseek-ai/dsh-tools'
|
|
5
|
+
import path from 'node:path'
|
|
6
|
+
import { mkdir, writeFile } from 'node:fs/promises'
|
|
7
|
+
import {
|
|
8
|
+
ASPECT_RATIOS,
|
|
9
|
+
buildSidecar,
|
|
10
|
+
makeProviders,
|
|
11
|
+
toLosslessJson,
|
|
12
|
+
saveAttachmentSafe,
|
|
13
|
+
} from '../providers.js'
|
|
14
|
+
import { resolveFallbackChain, executeWithFallback } from '../fallback-router.js'
|
|
15
|
+
import { renderToolOutput } from '../attachment-helper.js'
|
|
16
|
+
import {
|
|
17
|
+
CHARACTER_SHEET_LAYOUTS,
|
|
18
|
+
CHARACTER_SHEET_STYLES,
|
|
19
|
+
buildCharacterSheetPrompt,
|
|
20
|
+
} from '../character-sheet-helpers.js'
|
|
21
|
+
|
|
22
|
+
export function registerCharacterSheetTools(ctx, deps) {
|
|
23
|
+
const {
|
|
24
|
+
live,
|
|
25
|
+
slugify,
|
|
26
|
+
resolveApiKey,
|
|
27
|
+
} = deps
|
|
28
|
+
|
|
29
|
+
ctx.effect(() => {
|
|
30
|
+
ctx.tools.register(
|
|
31
|
+
defineTool({
|
|
32
|
+
name: 'generate_character_sheet',
|
|
33
|
+
description:
|
|
34
|
+
'Generate a multi-angle character model sheet or turnaround reference with high visual identity consistency. '
|
|
35
|
+
+ 'Supports horizontal turnarounds (front, profile, 3/4, back), 1x3 triptychs, 2x2 grids, and facial emotion sheets across concept, anime, 3D, and comic styles.',
|
|
36
|
+
parameters: {
|
|
37
|
+
prompt: {
|
|
38
|
+
type: 'string',
|
|
39
|
+
description: 'Core character description (e.g. "female cyborg detective with neon violet hair, yellow trenchcoat, cybernetic arm").',
|
|
40
|
+
},
|
|
41
|
+
layout: {
|
|
42
|
+
type: 'string',
|
|
43
|
+
enum: ['turnaround', '1x3', '2x2', 'emotions'],
|
|
44
|
+
description: 'Layout arrangement: "turnaround" (4 sequential horizontal views), "1x3" (front, side, back), "2x2" (quad angles), or "emotions" (4 facial expressions). Default: turnaround.',
|
|
45
|
+
},
|
|
46
|
+
style: {
|
|
47
|
+
type: 'string',
|
|
48
|
+
enum: ['concept_art', 'anime', '3d_animation', 'pixel_art', 'realistic', 'comic'],
|
|
49
|
+
description: 'Artistic rendering style. Default: concept_art.',
|
|
50
|
+
},
|
|
51
|
+
negativePrompt: {
|
|
52
|
+
type: 'string',
|
|
53
|
+
description: 'Negative prompt directives to suppress unwanted elements.',
|
|
54
|
+
},
|
|
55
|
+
seed: {
|
|
56
|
+
type: 'integer',
|
|
57
|
+
description: 'RNG seed for reproducible character generation across sessions.',
|
|
58
|
+
},
|
|
59
|
+
provider: {
|
|
60
|
+
type: 'string',
|
|
61
|
+
description: 'Optional provider override (e.g. fal, replicate, local). Defaults to active provider.',
|
|
62
|
+
},
|
|
63
|
+
},
|
|
64
|
+
output: {
|
|
65
|
+
schema: {
|
|
66
|
+
type: 'object',
|
|
67
|
+
additionalProperties: true,
|
|
68
|
+
properties: {
|
|
69
|
+
prompt: { type: 'string' },
|
|
70
|
+
layout: { type: 'string' },
|
|
71
|
+
style: { type: 'string' },
|
|
72
|
+
aspectRatio: { type: 'string' },
|
|
73
|
+
views: {
|
|
74
|
+
type: 'array',
|
|
75
|
+
items: { type: 'string' },
|
|
76
|
+
},
|
|
77
|
+
seed: { type: 'integer' },
|
|
78
|
+
provider: { type: 'string' },
|
|
79
|
+
attachment: {
|
|
80
|
+
type: 'object',
|
|
81
|
+
additionalProperties: true,
|
|
82
|
+
properties: {
|
|
83
|
+
attachmentId: { type: 'string' },
|
|
84
|
+
url: { type: 'string' },
|
|
85
|
+
mediaType: { type: 'string' },
|
|
86
|
+
filename: { type: 'string' },
|
|
87
|
+
byteSize: { type: 'integer' },
|
|
88
|
+
},
|
|
89
|
+
},
|
|
90
|
+
},
|
|
91
|
+
},
|
|
92
|
+
render: (result, renderCtx) =>
|
|
93
|
+
renderToolOutput(result, renderCtx, {
|
|
94
|
+
title: (res) => `Character Sheet (${res.layout || 'turnaround'}) — ${res.style || 'concept_art'}`,
|
|
95
|
+
subtitle: (res) => (res.views ? `Views: ${res.views.join(' | ')}` : ''),
|
|
96
|
+
}),
|
|
97
|
+
},
|
|
98
|
+
async execute(args, execCtx) {
|
|
99
|
+
const cfg = live.value
|
|
100
|
+
const activeProvider = args.provider || cfg.activeProvider || 'fal'
|
|
101
|
+
const seed = typeof args.seed === 'number' ? args.seed : Math.floor(Math.random() * 2147483647)
|
|
102
|
+
|
|
103
|
+
const sheetPlan = buildCharacterSheetPrompt({
|
|
104
|
+
prompt: args.prompt,
|
|
105
|
+
layout: args.layout,
|
|
106
|
+
style: args.style,
|
|
107
|
+
negativePrompt: args.negativePrompt,
|
|
108
|
+
})
|
|
109
|
+
|
|
110
|
+
const providers = makeProviders({
|
|
111
|
+
fetchImpl: fetch,
|
|
112
|
+
resolveKey: (id) => resolveApiKey(cfg, id),
|
|
113
|
+
cfg,
|
|
114
|
+
})
|
|
115
|
+
|
|
116
|
+
const fallbackChain = resolveFallbackChain(activeProvider, cfg)
|
|
117
|
+
|
|
118
|
+
const aspectPixels = ASPECT_RATIOS[sheetPlan.aspectRatio] || ASPECT_RATIOS['16:9']
|
|
119
|
+
|
|
120
|
+
const jobPayload = {
|
|
121
|
+
prompt: sheetPlan.prompt,
|
|
122
|
+
negativePrompt: sheetPlan.negativePrompt,
|
|
123
|
+
aspectRatio: sheetPlan.aspectRatio,
|
|
124
|
+
aspectPixels,
|
|
125
|
+
format: 'png',
|
|
126
|
+
seed,
|
|
127
|
+
quality: 'high',
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
const { result: genResult, provider: usedProvider } = await executeWithFallback(
|
|
131
|
+
fallbackChain,
|
|
132
|
+
providers,
|
|
133
|
+
jobPayload,
|
|
134
|
+
{ signal: execCtx?.signal }
|
|
135
|
+
)
|
|
136
|
+
|
|
137
|
+
const timestamp = Date.now()
|
|
138
|
+
const safeSlug = slugify(args.prompt.slice(0, 30)) || 'character-sheet'
|
|
139
|
+
const filename = `character-sheet-${sheetPlan.layout}-${safeSlug}-${timestamp}.png`
|
|
140
|
+
|
|
141
|
+
let attachment = null
|
|
142
|
+
if (genResult.bytes) {
|
|
143
|
+
const outDir = path.resolve(process.cwd(), cfg.outputDir || 'output')
|
|
144
|
+
await mkdir(outDir, { recursive: true })
|
|
145
|
+
const diskPath = path.join(outDir, filename)
|
|
146
|
+
await writeFile(diskPath, Buffer.from(genResult.bytes))
|
|
147
|
+
|
|
148
|
+
attachment = await saveAttachmentSafe(
|
|
149
|
+
ctx,
|
|
150
|
+
Buffer.from(genResult.bytes),
|
|
151
|
+
'image/png',
|
|
152
|
+
filename,
|
|
153
|
+
diskPath
|
|
154
|
+
)
|
|
155
|
+
} else if (genResult.url) {
|
|
156
|
+
attachment = {
|
|
157
|
+
url: genResult.url,
|
|
158
|
+
mediaType: 'image/png',
|
|
159
|
+
filename,
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
const response = {
|
|
164
|
+
prompt: sheetPlan.prompt,
|
|
165
|
+
characterDescription: args.prompt,
|
|
166
|
+
layout: sheetPlan.layout,
|
|
167
|
+
style: sheetPlan.style,
|
|
168
|
+
aspectRatio: sheetPlan.aspectRatio,
|
|
169
|
+
views: sheetPlan.views,
|
|
170
|
+
seed,
|
|
171
|
+
provider: usedProvider,
|
|
172
|
+
attachment,
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
if (attachment?.attachmentId) {
|
|
176
|
+
response.attachmentId = attachment.attachmentId
|
|
177
|
+
}
|
|
178
|
+
if (attachment?.url) {
|
|
179
|
+
response.url = attachment.url
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
return response
|
|
183
|
+
},
|
|
184
|
+
})
|
|
185
|
+
)
|
|
186
|
+
}, 'dsh-image-gen: tool generate_character_sheet')
|
|
187
|
+
}
|
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
// diagram.js — beautify_diagram tool (#185).
|
|
2
|
+
// Transforms raw Mermaid / Graphviz diagrams into high-end 3D isometric architectural illustrations.
|
|
3
|
+
|
|
4
|
+
import { defineTool } from '@deepseek-ai/dsh-tools'
|
|
5
|
+
import path from 'node:path'
|
|
6
|
+
import { mkdir, writeFile } from 'node:fs/promises'
|
|
7
|
+
import {
|
|
8
|
+
ASPECT_RATIOS,
|
|
9
|
+
makeProviders,
|
|
10
|
+
saveAttachmentSafe,
|
|
11
|
+
} from '../providers.js'
|
|
12
|
+
import { resolveFallbackChain, executeWithFallback } from '../fallback-router.js'
|
|
13
|
+
import { renderToolOutput } from '../attachment-helper.js'
|
|
14
|
+
import {
|
|
15
|
+
DIAGRAM_3D_STYLES,
|
|
16
|
+
buildDiagramIllustrationPrompt,
|
|
17
|
+
} from '../diagram-helpers.js'
|
|
18
|
+
|
|
19
|
+
export function registerDiagramTools(ctx, deps) {
|
|
20
|
+
const {
|
|
21
|
+
live,
|
|
22
|
+
slugify,
|
|
23
|
+
resolveApiKey,
|
|
24
|
+
} = deps
|
|
25
|
+
|
|
26
|
+
ctx.effect(() => {
|
|
27
|
+
ctx.tools.register(
|
|
28
|
+
defineTool({
|
|
29
|
+
name: 'beautify_diagram',
|
|
30
|
+
description:
|
|
31
|
+
'Transform raw Mermaid or Graphviz diagrams into stunning 3D isometric architectural illustrations '
|
|
32
|
+
+ 'with glowing data pipes, glassmorphism cards, and pristine technical lighting for documentation and hero blocks.',
|
|
33
|
+
parameters: {
|
|
34
|
+
diagram: {
|
|
35
|
+
type: 'string',
|
|
36
|
+
description: 'Raw Mermaid diagram source code, Graphviz DOT text, or image path/attachment.',
|
|
37
|
+
},
|
|
38
|
+
style: {
|
|
39
|
+
type: 'string',
|
|
40
|
+
enum: ['isometric_3d', 'cyber_blueprint', 'clay_minimal', 'glossy_dark'],
|
|
41
|
+
description: '3D rendering aesthetic style. Default: isometric_3d.',
|
|
42
|
+
},
|
|
43
|
+
title: {
|
|
44
|
+
type: 'string',
|
|
45
|
+
description: 'Optional architecture title or headline to feature in the composition.',
|
|
46
|
+
},
|
|
47
|
+
aspect_ratio: {
|
|
48
|
+
type: 'string',
|
|
49
|
+
enum: ['16:9', '1:1', '4:3', '21:9'],
|
|
50
|
+
description: 'Aspect ratio for documentation header or hero banner (default: 16:9).',
|
|
51
|
+
},
|
|
52
|
+
seed: {
|
|
53
|
+
type: 'integer',
|
|
54
|
+
description: 'Optional seed for reproducible diagram generation.',
|
|
55
|
+
},
|
|
56
|
+
provider: {
|
|
57
|
+
type: 'string',
|
|
58
|
+
description: 'Optional provider override. Defaults to active provider.',
|
|
59
|
+
},
|
|
60
|
+
},
|
|
61
|
+
output: {
|
|
62
|
+
schema: {
|
|
63
|
+
type: 'object',
|
|
64
|
+
additionalProperties: true,
|
|
65
|
+
properties: {
|
|
66
|
+
title: { type: 'string' },
|
|
67
|
+
style: { type: 'string' },
|
|
68
|
+
aspectRatio: { type: 'string' },
|
|
69
|
+
detectedNodes: {
|
|
70
|
+
type: 'array',
|
|
71
|
+
items: { type: 'string' },
|
|
72
|
+
},
|
|
73
|
+
prompt: { type: 'string' },
|
|
74
|
+
seed: { type: 'integer' },
|
|
75
|
+
provider: { type: 'string' },
|
|
76
|
+
attachment: {
|
|
77
|
+
type: 'object',
|
|
78
|
+
additionalProperties: true,
|
|
79
|
+
properties: {
|
|
80
|
+
attachmentId: { type: 'string' },
|
|
81
|
+
url: { type: 'string' },
|
|
82
|
+
mediaType: { type: 'string' },
|
|
83
|
+
filename: { type: 'string' },
|
|
84
|
+
},
|
|
85
|
+
},
|
|
86
|
+
},
|
|
87
|
+
},
|
|
88
|
+
render: (result, renderCtx) =>
|
|
89
|
+
renderToolOutput(result, renderCtx, {
|
|
90
|
+
title: (res) => `3D Architecture Diagram — ${res.title || res.style || 'Isometric'}`,
|
|
91
|
+
subtitle: (res) => (res.detectedNodes?.length ? `Nodes: ${res.detectedNodes.slice(0, 4).join(', ')}` : ''),
|
|
92
|
+
}),
|
|
93
|
+
},
|
|
94
|
+
async execute(args, execCtx) {
|
|
95
|
+
const cfg = live?.value || (typeof live === 'function' ? live() : {})
|
|
96
|
+
const activeProvider = args.provider || cfg.activeProvider || 'fal'
|
|
97
|
+
const seed = typeof args.seed === 'number' ? args.seed : Math.floor(Math.random() * 2147483647)
|
|
98
|
+
|
|
99
|
+
const plan = buildDiagramIllustrationPrompt({
|
|
100
|
+
diagram: args.diagram,
|
|
101
|
+
style: args.style,
|
|
102
|
+
title: args.title,
|
|
103
|
+
aspectRatio: args.aspect_ratio || '16:9',
|
|
104
|
+
})
|
|
105
|
+
|
|
106
|
+
const providers = makeProviders({
|
|
107
|
+
fetchImpl: fetch,
|
|
108
|
+
resolveKey: (id) => resolveApiKey(cfg, id),
|
|
109
|
+
cfg,
|
|
110
|
+
})
|
|
111
|
+
const fallbackChain = resolveFallbackChain(activeProvider, cfg)
|
|
112
|
+
const aspectPixels = ASPECT_RATIOS[plan.aspectRatio] || ASPECT_RATIOS['16:9']
|
|
113
|
+
|
|
114
|
+
const jobPayload = {
|
|
115
|
+
prompt: plan.prompt,
|
|
116
|
+
negativePrompt: plan.negativePrompt,
|
|
117
|
+
aspectRatio: plan.aspectRatio,
|
|
118
|
+
aspectPixels,
|
|
119
|
+
format: 'png',
|
|
120
|
+
seed,
|
|
121
|
+
quality: 'high',
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
const { result: genResult, provider: usedProvider } = await executeWithFallback(
|
|
125
|
+
fallbackChain,
|
|
126
|
+
providers,
|
|
127
|
+
jobPayload,
|
|
128
|
+
{ signal: execCtx?.signal }
|
|
129
|
+
)
|
|
130
|
+
|
|
131
|
+
const outDir = path.resolve(process.cwd(), cfg.outputDir || 'output')
|
|
132
|
+
await mkdir(outDir, { recursive: true })
|
|
133
|
+
const timestamp = Date.now()
|
|
134
|
+
const safeSlug = slugify(args.title || 'architecture-diagram') || 'diagram'
|
|
135
|
+
const filename = `diagram-3d-${safeSlug}-${timestamp}.png`
|
|
136
|
+
|
|
137
|
+
let attachment = null
|
|
138
|
+
if (genResult.bytes) {
|
|
139
|
+
const diskPath = path.join(outDir, filename)
|
|
140
|
+
await writeFile(diskPath, Buffer.from(genResult.bytes))
|
|
141
|
+
attachment = await saveAttachmentSafe(
|
|
142
|
+
ctx,
|
|
143
|
+
Buffer.from(genResult.bytes),
|
|
144
|
+
'image/png',
|
|
145
|
+
filename,
|
|
146
|
+
diskPath
|
|
147
|
+
)
|
|
148
|
+
} else if (genResult.url) {
|
|
149
|
+
attachment = {
|
|
150
|
+
url: genResult.url,
|
|
151
|
+
mediaType: 'image/png',
|
|
152
|
+
filename,
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
return {
|
|
157
|
+
title: args.title || 'System Architecture',
|
|
158
|
+
style: plan.style,
|
|
159
|
+
aspectRatio: plan.aspectRatio,
|
|
160
|
+
detectedNodes: plan.parsedNodes,
|
|
161
|
+
prompt: plan.prompt,
|
|
162
|
+
seed,
|
|
163
|
+
provider: usedProvider,
|
|
164
|
+
attachment,
|
|
165
|
+
url: attachment?.url,
|
|
166
|
+
attachmentId: attachment?.attachmentId,
|
|
167
|
+
}
|
|
168
|
+
},
|
|
169
|
+
})
|
|
170
|
+
)
|
|
171
|
+
}, 'dsh-image-gen: tool beautify_diagram')
|
|
172
|
+
}
|
package/lib/tools/generation.js
CHANGED
|
@@ -41,7 +41,7 @@ import { calculateGenerationCost, recordSpend, assertBudgetAvailable } from '../
|
|
|
41
41
|
import { trackAndAssertLoopGuard } from '../loop-guard.js'
|
|
42
42
|
import { getCachedGeneration, setCachedGeneration } from '../generation-cache.js'
|
|
43
43
|
import { resolveFallbackChain, executeWithFallback } from '../fallback-router.js'
|
|
44
|
-
import { getSessionAnchor, applyAnchorPromptHints } from '../anchor-helpers.js'
|
|
44
|
+
import { getSessionAnchor, applyAnchorPromptHints, buildFaceConsistencyPrompt } from '../anchor-helpers.js'
|
|
45
45
|
import { enhancePrompt } from '../prompt-enhancer.js'
|
|
46
46
|
import {
|
|
47
47
|
readHistory,
|
|
@@ -172,6 +172,14 @@ export function registerGenerationTools(ctx, deps) {
|
|
|
172
172
|
type: 'number',
|
|
173
173
|
description: 'Optional strength of reference anchor influence (0.1 to 1.0).',
|
|
174
174
|
},
|
|
175
|
+
face_reference: {
|
|
176
|
+
type: 'string',
|
|
177
|
+
description: 'Portrait reference image (URL, local path, or attachment id) for facial identity preservation via FaceID / IP-Adapter (#183).',
|
|
178
|
+
},
|
|
179
|
+
face_strength: {
|
|
180
|
+
type: 'number',
|
|
181
|
+
description: 'Facial identity similarity weight between 0.1 and 1.0 (default: 0.75).',
|
|
182
|
+
},
|
|
175
183
|
},
|
|
176
184
|
output: {
|
|
177
185
|
schema: {
|
|
@@ -345,7 +353,24 @@ export function registerGenerationTools(ctx, deps) {
|
|
|
345
353
|
const styleInfo = resolveStylePreset(effectiveStyle, args.negative_prompt, args.guidance_scale)
|
|
346
354
|
let effectivePrompt = styleInfo.promptSuffix ? `${enhanced.prompt}, ${styleInfo.promptSuffix}` : enhanced.prompt
|
|
347
355
|
const activeAnchor = getSessionAnchor(sessionId)
|
|
348
|
-
|
|
356
|
+
let resolvedFaceRef
|
|
357
|
+
const faceRefInput = args.face_reference || (activeAnchor && (activeAnchor.mode === 'face' || activeAnchor.mode === 'character') ? activeAnchor.image : undefined)
|
|
358
|
+
if (faceRefInput) {
|
|
359
|
+
try {
|
|
360
|
+
resolvedFaceRef = await resolveSource(ctx, exec, faceRefInput)
|
|
361
|
+
} catch {
|
|
362
|
+
resolvedFaceRef = { url: faceRefInput }
|
|
363
|
+
}
|
|
364
|
+
const facePlan = buildFaceConsistencyPrompt({
|
|
365
|
+
prompt: effectivePrompt,
|
|
366
|
+
faceStrength: args.face_strength || activeAnchor?.strength,
|
|
367
|
+
characterName: activeAnchor?.label,
|
|
368
|
+
})
|
|
369
|
+
effectivePrompt = facePlan.prompt
|
|
370
|
+
if (facePlan.negativePrompt) {
|
|
371
|
+
effectiveNegative = effectiveNegative ? `${effectiveNegative}, ${facePlan.negativePrompt}` : facePlan.negativePrompt
|
|
372
|
+
}
|
|
373
|
+
} else if (args.reference_image) {
|
|
349
374
|
effectivePrompt = `${effectivePrompt}, visual reference anchor`
|
|
350
375
|
} else if (activeAnchor) {
|
|
351
376
|
effectivePrompt = applyAnchorPromptHints(activeAnchor, effectivePrompt)
|
|
@@ -373,7 +398,7 @@ export function registerGenerationTools(ctx, deps) {
|
|
|
373
398
|
assertBudgetAvailable(estimatedCost, cfg.dailyBudgetUsd)
|
|
374
399
|
const providers = makeProviders(
|
|
375
400
|
{ fetchImpl: fetch, resolveKey: (ref) => resolveApiKey(ctx, ref), cfg, subscriptionImages },
|
|
376
|
-
{ prompt: effectivePrompt, size, format, seed: args.seed, signal: exec.signal, negativePrompt: effectiveNegative, guidanceScale: effectiveGuidance, source, mask, strength: args.strength, quality: args.quality, style: args.style, aspectPixels, aspectRatio: args.aspect_ratio },
|
|
401
|
+
{ prompt: effectivePrompt, size, format, seed: args.seed, signal: exec.signal, negativePrompt: effectiveNegative, guidanceScale: effectiveGuidance, source, mask, strength: args.strength, quality: args.quality, style: args.style, aspectPixels, aspectRatio: args.aspect_ratio, faceReference: resolvedFaceRef },
|
|
377
402
|
)
|
|
378
403
|
// Subscription providers return {ok:false, reason} instead of throwing:
|
|
379
404
|
// rejection reaches the model as text. Without this guard,
|
|
@@ -0,0 +1,281 @@
|
|
|
1
|
+
// vision-ocr.js — replace_image_text tool (#180).
|
|
2
|
+
// Seamless text replacement and OCR localization via dsh-vision-bridge & inpainting.
|
|
3
|
+
|
|
4
|
+
import { defineTool } from '@deepseek-ai/dsh-tools'
|
|
5
|
+
import path from 'node:path'
|
|
6
|
+
import { mkdir, writeFile } from 'node:fs/promises'
|
|
7
|
+
import {
|
|
8
|
+
ASPECT_RATIOS,
|
|
9
|
+
makeProviders,
|
|
10
|
+
saveAttachmentSafe,
|
|
11
|
+
} from '../providers.js'
|
|
12
|
+
import { resolveFallbackChain, executeWithFallback } from '../fallback-router.js'
|
|
13
|
+
import { renderToolOutput } from '../attachment-helper.js'
|
|
14
|
+
import {
|
|
15
|
+
normalizeBbox,
|
|
16
|
+
buildTextInpaintMaskSvg,
|
|
17
|
+
buildTextOverlaySvg,
|
|
18
|
+
} from '../vision-ocr-helpers.js'
|
|
19
|
+
|
|
20
|
+
export function registerVisionOcrTools(ctx, deps) {
|
|
21
|
+
const {
|
|
22
|
+
live,
|
|
23
|
+
resolveSource,
|
|
24
|
+
slugify,
|
|
25
|
+
resolveApiKey,
|
|
26
|
+
} = deps
|
|
27
|
+
|
|
28
|
+
ctx.effect(() => {
|
|
29
|
+
ctx.tools.register(
|
|
30
|
+
defineTool({
|
|
31
|
+
name: 'replace_image_text',
|
|
32
|
+
description:
|
|
33
|
+
'Locate text inscriptions on an image via vision OCR, erase them seamlessly via inpainting, '
|
|
34
|
+
+ 'and render translated or replaced typography precisely at the original coordinates.',
|
|
35
|
+
parameters: {
|
|
36
|
+
image: {
|
|
37
|
+
type: 'string',
|
|
38
|
+
description: 'Source image path, URL, or attachment id (sha256:...) containing text to replace.',
|
|
39
|
+
},
|
|
40
|
+
replacements: {
|
|
41
|
+
type: 'array',
|
|
42
|
+
description: 'Array of text replacements: [{ find: "Old Text", replace: "Новый Текст", bbox: [x1,y1,x2,y2] }].',
|
|
43
|
+
items: {
|
|
44
|
+
type: 'object',
|
|
45
|
+
additionalProperties: true,
|
|
46
|
+
properties: {
|
|
47
|
+
find: { type: 'string', description: 'Original text to locate' },
|
|
48
|
+
replace: { type: 'string', description: 'New replacement text to render' },
|
|
49
|
+
bbox: {
|
|
50
|
+
type: 'array',
|
|
51
|
+
items: { type: 'number' },
|
|
52
|
+
description: 'Optional normalized bounding box [x1, y1, x2, y2] (0-1000)',
|
|
53
|
+
},
|
|
54
|
+
},
|
|
55
|
+
},
|
|
56
|
+
},
|
|
57
|
+
target_language: {
|
|
58
|
+
type: 'string',
|
|
59
|
+
description: 'Language code (e.g. "ru", "en", "zh") for automated text localization.',
|
|
60
|
+
},
|
|
61
|
+
font_color: {
|
|
62
|
+
type: 'string',
|
|
63
|
+
description: 'Color of replacement typography (hex or CSS color, e.g. "#FFFFFF"). Default: #FFFFFF.',
|
|
64
|
+
},
|
|
65
|
+
font_family: {
|
|
66
|
+
type: 'string',
|
|
67
|
+
description: 'Font family to render (default: sans-serif).',
|
|
68
|
+
},
|
|
69
|
+
inpaint_prompt: {
|
|
70
|
+
type: 'string',
|
|
71
|
+
description: 'Prompt guiding the inpainting process to restore background behind erased text.',
|
|
72
|
+
},
|
|
73
|
+
provider: {
|
|
74
|
+
type: 'string',
|
|
75
|
+
description: 'Optional provider override for inpainting. Defaults to active provider.',
|
|
76
|
+
},
|
|
77
|
+
},
|
|
78
|
+
output: {
|
|
79
|
+
schema: {
|
|
80
|
+
type: 'object',
|
|
81
|
+
additionalProperties: true,
|
|
82
|
+
properties: {
|
|
83
|
+
replacedCount: { type: 'integer' },
|
|
84
|
+
bboxes: {
|
|
85
|
+
type: 'array',
|
|
86
|
+
items: {
|
|
87
|
+
type: 'array',
|
|
88
|
+
items: { type: 'number' },
|
|
89
|
+
},
|
|
90
|
+
},
|
|
91
|
+
inpaintPrompt: { type: 'string' },
|
|
92
|
+
attachment: {
|
|
93
|
+
type: 'object',
|
|
94
|
+
additionalProperties: true,
|
|
95
|
+
properties: {
|
|
96
|
+
attachmentId: { type: 'string' },
|
|
97
|
+
url: { type: 'string' },
|
|
98
|
+
mediaType: { type: 'string' },
|
|
99
|
+
filename: { type: 'string' },
|
|
100
|
+
},
|
|
101
|
+
},
|
|
102
|
+
maskAttachment: {
|
|
103
|
+
type: 'object',
|
|
104
|
+
additionalProperties: true,
|
|
105
|
+
},
|
|
106
|
+
overlayAttachment: {
|
|
107
|
+
type: 'object',
|
|
108
|
+
additionalProperties: true,
|
|
109
|
+
},
|
|
110
|
+
},
|
|
111
|
+
},
|
|
112
|
+
render: (result, renderCtx) =>
|
|
113
|
+
renderToolOutput(result, renderCtx, {
|
|
114
|
+
title: (res) => `Text Replaced (${res.replacedCount || 0} inscriptions)`,
|
|
115
|
+
subtitle: (res) => (res.bboxes?.length ? `Detected regions: ${res.bboxes.length}` : ''),
|
|
116
|
+
}),
|
|
117
|
+
},
|
|
118
|
+
async execute(args, execCtx) {
|
|
119
|
+
const cfg = live?.value || (typeof live === 'function' ? live() : {})
|
|
120
|
+
const activeProvider = args.provider || cfg.activeProvider || 'fal'
|
|
121
|
+
|
|
122
|
+
const source = await resolveSource(ctx, execCtx, args.image)
|
|
123
|
+
if (!source || !source.bytes) {
|
|
124
|
+
throw new Error(`Unable to resolve source image for text replacement: ${args.image}`)
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
const imgWidth = source.width || 1024
|
|
128
|
+
const imgHeight = source.height || 1024
|
|
129
|
+
|
|
130
|
+
// 1. Gather text items and bounding boxes
|
|
131
|
+
const replacements = Array.isArray(args.replacements) ? args.replacements : []
|
|
132
|
+
const bboxes = []
|
|
133
|
+
const overlayItems = []
|
|
134
|
+
|
|
135
|
+
for (const item of replacements) {
|
|
136
|
+
let bbox = item.bbox
|
|
137
|
+
if (!bbox && item.find && ctx.tools?.get) {
|
|
138
|
+
const groundTool = ctx.tools.get('vision_ground')
|
|
139
|
+
if (groundTool && typeof groundTool.execute === 'function' && source.ref) {
|
|
140
|
+
try {
|
|
141
|
+
const gRes = await groundTool.execute({
|
|
142
|
+
attachmentId: source.ref.startsWith('sha256:') ? source.ref : undefined,
|
|
143
|
+
target: `text "${item.find}"`,
|
|
144
|
+
}, execCtx)
|
|
145
|
+
if (gRes?.bbox && gRes.bbox.length === 4) {
|
|
146
|
+
bbox = gRes.bbox
|
|
147
|
+
}
|
|
148
|
+
} catch {
|
|
149
|
+
// non-fatal vision_ground fallback
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
// Fallback default bounding box if detection didn't locate exact coordinates
|
|
155
|
+
const normalizedBox = normalizeBbox(bbox || [200, 400, 800, 500], imgWidth, imgHeight)
|
|
156
|
+
if (normalizedBox) {
|
|
157
|
+
bboxes.push(normalizedBox)
|
|
158
|
+
overlayItems.push({
|
|
159
|
+
text: item.replace || item.find,
|
|
160
|
+
bbox: normalizedBox,
|
|
161
|
+
color: args.font_color || '#FFFFFF',
|
|
162
|
+
})
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
// If no replacements specified, check default center title box
|
|
167
|
+
if (bboxes.length === 0) {
|
|
168
|
+
const defaultBox = [150, 420, 850, 520]
|
|
169
|
+
bboxes.push(defaultBox)
|
|
170
|
+
overlayItems.push({
|
|
171
|
+
text: args.target_language === 'ru' ? 'Заголовок' : 'Localized Text',
|
|
172
|
+
bbox: defaultBox,
|
|
173
|
+
color: args.font_color || '#FFFFFF',
|
|
174
|
+
})
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
// 2. Generate inpaint mask
|
|
178
|
+
const maskSvg = buildTextInpaintMaskSvg(imgWidth, imgHeight, bboxes, 12)
|
|
179
|
+
const maskBuffer = Buffer.from(maskSvg, 'utf8')
|
|
180
|
+
|
|
181
|
+
// 3. Generate typography overlay SVG
|
|
182
|
+
const overlaySvg = buildTextOverlaySvg(imgWidth, imgHeight, overlayItems, args.font_family || 'sans-serif')
|
|
183
|
+
const overlayBuffer = Buffer.from(overlaySvg, 'utf8')
|
|
184
|
+
|
|
185
|
+
// 4. Inpaint to erase old text
|
|
186
|
+
const inpaintPrompt = args.inpaint_prompt || 'clean seamless surface, erase text artifacts, coherent background texture'
|
|
187
|
+
const providers = makeProviders({
|
|
188
|
+
fetchImpl: fetch,
|
|
189
|
+
resolveKey: (id) => resolveApiKey(cfg, id),
|
|
190
|
+
cfg,
|
|
191
|
+
})
|
|
192
|
+
const fallbackChain = resolveFallbackChain(activeProvider, cfg)
|
|
193
|
+
|
|
194
|
+
const inpaintJob = {
|
|
195
|
+
prompt: inpaintPrompt,
|
|
196
|
+
source: {
|
|
197
|
+
bytes: source.bytes,
|
|
198
|
+
mediaType: source.mediaType || 'image/png',
|
|
199
|
+
},
|
|
200
|
+
mask: {
|
|
201
|
+
bytes: maskBuffer,
|
|
202
|
+
mediaType: 'image/svg+xml',
|
|
203
|
+
},
|
|
204
|
+
strength: 0.95,
|
|
205
|
+
format: 'png',
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
let inpaintedBytes = null
|
|
209
|
+
let inpaintedUrl = null
|
|
210
|
+
try {
|
|
211
|
+
const { result: inpaintResult } = await executeWithFallback(
|
|
212
|
+
fallbackChain,
|
|
213
|
+
providers,
|
|
214
|
+
inpaintJob,
|
|
215
|
+
{ signal: execCtx?.signal }
|
|
216
|
+
)
|
|
217
|
+
inpaintedBytes = inpaintResult.bytes
|
|
218
|
+
inpaintedUrl = inpaintResult.url
|
|
219
|
+
} catch {
|
|
220
|
+
// If inpaint provider unavailable, keep source bytes and proceed with overlay
|
|
221
|
+
inpaintedBytes = source.bytes
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
const outDir = path.resolve(process.cwd(), cfg.outputDir || 'output')
|
|
225
|
+
await mkdir(outDir, { recursive: true })
|
|
226
|
+
const timestamp = Date.now()
|
|
227
|
+
const safeSlug = slugify(args.image.slice(0, 20)) || 'localized-image'
|
|
228
|
+
|
|
229
|
+
let attachment = null
|
|
230
|
+
if (inpaintedBytes) {
|
|
231
|
+
const filename = `text-replaced-${safeSlug}-${timestamp}.png`
|
|
232
|
+
const diskPath = path.join(outDir, filename)
|
|
233
|
+
await writeFile(diskPath, Buffer.from(inpaintedBytes))
|
|
234
|
+
attachment = await saveAttachmentSafe(
|
|
235
|
+
ctx,
|
|
236
|
+
Buffer.from(inpaintedBytes),
|
|
237
|
+
'image/png',
|
|
238
|
+
filename,
|
|
239
|
+
diskPath
|
|
240
|
+
)
|
|
241
|
+
} else if (inpaintedUrl) {
|
|
242
|
+
attachment = {
|
|
243
|
+
url: inpaintedUrl,
|
|
244
|
+
mediaType: 'image/png',
|
|
245
|
+
filename: `text-replaced-${safeSlug}-${timestamp}.png`,
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
const maskFilename = `mask-text-${safeSlug}-${timestamp}.svg`
|
|
250
|
+
const maskAttachment = await saveAttachmentSafe(
|
|
251
|
+
ctx,
|
|
252
|
+
maskBuffer,
|
|
253
|
+
'image/svg+xml',
|
|
254
|
+
maskFilename,
|
|
255
|
+
path.join(outDir, maskFilename)
|
|
256
|
+
)
|
|
257
|
+
|
|
258
|
+
const overlayFilename = `overlay-text-${safeSlug}-${timestamp}.svg`
|
|
259
|
+
const overlayAttachment = await saveAttachmentSafe(
|
|
260
|
+
ctx,
|
|
261
|
+
overlayBuffer,
|
|
262
|
+
'image/svg+xml',
|
|
263
|
+
overlayFilename,
|
|
264
|
+
path.join(outDir, overlayFilename)
|
|
265
|
+
)
|
|
266
|
+
|
|
267
|
+
return {
|
|
268
|
+
replacedCount: overlayItems.length,
|
|
269
|
+
bboxes,
|
|
270
|
+
inpaintPrompt,
|
|
271
|
+
attachment,
|
|
272
|
+
maskAttachment,
|
|
273
|
+
overlayAttachment,
|
|
274
|
+
url: attachment?.url,
|
|
275
|
+
attachmentId: attachment?.attachmentId,
|
|
276
|
+
}
|
|
277
|
+
},
|
|
278
|
+
})
|
|
279
|
+
)
|
|
280
|
+
}, 'dsh-image-gen: tool replace_image_text')
|
|
281
|
+
}
|