@goodandready/dsh-image-gen 0.11.21 → 0.11.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/attachment-helper.js +14 -0
- package/lib/commands.js +16 -8
- package/lib/config-schema.js +32 -4
- package/lib/generation-helpers.js +266 -250
- package/lib/index.js +12 -2
- package/lib/loop-guard.js +32 -8
- package/lib/provider-utils.js +40 -1
- package/lib/providers/backends/custom.js +15 -4
- package/lib/providers/backends/local.js +8 -1
- package/lib/providers/shared-helpers.js +2 -1
- package/lib/register-tools.js +42 -17
- package/lib/tools/character-sheet.js +98 -47
- package/lib/tools/diagram.js +101 -40
- package/lib/tools/editing.js +9 -0
- package/lib/tools/generation.js +505 -496
- package/lib/tools/processing-basic.js +6 -1
- package/lib/tools/style-matrix.js +16 -0
- package/lib/tools/theme-pair.js +80 -40
- package/lib/tools/ui-asset.js +15 -0
- package/lib/tools/vision-ocr.js +134 -71
- package/package.json +1 -1
|
@@ -15,7 +15,7 @@ import {
|
|
|
15
15
|
saveAttachmentSafe,
|
|
16
16
|
} from '../providers.js'
|
|
17
17
|
import { buildDualOutputMarkdown } from '../resolve-image.js'
|
|
18
|
-
import { assertBudgetAvailable } from '../cost-meter.js'
|
|
18
|
+
import { assertBudgetAvailable, recordSpend } from '../cost-meter.js'
|
|
19
19
|
import { trackAndAssertLoopGuard } from '../loop-guard.js'
|
|
20
20
|
import { sanitizeErrorAndLogs } from '../security.js'
|
|
21
21
|
|
|
@@ -55,6 +55,7 @@ export function registerProcessingTools(ctx, deps) {
|
|
|
55
55
|
const source = await resolveSource(ctx, exec, args.image)
|
|
56
56
|
if (!source || !source.bytes) throw new Error('Source image not found: ' + args.image)
|
|
57
57
|
const cfg = live()
|
|
58
|
+
assertBudgetAvailable(0.02, cfg.dailyBudgetUsd)
|
|
58
59
|
const res = await removeBackgroundFal(
|
|
59
60
|
{ fetchImpl: fetch, resolveKey: (ref) => resolveApiKey(ctx, ref), cfg },
|
|
60
61
|
{ imageBytes: source.bytes, mediaType: source.mediaType, model: args.model, signal: exec.signal }
|
|
@@ -67,6 +68,7 @@ export function registerProcessingTools(ctx, deps) {
|
|
|
67
68
|
await mkdir(outDir, { recursive: true })
|
|
68
69
|
const filePath = path.join(outDir, name)
|
|
69
70
|
await writeFile(filePath, res.bytes)
|
|
71
|
+
recordSpend(0.02, { ctx, meta: { tool: 'remove_background', model: args.model || 'fal-ai/birefnet' } })
|
|
70
72
|
const summary = buildDualOutputMarkdown({
|
|
71
73
|
action: 'background removed',
|
|
72
74
|
filePath,
|
|
@@ -124,6 +126,7 @@ export function registerProcessingTools(ctx, deps) {
|
|
|
124
126
|
const source = await resolveSource(ctx, exec, args.image)
|
|
125
127
|
if (!source || !source.bytes) throw new Error('Source image not found: ' + args.image)
|
|
126
128
|
const cfg = live()
|
|
129
|
+
assertBudgetAvailable(0.025, cfg.dailyBudgetUsd)
|
|
127
130
|
const res = await upscaleImageFal(
|
|
128
131
|
{ fetchImpl: fetch, resolveKey: (ref) => resolveApiKey(ctx, ref), cfg },
|
|
129
132
|
{ imageBytes: source.bytes, mediaType: source.mediaType, scale: args.scale, prompt: args.prompt, creativity: args.creativity, signal: exec.signal }
|
|
@@ -136,6 +139,7 @@ export function registerProcessingTools(ctx, deps) {
|
|
|
136
139
|
await mkdir(outDir, { recursive: true })
|
|
137
140
|
const filePath = path.join(outDir, name)
|
|
138
141
|
await writeFile(filePath, res.bytes)
|
|
142
|
+
recordSpend(0.025, { ctx, meta: { tool: 'upscale_image', scale: args.scale } })
|
|
139
143
|
const summary = buildDualOutputMarkdown({
|
|
140
144
|
action: `upscaled (${args.scale || 2}x)`,
|
|
141
145
|
filePath,
|
|
@@ -269,6 +273,7 @@ export function registerProcessingTools(ctx, deps) {
|
|
|
269
273
|
await mkdir(outDir, { recursive: true })
|
|
270
274
|
const filePath = path.join(outDir, name)
|
|
271
275
|
await writeFile(filePath, res.bytes)
|
|
276
|
+
recordSpend(0.05, { ctx, meta: { tool: 'blend_images' } })
|
|
272
277
|
const summary = buildDualOutputMarkdown({
|
|
273
278
|
action: 'blended composition',
|
|
274
279
|
filePath,
|
|
@@ -14,6 +14,7 @@ import {
|
|
|
14
14
|
saveAttachmentSafe,
|
|
15
15
|
} from '../providers.js'
|
|
16
16
|
import { resolveFallbackChain, executeWithFallback } from '../fallback-router.js'
|
|
17
|
+
import { assertBudgetAvailable, calculateGenerationCost, recordSpend } from '../cost-meter.js'
|
|
17
18
|
import {
|
|
18
19
|
normalizeMatrixStyles,
|
|
19
20
|
buildMatrixCellPrompt,
|
|
@@ -129,6 +130,15 @@ export function registerStyleMatrixTools(ctx, deps) {
|
|
|
129
130
|
let subscriptionImages
|
|
130
131
|
try { subscriptionImages = ctx.get && ctx.get('subscriptionImages') } catch (_err) { subscriptionImages = undefined }
|
|
131
132
|
|
|
133
|
+
const cellCount = activeStyles.length
|
|
134
|
+
const estimatedCost = calculateGenerationCost({
|
|
135
|
+
provider,
|
|
136
|
+
model: cfg.model || cfg.customModel || 'default',
|
|
137
|
+
size: cfg.defaultSize || '1024x1024',
|
|
138
|
+
count: cellCount,
|
|
139
|
+
})
|
|
140
|
+
assertBudgetAvailable(estimatedCost, cfg.dailyBudgetUsd)
|
|
141
|
+
|
|
132
142
|
const chain = resolveFallbackChain(provider, cfg.fallbackProviders, PROVIDER_KEYS)
|
|
133
143
|
const quadrantIds = ['A', 'B', 'C', 'D']
|
|
134
144
|
const matrixId = `matrix-${Date.now().toString(36)}`
|
|
@@ -168,6 +178,12 @@ export function registerStyleMatrixTools(ctx, deps) {
|
|
|
168
178
|
name: filename,
|
|
169
179
|
})
|
|
170
180
|
|
|
181
|
+
const cellCost = Number(gen.cost) || +(estimatedCost / cellCount).toFixed(4)
|
|
182
|
+
recordSpend(cellCost, {
|
|
183
|
+
ctx,
|
|
184
|
+
meta: { provider, model: cfg.model || cfg.customModel, prompt: cellPrompt, seed: cellSeed },
|
|
185
|
+
})
|
|
186
|
+
|
|
171
187
|
const filePath = path.join(outDir, filename)
|
|
172
188
|
await writeFile(filePath, gen.bytes)
|
|
173
189
|
|
package/lib/tools/theme-pair.js
CHANGED
|
@@ -1,18 +1,19 @@
|
|
|
1
|
-
|
|
2
|
-
//
|
|
3
|
-
// Synchronized dual generation of light and dark theme visuals with adaptive CSS snippets.
|
|
1
|
+
// theme-pair.js — generate_theme_pair tool (#189, #361).
|
|
2
|
+
// Generates complementary Light & Dark mode asset variants with synchronized compositions.
|
|
4
3
|
|
|
5
4
|
import { defineTool } from '@deepseek-ai/dsh-tools'
|
|
6
5
|
import path from 'node:path'
|
|
7
6
|
import { mkdir, writeFile } from 'node:fs/promises'
|
|
8
7
|
import {
|
|
9
8
|
ASPECT_RATIOS,
|
|
9
|
+
PROVIDER_KEYS,
|
|
10
10
|
buildSidecar,
|
|
11
11
|
makeProviders,
|
|
12
12
|
toLosslessJson,
|
|
13
13
|
saveAttachmentSafe,
|
|
14
14
|
} from '../providers.js'
|
|
15
15
|
import { resolveFallbackChain, executeWithFallback } from '../fallback-router.js'
|
|
16
|
+
import { assertBudgetAvailable, calculateGenerationCost, recordSpend } from '../cost-meter.js'
|
|
16
17
|
import { renderToolOutput } from '../attachment-helper.js'
|
|
17
18
|
import {
|
|
18
19
|
THEME_STYLES,
|
|
@@ -23,7 +24,7 @@ import {
|
|
|
23
24
|
export function registerThemePairTools(ctx, deps) {
|
|
24
25
|
const {
|
|
25
26
|
live,
|
|
26
|
-
slugify,
|
|
27
|
+
slugify = (s) => String(s || '').toLowerCase().replace(/[^a-z0-9]+/g, '-'),
|
|
27
28
|
resolveApiKey,
|
|
28
29
|
} = deps
|
|
29
30
|
|
|
@@ -32,23 +33,23 @@ export function registerThemePairTools(ctx, deps) {
|
|
|
32
33
|
defineTool({
|
|
33
34
|
name: 'generate_theme_pair',
|
|
34
35
|
description:
|
|
35
|
-
'Generate
|
|
36
|
-
+ '
|
|
36
|
+
'Generate matching Light and Dark mode UI illustration variants with synchronized seeds, '
|
|
37
|
+
+ 'consistent composition, and inverted contrast palettes. Outputs a responsive <picture> tag with prefers-color-scheme.',
|
|
37
38
|
parameters: {
|
|
38
39
|
prompt: {
|
|
39
40
|
type: 'string',
|
|
41
|
+
description: 'Core visual concept description (e.g. "cloud database topology architecture").',
|
|
40
42
|
required: true,
|
|
41
|
-
description: 'Core subject or illustration motif, e.g. "cloud server network topology" or "productivity dashboard banner".',
|
|
42
|
-
},
|
|
43
|
-
aspect_ratio: {
|
|
44
|
-
type: 'string',
|
|
45
|
-
enum: Object.keys(ASPECT_RATIOS),
|
|
46
|
-
description: 'Aspect ratio for both variants: "1:1", "16:9", "4:3", etc. (default: "16:9").',
|
|
47
43
|
},
|
|
48
44
|
style: {
|
|
49
45
|
type: 'string',
|
|
50
46
|
enum: THEME_STYLES,
|
|
51
|
-
description: '
|
|
47
|
+
description: 'Aesthetic visual direction: minimalist, neon_cyberpunk, editorial, gradient_glass, 3d_clay.',
|
|
48
|
+
},
|
|
49
|
+
aspect_ratio: {
|
|
50
|
+
type: 'string',
|
|
51
|
+
enum: ['16:9', '1:1', '4:3', '21:9'],
|
|
52
|
+
description: 'Asset aspect ratio (default: 16:9).',
|
|
52
53
|
},
|
|
53
54
|
dark_contrast_boost: {
|
|
54
55
|
type: 'number',
|
|
@@ -62,6 +63,10 @@ export function registerThemePairTools(ctx, deps) {
|
|
|
62
63
|
type: 'integer',
|
|
63
64
|
description: 'Base random seed to synchronize composition between variants.',
|
|
64
65
|
},
|
|
66
|
+
provider: {
|
|
67
|
+
type: 'string',
|
|
68
|
+
description: 'Optional provider override. Defaults to active provider.',
|
|
69
|
+
},
|
|
65
70
|
},
|
|
66
71
|
output: {
|
|
67
72
|
schema: {
|
|
@@ -99,7 +104,7 @@ export function registerThemePairTools(ctx, deps) {
|
|
|
99
104
|
},
|
|
100
105
|
isConcurrencySafe: () => false,
|
|
101
106
|
async execute(args, exec) {
|
|
102
|
-
const cfg = live()
|
|
107
|
+
const cfg = typeof live === 'function' ? live() : (live?.value || {})
|
|
103
108
|
if (cfg.enabled === false) {
|
|
104
109
|
throw new Error('Image generation is disabled in settings.')
|
|
105
110
|
}
|
|
@@ -120,8 +125,19 @@ export function registerThemePairTools(ctx, deps) {
|
|
|
120
125
|
darkContrastBoost,
|
|
121
126
|
})
|
|
122
127
|
|
|
123
|
-
const primaryProvider =
|
|
124
|
-
|
|
128
|
+
const primaryProvider = PROVIDER_KEYS.includes(args.provider || cfg.provider)
|
|
129
|
+
? (args.provider || cfg.provider)
|
|
130
|
+
: (PROVIDER_KEYS.includes(cfg.provider) ? cfg.provider : 'fal')
|
|
131
|
+
|
|
132
|
+
const estimatedCost = calculateGenerationCost({
|
|
133
|
+
provider: primaryProvider,
|
|
134
|
+
model: cfg.model || cfg.customModel || 'default',
|
|
135
|
+
size: cfg.defaultSize || '1024x1024',
|
|
136
|
+
count: 2,
|
|
137
|
+
})
|
|
138
|
+
assertBudgetAvailable(estimatedCost, cfg.dailyBudgetUsd)
|
|
139
|
+
|
|
140
|
+
const chain = resolveFallbackChain(primaryProvider, cfg.fallbackProviders, PROVIDER_KEYS)
|
|
125
141
|
const baseSeed = typeof args.seed === 'number' ? args.seed : Math.floor(Math.random() * 1000000)
|
|
126
142
|
|
|
127
143
|
const sessionCwd = exec.agent?.session?.header?.cwd
|
|
@@ -132,6 +148,9 @@ export function registerThemePairTools(ctx, deps) {
|
|
|
132
148
|
const slug = args.output_name ? slugify(args.output_name) : slugify(prompt).slice(0, 36)
|
|
133
149
|
const ts = Date.now().toString(36)
|
|
134
150
|
|
|
151
|
+
let subscriptionImages
|
|
152
|
+
try { subscriptionImages = ctx.get && ctx.get('subscriptionImages') } catch (_err) { subscriptionImages = undefined }
|
|
153
|
+
|
|
135
154
|
// 1. Generate Light theme variant
|
|
136
155
|
const lightJob = {
|
|
137
156
|
prompt: lightPrompt,
|
|
@@ -141,26 +160,34 @@ export function registerThemePairTools(ctx, deps) {
|
|
|
141
160
|
seed: baseSeed,
|
|
142
161
|
signal: exec.signal,
|
|
143
162
|
}
|
|
144
|
-
const
|
|
163
|
+
const lightProviders = makeProviders(
|
|
164
|
+
{ fetchImpl: fetch, resolveKey: (ref) => resolveApiKey(ctx, ref), cfg, subscriptionImages },
|
|
165
|
+
lightJob,
|
|
166
|
+
)
|
|
167
|
+
const lightGen = await executeWithFallback(
|
|
168
|
+
lightProviders,
|
|
145
169
|
chain,
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
})
|
|
170
|
+
baseSeed,
|
|
171
|
+
lightPrompt,
|
|
172
|
+
{ logger: ctx.logger },
|
|
173
|
+
)
|
|
151
174
|
|
|
152
175
|
const lightFilename = `${slug}-light-${ts}.png`
|
|
153
176
|
const lightPath = path.join(outDir, lightFilename)
|
|
154
|
-
await writeFile(lightPath, lightGen.bytes)
|
|
177
|
+
await writeFile(lightPath, Buffer.from(lightGen.bytes))
|
|
155
178
|
|
|
156
|
-
const lightAttachment = await saveAttachmentSafe(ctx, {
|
|
179
|
+
const { attachment: lightAttachment, localUrl: lightUrl } = await saveAttachmentSafe(ctx, {
|
|
157
180
|
bytes: lightGen.bytes,
|
|
158
181
|
mediaType: 'image/png',
|
|
159
|
-
|
|
182
|
+
name: lightFilename,
|
|
160
183
|
width: lightGen.width || aspectPixels[0],
|
|
161
184
|
height: lightGen.height || aspectPixels[1],
|
|
162
185
|
})
|
|
163
|
-
const
|
|
186
|
+
const halfCost = +(estimatedCost / 2).toFixed(4)
|
|
187
|
+
recordSpend(Number(lightGen.cost) || halfCost, {
|
|
188
|
+
ctx,
|
|
189
|
+
meta: { provider: primaryProvider, model: cfg.model || cfg.customModel, prompt: lightPrompt },
|
|
190
|
+
})
|
|
164
191
|
|
|
165
192
|
// 2. Generate Dark theme variant (using matched seed)
|
|
166
193
|
const darkJob = {
|
|
@@ -171,26 +198,39 @@ export function registerThemePairTools(ctx, deps) {
|
|
|
171
198
|
seed: baseSeed,
|
|
172
199
|
signal: exec.signal,
|
|
173
200
|
}
|
|
174
|
-
const
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
201
|
+
const darkProviders = makeProviders(
|
|
202
|
+
{ fetchImpl: fetch, resolveKey: (ref) => resolveApiKey(ctx, ref), cfg, subscriptionImages },
|
|
203
|
+
darkJob,
|
|
204
|
+
)
|
|
205
|
+
|
|
206
|
+
let darkGen
|
|
207
|
+
try {
|
|
208
|
+
darkGen = await executeWithFallback(
|
|
209
|
+
darkProviders,
|
|
210
|
+
chain,
|
|
211
|
+
baseSeed,
|
|
212
|
+
darkPrompt,
|
|
213
|
+
{ logger: ctx.logger },
|
|
214
|
+
)
|
|
215
|
+
} catch (darkErr) {
|
|
216
|
+
throw new Error(`Failed to generate dark theme variant: ${darkErr.message}`)
|
|
217
|
+
}
|
|
181
218
|
|
|
182
219
|
const darkFilename = `${slug}-dark-${ts}.png`
|
|
183
220
|
const darkPath = path.join(outDir, darkFilename)
|
|
184
|
-
await writeFile(darkPath, darkGen.bytes)
|
|
221
|
+
await writeFile(darkPath, Buffer.from(darkGen.bytes))
|
|
185
222
|
|
|
186
|
-
const darkAttachment = await saveAttachmentSafe(ctx, {
|
|
223
|
+
const { attachment: darkAttachment, localUrl: darkUrl } = await saveAttachmentSafe(ctx, {
|
|
187
224
|
bytes: darkGen.bytes,
|
|
188
225
|
mediaType: 'image/png',
|
|
189
|
-
|
|
226
|
+
name: darkFilename,
|
|
190
227
|
width: darkGen.width || aspectPixels[0],
|
|
191
228
|
height: darkGen.height || aspectPixels[1],
|
|
192
229
|
})
|
|
193
|
-
|
|
230
|
+
recordSpend(Number(darkGen.cost) || halfCost, {
|
|
231
|
+
ctx,
|
|
232
|
+
meta: { provider: primaryProvider, model: cfg.model || cfg.customModel, prompt: darkPrompt },
|
|
233
|
+
})
|
|
194
234
|
|
|
195
235
|
// Sidecars
|
|
196
236
|
await writeFile(
|
|
@@ -202,8 +242,8 @@ export function registerThemePairTools(ctx, deps) {
|
|
|
202
242
|
size: `${aspectPixels[0]}x${aspectPixels[1]}`,
|
|
203
243
|
format: 'png',
|
|
204
244
|
seed: baseSeed,
|
|
205
|
-
light: { path: lightPath, url: lightUrl, attachmentId: lightAttachment
|
|
206
|
-
dark: { path: darkPath, url: darkUrl, attachmentId: darkAttachment
|
|
245
|
+
light: { path: lightPath, url: lightUrl, attachmentId: lightAttachment?.attachmentId },
|
|
246
|
+
dark: { path: darkPath, url: darkUrl, attachmentId: darkAttachment?.attachmentId },
|
|
207
247
|
cost: (lightGen.cost || 0) + (darkGen.cost || 0),
|
|
208
248
|
}), null, 2),
|
|
209
249
|
)
|
|
@@ -248,4 +288,4 @@ export function registerThemePairTools(ctx, deps) {
|
|
|
248
288
|
}),
|
|
249
289
|
)
|
|
250
290
|
}, 'dsh-image-gen: tool generate_theme_pair')
|
|
251
|
-
}
|
|
291
|
+
}
|
package/lib/tools/ui-asset.js
CHANGED
|
@@ -15,6 +15,7 @@ import {
|
|
|
15
15
|
saveAttachmentSafe,
|
|
16
16
|
} from '../providers.js'
|
|
17
17
|
import { resolveFallbackChain, executeWithFallback } from '../fallback-router.js'
|
|
18
|
+
import { assertBudgetAvailable, calculateGenerationCost, recordSpend } from '../cost-meter.js'
|
|
18
19
|
import { buildDualOutputMarkdown } from '../resolve-image.js'
|
|
19
20
|
import {
|
|
20
21
|
UI_ASSET_TYPES,
|
|
@@ -161,6 +162,14 @@ export function registerUiAssetTools(ctx, deps) {
|
|
|
161
162
|
},
|
|
162
163
|
)
|
|
163
164
|
|
|
165
|
+
const estimatedCost = calculateGenerationCost({
|
|
166
|
+
provider,
|
|
167
|
+
model: cfg.model || cfg.customModel || 'default',
|
|
168
|
+
size: cfg.defaultSize || '1024x1024',
|
|
169
|
+
count: 1,
|
|
170
|
+
})
|
|
171
|
+
assertBudgetAvailable(estimatedCost, cfg.dailyBudgetUsd)
|
|
172
|
+
|
|
164
173
|
const chain = resolveFallbackChain(provider, cfg.fallbackProviders, PROVIDER_KEYS)
|
|
165
174
|
const seedVal = args.seed ?? Math.floor(Math.random() * 100000)
|
|
166
175
|
|
|
@@ -201,6 +210,12 @@ export function registerUiAssetTools(ctx, deps) {
|
|
|
201
210
|
name: filename,
|
|
202
211
|
})
|
|
203
212
|
|
|
213
|
+
const effectiveCost = Number(gen.cost) || estimatedCost
|
|
214
|
+
recordSpend(effectiveCost, {
|
|
215
|
+
ctx,
|
|
216
|
+
meta: { provider, model: cfg.model || cfg.customModel, prompt: synthesizedPrompt, seed: seedVal },
|
|
217
|
+
})
|
|
218
|
+
|
|
204
219
|
const sessionCwd = exec.agent?.session?.header?.cwd
|
|
205
220
|
const targetDir = cfg.outputDir || 'generated/images'
|
|
206
221
|
const outDir = path.resolve(sessionCwd || process.cwd(), targetDir)
|
package/lib/tools/vision-ocr.js
CHANGED
|
@@ -1,16 +1,18 @@
|
|
|
1
|
-
|
|
2
|
-
//
|
|
3
|
-
// Seamless text replacement and OCR localization via dsh-vision-bridge & inpainting.
|
|
1
|
+
// vision-ocr.js — replace_image_text tool (#180, #360).
|
|
2
|
+
// Grounding OCR, background inpainting, and typography re-rendering for visual localization.
|
|
4
3
|
|
|
5
4
|
import { defineTool } from '@deepseek-ai/dsh-tools'
|
|
6
5
|
import path from 'node:path'
|
|
7
6
|
import { mkdir, writeFile } from 'node:fs/promises'
|
|
8
7
|
import {
|
|
9
|
-
|
|
8
|
+
PROVIDER_KEYS,
|
|
10
9
|
makeProviders,
|
|
10
|
+
toLosslessJson,
|
|
11
11
|
saveAttachmentSafe,
|
|
12
12
|
} from '../providers.js'
|
|
13
|
+
import { buildDualOutputMarkdown } from '../resolve-image.js'
|
|
13
14
|
import { resolveFallbackChain, executeWithFallback } from '../fallback-router.js'
|
|
15
|
+
import { assertBudgetAvailable, calculateGenerationCost, recordSpend } from '../cost-meter.js'
|
|
14
16
|
import { renderToolOutput } from '../attachment-helper.js'
|
|
15
17
|
import {
|
|
16
18
|
normalizeBbox,
|
|
@@ -22,7 +24,7 @@ export function registerVisionOcrTools(ctx, deps) {
|
|
|
22
24
|
const {
|
|
23
25
|
live,
|
|
24
26
|
resolveSource,
|
|
25
|
-
slugify,
|
|
27
|
+
slugify = (s) => String(s || '').toLowerCase().replace(/[^a-z0-9]+/g, '-'),
|
|
26
28
|
resolveApiKey,
|
|
27
29
|
} = deps
|
|
28
30
|
|
|
@@ -31,22 +33,28 @@ export function registerVisionOcrTools(ctx, deps) {
|
|
|
31
33
|
defineTool({
|
|
32
34
|
name: 'replace_image_text',
|
|
33
35
|
description:
|
|
34
|
-
'Locate text
|
|
35
|
-
+ 'and render translated
|
|
36
|
+
'Locate text elements in an image using OCR/vision grounding, inpaint the background to erase them, '
|
|
37
|
+
+ 'and render crisp translated/replacement typography over the cleaned regions for visual localization.',
|
|
36
38
|
parameters: {
|
|
37
39
|
image: {
|
|
38
40
|
type: 'string',
|
|
39
|
-
description: 'Source image path,
|
|
41
|
+
description: 'Source image path, attachment ID, or URL containing text to replace.',
|
|
40
42
|
},
|
|
41
43
|
replacements: {
|
|
42
44
|
type: 'array',
|
|
43
|
-
description: '
|
|
45
|
+
description: 'List of text replacement instructions.',
|
|
44
46
|
items: {
|
|
45
47
|
type: 'object',
|
|
46
48
|
additionalProperties: true,
|
|
47
49
|
properties: {
|
|
48
|
-
find: {
|
|
49
|
-
|
|
50
|
+
find: {
|
|
51
|
+
type: 'string',
|
|
52
|
+
description: 'Original text string to search for and erase.',
|
|
53
|
+
},
|
|
54
|
+
replace: {
|
|
55
|
+
type: 'string',
|
|
56
|
+
description: 'Replacement text to render in place of the original.',
|
|
57
|
+
},
|
|
50
58
|
bbox: {
|
|
51
59
|
type: 'array',
|
|
52
60
|
items: { type: 'number' },
|
|
@@ -75,6 +83,10 @@ export function registerVisionOcrTools(ctx, deps) {
|
|
|
75
83
|
type: 'string',
|
|
76
84
|
description: 'Optional provider override for inpainting. Defaults to active provider.',
|
|
77
85
|
},
|
|
86
|
+
seed: {
|
|
87
|
+
type: 'integer',
|
|
88
|
+
description: 'Optional seed for reproducible inpainting.',
|
|
89
|
+
},
|
|
78
90
|
},
|
|
79
91
|
output: {
|
|
80
92
|
schema: {
|
|
@@ -90,6 +102,9 @@ export function registerVisionOcrTools(ctx, deps) {
|
|
|
90
102
|
},
|
|
91
103
|
},
|
|
92
104
|
inpaintPrompt: { type: 'string' },
|
|
105
|
+
targetLanguage: { type: 'string' },
|
|
106
|
+
path: { type: 'string' },
|
|
107
|
+
url: { type: 'string' },
|
|
93
108
|
attachment: {
|
|
94
109
|
type: 'object',
|
|
95
110
|
additionalProperties: true,
|
|
@@ -108,13 +123,23 @@ export function registerVisionOcrTools(ctx, deps) {
|
|
|
108
123
|
type: 'object',
|
|
109
124
|
additionalProperties: true,
|
|
110
125
|
},
|
|
126
|
+
_fallback: {
|
|
127
|
+
type: 'object',
|
|
128
|
+
additionalProperties: true,
|
|
129
|
+
},
|
|
111
130
|
},
|
|
112
131
|
},
|
|
113
132
|
render: (_args, value) => renderToolOutput(value),
|
|
114
133
|
},
|
|
115
134
|
async execute(args, execCtx) {
|
|
116
|
-
const cfg =
|
|
117
|
-
|
|
135
|
+
const cfg = typeof live === 'function' ? live() : (live?.value || {})
|
|
136
|
+
if (cfg.enabled === false) {
|
|
137
|
+
throw new Error('Image generation is disabled in settings.')
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
const provider = PROVIDER_KEYS.includes(args.provider || cfg.provider)
|
|
141
|
+
? (args.provider || cfg.provider)
|
|
142
|
+
: (PROVIDER_KEYS.includes(cfg.provider) ? cfg.provider : 'fal')
|
|
118
143
|
|
|
119
144
|
const source = await resolveSource(ctx, execCtx, args.image)
|
|
120
145
|
if (!source || !source.bytes) {
|
|
@@ -148,7 +173,6 @@ export function registerVisionOcrTools(ctx, deps) {
|
|
|
148
173
|
}
|
|
149
174
|
}
|
|
150
175
|
|
|
151
|
-
// Fallback default bounding box if detection didn't locate exact coordinates
|
|
152
176
|
const normalizedBox = normalizeBbox(bbox || [200, 400, 800, 500], imgWidth, imgHeight)
|
|
153
177
|
if (normalizedBox) {
|
|
154
178
|
bboxes.push(normalizedBox)
|
|
@@ -160,12 +184,11 @@ export function registerVisionOcrTools(ctx, deps) {
|
|
|
160
184
|
}
|
|
161
185
|
}
|
|
162
186
|
|
|
163
|
-
// If no replacements specified, check default center title box
|
|
164
187
|
if (bboxes.length === 0) {
|
|
165
188
|
const defaultBox = [150, 420, 850, 520]
|
|
166
189
|
bboxes.push(defaultBox)
|
|
167
190
|
overlayItems.push({
|
|
168
|
-
text: 'Localized Text',
|
|
191
|
+
text: args.target_language ? `[${args.target_language}] Localized Text` : 'Localized Text',
|
|
169
192
|
bbox: defaultBox,
|
|
170
193
|
color: args.font_color || '#FFFFFF',
|
|
171
194
|
})
|
|
@@ -181,12 +204,10 @@ export function registerVisionOcrTools(ctx, deps) {
|
|
|
181
204
|
|
|
182
205
|
// 4. Inpaint to erase old text
|
|
183
206
|
const inpaintPrompt = args.inpaint_prompt || 'clean seamless surface, erase text artifacts, coherent background texture'
|
|
184
|
-
const
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
})
|
|
189
|
-
const fallbackChain = resolveFallbackChain(activeProvider, cfg)
|
|
207
|
+
const seed = typeof args.seed === 'number' ? args.seed : Math.floor(Math.random() * 2147483647)
|
|
208
|
+
|
|
209
|
+
let subscriptionImages
|
|
210
|
+
try { subscriptionImages = ctx.get && ctx.get('subscriptionImages') } catch (_err) { subscriptionImages = undefined }
|
|
190
211
|
|
|
191
212
|
const inpaintJob = {
|
|
192
213
|
prompt: inpaintPrompt,
|
|
@@ -200,77 +221,119 @@ export function registerVisionOcrTools(ctx, deps) {
|
|
|
200
221
|
},
|
|
201
222
|
strength: 0.95,
|
|
202
223
|
format: 'png',
|
|
224
|
+
seed,
|
|
225
|
+
signal: execCtx?.signal,
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
const providers = makeProviders(
|
|
229
|
+
{ fetchImpl: fetch, resolveKey: (ref) => resolveApiKey(ctx, ref), cfg, subscriptionImages },
|
|
230
|
+
inpaintJob,
|
|
231
|
+
)
|
|
232
|
+
const fallbackChain = resolveFallbackChain(provider, cfg.fallbackProviders, PROVIDER_KEYS)
|
|
233
|
+
|
|
234
|
+
const estimatedCost = calculateGenerationCost({
|
|
235
|
+
provider,
|
|
236
|
+
model: cfg.model || cfg.customModel || 'default',
|
|
237
|
+
size: `${imgWidth}x${imgHeight}`,
|
|
238
|
+
count: 1,
|
|
239
|
+
})
|
|
240
|
+
assertBudgetAvailable(estimatedCost, cfg.dailyBudgetUsd)
|
|
241
|
+
|
|
242
|
+
const gen = await executeWithFallback(
|
|
243
|
+
providers,
|
|
244
|
+
fallbackChain,
|
|
245
|
+
seed,
|
|
246
|
+
inpaintPrompt,
|
|
247
|
+
{ logger: ctx.logger },
|
|
248
|
+
)
|
|
249
|
+
|
|
250
|
+
const inpaintedBytes = gen.bytes
|
|
251
|
+
if (!inpaintedBytes) {
|
|
252
|
+
throw new Error('Inpainting failed to return image bytes for text replacement')
|
|
203
253
|
}
|
|
204
254
|
|
|
205
|
-
|
|
206
|
-
let
|
|
255
|
+
// 5. Composite typography overlay on inpainted background
|
|
256
|
+
let finalBytes = inpaintedBytes
|
|
207
257
|
try {
|
|
208
|
-
const
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
} catch (_err) {
|
|
217
|
-
/* non-fatal: inpaint provider unavailable; use source bytes */
|
|
218
|
-
inpaintedBytes = source.bytes
|
|
258
|
+
const sharpMod = await import('sharp')
|
|
259
|
+
const sharp = sharpMod.default || sharpMod
|
|
260
|
+
finalBytes = await sharp(inpaintedBytes)
|
|
261
|
+
.composite([{ input: overlayBuffer, top: 0, left: 0 }])
|
|
262
|
+
.png()
|
|
263
|
+
.toBuffer()
|
|
264
|
+
} catch (sharpErr) {
|
|
265
|
+
if (ctx.logger?.warn) ctx.logger.warn(`Sharp compositing skipped: ${sharpErr.message}`)
|
|
219
266
|
}
|
|
220
267
|
|
|
221
|
-
const
|
|
268
|
+
const usedProvider = gen._fallback?.providerUsed || provider
|
|
269
|
+
const sessionCwd = execCtx?.agent?.session?.header?.cwd
|
|
270
|
+
const targetDir = cfg.outputDir || 'generated/images'
|
|
271
|
+
const outDir = path.resolve(sessionCwd || process.cwd(), targetDir)
|
|
222
272
|
await mkdir(outDir, { recursive: true })
|
|
223
273
|
const timestamp = Date.now()
|
|
224
|
-
const safeSlug = slugify(args.image.slice(0, 20)) || 'localized-image'
|
|
274
|
+
const safeSlug = slugify(String(args.image || '').slice(0, 20)) || 'localized-image'
|
|
225
275
|
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
filename,
|
|
236
|
-
diskPath
|
|
237
|
-
)
|
|
238
|
-
} else if (inpaintedUrl) {
|
|
239
|
-
attachment = {
|
|
240
|
-
url: inpaintedUrl,
|
|
241
|
-
mediaType: 'image/png',
|
|
242
|
-
filename: `text-replaced-${safeSlug}-${timestamp}.png`,
|
|
243
|
-
}
|
|
244
|
-
}
|
|
276
|
+
const filename = `text-replaced-${safeSlug}-${timestamp}.png`
|
|
277
|
+
const { attachment, localUrl } = await saveAttachmentSafe(ctx, {
|
|
278
|
+
bytes: finalBytes,
|
|
279
|
+
mediaType: 'image/png',
|
|
280
|
+
name: filename,
|
|
281
|
+
})
|
|
282
|
+
|
|
283
|
+
const diskPath = path.join(outDir, filename)
|
|
284
|
+
await writeFile(diskPath, Buffer.from(finalBytes))
|
|
245
285
|
|
|
246
286
|
const maskFilename = `mask-text-${safeSlug}-${timestamp}.svg`
|
|
247
|
-
const maskAttachment = await saveAttachmentSafe(
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
path.join(outDir, maskFilename)
|
|
253
|
-
)
|
|
287
|
+
const { attachment: maskAttachment } = await saveAttachmentSafe(ctx, {
|
|
288
|
+
bytes: maskBuffer,
|
|
289
|
+
mediaType: 'image/svg+xml',
|
|
290
|
+
name: maskFilename,
|
|
291
|
+
})
|
|
254
292
|
|
|
255
293
|
const overlayFilename = `overlay-text-${safeSlug}-${timestamp}.svg`
|
|
256
|
-
const overlayAttachment = await saveAttachmentSafe(
|
|
294
|
+
const { attachment: overlayAttachment } = await saveAttachmentSafe(ctx, {
|
|
295
|
+
bytes: overlayBuffer,
|
|
296
|
+
mediaType: 'image/svg+xml',
|
|
297
|
+
name: overlayFilename,
|
|
298
|
+
})
|
|
299
|
+
|
|
300
|
+
const effectiveCost = Number(gen.cost) || estimatedCost
|
|
301
|
+
recordSpend(effectiveCost, {
|
|
257
302
|
ctx,
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
overlayFilename,
|
|
261
|
-
path.join(outDir, overlayFilename)
|
|
262
|
-
)
|
|
303
|
+
meta: { provider: usedProvider, model: cfg.model || cfg.customModel, prompt: inpaintPrompt, seed },
|
|
304
|
+
})
|
|
263
305
|
|
|
264
|
-
|
|
306
|
+
const dualOutput = buildDualOutputMarkdown({
|
|
307
|
+
action: 'generated',
|
|
308
|
+
filePath: diskPath,
|
|
309
|
+
width: imgWidth,
|
|
310
|
+
height: imgHeight,
|
|
311
|
+
mediaType: 'image/png',
|
|
312
|
+
seed,
|
|
313
|
+
provider: usedProvider,
|
|
314
|
+
model: cfg.model || cfg.customModel || 'replace-image-text',
|
|
315
|
+
cost: gen.cost,
|
|
316
|
+
attachmentId: attachment?.attachmentId,
|
|
317
|
+
})
|
|
318
|
+
|
|
319
|
+
const response = {
|
|
320
|
+
summary: dualOutput,
|
|
265
321
|
replacedCount: overlayItems.length,
|
|
266
322
|
bboxes,
|
|
267
323
|
inpaintPrompt,
|
|
324
|
+
targetLanguage: args.target_language || '',
|
|
325
|
+
seed,
|
|
326
|
+
provider: usedProvider,
|
|
327
|
+
path: diskPath,
|
|
328
|
+
url: localUrl,
|
|
268
329
|
attachment,
|
|
269
330
|
maskAttachment,
|
|
270
331
|
overlayAttachment,
|
|
271
|
-
url: attachment?.url,
|
|
272
332
|
attachmentId: attachment?.attachmentId,
|
|
333
|
+
_fallback: gen._fallback,
|
|
273
334
|
}
|
|
335
|
+
|
|
336
|
+
return toLosslessJson(response)
|
|
274
337
|
},
|
|
275
338
|
})
|
|
276
339
|
)
|