@dickpy/dsh-imagegen 1.2.3 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -201
- package/README.md +203 -181
- package/cordis.patch.yml +8 -8
- package/docs/images/multi-model-comparison.png +0 -0
- package/lib/client.js +2711 -1318
- package/lib/client.js.map +1 -1
- package/lib/index.js +830 -155
- package/package.json +70 -68
- package/src/agent-image-tools.ts +418 -316
- package/src/client/ImageGenPanel.tsx +1703 -1476
- package/src/client/SettingsCard.tsx +936 -648
- package/src/client/TemplateLibrary.tsx +336 -336
- package/src/client/api.ts +193 -193
- package/src/client/channels-form.ts +263 -0
- package/src/client/controller.ts +46 -46
- package/src/client/conversation-sync.ts +14 -0
- package/src/client/css-modules.d.ts +5 -5
- package/src/client/helpers.ts +33 -33
- package/src/client/image-toolview.module.css +73 -73
- package/src/client/image-toolview.tsx +170 -152
- package/src/client/index.ts +32 -22
- package/src/client/locales.ts +610 -484
- package/src/client/mount.tsx +185 -96
- package/src/client/panel.module.css +1713 -1445
- package/src/client/settings-card.module.css +1023 -536
- package/src/client/settings-form.ts +336 -336
- package/src/client/settings-scope.ts +298 -250
- package/src/client/sidebar-entry.ts +148 -102
- package/src/client/templates.module.css +453 -453
- package/src/engine.ts +520 -464
- package/src/gallery-store.ts +286 -280
- package/src/generation-runtime.ts +79 -48
- package/src/history-store.ts +250 -238
- package/src/image-format.ts +11 -0
- package/src/image-models.ts +19 -19
- package/src/index.ts +318 -212
- package/src/model-catalog.ts +115 -0
- package/src/presets.ts +71 -0
- package/src/prompt-enhancer.ts +137 -79
- package/src/protocol.ts +338 -253
- package/src/routes.ts +916 -738
- package/src/task-queue.ts +113 -103
- package/src/templates/cases.json +10196 -10196
- package/src/templates-store.ts +278 -278
- package/src/updater.ts +117 -117
package/src/agent-image-tools.ts
CHANGED
|
@@ -1,316 +1,418 @@
|
|
|
1
|
-
/** Agent-facing image-generation tools backed by the shared host queue. */
|
|
2
|
-
|
|
3
|
-
import type { Context } from '@deepseek-ai/cordis'
|
|
4
|
-
import { defineTool } from '@deepseek-ai/dsh-tools'
|
|
5
|
-
import type {
|
|
6
|
-
import type {} from '@deepseek-ai/dsh-attachment'
|
|
7
|
-
import type {} from '@deepseek-ai/dsh-
|
|
8
|
-
import {
|
|
9
|
-
import {
|
|
10
|
-
import {
|
|
11
|
-
import
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
}
|
|
20
|
-
|
|
21
|
-
interface AgentImageRef {
|
|
22
|
-
attachment_id: string
|
|
23
|
-
media_type: string
|
|
24
|
-
bytes: number
|
|
25
|
-
width: number
|
|
26
|
-
height: number
|
|
27
|
-
name?: string
|
|
28
|
-
}
|
|
29
|
-
|
|
30
|
-
interface AgentTaskResult {
|
|
31
|
-
task_id: string
|
|
32
|
-
status: string
|
|
33
|
-
message: string
|
|
34
|
-
error?: string
|
|
35
|
-
images: AgentImageRef[]
|
|
36
|
-
}
|
|
37
|
-
|
|
38
|
-
const imageRefSchema = {
|
|
39
|
-
type: 'object',
|
|
40
|
-
additionalProperties: false,
|
|
41
|
-
properties: {
|
|
42
|
-
attachment_id: { type: 'string', required: true },
|
|
43
|
-
media_type: { type: 'string', required: true },
|
|
44
|
-
bytes: { type: 'integer', required: true },
|
|
45
|
-
width: { type: 'integer', required: true },
|
|
46
|
-
height: { type: 'integer', required: true },
|
|
47
|
-
name: { type: 'string' },
|
|
48
|
-
},
|
|
49
|
-
} as const
|
|
50
|
-
|
|
51
|
-
const taskResultSchema = {
|
|
52
|
-
type: 'object',
|
|
53
|
-
additionalProperties: false,
|
|
54
|
-
properties: {
|
|
55
|
-
task_id: { type: 'string', required: true },
|
|
56
|
-
status: { type: 'string', required: true },
|
|
57
|
-
message: { type: 'string', required: true },
|
|
58
|
-
error: { type: 'string' },
|
|
59
|
-
images: { type: 'array', required: true, items: imageRefSchema },
|
|
60
|
-
},
|
|
61
|
-
} as const
|
|
62
|
-
|
|
63
|
-
/** Agent calls stay pending until the provider and history write settle. */
|
|
64
|
-
const AGENT_GENERATION_TIMEOUT_MS = 300_000
|
|
65
|
-
|
|
66
|
-
function acceptedMediaType(value: string): value is ImageMediaType {
|
|
67
|
-
return value === 'image/png' || value === 'image/jpeg' || value === 'image/webp' || value === 'image/gif'
|
|
68
|
-
}
|
|
69
|
-
|
|
70
|
-
function projectRef(ref: ImageAttachmentRef): AgentImageRef {
|
|
71
|
-
return {
|
|
72
|
-
attachment_id: String(ref.attachmentId),
|
|
73
|
-
media_type: ref.mediaType,
|
|
74
|
-
bytes: ref.bytes,
|
|
75
|
-
width: ref.width,
|
|
76
|
-
height: ref.height,
|
|
77
|
-
...ref.name === undefined ? {} : { name: ref.name },
|
|
78
|
-
}
|
|
79
|
-
}
|
|
80
|
-
|
|
81
|
-
function restoreRef(value: AgentImageRef): ImageAttachmentRef {
|
|
82
|
-
if (!acceptedMediaType(value.media_type)) throw new ImageGenError('source_image.media_type is not a supported image type', 'bad-reference-image')
|
|
83
|
-
if (!Number.isInteger(value.bytes) || value.bytes < 1 || !Number.isInteger(value.width) || value.width < 1 || !Number.isInteger(value.height) || value.height < 1) {
|
|
84
|
-
throw new ImageGenError('source_image metadata is invalid', 'bad-reference-image')
|
|
85
|
-
}
|
|
86
|
-
return {
|
|
87
|
-
attachmentId: value.attachment_id as ImageAttachmentRef['attachmentId'],
|
|
88
|
-
mediaType: value.media_type,
|
|
89
|
-
bytes: value.bytes,
|
|
90
|
-
width: value.width,
|
|
91
|
-
height: value.height,
|
|
92
|
-
...value.name === undefined ? {} : { name: value.name },
|
|
93
|
-
}
|
|
94
|
-
}
|
|
95
|
-
|
|
96
|
-
function imageDataUrl(image: { data: Uint8Array; ref: ImageAttachmentRef }): string {
|
|
97
|
-
return `data:${image.ref.mediaType};base64,${Buffer.from(image.data).toString('base64')}`
|
|
98
|
-
}
|
|
99
|
-
|
|
100
|
-
function renderTaskResult(value: AgentTaskResult): Array<{ type: 'text'; text: string }
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
]
|
|
106
|
-
}
|
|
107
|
-
|
|
108
|
-
/**
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
}
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
}
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
}
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
1
|
+
/** Agent-facing image-generation tools backed by the shared host queue. */
|
|
2
|
+
|
|
3
|
+
import type { Context } from '@deepseek-ai/cordis'
|
|
4
|
+
import { defineTool } from '@deepseek-ai/dsh-tools'
|
|
5
|
+
import type { ToolResult, ToolResultView } from '@deepseek-ai/dsh-tools'
|
|
6
|
+
import type { ImageAttachmentRef, ImageMediaType } from '@deepseek-ai/dsh-attachment'
|
|
7
|
+
import type {} from '@deepseek-ai/dsh-attachment'
|
|
8
|
+
import type {} from '@deepseek-ai/dsh-tools'
|
|
9
|
+
import { ImageGenError } from './engine.ts'
|
|
10
|
+
import { ImageGenerationRuntime, type RuntimeChannel } from './generation-runtime.ts'
|
|
11
|
+
import { detectImageMime } from './image-format.ts'
|
|
12
|
+
import type { GenerationTask, GeneratedImage } from './protocol.ts'
|
|
13
|
+
|
|
14
|
+
export interface AgentImageToolConfig {
|
|
15
|
+
enabled: boolean
|
|
16
|
+
allowAgentImageGeneration: boolean
|
|
17
|
+
channels: RuntimeChannel[]
|
|
18
|
+
defaultChannelId: string
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
interface AgentImageRef {
|
|
22
|
+
attachment_id: string
|
|
23
|
+
media_type: string
|
|
24
|
+
bytes: number
|
|
25
|
+
width: number
|
|
26
|
+
height: number
|
|
27
|
+
name?: string
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
interface AgentTaskResult {
|
|
31
|
+
task_id: string
|
|
32
|
+
status: string
|
|
33
|
+
message: string
|
|
34
|
+
error?: string
|
|
35
|
+
images: AgentImageRef[]
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
const imageRefSchema = {
|
|
39
|
+
type: 'object',
|
|
40
|
+
additionalProperties: false,
|
|
41
|
+
properties: {
|
|
42
|
+
attachment_id: { type: 'string', required: true },
|
|
43
|
+
media_type: { type: 'string', required: true },
|
|
44
|
+
bytes: { type: 'integer', required: true },
|
|
45
|
+
width: { type: 'integer', required: true },
|
|
46
|
+
height: { type: 'integer', required: true },
|
|
47
|
+
name: { type: 'string' },
|
|
48
|
+
},
|
|
49
|
+
} as const
|
|
50
|
+
|
|
51
|
+
const taskResultSchema = {
|
|
52
|
+
type: 'object',
|
|
53
|
+
additionalProperties: false,
|
|
54
|
+
properties: {
|
|
55
|
+
task_id: { type: 'string', required: true },
|
|
56
|
+
status: { type: 'string', required: true },
|
|
57
|
+
message: { type: 'string', required: true },
|
|
58
|
+
error: { type: 'string' },
|
|
59
|
+
images: { type: 'array', required: true, items: imageRefSchema },
|
|
60
|
+
},
|
|
61
|
+
} as const
|
|
62
|
+
|
|
63
|
+
/** Agent calls stay pending until the provider and history write settle. */
|
|
64
|
+
const AGENT_GENERATION_TIMEOUT_MS = 300_000
|
|
65
|
+
|
|
66
|
+
function acceptedMediaType(value: string): value is ImageMediaType {
|
|
67
|
+
return value === 'image/png' || value === 'image/jpeg' || value === 'image/webp' || value === 'image/gif'
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
function projectRef(ref: ImageAttachmentRef): AgentImageRef {
|
|
71
|
+
return {
|
|
72
|
+
attachment_id: String(ref.attachmentId),
|
|
73
|
+
media_type: ref.mediaType,
|
|
74
|
+
bytes: ref.bytes,
|
|
75
|
+
width: ref.width,
|
|
76
|
+
height: ref.height,
|
|
77
|
+
...ref.name === undefined ? {} : { name: ref.name },
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
function restoreRef(value: AgentImageRef): ImageAttachmentRef {
|
|
82
|
+
if (!acceptedMediaType(value.media_type)) throw new ImageGenError('source_image.media_type is not a supported image type', 'bad-reference-image')
|
|
83
|
+
if (!Number.isInteger(value.bytes) || value.bytes < 1 || !Number.isInteger(value.width) || value.width < 1 || !Number.isInteger(value.height) || value.height < 1) {
|
|
84
|
+
throw new ImageGenError('source_image metadata is invalid', 'bad-reference-image')
|
|
85
|
+
}
|
|
86
|
+
return {
|
|
87
|
+
attachmentId: value.attachment_id as ImageAttachmentRef['attachmentId'],
|
|
88
|
+
mediaType: value.media_type,
|
|
89
|
+
bytes: value.bytes,
|
|
90
|
+
width: value.width,
|
|
91
|
+
height: value.height,
|
|
92
|
+
...value.name === undefined ? {} : { name: value.name },
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
function imageDataUrl(image: { data: Uint8Array; ref: ImageAttachmentRef }): string {
|
|
97
|
+
return `data:${image.ref.mediaType};base64,${Buffer.from(image.data).toString('base64')}`
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
function renderTaskResult(value: AgentTaskResult): Array<{ type: 'text'; text: string }> {
|
|
101
|
+
// Generated images are presentation output, not model input. Keeping the
|
|
102
|
+
// model-facing result textual lets image generation work with text-only
|
|
103
|
+
// conversation models while preserving the attachment references needed by
|
|
104
|
+
// edit_image.
|
|
105
|
+
return [{ type: 'text', text: JSON.stringify(value) }]
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/** The UI-only projection that keeps generated images beside the tool call. */
|
|
109
|
+
function imagePresentationMeta(value: AgentTaskResult): { images: Array<Record<string, string | number>> } {
|
|
110
|
+
return {
|
|
111
|
+
images: value.images.map(image => {
|
|
112
|
+
const ref: Record<string, string | number> = {
|
|
113
|
+
attachment_id: image.attachment_id,
|
|
114
|
+
media_type: image.media_type,
|
|
115
|
+
bytes: image.bytes,
|
|
116
|
+
width: image.width,
|
|
117
|
+
height: image.height,
|
|
118
|
+
}
|
|
119
|
+
if (image.name !== undefined) ref.name = image.name
|
|
120
|
+
return ref
|
|
121
|
+
}),
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
function imageBlocksFromMeta(meta: unknown): Array<{ type: 'image'; attachment: ImageAttachmentRef }> {
|
|
126
|
+
if (typeof meta !== 'object' || meta === null || Array.isArray(meta)) return []
|
|
127
|
+
const images = (meta as { images?: unknown }).images
|
|
128
|
+
if (!Array.isArray(images)) return []
|
|
129
|
+
return images.flatMap((value): Array<{ type: 'image'; attachment: ImageAttachmentRef }> => {
|
|
130
|
+
if (typeof value !== 'object' || value === null || Array.isArray(value)) return []
|
|
131
|
+
const raw = value as Record<string, unknown>
|
|
132
|
+
if (typeof raw.attachment_id !== 'string'
|
|
133
|
+
|| typeof raw.media_type !== 'string'
|
|
134
|
+
|| typeof raw.bytes !== 'number'
|
|
135
|
+
|| typeof raw.width !== 'number'
|
|
136
|
+
|| typeof raw.height !== 'number') return []
|
|
137
|
+
try {
|
|
138
|
+
return [{ type: 'image', attachment: restoreRef({
|
|
139
|
+
attachment_id: raw.attachment_id,
|
|
140
|
+
media_type: raw.media_type,
|
|
141
|
+
bytes: raw.bytes,
|
|
142
|
+
width: raw.width,
|
|
143
|
+
height: raw.height,
|
|
144
|
+
...typeof raw.name === 'string' ? { name: raw.name } : {},
|
|
145
|
+
}) }]
|
|
146
|
+
} catch {
|
|
147
|
+
return []
|
|
148
|
+
}
|
|
149
|
+
})
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
/** Rehydrate image attachments for the host-computed tool result view only. */
|
|
153
|
+
function presentImageResult(_args: unknown, result: ToolResult): ToolResultView | undefined {
|
|
154
|
+
const content = result.isError ? [] : imageBlocksFromMeta(result.meta)
|
|
155
|
+
return content.length === 0 ? undefined : { card: 'generic', content }
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
/** Register the global Agent tools and unregister them with the plugin lifecycle. */
|
|
159
|
+
export function registerAgentImageTools(ctx: Context, runtime: ImageGenerationRuntime, resolve: () => AgentImageToolConfig): () => void {
|
|
160
|
+
const attachmentRefs = new Map<string, Promise<AgentImageRef[]>>()
|
|
161
|
+
const ensureConfigured = (): void => {
|
|
162
|
+
const config = resolve()
|
|
163
|
+
if (!config.enabled) throw new ImageGenError('AI image generation is disabled. Open Settings > Plugins > AI Image and enable it.', 'plugin-disabled')
|
|
164
|
+
if (!config.allowAgentImageGeneration) throw new ImageGenError('Agent image generation is disabled in Settings > Plugins > AI Image.', 'agent-generation-disabled')
|
|
165
|
+
const usable = config.channels.some(channel => channel.apiUrl.trim() !== '' && channel.apiKey.trim() !== '')
|
|
166
|
+
if (!usable) throw new ImageGenError('Image API credentials are not configured. Open Settings > Plugins > AI Image, add a channel and fill in its API URL and API key.', 'image-api-not-configured')
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
/**
|
|
170
|
+
* Resolve the requested model alias onto a channel. Rules:
|
|
171
|
+
* - a named alias must exist in some channel's catalog (several channels
|
|
172
|
+
* may host it; the default channel wins);
|
|
173
|
+
* - with no alias, a single configured model is used directly, while
|
|
174
|
+
* multiple models require the Agent to ask the user first.
|
|
175
|
+
* @returns the channel plus the alias and its upstream id.
|
|
176
|
+
*/
|
|
177
|
+
const resolveModel = (requested: unknown): { channel: RuntimeChannel; alias: string; upstream: string } => {
|
|
178
|
+
const config = resolve()
|
|
179
|
+
const entries = config.channels.flatMap(channel => channel.models.map(model => ({ channel, alias: model.alias, upstream: model.id })))
|
|
180
|
+
if (entries.length === 0) {
|
|
181
|
+
throw new ImageGenError('No image models are configured. Open Settings > Plugins > AI Image and add a channel with at least one model.', 'no-models-configured')
|
|
182
|
+
}
|
|
183
|
+
const wanted = typeof requested === 'string' && requested.trim() !== '' ? requested.trim() : ''
|
|
184
|
+
if (wanted === '') {
|
|
185
|
+
if (entries.length === 1) return entries[0]!
|
|
186
|
+
const options = config.channels.flatMap(channel => channel.models.map(model => `"${channel.name} · ${model.alias}"`)).join(', ')
|
|
187
|
+
throw new ImageGenError(`Multiple image models are available — ask the user which channel and model to use, then call this tool again with that exact model name. Options: ${options}.`, 'model-choice-required')
|
|
188
|
+
}
|
|
189
|
+
const hosting = entries.filter(entry => entry.alias === wanted)
|
|
190
|
+
if (hosting.length === 0) {
|
|
191
|
+
const available = [...new Set(entries.map(entry => entry.alias))].join(', ')
|
|
192
|
+
throw new ImageGenError(`Image model "${wanted}" is not configured in any channel. Choose one of: ${available}.`, 'image-model-not-configured')
|
|
193
|
+
}
|
|
194
|
+
const preferred = hosting.find(entry => entry.channel.id === config.defaultChannelId)
|
|
195
|
+
return preferred ?? hosting[0]!
|
|
196
|
+
}
|
|
197
|
+
const materializeTaskImages = (task: GenerationTask): Promise<AgentImageRef[]> => {
|
|
198
|
+
if (task.status !== 'completed') return Promise.resolve([])
|
|
199
|
+
const existing = attachmentRefs.get(task.id)
|
|
200
|
+
if (existing !== undefined) return existing
|
|
201
|
+
const pending = ctx.attachments.saveImages((task.result?.images ?? []).map((image, index) => toSaveImage(image, task.id, index)))
|
|
202
|
+
.then(refs => refs.map(projectRef))
|
|
203
|
+
attachmentRefs.set(task.id, pending)
|
|
204
|
+
void pending.catch(() => {
|
|
205
|
+
if (attachmentRefs.get(task.id) === pending) attachmentRefs.delete(task.id)
|
|
206
|
+
})
|
|
207
|
+
return pending
|
|
208
|
+
}
|
|
209
|
+
const taskResult = async (task: GenerationTask): Promise<AgentTaskResult> => {
|
|
210
|
+
const images = await materializeTaskImages(task)
|
|
211
|
+
return {
|
|
212
|
+
task_id: task.id,
|
|
213
|
+
status: task.status,
|
|
214
|
+
message: task.status === 'completed'
|
|
215
|
+
? 'Generation completed. The images are shown beside this tool call and can be reused as source_image in edit_image.'
|
|
216
|
+
: task.status === 'failed'
|
|
217
|
+
? 'Generation failed.'
|
|
218
|
+
: task.status === 'cancelled'
|
|
219
|
+
? 'Generation was cancelled.'
|
|
220
|
+
: 'Generation is still running. Query the task again when you need its current status.',
|
|
221
|
+
...task.error === undefined ? {} : { error: task.error },
|
|
222
|
+
images,
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
const findTask = (id: string): GenerationTask => {
|
|
226
|
+
const task = runtime.queue.list().find(candidate => candidate.id === id)
|
|
227
|
+
if (task === undefined) throw new ImageGenError(`Image generation task ${id} was not found.`, 'task-not-found')
|
|
228
|
+
return task
|
|
229
|
+
}
|
|
230
|
+
const waitForTask = (id: string, signal: AbortSignal | undefined): Promise<GenerationTask> => new Promise((resolveTask, rejectTask) => {
|
|
231
|
+
let settled = false
|
|
232
|
+
let dispose = (): void => {}
|
|
233
|
+
let timer: ReturnType<typeof setTimeout> | undefined
|
|
234
|
+
let abort = (): void => {}
|
|
235
|
+
|
|
236
|
+
const cleanup = (): void => {
|
|
237
|
+
dispose()
|
|
238
|
+
if (timer !== undefined) clearTimeout(timer)
|
|
239
|
+
signal?.removeEventListener('abort', abort)
|
|
240
|
+
}
|
|
241
|
+
const resolve = (task: GenerationTask): void => {
|
|
242
|
+
if (settled) return
|
|
243
|
+
settled = true
|
|
244
|
+
cleanup()
|
|
245
|
+
resolveTask(task)
|
|
246
|
+
}
|
|
247
|
+
const reject = (error: unknown): void => {
|
|
248
|
+
if (settled) return
|
|
249
|
+
settled = true
|
|
250
|
+
cleanup()
|
|
251
|
+
rejectTask(error)
|
|
252
|
+
}
|
|
253
|
+
abort = (): void => {
|
|
254
|
+
if (settled) return
|
|
255
|
+
const reason = signal?.reason instanceof Error ? signal.reason : new Error('Image generation was cancelled.')
|
|
256
|
+
// Remove the listener before publishing cancellation so the queue event
|
|
257
|
+
// cannot turn an execution abort into a successful cancelled result.
|
|
258
|
+
settled = true
|
|
259
|
+
cleanup()
|
|
260
|
+
runtime.queue.cancel(id)
|
|
261
|
+
rejectTask(reason)
|
|
262
|
+
}
|
|
263
|
+
const onChange = (updated: GenerationTask): void => {
|
|
264
|
+
if (updated.id === id && isFinalTask(updated)) resolve(updated)
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
if (signal?.aborted === true) {
|
|
268
|
+
abort()
|
|
269
|
+
return
|
|
270
|
+
}
|
|
271
|
+
dispose = runtime.queue.subscribe(onChange)
|
|
272
|
+
signal?.addEventListener('abort', abort, { once: true })
|
|
273
|
+
timer = setTimeout(() => {
|
|
274
|
+
if (settled) return
|
|
275
|
+
const timeout = new ImageGenError(`Image generation task ${id} timed out after ${AGENT_GENERATION_TIMEOUT_MS / 1000} seconds.`, 'generation-timeout')
|
|
276
|
+
settled = true
|
|
277
|
+
cleanup()
|
|
278
|
+
runtime.queue.cancel(id)
|
|
279
|
+
rejectTask(timeout)
|
|
280
|
+
}, AGENT_GENERATION_TIMEOUT_MS)
|
|
281
|
+
let current: GenerationTask
|
|
282
|
+
try {
|
|
283
|
+
current = findTask(id)
|
|
284
|
+
} catch (error) {
|
|
285
|
+
reject(error)
|
|
286
|
+
return
|
|
287
|
+
}
|
|
288
|
+
if (isFinalTask(current)) resolve(current)
|
|
289
|
+
})
|
|
290
|
+
const disposers = [
|
|
291
|
+
ctx.tools.register(defineTool({
|
|
292
|
+
name: 'generate_image',
|
|
293
|
+
description: 'Generate an image. By default this tool call stays pending until the task reaches a final state; completed images are shown beside this tool call, while the model receives their attachment references, without creating a user message. Set wait_for_completion to false for background mode, then use get_image_generation_task explicitly. Only use models configured for this plugin; omit model to use the first configured image model.',
|
|
294
|
+
parameters: {
|
|
295
|
+
prompt: { type: 'string', required: true, description: 'Detailed image-generation prompt.' },
|
|
296
|
+
model: { type: 'string', description: 'One of the configured image models. Defaults to the first configured model.' },
|
|
297
|
+
size: { type: 'string', description: 'Aspect ratio such as 1:1, 16:9, 9:16, or auto.' },
|
|
298
|
+
quality: { type: 'string', description: 'auto, 1k, 2k, or 4k.' },
|
|
299
|
+
count: { type: 'integer', description: 'Number of images, 1 to 4. Defaults to 1.' },
|
|
300
|
+
detail: { type: 'string', description: 'Optional provider detail value, for example standard or high.' },
|
|
301
|
+
wait_for_completion: { type: 'boolean', description: 'Wait for images and return them in this tool result. Defaults to true; set false for background mode.' },
|
|
302
|
+
},
|
|
303
|
+
output: {
|
|
304
|
+
schema: taskResultSchema,
|
|
305
|
+
render: (_args, value) => renderTaskResult(value),
|
|
306
|
+
presentationMeta: (_args, value) => imagePresentationMeta(value),
|
|
307
|
+
},
|
|
308
|
+
presentResult: presentImageResult,
|
|
309
|
+
async execute(args, exec) {
|
|
310
|
+
ensureConfigured()
|
|
311
|
+
const picked = resolveModel(args.model)
|
|
312
|
+
const task = runtime.queue.submit({
|
|
313
|
+
mode: 'text',
|
|
314
|
+
model: picked.alias,
|
|
315
|
+
upstream: picked.upstream,
|
|
316
|
+
channelId: picked.channel.id,
|
|
317
|
+
channel: picked.channel.name,
|
|
318
|
+
prompt: args.prompt.trim(),
|
|
319
|
+
size: args.size ?? 'auto',
|
|
320
|
+
quality: args.quality ?? 'auto',
|
|
321
|
+
n: Math.min(4, Math.max(1, args.count ?? 1)),
|
|
322
|
+
detail: args.detail ?? '',
|
|
323
|
+
})
|
|
324
|
+
return taskResult(args.wait_for_completion === false ? task : await waitForTask(task.id, exec.signal))
|
|
325
|
+
},
|
|
326
|
+
})),
|
|
327
|
+
ctx.tools.register(defineTool({
|
|
328
|
+
name: 'edit_image',
|
|
329
|
+
description: 'Edit an image. By default this tool call stays pending until the task reaches a final state; completed images are shown beside this tool call, while the model receives their attachment references, without creating a user message. Set wait_for_completion to false for background mode, then use get_image_generation_task explicitly. source_image must be an image reference returned by a completed generation or get_image_generation_task; pass that entire object unchanged. Only configured image models are allowed; omit model to use the first configured model.',
|
|
330
|
+
parameters: {
|
|
331
|
+
prompt: { type: 'string', required: true, description: 'How to transform the source image.' },
|
|
332
|
+
source_image: { ...imageRefSchema, required: true, description: 'Image reference returned by get_image_generation_task.' },
|
|
333
|
+
model: { type: 'string', description: 'One of the configured image models. Defaults to the first configured model.' },
|
|
334
|
+
size: { type: 'string', description: 'Aspect ratio such as 1:1, 16:9, 9:16, or auto.' },
|
|
335
|
+
quality: { type: 'string', description: 'auto, 1k, 2k, or 4k.' },
|
|
336
|
+
count: { type: 'integer', description: 'Number of images, 1 to 4. Defaults to 1.' },
|
|
337
|
+
detail: { type: 'string', description: 'Optional provider detail value.' },
|
|
338
|
+
wait_for_completion: { type: 'boolean', description: 'Wait for images and return them in this tool result. Defaults to true; set false for background mode.' },
|
|
339
|
+
},
|
|
340
|
+
output: {
|
|
341
|
+
schema: taskResultSchema,
|
|
342
|
+
render: (_args, value) => renderTaskResult(value),
|
|
343
|
+
presentationMeta: (_args, value) => imagePresentationMeta(value),
|
|
344
|
+
},
|
|
345
|
+
presentResult: presentImageResult,
|
|
346
|
+
async execute(args, exec) {
|
|
347
|
+
ensureConfigured()
|
|
348
|
+
const reference = await ctx.attachments.readImage(restoreRef(args.source_image), exec.signal)
|
|
349
|
+
const picked = resolveModel(args.model)
|
|
350
|
+
const task = runtime.queue.submit({
|
|
351
|
+
mode: 'edit',
|
|
352
|
+
model: picked.alias,
|
|
353
|
+
upstream: picked.upstream,
|
|
354
|
+
channelId: picked.channel.id,
|
|
355
|
+
channel: picked.channel.name,
|
|
356
|
+
prompt: args.prompt.trim(),
|
|
357
|
+
size: args.size ?? 'auto',
|
|
358
|
+
quality: args.quality ?? 'auto',
|
|
359
|
+
n: Math.min(4, Math.max(1, args.count ?? 1)),
|
|
360
|
+
detail: args.detail ?? '',
|
|
361
|
+
image: imageDataUrl(reference),
|
|
362
|
+
...reference.ref.name === undefined ? {} : { refName: reference.ref.name },
|
|
363
|
+
})
|
|
364
|
+
return taskResult(args.wait_for_completion === false ? task : await waitForTask(task.id, exec.signal))
|
|
365
|
+
},
|
|
366
|
+
})),
|
|
367
|
+
ctx.tools.register(defineTool({
|
|
368
|
+
name: 'get_image_generation_task',
|
|
369
|
+
description: 'Check an image-generation task status. Completed tasks return image references; their images are shown beside this tool call and the references can be passed to edit_image. Generation tools normally wait for completion, so use this for explicit recovery or status checks.',
|
|
370
|
+
parameters: { task_id: { type: 'string', required: true, description: 'Task id returned by generate_image or edit_image.' } },
|
|
371
|
+
output: {
|
|
372
|
+
schema: taskResultSchema,
|
|
373
|
+
render: (_args, value) => renderTaskResult(value),
|
|
374
|
+
presentationMeta: (_args, value) => imagePresentationMeta(value),
|
|
375
|
+
},
|
|
376
|
+
presentResult: presentImageResult,
|
|
377
|
+
async execute(args) {
|
|
378
|
+
ensureConfigured()
|
|
379
|
+
return taskResult(findTask(args.task_id))
|
|
380
|
+
},
|
|
381
|
+
})),
|
|
382
|
+
ctx.tools.register(defineTool({
|
|
383
|
+
name: 'cancel_image_generation_task',
|
|
384
|
+
description: 'Cancel a queued or running image generation task.',
|
|
385
|
+
parameters: { task_id: { type: 'string', required: true, description: 'Task id returned by generate_image or edit_image.' } },
|
|
386
|
+
output: {
|
|
387
|
+
schema: taskResultSchema,
|
|
388
|
+
render: (_args, value) => renderTaskResult(value),
|
|
389
|
+
presentationMeta: (_args, value) => imagePresentationMeta(value),
|
|
390
|
+
},
|
|
391
|
+
presentResult: presentImageResult,
|
|
392
|
+
async execute(args) {
|
|
393
|
+
ensureConfigured()
|
|
394
|
+
const task = runtime.queue.cancel(args.task_id)
|
|
395
|
+
if (task === undefined) throw new ImageGenError(`Image generation task ${args.task_id} was not found.`, 'task-not-found')
|
|
396
|
+
return taskResult(task)
|
|
397
|
+
},
|
|
398
|
+
})),
|
|
399
|
+
]
|
|
400
|
+
return () => {
|
|
401
|
+
for (const dispose of disposers) dispose()
|
|
402
|
+
}
|
|
403
|
+
}
|
|
404
|
+
|
|
405
|
+
function isFinalTask(task: GenerationTask): boolean {
|
|
406
|
+
return task.status === 'completed' || task.status === 'failed' || task.status === 'cancelled'
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
function toSaveImage(image: GeneratedImage, taskId: string, index: number): { data: Uint8Array; mediaType: ImageMediaType; name: string } {
|
|
410
|
+
const data = Buffer.from(image.b64, 'base64')
|
|
411
|
+
const declaredMediaType = acceptedMediaType(image.mime) ? image.mime : 'image/png'
|
|
412
|
+
const mediaType = detectImageMime(data) ?? declaredMediaType
|
|
413
|
+
return {
|
|
414
|
+
data,
|
|
415
|
+
mediaType,
|
|
416
|
+
name: `imagegen-${taskId}-${index + 1}.${mediaType === 'image/jpeg' ? 'jpg' : mediaType.slice('image/'.length)}`,
|
|
417
|
+
}
|
|
418
|
+
}
|