@wxip/dsh-sub2api 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +158 -0
- package/README.zh.md +160 -0
- package/assets/icons/claude.svg +1 -0
- package/assets/icons/grok.svg +1 -0
- package/assets/icons/openai.svg +1 -0
- package/cordis.patch.yml +11 -0
- package/lib/client.cjs +1098 -0
- package/lib/client.js +1107 -0
- package/lib/image-tools.d.ts +18 -0
- package/lib/index.d.ts +148 -0
- package/lib/index.js +1220 -0
- package/lib/invariant.d.ts +13 -0
- package/lib/invariant.js +9 -0
- package/lib/pi-ai-patch.d.ts +45 -0
- package/lib/pi-ai.d.ts +50 -0
- package/lib/routes.d.ts +43 -0
- package/package.json +160 -0
- package/src/client/icons.tsx +38 -0
- package/src/client/index.tsx +31 -0
- package/src/client/settings.tsx +847 -0
- package/src/client/toolview.tsx +76 -0
- package/src/image-tools.ts +590 -0
- package/src/index.ts +339 -0
- package/src/invariant.ts +17 -0
- package/src/pi-ai-patch.ts +94 -0
- package/src/pi-ai.ts +185 -0
- package/src/routes.ts +442 -0
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Inline generated-image preview for the `generate_image` tool card.
|
|
3
|
+
*
|
|
4
|
+
* generate_image embeds an image content block (a durable attachment
|
|
5
|
+
* reference) in its tool result; the stock tool card only renders text and
|
|
6
|
+
* JSON blocks, so this keyed `tool.call.toolview` renders the attachment
|
|
7
|
+
* bytes as an inline `<img>` right inside the chat record. The image is
|
|
8
|
+
* served by the plugin's own `GET /plugins/dsh-sub2api/attachment` route
|
|
9
|
+
* (the request carries the full attachment ref; the store re-verifies the
|
|
10
|
+
* content digest, so only genuine generated images resolve).
|
|
11
|
+
*
|
|
12
|
+
* The tool UI owns the slot contract; import its public props so upstream
|
|
13
|
+
* changes are checked at build time.
|
|
14
|
+
*
|
|
15
|
+
* @module dsh-sub2api/client/toolview
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
import type { CSSProperties } from 'react'
|
|
19
|
+
import type { ToolCallViewProps } from '@deepseek-ai/dsh-client-ui-tool/client'
|
|
20
|
+
import type { ImageBlock } from '@deepseek-ai/dsh-llm'
|
|
21
|
+
|
|
22
|
+
type GenerateImageToolviewProps = ToolCallViewProps
|
|
23
|
+
|
|
24
|
+
const ROOT: CSSProperties = {
|
|
25
|
+
display: 'grid',
|
|
26
|
+
gap: '8px',
|
|
27
|
+
padding: '8px 10px',
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
const IMAGE: CSSProperties = {
|
|
31
|
+
maxWidth: '100%',
|
|
32
|
+
maxHeight: 420,
|
|
33
|
+
objectFit: 'contain',
|
|
34
|
+
borderRadius: 8,
|
|
35
|
+
background: 'var(--dsw-alias-bg-module-platform, #f2f2f2)',
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
const META: CSSProperties = {
|
|
39
|
+
fontSize: 11,
|
|
40
|
+
lineHeight: 1.5,
|
|
41
|
+
color: 'var(--dsw-alias-label-secondary, #6b6b6b)',
|
|
42
|
+
whiteSpace: 'pre-wrap',
|
|
43
|
+
wordBreak: 'break-word',
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/** Attachment-served URL for one image content block. */
|
|
47
|
+
function attachmentUrl(image: ImageBlock): string {
|
|
48
|
+
const ref = JSON.stringify(image.attachment)
|
|
49
|
+
return `/plugins/dsh-sub2api/attachment?ref=${encodeURIComponent(btoa(ref))}`
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
export function GenerateImageToolview(props: GenerateImageToolviewProps): JSX.Element {
|
|
53
|
+
const { block } = props
|
|
54
|
+
const content = 'content' in block ? block.content : []
|
|
55
|
+
const image = content.find((b) => b.type === 'image' && b.attachment !== undefined) as ImageBlock | undefined
|
|
56
|
+
const text = content.find((b) => b.type === 'text')
|
|
57
|
+
|
|
58
|
+
if (image === undefined && text === undefined) {
|
|
59
|
+
return (
|
|
60
|
+
<div style={{ ...META, padding: '8px 10px' }}>生成图片…</div>
|
|
61
|
+
)
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
return (
|
|
65
|
+
<div style={ROOT}>
|
|
66
|
+
{image !== undefined && (
|
|
67
|
+
<img
|
|
68
|
+
src={attachmentUrl(image)}
|
|
69
|
+
alt={image.attachment.name ?? 'generated image'}
|
|
70
|
+
style={IMAGE}
|
|
71
|
+
/>
|
|
72
|
+
)}
|
|
73
|
+
{text !== undefined && <div style={META}>{text.text}</div>}
|
|
74
|
+
</div>
|
|
75
|
+
)
|
|
76
|
+
}
|
|
@@ -0,0 +1,590 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Global image-generation tool.
|
|
3
|
+
*
|
|
4
|
+
* These call a configured Sub2API model independently of the current chat
|
|
5
|
+
* route, so a text-only session can still create images. Results include a workspace path and an image attachment.
|
|
6
|
+
*
|
|
7
|
+
* @module dsh-sub2api/image-tools
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import { mkdir, writeFile } from 'node:fs/promises'
|
|
11
|
+
import { basename, dirname, extname } from 'node:path'
|
|
12
|
+
import type { Context } from '@deepseek-ai/cordis'
|
|
13
|
+
import { defineTool } from '@deepseek-ai/dsh-tools'
|
|
14
|
+
import { attributionHeaders, LlmError } from '@deepseek-ai/dsh-llm'
|
|
15
|
+
import type { ContentBlock } from '@deepseek-ai/dsh-llm'
|
|
16
|
+
import type { ImageAttachmentRef } from '@deepseek-ai/dsh-attachment'
|
|
17
|
+
import type {} from '@deepseek-ai/dsh-tools'
|
|
18
|
+
import type {} from '@deepseek-ai/dsh-system-prompt'
|
|
19
|
+
import {
|
|
20
|
+
DEFAULT_MAX_TOKENS,
|
|
21
|
+
apiProtocolForKey,
|
|
22
|
+
gatewayAnthropicRoot,
|
|
23
|
+
gatewayApiRoot,
|
|
24
|
+
type ApiProtocol,
|
|
25
|
+
type Config,
|
|
26
|
+
type ProviderKey,
|
|
27
|
+
type ProviderProfile,
|
|
28
|
+
} from './index.ts'
|
|
29
|
+
|
|
30
|
+
interface ImageFsTarget {
|
|
31
|
+
displayPath: string
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
interface ImageFsInfo {
|
|
35
|
+
type: string
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
interface ImageFs {
|
|
39
|
+
resolve(path: string, opts?: { cwd?: string; signal?: AbortSignal }): Promise<ImageFsTarget>
|
|
40
|
+
stat(target: ImageFsTarget, signal?: AbortSignal): Promise<ImageFsInfo | undefined>
|
|
41
|
+
readBytes(target: ImageFsTarget, signal: AbortSignal | undefined, maxBytes: number): Promise<Uint8Array>
|
|
42
|
+
contains(parent: ImageFsTarget, child: ImageFsTarget): boolean
|
|
43
|
+
processPath(target: ImageFsTarget): string
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
function getFs(ctx: Context): ImageFs | undefined {
|
|
47
|
+
return (ctx as Context & { get(name: 'fs'): ImageFs | undefined }).get('fs')
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
export const GENERATE_IMAGE_NAME = 'generate_image'
|
|
51
|
+
export const DEFAULT_IMAGE_TOOL_TIMEOUT_MS = 180_000
|
|
52
|
+
export const DEFAULT_MAX_IMAGE_BYTES: number = 20 * 1024 * 1024
|
|
53
|
+
|
|
54
|
+
const IMAGE_EXTENSIONS: Record<string, ImageMediaType> = {
|
|
55
|
+
'.png': 'image/png',
|
|
56
|
+
'.jpg': 'image/jpeg',
|
|
57
|
+
'.jpeg': 'image/jpeg',
|
|
58
|
+
'.webp': 'image/webp',
|
|
59
|
+
'.gif': 'image/gif',
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
type ImageMediaType = 'image/png' | 'image/jpeg' | 'image/webp' | 'image/gif'
|
|
63
|
+
|
|
64
|
+
const GENERATE_SIZES: readonly string[] = [
|
|
65
|
+
'auto',
|
|
66
|
+
'256x256',
|
|
67
|
+
'512x512',
|
|
68
|
+
'1024x1024',
|
|
69
|
+
'1024x1536',
|
|
70
|
+
'1536x1024',
|
|
71
|
+
'1024x1792',
|
|
72
|
+
'1792x1024',
|
|
73
|
+
] as const
|
|
74
|
+
|
|
75
|
+
const GENERATE_QUALITIES: readonly string[] = ['auto', 'low', 'medium', 'high', 'standard', 'hd'] as const
|
|
76
|
+
|
|
77
|
+
export interface ImageToolHost {
|
|
78
|
+
config: () => Config
|
|
79
|
+
resolveApiKey: (route: string, profile: ProviderProfile) => Promise<string>
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
const PROVIDER_LABELS: Record<ProviderKey, string> = {
|
|
83
|
+
openai: 'OpenAI',
|
|
84
|
+
claude: 'Claude',
|
|
85
|
+
grok: 'Grok',
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
interface ResolvedToolModel {
|
|
89
|
+
route: string
|
|
90
|
+
label: string
|
|
91
|
+
profile: ProviderProfile
|
|
92
|
+
model: string
|
|
93
|
+
baseURL: string
|
|
94
|
+
/** Wire protocol the gateway speaks for this provider group (see DEFAULT_PROTOCOL). */
|
|
95
|
+
api: ApiProtocol
|
|
96
|
+
/** Output cap for the model, used where the wire protocol requires one. */
|
|
97
|
+
maxTokens: number
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
function isProviderKey(value: string): value is ProviderKey {
|
|
101
|
+
return value === 'openai' || value === 'claude' || value === 'grok'
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
function mediaTypeForPath(filePath: string): ImageMediaType | undefined {
|
|
105
|
+
return IMAGE_EXTENSIONS[extname(filePath).toLowerCase()]
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
function mediaTypeFromBytes(data: Uint8Array): ImageMediaType {
|
|
109
|
+
if (data.length >= 8 && data[0] === 0x89 && data[1] === 0x50 && data[2] === 0x4e && data[3] === 0x47) return 'image/png'
|
|
110
|
+
if (data.length >= 3 && data[0] === 0xff && data[1] === 0xd8 && data[2] === 0xff) return 'image/jpeg'
|
|
111
|
+
if (data.length >= 6 && data[0] === 0x47 && data[1] === 0x49 && data[2] === 0x46) return 'image/gif'
|
|
112
|
+
if (
|
|
113
|
+
data.length >= 12
|
|
114
|
+
&& data[0] === 0x52 && data[1] === 0x49 && data[2] === 0x46 && data[3] === 0x46
|
|
115
|
+
&& data[8] === 0x57 && data[9] === 0x45 && data[10] === 0x42 && data[11] === 0x50
|
|
116
|
+
) return 'image/webp'
|
|
117
|
+
return 'image/png'
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
function extensionForMediaType(mediaType: ImageMediaType): string {
|
|
121
|
+
switch (mediaType) {
|
|
122
|
+
case 'image/jpeg': return '.jpg'
|
|
123
|
+
case 'image/webp': return '.webp'
|
|
124
|
+
case 'image/gif': return '.gif'
|
|
125
|
+
default: return '.png'
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
function decodeDataUrl(value: string): { mediaType: ImageMediaType; data: Uint8Array } | undefined {
|
|
130
|
+
const match = /^data:(image\/(?:png|jpeg|jpg|webp|gif));base64,([A-Za-z0-9+/=\s]+)$/i.exec(value.trim())
|
|
131
|
+
if (match === null) return undefined
|
|
132
|
+
const declared = match[1]!.toLowerCase() === 'image/jpg' ? 'image/jpeg' : match[1]!.toLowerCase()
|
|
133
|
+
const mediaType = declared as ImageMediaType
|
|
134
|
+
return { mediaType, data: Buffer.from(match[2]!, 'base64') }
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
function sessionCwd(exec: { agent?: { session: { header: { cwd?: string } } } }): string | undefined {
|
|
138
|
+
return exec.agent?.session.header.cwd
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
function resolveToolModel(config: Config, kind: 'generate'): ResolvedToolModel {
|
|
142
|
+
const ref = config.tools?.[kind]
|
|
143
|
+
const label = '生图'
|
|
144
|
+
const provider = typeof ref?.provider === 'string' ? ref.provider.trim() : ''
|
|
145
|
+
const model = typeof ref?.model === 'string' ? ref.model.trim() : ''
|
|
146
|
+
if (provider.length === 0 || model.length === 0) {
|
|
147
|
+
throw new Error(`sub2api: 未配置${label}模型。打开设置 → Sub2API 模型,为「全局图像工具」指定一个模型后再试`)
|
|
148
|
+
}
|
|
149
|
+
if (!isProviderKey(provider)) {
|
|
150
|
+
throw new Error(`sub2api: ${label}模型的平台 "${provider}" 无效,应为 openai / claude / grok`)
|
|
151
|
+
}
|
|
152
|
+
const baseURL = config.baseURL.trim().replace(/\/+$/, '')
|
|
153
|
+
if (baseURL.length === 0) throw new Error('sub2api: baseURL is not configured')
|
|
154
|
+
const profile = config.providers[provider]
|
|
155
|
+
if (profile.apiKeyEnv === undefined) {
|
|
156
|
+
throw new Error(`sub2api: ${PROVIDER_LABELS[provider]} 未配置 API key,无法调用${label}模型`)
|
|
157
|
+
}
|
|
158
|
+
const catalogModel = profile.models?.find((entry) => entry.id === model)
|
|
159
|
+
// The settings store the bare gateway host; the wire endpoints live under the
|
|
160
|
+
// `/v1` root for OpenAI-style protocols, while the Anthropic SDK appends
|
|
161
|
+
// `/v1/messages` itself and needs the bare host.
|
|
162
|
+
const api = apiProtocolForKey(provider, profile)
|
|
163
|
+
const endpointRoot = api === 'anthropic-messages' ? gatewayAnthropicRoot(baseURL) : gatewayApiRoot(baseURL)
|
|
164
|
+
return {
|
|
165
|
+
route: `sub2api-${provider}`,
|
|
166
|
+
label: PROVIDER_LABELS[provider],
|
|
167
|
+
profile,
|
|
168
|
+
model,
|
|
169
|
+
baseURL: endpointRoot,
|
|
170
|
+
api,
|
|
171
|
+
maxTokens: catalogModel?.maxTokens ?? DEFAULT_MAX_TOKENS,
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
async function readErrorDetail(response: Response): Promise<string> {
|
|
176
|
+
try {
|
|
177
|
+
const parsed = await response.json() as { error?: { message?: unknown; code?: unknown; type?: unknown } }
|
|
178
|
+
const message = parsed.error?.message
|
|
179
|
+
if (typeof message === 'string' && message.length > 0) return message
|
|
180
|
+
} catch {
|
|
181
|
+
// keep the status fallback
|
|
182
|
+
}
|
|
183
|
+
return `HTTP ${response.status}`
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
async function gatewayFetch(
|
|
187
|
+
host: ImageToolHost,
|
|
188
|
+
resolved: ResolvedToolModel,
|
|
189
|
+
path: string,
|
|
190
|
+
body: Record<string, unknown>,
|
|
191
|
+
signal: AbortSignal | undefined,
|
|
192
|
+
accept: string,
|
|
193
|
+
): Promise<Response> {
|
|
194
|
+
const apiKey = await host.resolveApiKey(resolved.route, resolved.profile)
|
|
195
|
+
let response: Response
|
|
196
|
+
try {
|
|
197
|
+
response = await fetch(`${resolved.baseURL}${path}`, {
|
|
198
|
+
method: 'POST',
|
|
199
|
+
headers: {
|
|
200
|
+
authorization: `Bearer ${apiKey}`,
|
|
201
|
+
'content-type': 'application/json',
|
|
202
|
+
accept,
|
|
203
|
+
...attributionHeaders(),
|
|
204
|
+
},
|
|
205
|
+
body: JSON.stringify(body),
|
|
206
|
+
signal,
|
|
207
|
+
})
|
|
208
|
+
} catch (error) {
|
|
209
|
+
if (signal?.aborted) throw new LlmError('sub2api: request aborted', 'ABORTED', { cause: error })
|
|
210
|
+
throw new LlmError(`sub2api: API request to ${resolved.baseURL}${path} failed`, 'TRANSPORT', { cause: error })
|
|
211
|
+
}
|
|
212
|
+
if (!response.ok) {
|
|
213
|
+
throw new Error(`sub2api ${path}: ${await readErrorDetail(response)}`)
|
|
214
|
+
}
|
|
215
|
+
return response
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
async function collectSseText(response: Response): Promise<string> {
|
|
219
|
+
if (response.body === null) throw new Error('sub2api: API returned no response body')
|
|
220
|
+
const reader = response.body.getReader()
|
|
221
|
+
const decoder = new TextDecoder()
|
|
222
|
+
let buffer = ''
|
|
223
|
+
let text = ''
|
|
224
|
+
try {
|
|
225
|
+
while (true) {
|
|
226
|
+
const { done, value } = await reader.read()
|
|
227
|
+
if (done) break
|
|
228
|
+
buffer += decoder.decode(value, { stream: true })
|
|
229
|
+
while (true) {
|
|
230
|
+
const idx = buffer.indexOf('\n\n')
|
|
231
|
+
if (idx === -1) break
|
|
232
|
+
const raw = buffer.slice(0, idx)
|
|
233
|
+
buffer = buffer.slice(idx + 2)
|
|
234
|
+
const dataLines = raw.split('\n').filter((line) => line.startsWith('data:')).map((line) => line.slice(5).trimStart())
|
|
235
|
+
if (dataLines.length === 0) continue
|
|
236
|
+
const joined = dataLines.join('\n')
|
|
237
|
+
if (joined === '[DONE]') return text
|
|
238
|
+
for (const line of dataLines) {
|
|
239
|
+
if (line === '[DONE]') return text
|
|
240
|
+
let chunk: { choices?: Array<{ delta?: { content?: unknown }; message?: { content?: unknown } }> }
|
|
241
|
+
try {
|
|
242
|
+
chunk = JSON.parse(line) as typeof chunk
|
|
243
|
+
} catch {
|
|
244
|
+
continue
|
|
245
|
+
}
|
|
246
|
+
for (const choice of chunk.choices ?? []) {
|
|
247
|
+
const delta = choice.delta?.content ?? choice.message?.content
|
|
248
|
+
if (typeof delta === 'string') text += delta
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
} finally {
|
|
254
|
+
try {
|
|
255
|
+
reader.releaseLock()
|
|
256
|
+
} catch {
|
|
257
|
+
// lock already released
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
return text
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
function flattenMessageContent(content: unknown): string {
|
|
264
|
+
if (typeof content === 'string') return content
|
|
265
|
+
if (!Array.isArray(content)) return ''
|
|
266
|
+
return content
|
|
267
|
+
.map((part) => {
|
|
268
|
+
if (typeof part === 'string') return part
|
|
269
|
+
if (typeof part !== 'object' || part === null) return ''
|
|
270
|
+
const record = part as Record<string, unknown>
|
|
271
|
+
if (typeof record.text === 'string') return record.text
|
|
272
|
+
return ''
|
|
273
|
+
})
|
|
274
|
+
.join('')
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
async function collectChatText(response: Response): Promise<string> {
|
|
278
|
+
const contentType = response.headers.get('content-type') ?? ''
|
|
279
|
+
if (contentType.includes('text/event-stream')) {
|
|
280
|
+
const text = (await collectSseText(response)).trim()
|
|
281
|
+
if (text.length === 0) throw new Error('sub2api: image model returned no text')
|
|
282
|
+
return text
|
|
283
|
+
}
|
|
284
|
+
const payload = await response.json() as { choices?: Array<{ message?: { content?: unknown } }> }
|
|
285
|
+
const text = flattenMessageContent(payload.choices?.[0]?.message?.content).trim()
|
|
286
|
+
if (text.length === 0) throw new Error('sub2api: image model returned no text')
|
|
287
|
+
return text
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
async function loadRemoteImage(url: string, signal: AbortSignal | undefined, maxBytes: number): Promise<{ mediaType: ImageMediaType; data: Uint8Array }> {
|
|
291
|
+
const encoded = decodeDataUrl(url)
|
|
292
|
+
if (encoded !== undefined) return encoded
|
|
293
|
+
let response: Response
|
|
294
|
+
try {
|
|
295
|
+
response = await fetch(url, { method: 'GET', signal, redirect: 'follow' })
|
|
296
|
+
} catch (error) {
|
|
297
|
+
if (signal?.aborted) throw new LlmError('sub2api: request aborted', 'ABORTED', { cause: error })
|
|
298
|
+
throw new Error(`sub2api: failed to download image URL: ${error instanceof Error ? error.message : String(error)}`)
|
|
299
|
+
}
|
|
300
|
+
if (!response.ok) throw new Error(`sub2api: image URL returned HTTP ${response.status}`)
|
|
301
|
+
const buffer = new Uint8Array(await response.arrayBuffer())
|
|
302
|
+
if (buffer.byteLength > maxBytes) throw new Error(`sub2api: image URL exceeds ${maxBytes} bytes`)
|
|
303
|
+
const headerType = (response.headers.get('content-type') ?? '').split(';')[0]?.trim().toLowerCase()
|
|
304
|
+
const mediaType = headerType === 'image/png' || headerType === 'image/jpeg' || headerType === 'image/webp' || headerType === 'image/gif'
|
|
305
|
+
? headerType
|
|
306
|
+
: mediaTypeFromBytes(buffer)
|
|
307
|
+
return { mediaType, data: buffer }
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
function extractGeneratedImage(payload: unknown): { data?: Uint8Array; mediaType?: ImageMediaType; url?: string; revisedPrompt?: string } | undefined {
|
|
311
|
+
if (typeof payload !== 'object' || payload === null) return undefined
|
|
312
|
+
const record = payload as Record<string, unknown>
|
|
313
|
+
const items = Array.isArray(record.data) ? record.data : []
|
|
314
|
+
for (const item of items) {
|
|
315
|
+
if (typeof item !== 'object' || item === null) continue
|
|
316
|
+
const row = item as Record<string, unknown>
|
|
317
|
+
const revised = typeof row.revised_prompt === 'string' ? row.revised_prompt : undefined
|
|
318
|
+
if (typeof row.b64_json === 'string' && row.b64_json.length > 0) {
|
|
319
|
+
const data = Buffer.from(row.b64_json, 'base64')
|
|
320
|
+
return { data, mediaType: mediaTypeFromBytes(data), ...(revised !== undefined ? { revisedPrompt: revised } : {}) }
|
|
321
|
+
}
|
|
322
|
+
if (typeof row.url === 'string' && row.url.length > 0) {
|
|
323
|
+
if (row.url.startsWith('data:')) {
|
|
324
|
+
const decoded = decodeDataUrl(row.url)
|
|
325
|
+
if (decoded !== undefined) return { ...decoded, ...(revised !== undefined ? { revisedPrompt: revised } : {}) }
|
|
326
|
+
}
|
|
327
|
+
return { url: row.url, ...(revised !== undefined ? { revisedPrompt: revised } : {}) }
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
return undefined
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
function extractImageFromText(text: string): { kind: 'data'; mediaType: ImageMediaType; data: Uint8Array } | { kind: 'url'; url: string } | undefined {
|
|
334
|
+
const dataMatch = /data:image\/(?:png|jpeg|jpg|webp|gif);base64,[A-Za-z0-9+/=\s]+/i.exec(text)
|
|
335
|
+
if (dataMatch !== null) {
|
|
336
|
+
const decoded = decodeDataUrl(dataMatch[0]!)
|
|
337
|
+
if (decoded !== undefined) return { kind: 'data', ...decoded }
|
|
338
|
+
}
|
|
339
|
+
const urlMatch = /https?:\/\/\S+\.(?:png|jpe?g|webp|gif)(?:\?\S*)?/i.exec(text)
|
|
340
|
+
if (urlMatch !== null) return { kind: 'url', url: urlMatch[0]!.replace(/[),.;]+$/, '') }
|
|
341
|
+
const markdown = /!\[[^\]]*]\((https?:\/\/[^)\s]+)\)/i.exec(text)
|
|
342
|
+
if (markdown !== null) return { kind: 'url', url: markdown[1]! }
|
|
343
|
+
return undefined
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
function defaultOutputName(mediaType: ImageMediaType): string {
|
|
347
|
+
const stamp = new Date().toISOString().replace(/[:.]/g, '-').replace('T', '-').slice(0, 19)
|
|
348
|
+
return `generated-${stamp}${extensionForMediaType(mediaType)}`
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
async function writeGeneratedFile(
|
|
352
|
+
ctx: Context,
|
|
353
|
+
exec: { signal: AbortSignal; agent?: { session: { header: { cwd?: string } } } },
|
|
354
|
+
requestedPath: string | undefined,
|
|
355
|
+
data: Uint8Array,
|
|
356
|
+
mediaType: ImageMediaType,
|
|
357
|
+
): Promise<string> {
|
|
358
|
+
const fs = getFs(ctx)
|
|
359
|
+
if (fs === undefined) throw new Error('cannot write generated image: filesystem service is not mounted')
|
|
360
|
+
const cwd = sessionCwd(exec)
|
|
361
|
+
const rawPath = requestedPath !== undefined && requestedPath.trim().length > 0
|
|
362
|
+
? requestedPath.trim()
|
|
363
|
+
: defaultOutputName(mediaType)
|
|
364
|
+
const withExt = extname(rawPath).length === 0 ? `${rawPath}${extensionForMediaType(mediaType)}` : rawPath
|
|
365
|
+
const target = await fs.resolve(withExt, { ...(cwd !== undefined ? { cwd } : {}), signal: exec.signal })
|
|
366
|
+
if (cwd !== undefined) {
|
|
367
|
+
const root = await fs.resolve('.', { cwd, signal: exec.signal })
|
|
368
|
+
if (!fs.contains(root, target)) {
|
|
369
|
+
throw new Error(`generate_image can only write inside the session workspace; refused "${target.displayPath}"`)
|
|
370
|
+
}
|
|
371
|
+
}
|
|
372
|
+
const abs = fs.processPath(target)
|
|
373
|
+
await mkdir(dirname(abs), { recursive: true })
|
|
374
|
+
await writeFile(abs, data)
|
|
375
|
+
return target.displayPath
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
async function generateViaImagesApi(
|
|
379
|
+
host: ImageToolHost,
|
|
380
|
+
resolved: ResolvedToolModel,
|
|
381
|
+
args: { prompt: string; size?: string; quality?: string },
|
|
382
|
+
signal: AbortSignal | undefined,
|
|
383
|
+
maxBytes: number,
|
|
384
|
+
): Promise<{ data: Uint8Array; mediaType: ImageMediaType; revisedPrompt?: string }> {
|
|
385
|
+
const response = await gatewayFetch(host, resolved, '/images/generations', {
|
|
386
|
+
model: resolved.model,
|
|
387
|
+
prompt: args.prompt,
|
|
388
|
+
n: 1,
|
|
389
|
+
response_format: 'b64_json',
|
|
390
|
+
...(args.size !== undefined && args.size !== 'auto' ? { size: args.size } : {}),
|
|
391
|
+
...(args.quality !== undefined && args.quality !== 'auto' ? { quality: args.quality } : {}),
|
|
392
|
+
}, signal, 'application/json')
|
|
393
|
+
const payload: unknown = await response.json()
|
|
394
|
+
const image = extractGeneratedImage(payload)
|
|
395
|
+
if (image === undefined) throw new Error('sub2api: image API returned no image data')
|
|
396
|
+
if (image.data !== undefined && image.mediaType !== undefined) {
|
|
397
|
+
return { data: image.data, mediaType: image.mediaType, ...(image.revisedPrompt !== undefined ? { revisedPrompt: image.revisedPrompt } : {}) }
|
|
398
|
+
}
|
|
399
|
+
if (image.url !== undefined) {
|
|
400
|
+
const downloaded = await loadRemoteImage(image.url, signal, maxBytes)
|
|
401
|
+
return { ...downloaded, ...(image.revisedPrompt !== undefined ? { revisedPrompt: image.revisedPrompt } : {}) }
|
|
402
|
+
}
|
|
403
|
+
throw new Error('sub2api: image API returned no image data')
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
async function generateViaChat(
|
|
407
|
+
host: ImageToolHost,
|
|
408
|
+
resolved: ResolvedToolModel,
|
|
409
|
+
prompt: string,
|
|
410
|
+
signal: AbortSignal | undefined,
|
|
411
|
+
maxBytes: number,
|
|
412
|
+
): Promise<{ data: Uint8Array; mediaType: ImageMediaType; revisedPrompt?: string }> {
|
|
413
|
+
const response = await gatewayFetch(host, resolved, '/chat/completions', {
|
|
414
|
+
model: resolved.model,
|
|
415
|
+
messages: [{
|
|
416
|
+
role: 'user',
|
|
417
|
+
content: `Generate an image for this prompt and return the image itself (as a data URL or a direct image URL), not a description:\n\n${prompt}`,
|
|
418
|
+
}],
|
|
419
|
+
stream: true,
|
|
420
|
+
stream_options: { include_usage: true },
|
|
421
|
+
}, signal, 'text/event-stream')
|
|
422
|
+
const text = await collectChatText(response)
|
|
423
|
+
const extracted = extractImageFromText(text)
|
|
424
|
+
if (extracted === undefined) {
|
|
425
|
+
throw new Error(`sub2api: chat image model did not return image data. Response preview: ${text.slice(0, 240)}`)
|
|
426
|
+
}
|
|
427
|
+
if (extracted.kind === 'data') return { data: extracted.data, mediaType: extracted.mediaType }
|
|
428
|
+
return await loadRemoteImage(extracted.url, signal, maxBytes)
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
export function registerImageTools(ctx: Context, host: ImageToolHost): void {
|
|
432
|
+
ctx.inject(['tools', 'systemPrompt'], (toolCtx) => {
|
|
433
|
+
toolCtx.systemPrompt.section({
|
|
434
|
+
name: 'tool:generate_image',
|
|
435
|
+
order: 119,
|
|
436
|
+
text: 'Use the generate_image tool to create an image with the configured image model and write it to the workspace. Call it when the current chat model cannot generate images. The tool returns the saved file path, not the image bytes.',
|
|
437
|
+
})
|
|
438
|
+
|
|
439
|
+
toolCtx.tools.register(defineTool({
|
|
440
|
+
name: GENERATE_IMAGE_NAME,
|
|
441
|
+
description: 'Generate an image with the configured image model and write it to the workspace. Use this when the current chat model cannot generate images. Returns the saved file path.',
|
|
442
|
+
parameters: {
|
|
443
|
+
prompt: {
|
|
444
|
+
type: 'string',
|
|
445
|
+
required: true,
|
|
446
|
+
description: 'Image generation prompt.',
|
|
447
|
+
},
|
|
448
|
+
file_path: {
|
|
449
|
+
type: 'string',
|
|
450
|
+
description: 'Workspace path to write. Defaults to generated-<timestamp>.png in the session cwd.',
|
|
451
|
+
},
|
|
452
|
+
size: {
|
|
453
|
+
type: 'string',
|
|
454
|
+
enum: [...GENERATE_SIZES],
|
|
455
|
+
description: 'Requested size when the image API supports it.',
|
|
456
|
+
},
|
|
457
|
+
quality: {
|
|
458
|
+
type: 'string',
|
|
459
|
+
enum: [...GENERATE_QUALITIES],
|
|
460
|
+
description: 'Requested quality when the image API supports it.',
|
|
461
|
+
},
|
|
462
|
+
},
|
|
463
|
+
output: {
|
|
464
|
+
schema: {
|
|
465
|
+
type: 'object',
|
|
466
|
+
additionalProperties: false,
|
|
467
|
+
properties: {
|
|
468
|
+
path: { type: 'string', required: true },
|
|
469
|
+
model: { type: 'string', required: true },
|
|
470
|
+
mediaType: { type: 'string', required: true },
|
|
471
|
+
bytes: { type: 'integer', required: true },
|
|
472
|
+
revisedPrompt: { type: 'string' },
|
|
473
|
+
attachment: {
|
|
474
|
+
type: 'object',
|
|
475
|
+
additionalProperties: false,
|
|
476
|
+
properties: {
|
|
477
|
+
attachmentId: { type: 'string', required: true },
|
|
478
|
+
mediaType: { type: 'string', required: true },
|
|
479
|
+
bytes: { type: 'integer', required: true },
|
|
480
|
+
width: { type: 'integer', required: true },
|
|
481
|
+
height: { type: 'integer', required: true },
|
|
482
|
+
name: { type: 'string' },
|
|
483
|
+
originalDimensions: {
|
|
484
|
+
type: 'object',
|
|
485
|
+
additionalProperties: false,
|
|
486
|
+
properties: {
|
|
487
|
+
width: { type: 'integer', required: true },
|
|
488
|
+
height: { type: 'integer', required: true },
|
|
489
|
+
},
|
|
490
|
+
},
|
|
491
|
+
},
|
|
492
|
+
},
|
|
493
|
+
},
|
|
494
|
+
},
|
|
495
|
+
render: (_args, value) => {
|
|
496
|
+
const blocks: ContentBlock[] = []
|
|
497
|
+
const attachment = value.attachment
|
|
498
|
+
if (attachment !== undefined && typeof attachment === 'object' && attachment !== null) {
|
|
499
|
+
// Inline the generated image into the tool result so the chat UI
|
|
500
|
+
// renders it (image content blocks render as a gallery, exactly
|
|
501
|
+
// like pasted images). Text blocks are kept alongside so the
|
|
502
|
+
// model still sees the path and metadata on later turns.
|
|
503
|
+
blocks.push({ type: 'image', attachment: attachment as ImageAttachmentRef })
|
|
504
|
+
}
|
|
505
|
+
blocks.push({
|
|
506
|
+
type: 'text',
|
|
507
|
+
text: [
|
|
508
|
+
`<path>${value.path}</path>`,
|
|
509
|
+
'<type>image</type>',
|
|
510
|
+
'<content>',
|
|
511
|
+
`${value.mediaType}, ${value.bytes} bytes, model ${value.model}`,
|
|
512
|
+
value.revisedPrompt !== undefined ? `revised prompt: ${value.revisedPrompt}` : '',
|
|
513
|
+
'</content>',
|
|
514
|
+
].filter((line) => line.length > 0).join('\n'),
|
|
515
|
+
})
|
|
516
|
+
return blocks
|
|
517
|
+
},
|
|
518
|
+
},
|
|
519
|
+
timeoutMs: DEFAULT_IMAGE_TOOL_TIMEOUT_MS,
|
|
520
|
+
async execute(args, exec) {
|
|
521
|
+
const prompt = args.prompt.trim()
|
|
522
|
+
if (prompt.length === 0) throw new Error('prompt must be a non-empty string')
|
|
523
|
+
const resolved = resolveToolModel(host.config(), 'generate')
|
|
524
|
+
const maxBytes = ctx.get('attachments')?.imageLimits.maxImageBytes ?? DEFAULT_MAX_IMAGE_BYTES
|
|
525
|
+
let image: { data: Uint8Array; mediaType: ImageMediaType; revisedPrompt?: string }
|
|
526
|
+
try {
|
|
527
|
+
image = await generateViaImagesApi(host, resolved, {
|
|
528
|
+
prompt,
|
|
529
|
+
...(args.size !== undefined ? { size: args.size } : {}),
|
|
530
|
+
...(args.quality !== undefined ? { quality: args.quality } : {}),
|
|
531
|
+
}, exec.signal, maxBytes)
|
|
532
|
+
} catch (error) {
|
|
533
|
+
const message = error instanceof Error ? error.message : String(error)
|
|
534
|
+
if (!/HTTP 404|HTTP 405|HTTP 501|not found|unknown endpoint|does not exist|not implemented/i.test(message)) throw error
|
|
535
|
+
image = await generateViaChat(host, resolved, prompt, exec.signal, maxBytes)
|
|
536
|
+
}
|
|
537
|
+
const path = await writeGeneratedFile(ctx, exec, args.file_path, image.data, image.mediaType)
|
|
538
|
+
// Save the generated bytes as a durable attachment so the tool result
|
|
539
|
+
// can carry an image content block that the chat UI renders inline.
|
|
540
|
+
let attachment: ImageAttachmentRef | undefined
|
|
541
|
+
const attachmentStore = ctx.get('attachments')
|
|
542
|
+
if (attachmentStore !== undefined) {
|
|
543
|
+
try {
|
|
544
|
+
attachment = await attachmentStore.saveImage({
|
|
545
|
+
data: image.data,
|
|
546
|
+
mediaType: image.mediaType,
|
|
547
|
+
...(path.length > 0 ? { name: basename(path) } : {}),
|
|
548
|
+
})
|
|
549
|
+
} catch {
|
|
550
|
+
// Attachment storage is best-effort: fall back to a text-only
|
|
551
|
+
// result (the workspace file is still there for the model/UI).
|
|
552
|
+
attachment = undefined
|
|
553
|
+
}
|
|
554
|
+
}
|
|
555
|
+
return {
|
|
556
|
+
path,
|
|
557
|
+
model: `${resolved.route}/${resolved.model}`,
|
|
558
|
+
mediaType: image.mediaType,
|
|
559
|
+
bytes: image.data.byteLength,
|
|
560
|
+
...(image.revisedPrompt !== undefined ? { revisedPrompt: image.revisedPrompt } : {}),
|
|
561
|
+
...(attachment !== undefined ? { attachment } : {}),
|
|
562
|
+
}
|
|
563
|
+
},
|
|
564
|
+
presentCall(args) {
|
|
565
|
+
return {
|
|
566
|
+
card: 'generic',
|
|
567
|
+
title: `Generate image${args.file_path !== undefined ? ` ${args.file_path}` : ''}`,
|
|
568
|
+
...(args.file_path !== undefined ? { locations: [{ path: args.file_path }] } : {}),
|
|
569
|
+
}
|
|
570
|
+
},
|
|
571
|
+
presentResult(_args, result) {
|
|
572
|
+
// Show only the image in the UI; the XML text block is for the model and
|
|
573
|
+
// stays in the model-facing content unchanged.
|
|
574
|
+
const imageBlock = result.content.find((b) => b.type === 'image')
|
|
575
|
+
if (imageBlock !== undefined) {
|
|
576
|
+
return { card: 'generic' as const, content: [imageBlock] }
|
|
577
|
+
}
|
|
578
|
+
// No attachment (read-only install fallback): show just the path line.
|
|
579
|
+
const textBlock = result.content.find((b) => b.type === 'text')
|
|
580
|
+
if (textBlock !== undefined && textBlock.type === 'text') {
|
|
581
|
+
const pathLine = textBlock.text.match(/<path>(.*?)<\/path>/)?.[1]
|
|
582
|
+
if (pathLine !== undefined) {
|
|
583
|
+
return { card: 'generic' as const, content: [{ type: 'text', text: pathLine }] }
|
|
584
|
+
}
|
|
585
|
+
}
|
|
586
|
+
return undefined
|
|
587
|
+
},
|
|
588
|
+
}))
|
|
589
|
+
})
|
|
590
|
+
}
|