@dickpy/dsh-imagegen 1.0.20 → 1.2.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@dickpy/dsh-imagegen",
   "description": "AI 鐢熷浘 (image generation) plugin for the dsh web GUI: text-to-image and image-to-image through a configurable OpenAI-compatible endpoint (gpt-image-2 / grok-imagine-image / dall-e-3, with native xAI Grok Imagine request shaping), with a settings card for api_url / api_key and a sidebar entry opening a split-pane generation studio.",
-  "version": "1.0.20",
+  "version": "1.2.0",
   "type": "module",
   "main": "lib/index.js",
   "exports": {
@@ -27,6 +27,7 @@
   },
   "devDependencies": {
     "@deepseek-ai/cordis": "^4.0.1",
+    "@deepseek-ai/dsh-attachment": "0.1.0-rc.7",
     "@deepseek-ai/dsh-client-connection": "0.1.0-rc.7",
     "@deepseek-ai/dsh-client-locale": "0.1.0-rc.7",
     "@deepseek-ai/dsh-client-runtime": "0.1.0-rc.7",
@@ -34,8 +35,10 @@
     "@deepseek-ai/dsh-client-ui-settings": "0.1.0-rc.7",
     "@deepseek-ai/dsh-client-ui-slots": "0.1.0-rc.7",
     "@deepseek-ai/dsh-host-webserver": "0.1.0-rc.7",
+    "@deepseek-ai/dsh-llm": "0.1.0-rc.7",
     "@deepseek-ai/dsh-settings": "0.1.0-rc.7",
     "@deepseek-ai/dsh-system-prompt": "0.1.0-rc.7",
+    "@deepseek-ai/dsh-tools": "0.1.0-rc.7",
     "@types/node": "^22.20.0",
     "@types/react": "~18.3.1",
     "@types/react-dom": "^18.3.5",

package/src/agent-image-tools.ts

@@ -0,0 +1,289 @@
+/** Agent-facing image-generation tools backed by the shared host queue. */
+
+import type { Context } from '@deepseek-ai/cordis'
+import { createUserMessage } from '@deepseek-ai/dsh-llm/message'
+import { defineTool } from '@deepseek-ai/dsh-tools'
+import type { ImageAttachmentRef, ImageMediaType } from '@deepseek-ai/dsh-attachment'
+import type {} from '@deepseek-ai/dsh-attachment'
+import type {} from '@deepseek-ai/dsh-tools'
+import { ImageGenError } from './engine.ts'
+import { ImageGenerationRuntime } from './generation-runtime.ts'
+import { normalizeImageModels } from './image-models.ts'
+import type { GenerationTask, GeneratedImage } from './protocol.ts'
+
+export interface AgentImageToolConfig {
+  enabled: boolean
+  allowAgentImageGeneration: boolean
+  apiUrl: string
+  apiKey: string
+  imageModels: string[]
+}
+
+interface AgentImageRef {
+  attachment_id: string
+  media_type: string
+  bytes: number
+  width: number
+  height: number
+  name?: string
+}
+
+interface AgentTaskResult {
+  task_id: string
+  status: string
+  message: string
+  error?: string
+  images: AgentImageRef[]
+}
+
+const imageRefSchema = {
+  type: 'object',
+  additionalProperties: false,
+  properties: {
+    attachment_id: { type: 'string', required: true },
+    media_type: { type: 'string', required: true },
+    bytes: { type: 'integer', required: true },
+    width: { type: 'integer', required: true },
+    height: { type: 'integer', required: true },
+    name: { type: 'string' },
+  },
+} as const
+
+const taskResultSchema = {
+  type: 'object',
+  additionalProperties: false,
+  properties: {
+    task_id: { type: 'string', required: true },
+    status: { type: 'string', required: true },
+    message: { type: 'string', required: true },
+    error: { type: 'string' },
+    images: { type: 'array', required: true, items: imageRefSchema },
+  },
+} as const
+
+function acceptedMediaType(value: string): value is ImageMediaType {
+  return value === 'image/png' || value === 'image/jpeg' || value === 'image/webp' || value === 'image/gif'
+}
+
+function projectRef(ref: ImageAttachmentRef): AgentImageRef {
+  return {
+    attachment_id: String(ref.attachmentId),
+    media_type: ref.mediaType,
+    bytes: ref.bytes,
+    width: ref.width,
+    height: ref.height,
+    ...ref.name === undefined ? {} : { name: ref.name },
+  }
+}
+
+function restoreRef(value: AgentImageRef): ImageAttachmentRef {
+  if (!acceptedMediaType(value.media_type)) throw new ImageGenError('source_image.media_type is not a supported image type', 'bad-reference-image')
+  if (!Number.isInteger(value.bytes) || value.bytes < 1 || !Number.isInteger(value.width) || value.width < 1 || !Number.isInteger(value.height) || value.height < 1) {
+    throw new ImageGenError('source_image metadata is invalid', 'bad-reference-image')
+  }
+  return {
+    attachmentId: value.attachment_id as ImageAttachmentRef['attachmentId'],
+    mediaType: value.media_type,
+    bytes: value.bytes,
+    width: value.width,
+    height: value.height,
+    ...value.name === undefined ? {} : { name: value.name },
+  }
+}
+
+function imageDataUrl(image: { data: Uint8Array; ref: ImageAttachmentRef }): string {
+  return `data:${image.ref.mediaType};base64,${Buffer.from(image.data).toString('base64')}`
+}
+
+function renderTaskResult(value: AgentTaskResult): Array<{ type: 'text'; text: string } | { type: 'image'; attachment: ImageAttachmentRef }> {
+  const text = JSON.stringify(value)
+  return [
+    { type: 'text', text },
+    ...value.images.map(image => ({ type: 'image' as const, attachment: restoreRef(image) })),
+  ]
+}
+
+/** Register the global Agent tools and unregister them with the plugin lifecycle. */
+export function registerAgentImageTools(ctx: Context, runtime: ImageGenerationRuntime, resolve: () => AgentImageToolConfig): () => void {
+  const attachmentRefs = new Map<string, Promise<AgentImageRef[]>>()
+  const taskSubscriptions = new Set<() => void>()
+  const ensureConfigured = (): void => {
+    const config = resolve()
+    if (!config.enabled) throw new ImageGenError('AI image generation is disabled. Open Settings > Plugins > AI Image and enable it.', 'plugin-disabled')
+    if (!config.allowAgentImageGeneration) throw new ImageGenError('Agent image generation is disabled in Settings > Plugins > AI Image.', 'agent-generation-disabled')
+    if (config.apiUrl.trim() === '' || config.apiKey.trim() === '') throw new ImageGenError('Image API credentials are not configured. Open Settings > Plugins > AI Image and fill in API URL and API key.', 'image-api-not-configured')
+  }
+  const selectedModel = (requested: unknown): string => {
+    const models = normalizeImageModels(resolve().imageModels)
+    const model = typeof requested === 'string' && requested.trim() !== '' ? requested.trim() : models[0]
+    if (!models.includes(model)) {
+      throw new ImageGenError(`Image model "${model}" is not configured. Choose one of: ${models.join(', ')}.`, 'image-model-not-configured')
+    }
+    return model
+  }
+  const materializeTaskImages = (task: GenerationTask): Promise<AgentImageRef[]> => {
+    if (task.status !== 'completed') return Promise.resolve([])
+    const existing = attachmentRefs.get(task.id)
+    if (existing !== undefined) return existing
+    const pending = ctx.attachments.saveImages((task.result?.images ?? []).map((image, index) => toSaveImage(image, task.id, index)))
+      .then(refs => refs.map(projectRef))
+    attachmentRefs.set(task.id, pending)
+    void pending.catch(() => {
+      if (attachmentRefs.get(task.id) === pending) attachmentRefs.delete(task.id)
+    })
+    return pending
+  }
+  const taskResult = async (task: GenerationTask): Promise<AgentTaskResult> => {
+    const images = await materializeTaskImages(task)
+    return {
+      task_id: task.id,
+      status: task.status,
+      message: task.status === 'completed'
+        ? 'Generation completed. The images are attached below and can be reused as source_image in edit_image.'
+        : task.status === 'failed'
+          ? 'Generation failed.'
+          : task.status === 'cancelled'
+            ? 'Generation was cancelled.'
+            : 'Generation is still running. Completion will be delivered to the conversation automatically with image attachments.',
+      ...task.error === undefined ? {} : { error: task.error },
+      images,
+    }
+  }
+  const findTask = (id: string): GenerationTask => {
+    const task = runtime.queue.list().find(candidate => candidate.id === id)
+    if (task === undefined) throw new ImageGenError(`Image generation task ${id} was not found.`, 'task-not-found')
+    return task
+  }
+  const notifyCompletion = async (agent: { send: (message: ReturnType<typeof createUserMessage>, target: 'next-turn', wakeup: true) => void }, task: GenerationTask): Promise<void> => {
+    const result = await taskResult(task)
+    const completed = task.status === 'completed'
+    const text = completed
+      ? `图像生成任务已完成（${task.id}）。图片已附在这条消息中，可以直接查看、下载或作为后续图生图的参考。`
+      : task.status === 'cancelled'
+        ? `图像生成任务已取消（${task.id}）。`
+        : `图像生成任务失败（${task.id}）：${task.error ?? '未知错误'}`
+    agent.send(createUserMessage({
+      content: [
+        { type: 'text', text },
+        ...completed ? result.images.map(image => ({ type: 'image' as const, attachment: restoreRef(image) })) : [],
+      ],
+      // The stock conversation's plugin-context row renders text only; a
+      // user-role message is the supported path that renders image attachments.
+      source: { kind: 'user' },
+    }), 'next-turn', true)
+  }
+  const watchTask = (task: GenerationTask, agent: { send: (message: ReturnType<typeof createUserMessage>, target: 'next-turn', wakeup: true) => void } | undefined): void => {
+    if (agent === undefined) return
+    let dispose: (() => void) | undefined
+    const onChange = (updated: GenerationTask): void => {
+      if (updated.id !== task.id || !isFinalTask(updated)) return
+      dispose?.()
+      if (dispose !== undefined) taskSubscriptions.delete(dispose)
+      void notifyCompletion(agent, updated).catch(() => {})
+    }
+    dispose = runtime.queue.subscribe(onChange)
+    taskSubscriptions.add(dispose)
+    const current = findTask(task.id)
+    if (isFinalTask(current)) onChange(current)
+  }
+  const disposers = [
+    ctx.tools.register(defineTool({
+      name: 'generate_image',
+      description: 'Queue a text-to-image generation request. This returns immediately with a task id. When it finishes, the conversation automatically receives a visible image-attachment notification; do not repeatedly poll. Only use models configured for this plugin; omit model to use the first configured image model. Use get_image_generation_task only for an explicit status check or recovery.',
+      parameters: {
+        prompt: { type: 'string', required: true, description: 'Detailed image-generation prompt.' },
+        model: { type: 'string', description: 'One of the configured image models. Defaults to the first configured model.' },
+        size: { type: 'string', description: 'Aspect ratio such as 1:1, 16:9, 9:16, or auto.' },
+        quality: { type: 'string', description: 'auto, 1k, 2k, or 4k.' },
+        count: { type: 'integer', description: 'Number of images, 1 to 4. Defaults to 1.' },
+        detail: { type: 'string', description: 'Optional provider detail value, for example standard or high.' },
+      },
+      output: { schema: taskResultSchema, render: (_args, value) => renderTaskResult(value) },
+      async execute(args, exec) {
+        ensureConfigured()
+        const task = runtime.queue.submit({
+          mode: 'text',
+          model: selectedModel(args.model),
+          prompt: args.prompt.trim(),
+          size: args.size ?? 'auto',
+          quality: args.quality ?? 'auto',
+          n: Math.min(4, Math.max(1, args.count ?? 1)),
+          detail: args.detail ?? '',
+        })
+        watchTask(task, exec.agent)
+        return taskResult(task)
+      },
+    })),
+    ctx.tools.register(defineTool({
+      name: 'edit_image',
+      description: 'Queue an image-to-image edit. source_image must be an image reference returned by a completed generation notification or get_image_generation_task; pass that entire object unchanged. Only configured image models are allowed; omit model to use the first configured model. Completion is automatically delivered to the conversation with visible image attachments; do not repeatedly poll.',
+      parameters: {
+        prompt: { type: 'string', required: true, description: 'How to transform the source image.' },
+        source_image: { ...imageRefSchema, required: true, description: 'Image reference returned by get_image_generation_task.' },
+        model: { type: 'string', description: 'One of the configured image models. Defaults to the first configured model.' },
+        size: { type: 'string', description: 'Aspect ratio such as 1:1, 16:9, 9:16, or auto.' },
+        quality: { type: 'string', description: 'auto, 1k, 2k, or 4k.' },
+        count: { type: 'integer', description: 'Number of images, 1 to 4. Defaults to 1.' },
+        detail: { type: 'string', description: 'Optional provider detail value.' },
+      },
+      output: { schema: taskResultSchema, render: (_args, value) => renderTaskResult(value) },
+      async execute(args, exec) {
+        ensureConfigured()
+        const reference = await ctx.attachments.readImage(restoreRef(args.source_image), exec.signal)
+        const task = runtime.queue.submit({
+          mode: 'edit',
+          model: selectedModel(args.model),
+          prompt: args.prompt.trim(),
+          size: args.size ?? 'auto',
+          quality: args.quality ?? 'auto',
+          n: Math.min(4, Math.max(1, args.count ?? 1)),
+          detail: args.detail ?? '',
+          image: imageDataUrl(reference),
+          ...reference.ref.name === undefined ? {} : { refName: reference.ref.name },
+        })
+        watchTask(task, exec.agent)
+        return taskResult(task)
+      },
+    })),
+    ctx.tools.register(defineTool({
+      name: 'get_image_generation_task',
+      description: 'Optionally check an image-generation task status. Completed tasks return image references for edit_image, but the conversation already receives a visible completion notification automatically; do not poll repeatedly.',
+      parameters: { task_id: { type: 'string', required: true, description: 'Task id returned by generate_image or edit_image.' } },
+      output: { schema: taskResultSchema, render: (_args, value) => renderTaskResult(value) },
+      async execute(args) {
+        ensureConfigured()
+        return taskResult(findTask(args.task_id))
+      },
+    })),
+    ctx.tools.register(defineTool({
+      name: 'cancel_image_generation_task',
+      description: 'Cancel a queued or running image generation task.',
+      parameters: { task_id: { type: 'string', required: true, description: 'Task id returned by generate_image or edit_image.' } },
+      output: { schema: taskResultSchema, render: (_args, value) => renderTaskResult(value) },
+      async execute(args) {
+        ensureConfigured()
+        const task = runtime.queue.cancel(args.task_id)
+        if (task === undefined) throw new ImageGenError(`Image generation task ${args.task_id} was not found.`, 'task-not-found')
+        return taskResult(task)
+      },
+    })),
+  ]
+  return () => {
+    for (const dispose of taskSubscriptions) dispose()
+    taskSubscriptions.clear()
+    for (const dispose of disposers) dispose()
+  }
+}
+
+function isFinalTask(task: GenerationTask): boolean {
+  return task.status === 'completed' || task.status === 'failed' || task.status === 'cancelled'
+}
+
+function toSaveImage(image: GeneratedImage, taskId: string, index: number): { data: Uint8Array; mediaType: ImageMediaType; name: string } {
+  const mediaType = acceptedMediaType(image.mime) ? image.mime : 'image/png'
+  return {
+    data: Buffer.from(image.b64, 'base64'),
+    mediaType,
+    name: `imagegen-${taskId}-${index + 1}.${mediaType === 'image/jpeg' ? 'jpg' : mediaType.slice('image/'.length)}`,
+  }
+}