@dickpy/dsh-imagegen 1.1.0 → 1.2.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@dickpy/dsh-imagegen",
   "description": "AI 鐢熷浘 (image generation) plugin for the dsh web GUI: text-to-image and image-to-image through a configurable OpenAI-compatible endpoint (gpt-image-2 / grok-imagine-image / dall-e-3, with native xAI Grok Imagine request shaping), with a settings card for api_url / api_key and a sidebar entry opening a split-pane generation studio.",
-  "version": "1.1.0",
+  "version": "1.2.0",
   "type": "module",
   "main": "lib/index.js",
   "exports": {
@@ -27,6 +27,7 @@
   },
   "devDependencies": {
     "@deepseek-ai/cordis": "^4.0.1",
+    "@deepseek-ai/dsh-attachment": "0.1.0-rc.7",
     "@deepseek-ai/dsh-client-connection": "0.1.0-rc.7",
     "@deepseek-ai/dsh-client-locale": "0.1.0-rc.7",
     "@deepseek-ai/dsh-client-runtime": "0.1.0-rc.7",
@@ -34,8 +35,10 @@
     "@deepseek-ai/dsh-client-ui-settings": "0.1.0-rc.7",
     "@deepseek-ai/dsh-client-ui-slots": "0.1.0-rc.7",
     "@deepseek-ai/dsh-host-webserver": "0.1.0-rc.7",
+    "@deepseek-ai/dsh-llm": "0.1.0-rc.7",
     "@deepseek-ai/dsh-settings": "0.1.0-rc.7",
     "@deepseek-ai/dsh-system-prompt": "0.1.0-rc.7",
+    "@deepseek-ai/dsh-tools": "0.1.0-rc.7",
     "@types/node": "^22.20.0",
     "@types/react": "~18.3.1",
     "@types/react-dom": "^18.3.5",

package/src/agent-image-tools.ts

@@ -0,0 +1,289 @@
+/** Agent-facing image-generation tools backed by the shared host queue. */
+
+import type { Context } from '@deepseek-ai/cordis'
+import { createUserMessage } from '@deepseek-ai/dsh-llm/message'
+import { defineTool } from '@deepseek-ai/dsh-tools'
+import type { ImageAttachmentRef, ImageMediaType } from '@deepseek-ai/dsh-attachment'
+import type {} from '@deepseek-ai/dsh-attachment'
+import type {} from '@deepseek-ai/dsh-tools'
+import { ImageGenError } from './engine.ts'
+import { ImageGenerationRuntime } from './generation-runtime.ts'
+import { normalizeImageModels } from './image-models.ts'
+import type { GenerationTask, GeneratedImage } from './protocol.ts'
+
+export interface AgentImageToolConfig {
+  enabled: boolean
+  allowAgentImageGeneration: boolean
+  apiUrl: string
+  apiKey: string
+  imageModels: string[]
+}
+
+interface AgentImageRef {
+  attachment_id: string
+  media_type: string
+  bytes: number
+  width: number
+  height: number
+  name?: string
+}
+
+interface AgentTaskResult {
+  task_id: string
+  status: string
+  message: string
+  error?: string
+  images: AgentImageRef[]
+}
+
+const imageRefSchema = {
+  type: 'object',
+  additionalProperties: false,
+  properties: {
+    attachment_id: { type: 'string', required: true },
+    media_type: { type: 'string', required: true },
+    bytes: { type: 'integer', required: true },
+    width: { type: 'integer', required: true },
+    height: { type: 'integer', required: true },
+    name: { type: 'string' },
+  },
+} as const
+
+const taskResultSchema = {
+  type: 'object',
+  additionalProperties: false,
+  properties: {
+    task_id: { type: 'string', required: true },
+    status: { type: 'string', required: true },
+    message: { type: 'string', required: true },
+    error: { type: 'string' },
+    images: { type: 'array', required: true, items: imageRefSchema },
+  },
+} as const
+
+function acceptedMediaType(value: string): value is ImageMediaType {
+  return value === 'image/png' || value === 'image/jpeg' || value === 'image/webp' || value === 'image/gif'
+}
+
+function projectRef(ref: ImageAttachmentRef): AgentImageRef {
+  return {
+    attachment_id: String(ref.attachmentId),
+    media_type: ref.mediaType,
+    bytes: ref.bytes,
+    width: ref.width,
+    height: ref.height,
+    ...ref.name === undefined ? {} : { name: ref.name },
+  }
+}
+
+function restoreRef(value: AgentImageRef): ImageAttachmentRef {
+  if (!acceptedMediaType(value.media_type)) throw new ImageGenError('source_image.media_type is not a supported image type', 'bad-reference-image')
+  if (!Number.isInteger(value.bytes) || value.bytes < 1 || !Number.isInteger(value.width) || value.width < 1 || !Number.isInteger(value.height) || value.height < 1) {
+    throw new ImageGenError('source_image metadata is invalid', 'bad-reference-image')
+  }
+  return {
+    attachmentId: value.attachment_id as ImageAttachmentRef['attachmentId'],
+    mediaType: value.media_type,
+    bytes: value.bytes,
+    width: value.width,
+    height: value.height,
+    ...value.name === undefined ? {} : { name: value.name },
+  }
+}
+
+function imageDataUrl(image: { data: Uint8Array; ref: ImageAttachmentRef }): string {
+  return `data:${image.ref.mediaType};base64,${Buffer.from(image.data).toString('base64')}`
+}
+
+function renderTaskResult(value: AgentTaskResult): Array<{ type: 'text'; text: string } | { type: 'image'; attachment: ImageAttachmentRef }> {
+  const text = JSON.stringify(value)
+  return [
+    { type: 'text', text },
+    ...value.images.map(image => ({ type: 'image' as const, attachment: restoreRef(image) })),
+  ]
+}
+
+/** Register the global Agent tools and unregister them with the plugin lifecycle. */
+export function registerAgentImageTools(ctx: Context, runtime: ImageGenerationRuntime, resolve: () => AgentImageToolConfig): () => void {
+  const attachmentRefs = new Map<string, Promise<AgentImageRef[]>>()
+  const taskSubscriptions = new Set<() => void>()
+  const ensureConfigured = (): void => {
+    const config = resolve()
+    if (!config.enabled) throw new ImageGenError('AI image generation is disabled. Open Settings > Plugins > AI Image and enable it.', 'plugin-disabled')
+    if (!config.allowAgentImageGeneration) throw new ImageGenError('Agent image generation is disabled in Settings > Plugins > AI Image.', 'agent-generation-disabled')
+    if (config.apiUrl.trim() === '' || config.apiKey.trim() === '') throw new ImageGenError('Image API credentials are not configured. Open Settings > Plugins > AI Image and fill in API URL and API key.', 'image-api-not-configured')
+  }
+  const selectedModel = (requested: unknown): string => {
+    const models = normalizeImageModels(resolve().imageModels)
+    const model = typeof requested === 'string' && requested.trim() !== '' ? requested.trim() : models[0]
+    if (!models.includes(model)) {
+      throw new ImageGenError(`Image model "${model}" is not configured. Choose one of: ${models.join(', ')}.`, 'image-model-not-configured')
+    }
+    return model
+  }
+  const materializeTaskImages = (task: GenerationTask): Promise<AgentImageRef[]> => {
+    if (task.status !== 'completed') return Promise.resolve([])
+    const existing = attachmentRefs.get(task.id)
+    if (existing !== undefined) return existing
+    const pending = ctx.attachments.saveImages((task.result?.images ?? []).map((image, index) => toSaveImage(image, task.id, index)))
+      .then(refs => refs.map(projectRef))
+    attachmentRefs.set(task.id, pending)
+    void pending.catch(() => {
+      if (attachmentRefs.get(task.id) === pending) attachmentRefs.delete(task.id)
+    })
+    return pending
+  }
+  const taskResult = async (task: GenerationTask): Promise<AgentTaskResult> => {
+    const images = await materializeTaskImages(task)
+    return {
+      task_id: task.id,
+      status: task.status,
+      message: task.status === 'completed'
+        ? 'Generation completed. The images are attached below and can be reused as source_image in edit_image.'
+        : task.status === 'failed'
+          ? 'Generation failed.'
+          : task.status === 'cancelled'
+            ? 'Generation was cancelled.'
+            : 'Generation is still running. Completion will be delivered to the conversation automatically with image attachments.',
+      ...task.error === undefined ? {} : { error: task.error },
+      images,
+    }
+  }
+  const findTask = (id: string): GenerationTask => {
+    const task = runtime.queue.list().find(candidate => candidate.id === id)
+    if (task === undefined) throw new ImageGenError(`Image generation task ${id} was not found.`, 'task-not-found')
+    return task
+  }
+  const notifyCompletion = async (agent: { send: (message: ReturnType<typeof createUserMessage>, target: 'next-turn', wakeup: true) => void }, task: GenerationTask): Promise<void> => {
+    const result = await taskResult(task)
+    const completed = task.status === 'completed'
+    const text = completed
+      ? `图像生成任务已完成（${task.id}）。图片已附在这条消息中，可以直接查看、下载或作为后续图生图的参考。`
+      : task.status === 'cancelled'
+        ? `图像生成任务已取消（${task.id}）。`
+        : `图像生成任务失败（${task.id}）：${task.error ?? '未知错误'}`
+    agent.send(createUserMessage({
+      content: [
+        { type: 'text', text },
+        ...completed ? result.images.map(image => ({ type: 'image' as const, attachment: restoreRef(image) })) : [],
+      ],
+      // The stock conversation's plugin-context row renders text only; a
+      // user-role message is the supported path that renders image attachments.
+      source: { kind: 'user' },
+    }), 'next-turn', true)
+  }
+  const watchTask = (task: GenerationTask, agent: { send: (message: ReturnType<typeof createUserMessage>, target: 'next-turn', wakeup: true) => void } | undefined): void => {
+    if (agent === undefined) return
+    let dispose: (() => void) | undefined
+    const onChange = (updated: GenerationTask): void => {
+      if (updated.id !== task.id || !isFinalTask(updated)) return
+      dispose?.()
+      if (dispose !== undefined) taskSubscriptions.delete(dispose)
+      void notifyCompletion(agent, updated).catch(() => {})
+    }
+    dispose = runtime.queue.subscribe(onChange)
+    taskSubscriptions.add(dispose)
+    const current = findTask(task.id)
+    if (isFinalTask(current)) onChange(current)
+  }
+  const disposers = [
+    ctx.tools.register(defineTool({
+      name: 'generate_image',
+      description: 'Queue a text-to-image generation request. This returns immediately with a task id. When it finishes, the conversation automatically receives a visible image-attachment notification; do not repeatedly poll. Only use models configured for this plugin; omit model to use the first configured image model. Use get_image_generation_task only for an explicit status check or recovery.',
+      parameters: {
+        prompt: { type: 'string', required: true, description: 'Detailed image-generation prompt.' },
+        model: { type: 'string', description: 'One of the configured image models. Defaults to the first configured model.' },
+        size: { type: 'string', description: 'Aspect ratio such as 1:1, 16:9, 9:16, or auto.' },
+        quality: { type: 'string', description: 'auto, 1k, 2k, or 4k.' },
+        count: { type: 'integer', description: 'Number of images, 1 to 4. Defaults to 1.' },
+        detail: { type: 'string', description: 'Optional provider detail value, for example standard or high.' },
+      },
+      output: { schema: taskResultSchema, render: (_args, value) => renderTaskResult(value) },
+      async execute(args, exec) {
+        ensureConfigured()
+        const task = runtime.queue.submit({
+          mode: 'text',
+          model: selectedModel(args.model),
+          prompt: args.prompt.trim(),
+          size: args.size ?? 'auto',
+          quality: args.quality ?? 'auto',
+          n: Math.min(4, Math.max(1, args.count ?? 1)),
+          detail: args.detail ?? '',
+        })
+        watchTask(task, exec.agent)
+        return taskResult(task)
+      },
+    })),
+    ctx.tools.register(defineTool({
+      name: 'edit_image',
+      description: 'Queue an image-to-image edit. source_image must be an image reference returned by a completed generation notification or get_image_generation_task; pass that entire object unchanged. Only configured image models are allowed; omit model to use the first configured model. Completion is automatically delivered to the conversation with visible image attachments; do not repeatedly poll.',
+      parameters: {
+        prompt: { type: 'string', required: true, description: 'How to transform the source image.' },
+        source_image: { ...imageRefSchema, required: true, description: 'Image reference returned by get_image_generation_task.' },
+        model: { type: 'string', description: 'One of the configured image models. Defaults to the first configured model.' },
+        size: { type: 'string', description: 'Aspect ratio such as 1:1, 16:9, 9:16, or auto.' },
+        quality: { type: 'string', description: 'auto, 1k, 2k, or 4k.' },
+        count: { type: 'integer', description: 'Number of images, 1 to 4. Defaults to 1.' },
+        detail: { type: 'string', description: 'Optional provider detail value.' },
+      },
+      output: { schema: taskResultSchema, render: (_args, value) => renderTaskResult(value) },
+      async execute(args, exec) {
+        ensureConfigured()
+        const reference = await ctx.attachments.readImage(restoreRef(args.source_image), exec.signal)
+        const task = runtime.queue.submit({
+          mode: 'edit',
+          model: selectedModel(args.model),
+          prompt: args.prompt.trim(),
+          size: args.size ?? 'auto',
+          quality: args.quality ?? 'auto',
+          n: Math.min(4, Math.max(1, args.count ?? 1)),
+          detail: args.detail ?? '',
+          image: imageDataUrl(reference),
+          ...reference.ref.name === undefined ? {} : { refName: reference.ref.name },
+        })
+        watchTask(task, exec.agent)
+        return taskResult(task)
+      },
+    })),
+    ctx.tools.register(defineTool({
+      name: 'get_image_generation_task',
+      description: 'Optionally check an image-generation task status. Completed tasks return image references for edit_image, but the conversation already receives a visible completion notification automatically; do not poll repeatedly.',
+      parameters: { task_id: { type: 'string', required: true, description: 'Task id returned by generate_image or edit_image.' } },
+      output: { schema: taskResultSchema, render: (_args, value) => renderTaskResult(value) },
+      async execute(args) {
+        ensureConfigured()
+        return taskResult(findTask(args.task_id))
+      },
+    })),
+    ctx.tools.register(defineTool({
+      name: 'cancel_image_generation_task',
+      description: 'Cancel a queued or running image generation task.',
+      parameters: { task_id: { type: 'string', required: true, description: 'Task id returned by generate_image or edit_image.' } },
+      output: { schema: taskResultSchema, render: (_args, value) => renderTaskResult(value) },
+      async execute(args) {
+        ensureConfigured()
+        const task = runtime.queue.cancel(args.task_id)
+        if (task === undefined) throw new ImageGenError(`Image generation task ${args.task_id} was not found.`, 'task-not-found')
+        return taskResult(task)
+      },
+    })),
+  ]
+  return () => {
+    for (const dispose of taskSubscriptions) dispose()
+    taskSubscriptions.clear()
+    for (const dispose of disposers) dispose()
+  }
+}
+
+function isFinalTask(task: GenerationTask): boolean {
+  return task.status === 'completed' || task.status === 'failed' || task.status === 'cancelled'
+}
+
+function toSaveImage(image: GeneratedImage, taskId: string, index: number): { data: Uint8Array; mediaType: ImageMediaType; name: string } {
+  const mediaType = acceptedMediaType(image.mime) ? image.mime : 'image/png'
+  return {
+    data: Buffer.from(image.b64, 'base64'),
+    mediaType,
+    name: `imagegen-${taskId}-${index + 1}.${mediaType === 'image/jpeg' ? 'jpg' : mediaType.slice('image/'.length)}`,
+  }
+}

package/src/client/ImageGenPanel.tsx

@@ -16,13 +16,9 @@ import { errorMessage, tt } from './helpers.ts'
 import { TemplateLibrary } from './TemplateLibrary.tsx'
 import type { GeneratedImage, GenerateMode, GenerateRequest, GenerationTask, HistoryEntry, HistoryImageRef, UpdateInfo } from '../protocol.ts'
 import type { ImageGenConfig, ImageGenScope } from './settings-scope.ts'
+import { DEFAULT_IMAGE_MODELS, normalizeImageModels } from '../image-models.ts'
 import css from './panel.module.css'
 
-/** Models offered by the dropdown. Anything OpenAI-compatible that answers
- *  /images/generations (+ /images/edits) works; grok-imagine-image is handled
- *  specially host-side (JSON /images/edits, aspect_ratio, b64_json). */
-const MODELS = ['gpt-image-2', 'grok-imagine-image'] as const
-
 /** Size options, presented as aspect ratios (auto = let the model decide).
  *  The host maps each ratio onto the model's own vocabulary: aspect_ratio for
  *  Grok Imagine, the closest pixel size for OpenAI-compatible endpoints. */
@@ -153,7 +149,7 @@ function formatTime(timestamp: number): string {
 /** Studio tabs: the two generation modes plus the gallery view. */
 type PanelTab = GenerateMode | 'gallery'
 
-type GalleryFilter = 'all' | 'text' | 'edit' | 'gpt-image-2' | 'grok-imagine-image'
+type GalleryFilter = string
 type ComparisonSession = { taskIds: string[]; prompt: string }
 
 /** Render the studio. */
@@ -169,6 +165,7 @@ export function ImageGenPanel(props: {
   const apiKeySet = useSecretSet(scope, 'apiKey')
   const promptKeySet = useSecretSet(scope, 'promptApiKey')
   const connected = enabled && configured && apiKeySet
+  const imageModels = normalizeImageModels(config?.imageModels)
 
   const [tab, setTab] = useState<PanelTab>('text')
   const [prompt, setPrompt] = useState('')
@@ -176,9 +173,9 @@ export function ImageGenPanel(props: {
   const [quality, setQuality] = useState<string>('auto')
   const [count, setCount] = useState(1)
   const [detail, setDetail] = useState('')
-  const [model, setModel] = useState<string>(MODELS[0])
+  const [model, setModel] = useState<string>(DEFAULT_IMAGE_MODELS[0])
   const [compareEnabled, setCompareEnabled] = useState(false)
-  const [compareModels, setCompareModels] = useState<string[]>([...MODELS])
+  const [compareModels, setCompareModels] = useState<string[]>([...DEFAULT_IMAGE_MODELS])
   const [modelOpen, setModelOpen] = useState(false)
   const [refImage, setRefImage] = useState<{ dataUrl: string; name: string } | null>(null)
   const [images, setImages] = useState<GeneratedImage[]>([])
@@ -222,6 +219,17 @@ export function ImageGenPanel(props: {
   const previewStage = useRef<HTMLDivElement>(null)
   const elapsed = useElapsed(generating, startedAt)
 
+  // A saved settings change is authoritative. Keep the active selection and
+  // comparison choices in that allow-list without disturbing valid choices.
+  const imageModelKey = imageModels.join('\u0000')
+  useEffect(() => {
+    setModel(previous => imageModels.includes(previous) ? previous : imageModels[0])
+    setCompareModels(previous => {
+      const retained = previous.filter(candidate => imageModels.includes(candidate))
+      return retained.length > 0 ? retained : [imageModels[0]]
+    })
+  }, [imageModelKey])
+
   const filteredGallery = gallery
     .filter(entry => {
       if (galleryFilter === 'all') return true
@@ -235,6 +243,7 @@ export function ImageGenPanel(props: {
     .sort((a, b) => gallerySort === 'newest' ? b.createdAt - a.createdAt : a.createdAt - b.createdAt)
 
   const galleryTagOptions = [...new Set(gallery.flatMap(entry => entry.tags ?? []))].sort((a, b) => a.localeCompare(b))
+  const galleryModels = [...new Set([...imageModels, ...gallery.map(entry => entry.model)])]
 
   const filteredHistory = history.filter(entry => {
     const query = historyQuery.trim().toLocaleLowerCase()
@@ -399,7 +408,7 @@ export function ImageGenPanel(props: {
     }
     const request: GenerateRequest = {
       mode: tab === 'gallery' ? 'text' : tab,
-      model,
+      model: imageModels.includes(model) ? model : imageModels[0],
       prompt: promptText,
       size,
       quality,
@@ -410,7 +419,7 @@ export function ImageGenPanel(props: {
     }
     setError(null)
     try {
-      const targetModels = compareEnabled ? compareModels : [model]
+      const targetModels = (compareEnabled ? compareModels : [request.model]).filter(candidate => imageModels.includes(candidate))
       if (targetModels.length === 0) {
         setError(tt('compare.selectRequired'))
         return
@@ -497,7 +506,7 @@ export function ImageGenPanel(props: {
       setQuality(normalizeQuality(entry.quality))
       setDetail((DETAILS as readonly string[]).includes(entry.detail) ? entry.detail : '')
       setCount(entry.n >= 1 && entry.n <= 4 ? entry.n : 1)
-      setModel((MODELS as readonly string[]).includes(entry.model) ? entry.model : MODELS[0])
+      setModel(imageModels.includes(entry.model) ? entry.model : imageModels[0])
       setRefImage(null)
       setImages(restored)
       setError(null)
@@ -607,7 +616,7 @@ export function ImageGenPanel(props: {
       setQuality(normalizeQuality(entry.quality))
       setDetail((DETAILS as readonly string[]).includes(entry.detail) ? entry.detail : '')
       setCount(entry.n >= 1 && entry.n <= 4 ? entry.n : 1)
-      setModel((MODELS as readonly string[]).includes(entry.model) ? entry.model : MODELS[0])
+      setModel(imageModels.includes(entry.model) ? entry.model : imageModels[0])
       setRefImage(null)
       setImages(restored)
       setError(null)
@@ -794,13 +803,12 @@ export function ImageGenPanel(props: {
           {tab === 'gallery' ? (
             <div className={css.galleryFilters}>
               <div className={css.galleryFilterHeading}>{tt('gallery.categories')}</div>
-              {([
-                ['all', 'gallery.all'],
-                ['text', 'mode.text'],
-                ['edit', 'mode.edit'],
-                ['gpt-image-2', 'gallery.gpt'],
-                ['grok-imagine-image', 'gallery.grok'],
-              ] as const).map(([value, label]) => (
+              {[
+                ['all', tt('gallery.all')],
+                ['text', tt('mode.text')],
+                ['edit', tt('mode.edit')],
+                ...galleryModels.map(value => [value, value]),
+              ].map(([value, label]) => (
                 <button
                   key={value}
                   type="button"
@@ -808,7 +816,7 @@ export function ImageGenPanel(props: {
                   data-active={galleryFilter === value ? '' : undefined}
                   onClick={() => { setGalleryFilter(value) }}
                 >
-                  <span>{tt(label as never)}</span>
+                  <span>{label}</span>
                   <span className={css.galleryFilterCount}>{gallery.filter(entry => value === 'all' || value === 'text' || value === 'edit' ? (value === 'all' ? true : entry.mode === value) : entry.model === value).length}</span>
                 </button>
               ))}
@@ -1012,7 +1020,7 @@ export function ImageGenPanel(props: {
                 </button>
                 {modelOpen ? (
                   <div className={css.modelMenuList} role="listbox" aria-label={tt('model.label')}>
-                    {MODELS.map(option => (
+                    {imageModels.map(option => (
                       <button
                         key={option}
                         type="button"
@@ -1036,7 +1044,7 @@ export function ImageGenPanel(props: {
               </label>
               {compareEnabled ? (
                 <div className={css.compareModelChoices} role="group" aria-label={tt('compare.models')}>
-                  {MODELS.map(option => (
+                  {imageModels.map(option => (
                     <label key={option}>
                       <input type="checkbox" checked={compareModels.includes(option)} onChange={() => { setCompareModels(previous => previous.includes(option) ? previous.filter(value => value !== option) : [...previous, option]) }} />
                       <span>{option}</span>