@anionex/dsh-vision-toolkit 0.1.7 → 0.1.9

package/src/client/paste-images.tsx

@@ -8,9 +8,12 @@ import type { PropsRuntime } from '@deepseek-ai/dsh-client-ui-slots'
 
 const SOURCE = 'vision-toolkit-pasted-image'
 export const PASTE_IMAGES_ROUTE = '/_dsh/vision-toolkit/paste-images'
+export const PASTE_POLICY_ROUTE = '/_dsh/vision-toolkit/paste-policy'
 const MAX_IMAGES = 20
 const MAX_IMAGE_BYTES = 20 * 1024 * 1024
 const MAX_BATCH_BYTES = 80 * 1024 * 1024
+/** A confirmed paste verdict older than this is unknown again, even while a refresh is in flight. */
+const VERDICT_MAX_AGE_MS = 15000
 
 interface PasteRecord {
   ref: string
@@ -28,6 +31,20 @@ interface PasteBatch {
   unsubscribe?: (() => void) | undefined
 }
 
+/** One model route the host asks the browser to switch to before the native paste flow. */
+interface PasteSwitchRoute {
+  provider: string
+  model: string
+  label: string
+  reasoningEffort?: string
+}
+
+/** A fresh host verdict for one Session and model label. */
+interface PasteVerdictValue {
+  takeOver: boolean
+  autoSwitch?: PasteSwitchRoute
+}
+
 interface PasteResponse {
   ok: boolean
   value?: { absolutePath?: string }
@@ -105,6 +122,26 @@ function imageFiles(data: DataTransfer | null): File[] {
   return candidates.filter(file => file.type.toLowerCase().startsWith('image/'))
 }
 
+/**
+ * The selector label the model picker currently shows, or '' when none is
+ * readable. Matches the host ModelSelect trigger aria-labels ("Select model,
+ * current …" / "选择模型，当前 …"); any other label wording falls back to the
+ * session-header verdict, which is stale until the next request.
+ */
+function currentModelLabel(): string {
+  const buttons = document.querySelectorAll('button[aria-label]')
+  for (const button of buttons) {
+    const label = button.getAttribute('aria-label') ?? ''
+    if (/select model|current model|选择模型/iu.test(label)) return label
+  }
+  return ''
+}
+
+/** Verdict cache key: the model label is part of the answer, so a switch invalidates it. */
+function verdictKey(sessionId: string, modelLabel: string): string {
+  return `${sessionId}|${modelLabel}`
+}
+
 function validateImages(files: readonly File[]): void {
   if (files.length > MAX_IMAGES) throw new Error(`Paste at most ${MAX_IMAGES} images at a time`)
   let total = 0
@@ -132,6 +169,14 @@ export class PasteImageController {
   private readonly records = new Map<string, PasteRecord>()
   private readonly listeners = new Set<() => void>()
   private revision = 0
+  private readonly verdicts = new Map<string, {
+    takeOver: boolean
+    autoSwitch?: PasteSwitchRoute
+    at: number
+    pending: boolean
+  }>()
+  /** Guards the synthetic replay paste from re-entering capture interception. */
+  private replaying = false
 
   constructor(private readonly ctx: ClientContext) {}
 
@@ -234,29 +279,254 @@ export class PasteImageController {
     }
   }
 
-  handlePaste(event: ClipboardEvent): boolean {
-    const files = imageFiles(event.clipboardData)
-    if (files.length === 0) return false
-    const target = event.target
-    if (!(target instanceof HTMLTextAreaElement) || target.closest('[data-composer-card]') === null) return false
+  /**
+   * The host's verdict for one Session and selector label, when fresh. The
+   * last CONFIRMED answer is authoritative while a background refresh is in
+   * flight (the paste acts on what the host last said; the refresh only
+   * covers the next paste). A label that changed since the confirmation
+   * answers undefined, so the native attachment flow stays the default.
+   * @param sessionId - the live Session the paste belongs to.
+   * @param modelLabel - the model-selector label currently shown.
+   * @returns the fresh confirmed verdict, or undefined when unconfirmed.
+   */
+  private verdictFor(sessionId: string, modelLabel: string): PasteVerdictValue | undefined {
+    const entry = this.verdicts.get(verdictKey(sessionId, modelLabel))
+    if (entry === undefined || entry.at === 0) return undefined
+    if (Date.now() - entry.at > VERDICT_MAX_AGE_MS) return undefined
+    return { takeOver: entry.takeOver, ...(entry.autoSwitch === undefined ? {} : { autoSwitch: entry.autoSwitch }) }
+  }
 
-    event.preventDefault()
-    event.stopPropagation()
-    event.stopImmediatePropagation()
+  /**
+   * The exact model route the live model catalog reports for one Session.
+   * Unreadable routes answer undefined, so the verdict falls back to the
+   * selector label alone.
+   * @param sessionId - the live Session id.
+   * @returns the current provider/model selection, when readable.
+   */
+  private async readSelection(sessionId: string): Promise<{ provider: string; model: string; reasoningEffort?: string } | undefined> {
+    const connection = this.ctx.get('connection') as { api: { sessions: {
+      models(request: { sessionId: string }): Promise<{ result: {
+        ok: true
+        value: { current?: { provider: string; model: string; reasoningEffort?: string } | null }
+      } | { ok: false; error: { code: string; message: string } } }>
+    } } } | undefined
+    if (connection === undefined) return undefined
+    try {
+      const { result } = await connection.api.sessions.models({ sessionId })
+      if (!result.ok) return undefined
+      const current = result.value.current
+      if (current === undefined || current === null || current.provider === '' || current.model === '') return undefined
+      return {
+        provider: current.provider,
+        model: current.model,
+        ...(current.reasoningEffort === undefined ? {} : { reasoningEffort: current.reasoningEffort }),
+      }
+    } catch {
+      return undefined
+    }
+  }
 
-    const sessionId = this.ctx.sessions.list.getSnapshot().current
-    if (sessionId === undefined) return true
+  /**
+   * Ask the host what to do with a paste for the current model, and cache the
+   * answer per Session and selector label. A model switch changes the label,
+   * which changes the cache key, so a stale verdict never outlives the model
+   * it described. The exact selection rides along when the live model catalog
+   * is readable, so the host can answer with an auto-switch route; a 404
+   * simply leaves the verdict unconfirmed; the next focus or paste retries.
+   * @param sessionId - the live Session to ask about.
+   * @param modelLabel - the model-selector label currently shown.
+   */
+  refreshVerdict(sessionId: string, modelLabel: string): void {
+    const key = verdictKey(sessionId, modelLabel)
+    const cached = this.verdicts.get(key)
+    // Dedupe only on an in-flight request, never on freshness: the host's
+    // model route can change under an unchanged Session id.
+    if (cached?.pending) return
+    const entry = {
+      pending: true,
+      takeOver: cached ? cached.takeOver : false,
+      at: cached ? cached.at : 0,
+      ...(cached?.autoSwitch === undefined ? {} : { autoSwitch: cached.autoSwitch }),
+    }
+    this.verdicts.set(key, entry)
+    void (async () => {
+      const selection = await this.readSelection(sessionId)
+      const query = new URLSearchParams({ sessionId })
+      if (modelLabel !== '') query.set('model', modelLabel)
+      if (selection !== undefined) {
+        query.set('provider', selection.provider)
+        query.set('modelId', selection.model)
+        if (selection.reasoningEffort !== undefined) query.set('reasoningEffort', selection.reasoningEffort)
+      }
+      let request: Promise<Response>
+      try {
+        request = fetch(`${PASTE_POLICY_ROUTE}?${query.toString()}`)
+      } catch {
+        // No fetch surface (test runtime, pre-fetch bootstrap): leave the
+        // verdict unconfirmed rather than letting the paste listener die.
+        entry.pending = false
+        return
+      }
+      request
+        .then((response) => {
+          if (response.status === 404) {
+            // Route not mounted yet (plugin load race, hot reload): forget every
+            // verdict and retry on the next focus or paste instead of standing
+            // down for the page lifetime.
+            this.verdicts.clear()
+            return null
+          }
+          if (!response.ok) throw new Error(`paste policy ${response.status}`)
+          return response.json() as Promise<{ ok: true; value: PasteVerdictValue }>
+        })
+        .then((body) => {
+          entry.pending = false
+          if (body !== null) {
+            entry.takeOver = body.value.takeOver === true
+            if (body.value.autoSwitch !== undefined) entry.autoSwitch = body.value.autoSwitch
+            else delete entry.autoSwitch
+            entry.at = Date.now()
+          }
+        })
+        .catch(() => {
+          entry.pending = false
+        })
+    })()
+  }
+
+  /**
+   * Switch one Session to the route the host validated, through the same
+   * model-directory seat the selector uses when present (so the shared UI
+   * state moves with the session), falling back to the raw RPC.
+   * @param sessionId - the live Session id.
+   * @param route - the validated variant route.
+   */
+  private async switchModel(sessionId: string, route: PasteSwitchRoute): Promise<void> {
+    const directories = this.ctx.get('modelDirectories') as {
+      directoryFor(id: string): { select(selection: { provider: string; model: string; reasoningEffort?: string }): Promise<void> }
+    } | undefined
+    if (directories !== undefined) {
+      // The label is a display hint; the seat only needs the exact route.
+      await directories.directoryFor(sessionId).select({
+        provider: route.provider,
+        model: route.model,
+        ...(route.reasoningEffort === undefined ? {} : { reasoningEffort: route.reasoningEffort }),
+      })
+      return
+    }
+    const connection = this.ctx.get('connection') as { api: { sessions: {
+      selectModel(request: {
+        sessionId: string
+        provider: string
+        model: string
+        reasoningEffort?: string
+      }): Promise<{ result: { ok: boolean; error?: { code: string; message: string } } }>
+    } } } | undefined
+    if (connection === undefined) throw new Error('No model switch channel is available in this Web application')
+    const { result } = await connection.api.sessions.selectModel({
+      sessionId,
+      provider: route.provider,
+      model: route.model,
+      ...(route.reasoningEffort === undefined ? {} : { reasoningEffort: route.reasoningEffort }),
+    })
+    if (!result.ok) throw new Error(`${result.error?.code ?? 'select-model-failed'}: ${result.error?.message ?? 'unknown error'}`)
+  }
+
+  /**
+   * Replay a swallowed paste as a synthetic clipboard event so the composer's
+   * own intake (limits, thumbnails, keyboard) runs with the captured files.
+   * @returns false when the environment cannot construct a clipboard payload.
+   */
+  private replayPaste(target: HTMLTextAreaElement, files: readonly File[], text: string): boolean {
+    let data: DataTransfer
+    try {
+      data = new DataTransfer()
+      for (const file of files) data.items.add(file)
+      if (text !== '') data.setData('text/plain', text)
+    } catch {
+      return false
+    }
+    let event: ClipboardEvent
+    try {
+      event = new ClipboardEvent('paste', {
+        clipboardData: data,
+        bubbles: true,
+        cancelable: true,
+      } as ClipboardEventInit)
+    } catch {
+      return false
+    }
+    if (event.clipboardData === null || event.clipboardData.files.length === 0) return false
+    this.replaying = true
+    try {
+      target.dispatchEvent(event)
+    } finally {
+      this.replaying = false
+    }
+    return true
+  }
+
+  /**
+   * Auto-switch flow: switch the Session to the image-input variant, announce
+   * it, then replay the paste into the composer's native intake. A failed
+   * switch, or an environment that cannot replay clipboard bytes, degrades to
+   * the path takeover with the same files.
+   * @param sessionId - the live Session id.
+   * @param target - the composer textarea the paste landed on.
+   * @param files - the captured image files.
+   * @param text - same-paste text, replayed alongside the files.
+   * @param route - the validated variant route to switch to.
+   */
+  private async autoSwitchPaste(
+    sessionId: string,
+    target: HTMLTextAreaElement,
+    files: readonly File[],
+    text: string,
+    route: PasteSwitchRoute,
+  ): Promise<void> {
     const input = this.inputFor(sessionId)
-    const snapshot = input.state.getSnapshot()
-    if (snapshot.phase !== 'plain') return true
+    try {
+      await this.switchModel(sessionId, route)
+      input.notify('info', `Switched to ${route.label || `${route.model} (Vision Toolkit)`}; pasted images now keep the native attachment flow`)
+    } catch (error) {
+      input.notify('error', `Model switch failed; images will be sent as workspace paths: ${message(error)}`)
+      this.takeoverPaste(sessionId, target, files, text)
+      return
+    }
+    // Replaying lets the composer's own intake run (thumbnail, limits,
+    // keyboard); if the environment cannot replay clipboard bytes, the
+    // images still land as workspace paths.
+    const before = input.state.getSnapshot().imageIds.length
+    const replayed = this.replayPaste(target, files, text)
+    const after = input.state.getSnapshot().imageIds.length
+    if (!replayed || after <= before) {
+      this.takeoverPaste(sessionId, target, files, text)
+    }
+  }
 
+  /**
+   * Path-takeover flow: insert the same-paste text and every image as a text
+   * reference that serializes to the image's workspace path on send.
+   * @param sessionId - the live Session id.
+   * @param target - the composer textarea the paste landed on.
+   * @param files - the captured image files.
+   * @param text - same-paste text.
+   */
+  private takeoverPaste(
+    sessionId: string,
+    target: HTMLTextAreaElement,
+    files: readonly File[],
+    text: string,
+  ): void {
+    const input = this.inputFor(sessionId)
+    const snapshot = input.state.getSnapshot()
+    if (snapshot.phase !== 'plain') return
     const start = Math.max(0, Math.min(target.selectionStart ?? snapshot.draft.length, snapshot.draft.length))
     const end = Math.max(start, Math.min(target.selectionEnd ?? start, snapshot.draft.length))
-    const text = (event.clipboardData?.getData('text/plain') ?? '').replaceAll('\uFFFC', '')
     try {
       let cursor = this.insertText(input, text, start, end)
       validateImages(files)
-      cursor = this.insertRecords(String(sessionId), input, files, cursor)
+      cursor = this.insertRecords(sessionId, input, files, cursor)
       requestAnimationFrame(() => {
         target.focus({ preventScroll: true })
         target.setSelectionRange(cursor, cursor)
@@ -264,6 +534,43 @@ export class PasteImageController {
     } catch (error) {
       input.notify('error', message(error))
     }
+  }
+
+  handlePaste(event: ClipboardEvent): boolean {
+    if (this.replaying) return false
+    const files = imageFiles(event.clipboardData)
+    if (files.length === 0) return false
+    const target = event.target
+    if (!(target instanceof HTMLTextAreaElement) || target.closest('[data-composer-card]') === null) return false
+
+    const sessionId = this.ctx.sessions.list.getSnapshot().current
+    if (sessionId === undefined) return false
+    const modelLabel = currentModelLabel()
+    this.refreshVerdict(sessionId, modelLabel)
+    // Only a fresh host verdict acts; the native attachment flow stays the
+    // default while the host is unconfirmed.
+    const verdict = this.verdictFor(sessionId, modelLabel)
+    if (verdict === undefined) return false
+    // An image-capable model (the variant routes included) keeps its native
+    // paste: no switch, no takeover.
+    if (verdict.takeOver === false && verdict.autoSwitch === undefined) return false
+
+    event.preventDefault()
+    event.stopPropagation()
+    event.stopImmediatePropagation()
+
+    const input = this.inputFor(sessionId)
+    if (input.state.getSnapshot().phase !== 'plain') return true
+
+    const text = (event.clipboardData?.getData('text/plain') ?? '').replaceAll('\uFFFC', '')
+    if (verdict.autoSwitch !== undefined) {
+      // The text-only model has an image-input variant: switch first, then
+      // let the paste flow natively so the thumbnail and durable session
+      // image are preserved.
+      void this.autoSwitchPaste(sessionId, target, files, text, verdict.autoSwitch)
+      return true
+    }
+    this.takeoverPaste(sessionId, target, files, text)
     return true
   }
 
@@ -411,8 +718,17 @@ export function installPasteImages(ctx: ClientContext): void {
   })
   ctx.effect(() => {
     const listener = (event: ClipboardEvent): void => { controller.handlePaste(event) }
+    // A focus-time prefetch has the verdict ready before the first paste can land.
+    const onFocusIn = (): void => {
+      const sessionId = ctx.sessions.list.getSnapshot().current
+      if (sessionId !== undefined) controller.refreshVerdict(String(sessionId), currentModelLabel())
+    }
     document.addEventListener('paste', listener, true)
-    return () => { document.removeEventListener('paste', listener, true) }
+    document.addEventListener('focusin', onFocusIn, true)
+    return () => {
+      document.removeEventListener('paste', listener, true)
+      document.removeEventListener('focusin', onFocusIn, true)
+    }
   }, 'dsh-vision-toolkit: clipboard image capture')
   ctx.slots.inject('conversation.input.dock', () => ctx.slots.register({
     name: 'conversation.input.dock',

package/src/config.ts

@@ -54,6 +54,27 @@ export interface VisionToolkitConfig {
   }
   /** Extra directories (besides the workspace) inputs may come from. */
   allowedDirs?: string[]
+  /**
+   * Image-input variants: sibling model-selector entries for every model the
+   * host positively declares text-only. A variant declares image input, so
+   * pasted images keep the native attachment flow (composer thumbnail and
+   * durable session image), and the plugin rewrites image blocks into Vision
+   * Toolkit descriptions only on the wire to the model.
+   */
+  imageInputVariants?: {
+    /** Whether variant routes are registered at all (default true). */
+    enabled?: boolean
+    /** Restrict wrapped upstream routes by provider id; empty wraps every eligible route. */
+    providers?: string[]
+    /**
+     * Whether the browser paste integration automatically switches the Session
+     * to the image-input variant of a text-only model before the paste, so
+     * pasted images keep the native attachment flow with no manual model
+     * change. Off means a text-only model keeps the path takeover (default
+     * true).
+     */
+    autoSwitch?: boolean
+  }
 }
 
 /** Configuration schema with the documented P0 defaults. */
@@ -77,6 +98,11 @@ export const Config: Schema<VisionToolkitConfig> = z.object({
     python: z.string(),
   }),
   allowedDirs: z.array(z.string()).default([]),
+  imageInputVariants: z.object({
+    enabled: z.boolean().default(true),
+    providers: z.array(z.string()).default([]),
+    autoSwitch: z.boolean().default(true),
+  }),
 })
 
 /** Configuration after static validation, with every default materialized. */
@@ -100,6 +126,11 @@ export interface ResolvedVisionToolkitConfig {
     python?: string
   }
   allowedDirs: string[]
+  imageInputVariants: {
+    enabled: boolean
+    providers: string[]
+    autoSwitch: boolean
+  }
 }
 
 const MAX_TIMEOUT_MS = 600000
@@ -187,6 +218,10 @@ export function resolveConfig(config: VisionToolkitConfig = {}): ResolvedVisionT
     throw new VisionToolkitError('config', 'runtime.python must not be empty')
   }
   const allowedDirs = (config.allowedDirs ?? []).map(dir => dir.trim()).filter(dir => dir.length > 0)
+  const imageInputVariants = config.imageInputVariants ?? {}
+  const variantProviders = (imageInputVariants.providers ?? [])
+    .map(provider => provider.trim())
+    .filter(provider => provider.length > 0)
   return {
     provider: { baseUrl, credential, model, protocol, anthropicThinking, userAgent },
     language,
@@ -200,5 +235,10 @@ export function resolveConfig(config: VisionToolkitConfig = {}): ResolvedVisionT
       ...(python !== undefined ? { python } : {}),
     },
     allowedDirs,
+    imageInputVariants: {
+      enabled: imageInputVariants.enabled ?? true,
+      providers: variantProviders,
+      autoSwitch: imageInputVariants.autoSwitch ?? true,
+    },
   }
 }