@mengruo/dsh-vision-toolkit 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (216) hide show
  1. package/LICENSE +21 -0
  2. package/README.i18n.yaml +6 -0
  3. package/README.md +356 -0
  4. package/README.zh.md +354 -0
  5. package/assets/community-group-qr.png +0 -0
  6. package/assets/dsh-conversation-artifact.png +0 -0
  7. package/assets/dsh-conversation-image-qa-top.png +0 -0
  8. package/assets/dsh-conversation-image-qa.png +0 -0
  9. package/assets/dsh-conversation-pixel-diff.png +0 -0
  10. package/assets/dsh-conversation-screenshot-debugging-top.png +0 -0
  11. package/assets/dsh-conversation-screenshot-debugging.png +0 -0
  12. package/assets/dsh-conversation-tool-call.png +0 -0
  13. package/assets/dsh-conversation-vision-trace.png +0 -0
  14. package/assets/dsh-view-example.png +0 -0
  15. package/assets/hero-v2.png +0 -0
  16. package/assets/logo_aihubmix.png +0 -0
  17. package/assets/python-bootstrap.json +48 -0
  18. package/assets/skill/SKILL.md +329 -0
  19. package/assets/skill/UPSTREAM.json +71 -0
  20. package/assets/skill/references/gui.md +88 -0
  21. package/assets/skill/references/long-screenshot-ocr.md +77 -0
  22. package/assets/skill/references/restore-graphic.md +84 -0
  23. package/assets/skill/references/restore-structure.md +45 -0
  24. package/assets/skill/references/restore-ui.md +202 -0
  25. package/assets/social-preview.png +0 -0
  26. package/assets/upstream/README.md +18 -0
  27. package/assets/upstream/focus-hint-comparison-1.webp +0 -0
  28. package/assets/upstream/focus-hint-comparison-2.webp +0 -0
  29. package/assets/upstream/infographic-reference.webp +0 -0
  30. package/assets/upstream/infographic-result.webp +0 -0
  31. package/assets/upstream/ui-fast-restore-reference.webp +0 -0
  32. package/assets/upstream/ui-fast-restore-result.webp +0 -0
  33. package/assets/upstream/ui-result.webp +0 -0
  34. package/assets/upstream/ui-sketch.webp +0 -0
  35. package/assets/vision-model-test.png +0 -0
  36. package/assets/vision-settings.png +0 -0
  37. package/assets/wechat-reward.png +0 -0
  38. package/cordis.patch.yml +6 -0
  39. package/docs/aihubmix-gemini-vision.i18n.yaml +6 -0
  40. package/docs/aihubmix-gemini-vision.md +138 -0
  41. package/docs/aihubmix-gemini-vision.zh.md +138 -0
  42. package/docs/assets/aihubmix-api-keys.png +0 -0
  43. package/docs/assets/aihubmix-create-key.png +0 -0
  44. package/docs/assets/aihubmix-free-vision-model.png +0 -0
  45. package/docs/assets/aihubmix-home.png +0 -0
  46. package/docs/assets/aihubmix-sign-up.png +0 -0
  47. package/docs/assets/groq-console-home.png +0 -0
  48. package/docs/assets/groq-console-keys-login.png +0 -0
  49. package/docs/assets/groq-docs-quickstart.png +0 -0
  50. package/docs/assets/groq-docs-qwen3.6.png +0 -0
  51. package/docs/assets/groq-docs-vision.png +0 -0
  52. package/docs/assets/vision-settings.png +0 -0
  53. package/docs/dsh-desktop-install.md +67 -0
  54. package/docs/dsh-desktop-install.zh.md +67 -0
  55. package/docs/groq-qwen3.6-vision.i18n.yaml +6 -0
  56. package/docs/groq-qwen3.6-vision.md +236 -0
  57. package/docs/groq-qwen3.6-vision.zh.md +236 -0
  58. package/docs/python-runtime.i18n.yaml +6 -0
  59. package/docs/python-runtime.md +89 -0
  60. package/docs/python-runtime.zh.md +89 -0
  61. package/docs/requirements-traceability/README.i18n.yaml +6 -0
  62. package/docs/requirements-traceability/README.md +75 -0
  63. package/docs/requirements-traceability/README.zh.md +75 -0
  64. package/examples/ui-restoration/README.i18n.yaml +6 -0
  65. package/examples/ui-restoration/README.md +70 -0
  66. package/examples/ui-restoration/README.zh.md +70 -0
  67. package/examples/ui-restoration/assets/final-heatmap.png +0 -0
  68. package/examples/ui-restoration/assets/final-report.json +83 -0
  69. package/examples/ui-restoration/assets/implementation.png +0 -0
  70. package/examples/ui-restoration/assets/initial-heatmap.png +0 -0
  71. package/examples/ui-restoration/assets/initial-report.json +83 -0
  72. package/examples/ui-restoration/assets/initial.png +0 -0
  73. package/examples/ui-restoration/assets/metrics.json +12 -0
  74. package/examples/ui-restoration/assets/reference.png +0 -0
  75. package/examples/ui-restoration/implementation.html +94 -0
  76. package/examples/ui-restoration/initial.html +57 -0
  77. package/lib/artifact-access.js +369 -0
  78. package/lib/artifact-access.js.map +1 -0
  79. package/lib/artifacts.js +56 -0
  80. package/lib/artifacts.js.map +1 -0
  81. package/lib/client.js +2099 -0
  82. package/lib/client.js.map +1 -0
  83. package/lib/config.js +172 -0
  84. package/lib/config.js.map +1 -0
  85. package/lib/defaults.js +6 -0
  86. package/lib/defaults.js.map +1 -0
  87. package/lib/errors.js +56 -0
  88. package/lib/errors.js.map +1 -0
  89. package/lib/evidence-cache.js +350 -0
  90. package/lib/evidence-cache.js.map +1 -0
  91. package/lib/exposure.js +247 -0
  92. package/lib/exposure.js.map +1 -0
  93. package/lib/image-input-variants.js +882 -0
  94. package/lib/image-input-variants.js.map +1 -0
  95. package/lib/index.js +105 -0
  96. package/lib/index.js.map +1 -0
  97. package/lib/paste-images.js +217 -0
  98. package/lib/paste-images.js.map +1 -0
  99. package/lib/paths.js +354 -0
  100. package/lib/paths.js.map +1 -0
  101. package/lib/plugin-update.js +1003 -0
  102. package/lib/plugin-update.js.map +1 -0
  103. package/lib/runtime-install.js +964 -0
  104. package/lib/runtime-install.js.map +1 -0
  105. package/lib/runtime-manager.js +133 -0
  106. package/lib/runtime-manager.js.map +1 -0
  107. package/lib/runtime.js +1678 -0
  108. package/lib/runtime.js.map +1 -0
  109. package/lib/skill.js +25 -0
  110. package/lib/skill.js.map +1 -0
  111. package/lib/tools.js +549 -0
  112. package/lib/tools.js.map +1 -0
  113. package/lib/types/artifact-access.d.ts +61 -0
  114. package/lib/types/artifact-access.d.ts.map +1 -0
  115. package/lib/types/artifacts.d.ts +42 -0
  116. package/lib/types/artifacts.d.ts.map +1 -0
  117. package/lib/types/client/display-config.d.ts +24 -0
  118. package/lib/types/client/display-config.d.ts.map +1 -0
  119. package/lib/types/client/index.d.ts +350 -0
  120. package/lib/types/client/index.d.ts.map +1 -0
  121. package/lib/types/client/model-variants-hider.d.ts +40 -0
  122. package/lib/types/client/model-variants-hider.d.ts.map +1 -0
  123. package/lib/types/client/paste-images.d.ts +126 -0
  124. package/lib/types/client/paste-images.d.ts.map +1 -0
  125. package/lib/types/config.d.ts +123 -0
  126. package/lib/types/config.d.ts.map +1 -0
  127. package/lib/types/defaults.d.ts +6 -0
  128. package/lib/types/defaults.d.ts.map +1 -0
  129. package/lib/types/errors.d.ts +35 -0
  130. package/lib/types/errors.d.ts.map +1 -0
  131. package/lib/types/evidence-cache.d.ts +91 -0
  132. package/lib/types/evidence-cache.d.ts.map +1 -0
  133. package/lib/types/exposure.d.ts +50 -0
  134. package/lib/types/exposure.d.ts.map +1 -0
  135. package/lib/types/image-input-variants.d.ts +144 -0
  136. package/lib/types/image-input-variants.d.ts.map +1 -0
  137. package/lib/types/index.d.ts +19 -0
  138. package/lib/types/index.d.ts.map +1 -0
  139. package/lib/types/paste-images.d.ts +73 -0
  140. package/lib/types/paste-images.d.ts.map +1 -0
  141. package/lib/types/paths.d.ts +117 -0
  142. package/lib/types/paths.d.ts.map +1 -0
  143. package/lib/types/plugin-update.d.ts +119 -0
  144. package/lib/types/plugin-update.d.ts.map +1 -0
  145. package/lib/types/runtime-install.d.ts +92 -0
  146. package/lib/types/runtime-install.d.ts.map +1 -0
  147. package/lib/types/runtime-manager.d.ts +60 -0
  148. package/lib/types/runtime-manager.d.ts.map +1 -0
  149. package/lib/types/runtime.d.ts +412 -0
  150. package/lib/types/runtime.d.ts.map +1 -0
  151. package/lib/types/skill.d.ts +14 -0
  152. package/lib/types/skill.d.ts.map +1 -0
  153. package/lib/types/tools.d.ts +35 -0
  154. package/lib/types/tools.d.ts.map +1 -0
  155. package/lib/types/upstream.d.ts +233 -0
  156. package/lib/types/upstream.d.ts.map +1 -0
  157. package/lib/types/version.d.ts +15 -0
  158. package/lib/types/version.d.ts.map +1 -0
  159. package/lib/types/web-request.d.ts +11 -0
  160. package/lib/types/web-request.d.ts.map +1 -0
  161. package/lib/types/web.d.ts +118 -0
  162. package/lib/types/web.d.ts.map +1 -0
  163. package/lib/upstream.js +860 -0
  164. package/lib/upstream.js.map +1 -0
  165. package/lib/version.js +18 -0
  166. package/lib/version.js.map +1 -0
  167. package/lib/web-request.js +29 -0
  168. package/lib/web-request.js.map +1 -0
  169. package/lib/web.js +462 -0
  170. package/lib/web.js.map +1 -0
  171. package/package.json +149 -0
  172. package/patches/vision-tools-dsh.patch +970 -0
  173. package/runtime/requirements.lock +3 -0
  174. package/src/artifact-access.ts +386 -0
  175. package/src/artifacts.ts +85 -0
  176. package/src/client/display-config.ts +62 -0
  177. package/src/client/index.tsx +1580 -0
  178. package/src/client/model-variants-hider.ts +159 -0
  179. package/src/client/paste-images.tsx +747 -0
  180. package/src/config.ts +290 -0
  181. package/src/defaults.ts +5 -0
  182. package/src/errors.ts +62 -0
  183. package/src/evidence-cache.ts +418 -0
  184. package/src/exposure.ts +265 -0
  185. package/src/image-input-variants.ts +996 -0
  186. package/src/index.ts +141 -0
  187. package/src/paste-images.ts +286 -0
  188. package/src/paths.ts +383 -0
  189. package/src/plugin-update.ts +1143 -0
  190. package/src/runtime-install.ts +1161 -0
  191. package/src/runtime-manager.ts +174 -0
  192. package/src/runtime.ts +2189 -0
  193. package/src/skill.ts +35 -0
  194. package/src/tools.ts +691 -0
  195. package/src/upstream.ts +1078 -0
  196. package/src/version.ts +37 -0
  197. package/src/web-request.ts +27 -0
  198. package/src/web.ts +594 -0
  199. package/vendor/agent-vision-toolkit/CHANGELOG.md +30 -0
  200. package/vendor/agent-vision-toolkit/LICENSE +21 -0
  201. package/vendor/agent-vision-toolkit/README.md +405 -0
  202. package/vendor/agent-vision-toolkit/UPSTREAM_MANIFEST.json +94 -0
  203. package/vendor/agent-vision-toolkit/bin/crop +90 -0
  204. package/vendor/agent-vision-toolkit/bin/detect +13 -0
  205. package/vendor/agent-vision-toolkit/bin/glance +93 -0
  206. package/vendor/agent-vision-toolkit/bin/ground +13 -0
  207. package/vendor/agent-vision-toolkit/bin/trace +129 -0
  208. package/vendor/agent-vision-toolkit/detect.py +60 -0
  209. package/vendor/agent-vision-toolkit/ground.py +267 -0
  210. package/vendor/agent-vision-toolkit/skills/vision-tools/scripts/dominant_colors.py +224 -0
  211. package/vendor/agent-vision-toolkit/skills/vision-tools/scripts/extract_fg.py +278 -0
  212. package/vendor/agent-vision-toolkit/skills/vision-tools/scripts/html_shot.py +420 -0
  213. package/vendor/agent-vision-toolkit/skills/vision-tools/scripts/long_screenshot_ocr.py +1245 -0
  214. package/vendor/agent-vision-toolkit/skills/vision-tools/scripts/pixel_diff.py +88 -0
  215. package/vendor/agent-vision-toolkit/tests/test_vision_client.py +450 -0
  216. package/vendor/agent-vision-toolkit/vision_client.py +310 -0
@@ -0,0 +1,747 @@
1
+ /** Clipboard-only multi-image input for DSH Web. */
2
+
3
+ import { useSyncExternalStore, type ReactNode } from 'react'
4
+ import type { ClientContext } from '@deepseek-ai/dsh-client-runtime/client'
5
+ import type {} from '@deepseek-ai/dsh-client-ui-conversation/client'
6
+ import type { InputTriggerSource } from '@deepseek-ai/dsh-client-ui-input-trigger/client'
7
+ import type { PropsRuntime } from '@deepseek-ai/dsh-client-ui-slots'
8
+ import { readDisplayConfig } from './display-config.ts'
9
+
10
+ const SOURCE = 'vision-toolkit-pasted-image'
11
+ export const PASTE_IMAGES_ROUTE = '/_dsh/vision-toolkit/paste-images'
12
+ export const PASTE_POLICY_ROUTE = '/_dsh/vision-toolkit/paste-policy'
13
+ const MAX_IMAGES = 20
14
+ /** Hard per-image paste ceiling; must match MAX_PASTE_IMAGE_BYTES on the server. */
15
+ const MAX_IMAGE_BYTES = 20 * 1024 * 1024
16
+ const MAX_BATCH_BYTES = 80 * 1024 * 1024
17
+ /** A confirmed paste verdict older than this is unknown again, even while a refresh is in flight. */
18
+ const VERDICT_MAX_AGE_MS = 15000
19
+
20
+ interface PasteRecord {
21
+ ref: string
22
+ file: File
23
+ batch: PasteBatch
24
+ status: 'ready' | 'copying' | 'copied' | 'error'
25
+ error?: string | undefined
26
+ absolutePath?: string | undefined
27
+ }
28
+
29
+ interface PasteBatch {
30
+ sessionId: string
31
+ records: PasteRecord[]
32
+ inflight?: Promise<void> | undefined
33
+ unsubscribe?: (() => void) | undefined
34
+ }
35
+
36
+ /** One model route the host asks the browser to switch to before the native paste flow. */
37
+ interface PasteSwitchRoute {
38
+ provider: string
39
+ model: string
40
+ label: string
41
+ reasoningEffort?: string
42
+ }
43
+
44
+ /** A fresh host verdict for one Session and model label. */
45
+ interface PasteVerdictValue {
46
+ takeOver: boolean
47
+ autoSwitch?: PasteSwitchRoute
48
+ }
49
+
50
+ interface PasteResponse {
51
+ ok: boolean
52
+ value?: { absolutePath?: string }
53
+ error?: { message?: string }
54
+ }
55
+
56
+ interface PasteOccurrence {
57
+ occurrenceId: number
58
+ source: string
59
+ ref: string
60
+ offset: number
61
+ label: string
62
+ }
63
+
64
+ type PasteDockProps = PropsRuntime<'conversation.input.dock'> & {
65
+ controller: PasteImageController
66
+ remove: (occurrence: PasteOccurrence) => void
67
+ }
68
+
69
+ interface ReferenceSourceRegistry {
70
+ registerSource: (source: InputTriggerSource) => () => void
71
+ }
72
+
73
+ interface ReferenceSourceRegistration {
74
+ dispose: () => void
75
+ owners: number
76
+ }
77
+
78
+ interface LegacyTriggerContext {
79
+ inputTriggers: ReferenceSourceRegistry
80
+ }
81
+
82
+ interface LegacySlashContext {
83
+ slash: ReferenceSourceRegistry
84
+ }
85
+
86
+ const CORDIS_ORIGINAL = Symbol.for('cordis.original')
87
+
88
+ function registryIdentity(registry: ReferenceSourceRegistry): object {
89
+ let current: object = registry
90
+ while (true) {
91
+ const original = (current as Record<symbol, unknown>)[CORDIS_ORIGINAL]
92
+ if ((typeof original !== 'object' && typeof original !== 'function') || original === null || original === current) {
93
+ return current
94
+ }
95
+ current = original
96
+ }
97
+ }
98
+
99
+ let fallbackId = 0
100
+
101
+ function id(): string {
102
+ if (typeof globalThis.crypto?.randomUUID === 'function') return globalThis.crypto.randomUUID()
103
+ fallbackId += 1
104
+ return `paste-${Date.now()}-${fallbackId}`
105
+ }
106
+
107
+ function humanBytes(bytes: number): string {
108
+ if (bytes < 1024) return `${bytes} B`
109
+ if (bytes < 1024 ** 2) return `${(bytes / 1024).toFixed(1)} KB`
110
+ return `${(bytes / 1024 ** 2).toFixed(1)} MB`
111
+ }
112
+
113
+ function message(error: unknown): string {
114
+ return error instanceof Error ? error.message : String(error)
115
+ }
116
+
117
+ function imageFiles(data: DataTransfer | null): File[] {
118
+ if (data === null) return []
119
+ const itemFiles = Array.from(data.items)
120
+ .filter(item => item.kind === 'file')
121
+ .map(item => item.getAsFile())
122
+ .filter((file): file is File => file !== null)
123
+ const candidates = itemFiles.length > 0 ? itemFiles : Array.from(data.files)
124
+ return candidates.filter(file => file.type.toLowerCase().startsWith('image/'))
125
+ }
126
+
127
+ /**
128
+ * The selector label the model picker currently shows, or '' when none is
129
+ * readable. Matches the host ModelSelect trigger aria-labels ("Select model,
130
+ * current …" / "选择模型,当前 …"); any other label wording falls back to the
131
+ * session-header verdict, which is stale until the next request.
132
+ */
133
+ function currentModelLabel(): string {
134
+ const buttons = document.querySelectorAll('button[aria-label]')
135
+ for (const button of buttons) {
136
+ const label = button.getAttribute('aria-label') ?? ''
137
+ if (/select model|current model|选择模型/iu.test(label)) return label
138
+ }
139
+ return ''
140
+ }
141
+
142
+ /** Verdict cache key: the model label is part of the answer, so a switch invalidates it. */
143
+ function verdictKey(sessionId: string, modelLabel: string): string {
144
+ return `${sessionId}|${modelLabel}`
145
+ }
146
+
147
+ function validateImages(files: readonly File[]): void {
148
+ if (files.length > MAX_IMAGES) throw new Error(`Paste at most ${MAX_IMAGES} images at a time`)
149
+ let total = 0
150
+ for (const file of files) {
151
+ if (!file.type.toLowerCase().startsWith('image/')) throw new Error(`${file.name || 'clipboard item'} is not an image`)
152
+ if (file.size <= 0) throw new Error(`${file.name || 'clipboard image'} is empty`)
153
+ if (file.size > MAX_IMAGE_BYTES) throw new Error(`${file.name || 'clipboard image'} exceeds ${humanBytes(MAX_IMAGE_BYTES)}`)
154
+ total += file.size
155
+ }
156
+ if (total > MAX_BATCH_BYTES) throw new Error(`Pasted images exceed ${humanBytes(MAX_BATCH_BYTES)} in total`)
157
+ }
158
+
159
+ async function responseJson(response: Response): Promise<PasteResponse> {
160
+ const body = await response.json() as PasteResponse
161
+ if (!response.ok || body.ok !== true) throw new Error(body.error?.message ?? `Image copy failed (${response.status})`)
162
+ return body
163
+ }
164
+
165
+ function pasteLabel(file: File, index: number): string {
166
+ return file.name.trim() || `clipboard-image-${index + 1}`
167
+ }
168
+
169
+ /** Owns browser File objects until DSH serializes the corresponding text references. */
170
+ export class PasteImageController {
171
+ private readonly records = new Map<string, PasteRecord>()
172
+ private readonly listeners = new Set<() => void>()
173
+ private revision = 0
174
+ private readonly verdicts = new Map<string, {
175
+ takeOver: boolean
176
+ autoSwitch?: PasteSwitchRoute
177
+ at: number
178
+ pending: boolean
179
+ }>()
180
+ /** Guards the synthetic replay paste from re-entering capture interception. */
181
+ private replaying = false
182
+
183
+ constructor(private readonly ctx: ClientContext) {}
184
+
185
+ subscribe = (listener: () => void): (() => void) => {
186
+ this.listeners.add(listener)
187
+ return () => { this.listeners.delete(listener) }
188
+ }
189
+
190
+ snapshot = (): number => this.revision
191
+
192
+ private changed(): void {
193
+ this.revision += 1
194
+ for (const listener of this.listeners) listener()
195
+ }
196
+
197
+ source(): InputTriggerSource {
198
+ return {
199
+ trigger: '@',
200
+ name: SOURCE,
201
+ order: 1000,
202
+ candidates: () => Promise.resolve([]),
203
+ onPick: () => undefined,
204
+ codec: {
205
+ clipboardText: ref => `[pasted image: ${this.records.get(ref)?.file.name ?? ref}]`,
206
+ serialize: (ref, signal) => this.serialize(ref, signal),
207
+ },
208
+ }
209
+ }
210
+
211
+ recordsFor(occurrences: readonly PasteOccurrence[]): PasteRecord[] {
212
+ return occurrences
213
+ .filter(occurrence => occurrence.source === SOURCE)
214
+ .map(occurrence => this.records.get(occurrence.ref))
215
+ .filter((record): record is PasteRecord => record !== undefined)
216
+ }
217
+
218
+ private inputFor(sessionId: string) {
219
+ const actx = this.ctx.sessions.scope(sessionId as never)
220
+ if (actx === undefined) throw new Error('Open a live session before pasting images')
221
+ return this.ctx.conversation.input.for(actx)
222
+ }
223
+
224
+ private insertText(input: ReturnType<PasteImageController['inputFor']>, text: string, start: number, end = start): number {
225
+ if (text === '') return start
226
+ const snapshot = input.state.getSnapshot()
227
+ input.setDraft(snapshot.draft.slice(0, start) + text + snapshot.draft.slice(end))
228
+ return start + text.length
229
+ }
230
+
231
+ private insertRecords(
232
+ sessionId: string,
233
+ input: ReturnType<PasteImageController['inputFor']>,
234
+ files: readonly File[],
235
+ cursor: number,
236
+ ): number {
237
+ const batch: PasteBatch = { sessionId, records: [] }
238
+ const draftBeforeReferences = input.state.getSnapshot().draft
239
+ try {
240
+ const before = input.state.getSnapshot().draft.slice(0, cursor)
241
+ if (before !== '' && !/\s$/u.test(before)) cursor = this.insertText(input, ' ', cursor)
242
+ for (const [index, file] of files.entries()) {
243
+ const ref = id()
244
+ const record: PasteRecord = { ref, file, batch, status: 'ready' }
245
+ batch.records.push(record)
246
+ this.records.set(ref, record)
247
+ const snapshot = input.state.getSnapshot()
248
+ const accepted = input.insertReference({
249
+ source: SOURCE,
250
+ ref,
251
+ label: pasteLabel(file, index),
252
+ clipboardText: `[pasted image: ${pasteLabel(file, index)}]`,
253
+ }, { start: cursor, end: cursor, draftRev: snapshot.draftRev })
254
+ if (!accepted) throw new Error('The composer changed before pasted images could be inserted')
255
+ cursor += 1
256
+ const hasNext = index + 1 < files.length
257
+ const suffix = input.state.getSnapshot().draft.slice(cursor)
258
+ if (hasNext || (suffix !== '' && !/^\s/u.test(suffix))) cursor = this.insertText(input, ' ', cursor)
259
+ }
260
+ batch.unsubscribe = input.state.subscribe(() => {
261
+ const alive = new Set(input.state.getSnapshot().occurrences
262
+ .filter(occurrence => occurrence.source === SOURCE)
263
+ .map(occurrence => occurrence.ref))
264
+ let changed = false
265
+ for (const record of batch.records) {
266
+ if (alive.has(record.ref) || record.batch.inflight !== undefined) continue
267
+ changed = this.records.delete(record.ref) || changed
268
+ }
269
+ if (batch.records.every(record => !this.records.has(record.ref)) && batch.inflight === undefined) {
270
+ batch.unsubscribe?.()
271
+ batch.unsubscribe = undefined
272
+ }
273
+ if (changed) this.changed()
274
+ })
275
+ this.changed()
276
+ return cursor
277
+ } catch (error) {
278
+ input.setDraft(draftBeforeReferences)
279
+ for (const record of batch.records) this.records.delete(record.ref)
280
+ throw error
281
+ }
282
+ }
283
+
284
+ /**
285
+ * The host's verdict for one Session and selector label, when fresh. The
286
+ * last CONFIRMED answer is authoritative while a background refresh is in
287
+ * flight (the paste acts on what the host last said; the refresh only
288
+ * covers the next paste). A label that changed since the confirmation
289
+ * answers undefined, so the native attachment flow stays the default.
290
+ * @param sessionId - the live Session the paste belongs to.
291
+ * @param modelLabel - the model-selector label currently shown.
292
+ * @returns the fresh confirmed verdict, or undefined when unconfirmed.
293
+ */
294
+ private verdictFor(sessionId: string, modelLabel: string): PasteVerdictValue | undefined {
295
+ const entry = this.verdicts.get(verdictKey(sessionId, modelLabel))
296
+ if (entry === undefined || entry.at === 0) return undefined
297
+ if (Date.now() - entry.at > VERDICT_MAX_AGE_MS) return undefined
298
+ return { takeOver: entry.takeOver, ...(entry.autoSwitch === undefined ? {} : { autoSwitch: entry.autoSwitch }) }
299
+ }
300
+
301
+ /**
302
+ * The exact model route the live model catalog reports for one Session.
303
+ * Unreadable routes answer undefined, so the verdict falls back to the
304
+ * selector label alone.
305
+ * @param sessionId - the live Session id.
306
+ * @returns the current provider/model selection, when readable.
307
+ */
308
+ private async readSelection(sessionId: string): Promise<{ provider: string; model: string; reasoningEffort?: string } | undefined> {
309
+ const connection = this.ctx.get('connection') as { api: { sessions: {
310
+ models(request: { sessionId: string }): Promise<{ result: {
311
+ ok: true
312
+ value: { current?: { provider: string; model: string; reasoningEffort?: string } | null }
313
+ } | { ok: false; error: { code: string; message: string } } }>
314
+ } } } | undefined
315
+ if (connection === undefined) return undefined
316
+ try {
317
+ const { result } = await connection.api.sessions.models({ sessionId })
318
+ if (!result.ok) return undefined
319
+ const current = result.value.current
320
+ if (current === undefined || current === null || current.provider === '' || current.model === '') return undefined
321
+ return {
322
+ provider: current.provider,
323
+ model: current.model,
324
+ ...(current.reasoningEffort === undefined ? {} : { reasoningEffort: current.reasoningEffort }),
325
+ }
326
+ } catch {
327
+ return undefined
328
+ }
329
+ }
330
+
331
+ /**
332
+ * Ask the host what to do with a paste for the current model, and cache the
333
+ * answer per Session and selector label. A model switch changes the label,
334
+ * which changes the cache key, so a stale verdict never outlives the model
335
+ * it described. The exact selection rides along when the live model catalog
336
+ * is readable, so the host can answer with an auto-switch route; a 404
337
+ * simply leaves the verdict unconfirmed; the next focus or paste retries.
338
+ * @param sessionId - the live Session to ask about.
339
+ * @param modelLabel - the model-selector label currently shown.
340
+ */
341
+ refreshVerdict(sessionId: string, modelLabel: string): void {
342
+ const key = verdictKey(sessionId, modelLabel)
343
+ const cached = this.verdicts.get(key)
344
+ // Dedupe only on an in-flight request, never on freshness: the host's
345
+ // model route can change under an unchanged Session id.
346
+ if (cached?.pending) return
347
+ const entry = {
348
+ pending: true,
349
+ takeOver: cached ? cached.takeOver : false,
350
+ at: cached ? cached.at : 0,
351
+ ...(cached?.autoSwitch === undefined ? {} : { autoSwitch: cached.autoSwitch }),
352
+ }
353
+ this.verdicts.set(key, entry)
354
+ void (async () => {
355
+ const selection = await this.readSelection(sessionId)
356
+ const query = new URLSearchParams({ sessionId })
357
+ if (modelLabel !== '') query.set('model', modelLabel)
358
+ if (selection !== undefined) {
359
+ query.set('provider', selection.provider)
360
+ query.set('modelId', selection.model)
361
+ if (selection.reasoningEffort !== undefined) query.set('reasoningEffort', selection.reasoningEffort)
362
+ }
363
+ let request: Promise<Response>
364
+ try {
365
+ request = fetch(`${PASTE_POLICY_ROUTE}?${query.toString()}`)
366
+ } catch {
367
+ // No fetch surface (test runtime, pre-fetch bootstrap): leave the
368
+ // verdict unconfirmed rather than letting the paste listener die.
369
+ entry.pending = false
370
+ return
371
+ }
372
+ request
373
+ .then((response) => {
374
+ if (response.status === 404) {
375
+ // Route not mounted yet (plugin load race, hot reload): forget every
376
+ // verdict and retry on the next focus or paste instead of standing
377
+ // down for the page lifetime.
378
+ this.verdicts.clear()
379
+ return null
380
+ }
381
+ if (!response.ok) throw new Error(`paste policy ${response.status}`)
382
+ return response.json() as Promise<{ ok: true; value: PasteVerdictValue }>
383
+ })
384
+ .then((body) => {
385
+ entry.pending = false
386
+ if (body !== null) {
387
+ entry.takeOver = body.value.takeOver === true
388
+ if (body.value.autoSwitch !== undefined) entry.autoSwitch = body.value.autoSwitch
389
+ else delete entry.autoSwitch
390
+ entry.at = Date.now()
391
+ }
392
+ })
393
+ .catch(() => {
394
+ entry.pending = false
395
+ })
396
+ })()
397
+ }
398
+
399
+ /**
400
+ * Switch one Session to the route the host validated, through the same
401
+ * model-directory seat the selector uses when present (so the shared UI
402
+ * state moves with the session), falling back to the raw RPC.
403
+ * @param sessionId - the live Session id.
404
+ * @param route - the validated variant route.
405
+ */
406
+ private async switchModel(sessionId: string, route: PasteSwitchRoute): Promise<void> {
407
+ const directories = this.ctx.get('modelDirectories') as {
408
+ directoryFor(id: string): { select(selection: { provider: string; model: string; reasoningEffort?: string }): Promise<void> }
409
+ } | undefined
410
+ if (directories !== undefined) {
411
+ // The label is a display hint; the seat only needs the exact route.
412
+ await directories.directoryFor(sessionId).select({
413
+ provider: route.provider,
414
+ model: route.model,
415
+ ...(route.reasoningEffort === undefined ? {} : { reasoningEffort: route.reasoningEffort }),
416
+ })
417
+ return
418
+ }
419
+ const connection = this.ctx.get('connection') as { api: { sessions: {
420
+ selectModel(request: {
421
+ sessionId: string
422
+ provider: string
423
+ model: string
424
+ reasoningEffort?: string
425
+ }): Promise<{ result: { ok: boolean; error?: { code: string; message: string } } }>
426
+ } } } | undefined
427
+ if (connection === undefined) throw new Error('No model switch channel is available in this Web application')
428
+ const { result } = await connection.api.sessions.selectModel({
429
+ sessionId,
430
+ provider: route.provider,
431
+ model: route.model,
432
+ ...(route.reasoningEffort === undefined ? {} : { reasoningEffort: route.reasoningEffort }),
433
+ })
434
+ if (!result.ok) throw new Error(`${result.error?.code ?? 'select-model-failed'}: ${result.error?.message ?? 'unknown error'}`)
435
+ }
436
+
437
+ /**
438
+ * Replay a swallowed paste as a synthetic clipboard event so the composer's
439
+ * own intake (limits, thumbnails, keyboard) runs with the captured files.
440
+ * @returns false when the environment cannot construct a clipboard payload.
441
+ */
442
+ private replayPaste(target: HTMLTextAreaElement, files: readonly File[], text: string): boolean {
443
+ let data: DataTransfer
444
+ try {
445
+ data = new DataTransfer()
446
+ for (const file of files) data.items.add(file)
447
+ if (text !== '') data.setData('text/plain', text)
448
+ } catch {
449
+ return false
450
+ }
451
+ let event: ClipboardEvent
452
+ try {
453
+ event = new ClipboardEvent('paste', {
454
+ clipboardData: data,
455
+ bubbles: true,
456
+ cancelable: true,
457
+ } as ClipboardEventInit)
458
+ } catch {
459
+ return false
460
+ }
461
+ if (event.clipboardData === null || event.clipboardData.files.length === 0) return false
462
+ this.replaying = true
463
+ try {
464
+ target.dispatchEvent(event)
465
+ } finally {
466
+ this.replaying = false
467
+ }
468
+ return true
469
+ }
470
+
471
+ /**
472
+ * Auto-switch flow: switch the Session to the image-input variant, announce
473
+ * it, then replay the paste into the composer's native intake. A failed
474
+ * switch, or an environment that cannot replay clipboard bytes, degrades to
475
+ * the path takeover with the same files.
476
+ * @param sessionId - the live Session id.
477
+ * @param target - the composer textarea the paste landed on.
478
+ * @param files - the captured image files.
479
+ * @param text - same-paste text, replayed alongside the files.
480
+ * @param route - the validated variant route to switch to.
481
+ */
482
+ private async autoSwitchPaste(
483
+ sessionId: string,
484
+ target: HTMLTextAreaElement,
485
+ files: readonly File[],
486
+ text: string,
487
+ route: PasteSwitchRoute,
488
+ ): Promise<void> {
489
+ const input = this.inputFor(sessionId)
490
+ try {
491
+ await this.switchModel(sessionId, route)
492
+ const { hidden } = await readDisplayConfig()
493
+ input.notify('info', hidden
494
+ ? 'Visual enhancement active: pasted images keep the native attachment flow'
495
+ : `Switched to ${route.label || `${route.model} (Vision Toolkit)`}; pasted images now keep the native attachment flow`)
496
+ } catch (error) {
497
+ input.notify('error', `Model switch failed; images will be sent as workspace paths: ${message(error)}`)
498
+ this.takeoverPaste(sessionId, target, files, text)
499
+ return
500
+ }
501
+ // Replaying lets the composer's own intake run (thumbnail, limits,
502
+ // keyboard); if the environment cannot replay clipboard bytes, the
503
+ // images still land as workspace paths.
504
+ const before = input.state.getSnapshot().imageIds.length
505
+ const replayed = this.replayPaste(target, files, text)
506
+ const after = input.state.getSnapshot().imageIds.length
507
+ if (!replayed || after <= before) {
508
+ this.takeoverPaste(sessionId, target, files, text)
509
+ }
510
+ }
511
+
512
+ /**
513
+ * Path-takeover flow: insert the same-paste text and every image as a text
514
+ * reference that serializes to the image's workspace path on send.
515
+ * @param sessionId - the live Session id.
516
+ * @param target - the composer textarea the paste landed on.
517
+ * @param files - the captured image files.
518
+ * @param text - same-paste text.
519
+ */
520
+ private takeoverPaste(
521
+ sessionId: string,
522
+ target: HTMLTextAreaElement,
523
+ files: readonly File[],
524
+ text: string,
525
+ ): void {
526
+ const input = this.inputFor(sessionId)
527
+ const snapshot = input.state.getSnapshot()
528
+ if (snapshot.phase !== 'plain') return
529
+ const start = Math.max(0, Math.min(target.selectionStart ?? snapshot.draft.length, snapshot.draft.length))
530
+ const end = Math.max(start, Math.min(target.selectionEnd ?? start, snapshot.draft.length))
531
+ try {
532
+ let cursor = this.insertText(input, text, start, end)
533
+ validateImages(files)
534
+ cursor = this.insertRecords(sessionId, input, files, cursor)
535
+ requestAnimationFrame(() => {
536
+ target.focus({ preventScroll: true })
537
+ target.setSelectionRange(cursor, cursor)
538
+ })
539
+ } catch (error) {
540
+ input.notify('error', message(error))
541
+ }
542
+ }
543
+
544
+ handlePaste(event: ClipboardEvent): boolean {
545
+ if (this.replaying) return false
546
+ const files = imageFiles(event.clipboardData)
547
+ if (files.length === 0) return false
548
+ const target = event.target
549
+ if (!(target instanceof HTMLTextAreaElement) || target.closest('[data-composer-card]') === null) return false
550
+
551
+ const sessionId = this.ctx.sessions.list.getSnapshot().current
552
+ if (sessionId === undefined) return false
553
+ const modelLabel = currentModelLabel()
554
+ this.refreshVerdict(sessionId, modelLabel)
555
+ // Only a fresh host verdict acts; the native attachment flow stays the
556
+ // default while the host is unconfirmed.
557
+ const verdict = this.verdictFor(sessionId, modelLabel)
558
+ if (verdict === undefined) return false
559
+ // An image-capable model (the variant routes included) keeps its native
560
+ // paste: no switch, no takeover.
561
+ if (verdict.takeOver === false && verdict.autoSwitch === undefined) return false
562
+
563
+ event.preventDefault()
564
+ event.stopPropagation()
565
+ event.stopImmediatePropagation()
566
+
567
+ const input = this.inputFor(sessionId)
568
+ if (input.state.getSnapshot().phase !== 'plain') return true
569
+
570
+ const text = (event.clipboardData?.getData('text/plain') ?? '').replaceAll('\uFFFC', '')
571
+ if (verdict.autoSwitch !== undefined) {
572
+ // The text-only model has an image-input variant: switch first, then
573
+ // let the paste flow natively so the thumbnail and durable session
574
+ // image are preserved.
575
+ void this.autoSwitchPaste(sessionId, target, files, text, verdict.autoSwitch)
576
+ return true
577
+ }
578
+ this.takeoverPaste(sessionId, target, files, text)
579
+ return true
580
+ }
581
+
582
+ remove(sessionId: string, occurrence: PasteOccurrence): void {
583
+ const record = this.records.get(occurrence.ref)
584
+ if (record?.batch.inflight !== undefined) return
585
+ const input = this.inputFor(sessionId)
586
+ const snapshot = input.state.getSnapshot()
587
+ if (snapshot.phase !== 'plain') return
588
+ const current = snapshot.occurrences.find(candidate =>
589
+ candidate.source === SOURCE
590
+ && candidate.occurrenceId === occurrence.occurrenceId
591
+ && candidate.ref === occurrence.ref)
592
+ if (current === undefined) return
593
+ const accepted = (input as typeof input & {
594
+ insertText: (text: string, span: { start: number; end: number; draftRev: number }) => boolean
595
+ }).insertText('', {
596
+ start: current.offset,
597
+ end: current.offset + 1,
598
+ draftRev: snapshot.draftRev,
599
+ })
600
+ if (!accepted) return
601
+ this.records.delete(occurrence.ref)
602
+ this.changed()
603
+ }
604
+
605
+ private async upload(batch: PasteBatch, signal: AbortSignal): Promise<void> {
606
+ if (batch.inflight !== undefined) return batch.inflight
607
+ const active = batch.records.filter(record => this.records.get(record.ref) === record)
608
+ if (active.length === 0) throw new Error('Pasted images were removed before sending')
609
+ const pending = active.filter(record => record.absolutePath === undefined)
610
+ if (pending.length === 0) return
611
+ const task = (async () => {
612
+ for (const record of pending) {
613
+ record.status = 'copying'
614
+ record.error = undefined
615
+ }
616
+ this.changed()
617
+ try {
618
+ const failures = await Promise.all(pending.map(async (record) => {
619
+ try {
620
+ if (signal.aborted) throw signal.reason ?? new DOMException('Aborted', 'AbortError')
621
+ const query = new URLSearchParams({
622
+ sessionId: batch.sessionId,
623
+ name: record.file.name || 'clipboard-image',
624
+ size: String(record.file.size),
625
+ })
626
+ const body = await responseJson(await fetch(`${PASTE_IMAGES_ROUTE}?${query.toString()}`, {
627
+ method: 'POST',
628
+ headers: { 'Content-Type': record.file.type },
629
+ body: record.file,
630
+ signal,
631
+ }))
632
+ const absolutePath = body.value?.absolutePath
633
+ if (typeof absolutePath !== 'string' || absolutePath === '') {
634
+ throw new Error('Image copy response contained an invalid path')
635
+ }
636
+ record.absolutePath = absolutePath
637
+ record.status = 'copied'
638
+ record.error = undefined
639
+ return undefined
640
+ } catch (error) {
641
+ const failure = error instanceof Error ? error : new Error(message(error))
642
+ record.status = 'error'
643
+ record.error = failure.message
644
+ return failure
645
+ }
646
+ }))
647
+ this.changed()
648
+ const failure = failures.find((error): error is Error => error !== undefined)
649
+ if (failure !== undefined) throw failure
650
+ } finally {
651
+ batch.inflight = undefined
652
+ this.changed()
653
+ }
654
+ })()
655
+ batch.inflight = task
656
+ return task
657
+ }
658
+
659
+ private async serialize(ref: string, signal: AbortSignal): Promise<string> {
660
+ const record = this.records.get(ref)
661
+ if (record === undefined) throw new Error('Pasted image is no longer available in this browser tab')
662
+ await this.upload(record.batch, signal)
663
+ if (record.absolutePath === undefined) throw new Error('Pasted image was not copied into the workspace')
664
+ return `[Pasted image available at absolute path: ${JSON.stringify(record.absolutePath)}]`
665
+ }
666
+ }
667
+
668
+ /** Minimal per-image progress, failure, and removal feedback above the composer. */
669
+ export function PasteImageDock(props: PasteDockProps): ReactNode {
670
+ useSyncExternalStore(props.controller.subscribe, props.controller.snapshot)
671
+ const occurrences = props.input.occurrences.filter(occurrence => occurrence.source === SOURCE)
672
+ const records = props.controller.recordsFor(occurrences)
673
+ if (records.length === 0) return null
674
+ return <div className="dvt-paste-dock" role="status" aria-label="Pasted images">
675
+ {occurrences.map((occurrence) => {
676
+ const record = props.controller.recordsFor([occurrence])[0]
677
+ if (record === undefined) return null
678
+ const detail = record.status === 'copying' ? 'copying…'
679
+ : record.status === 'copied' ? 'copied'
680
+ : record.status === 'error' ? record.error ?? 'copy failed'
681
+ : humanBytes(record.file.size)
682
+ return <div className="dvt-paste-chip" data-status={record.status} key={occurrence.occurrenceId}>
683
+ <span className="dvt-paste-name" title={record.file.name}>{record.file.name || 'clipboard image'}</span>
684
+ <span className="dvt-paste-detail" title={record.error}>{detail}</span>
685
+ <button
686
+ type="button"
687
+ aria-label={`Remove ${record.file.name || 'clipboard image'}`}
688
+ disabled={props.input.phase !== 'plain' || record.status === 'copying'}
689
+ onClick={() => { props.remove(occurrence) }}
690
+ >×</button>
691
+ </div>
692
+ })}
693
+ </div>
694
+ }
695
+
696
+ /** Install capture interception, the text-reference codec, and composer feedback. */
697
+ export function installPasteImages(ctx: ClientContext): void {
698
+ const controller = new PasteImageController(ctx)
699
+ const registered = new WeakMap<object, ReferenceSourceRegistration>()
700
+ const register = (scope: ClientContext, registry: ReferenceSourceRegistry): void => {
701
+ scope.effect(() => {
702
+ const identity = registryIdentity(registry)
703
+ let registration = registered.get(identity)
704
+ if (registration === undefined) {
705
+ registration = { dispose: registry.registerSource(controller.source()), owners: 0 }
706
+ registered.set(identity, registration)
707
+ }
708
+ registration.owners += 1
709
+ return () => {
710
+ if (registered.get(identity) !== registration) return
711
+ registration.owners -= 1
712
+ if (registration.owners > 0) return
713
+ registered.delete(identity)
714
+ registration.dispose()
715
+ }
716
+ }, 'dsh-vision-toolkit: pasted image reference codec')
717
+ }
718
+ ctx.inject(['slash'], (scope: ClientContext) => {
719
+ register(scope, (scope as unknown as LegacySlashContext).slash)
720
+ })
721
+ ctx.inject(['inputTriggers'], (scope: ClientContext) => {
722
+ register(scope, (scope as unknown as LegacyTriggerContext).inputTriggers)
723
+ })
724
+ ctx.effect(() => {
725
+ const listener = (event: ClipboardEvent): void => { controller.handlePaste(event) }
726
+ // A focus-time prefetch has the verdict ready before the first paste can land.
727
+ const onFocusIn = (): void => {
728
+ const sessionId = ctx.sessions.list.getSnapshot().current
729
+ if (sessionId !== undefined) controller.refreshVerdict(String(sessionId), currentModelLabel())
730
+ }
731
+ document.addEventListener('paste', listener, true)
732
+ document.addEventListener('focusin', onFocusIn, true)
733
+ return () => {
734
+ document.removeEventListener('paste', listener, true)
735
+ document.removeEventListener('focusin', onFocusIn, true)
736
+ }
737
+ }, 'dsh-vision-toolkit: clipboard image capture')
738
+ ctx.slots.inject('conversation.input.dock', () => ctx.slots.register({
739
+ name: 'conversation.input.dock',
740
+ id: 'vision-toolkit-pasted-images',
741
+ order: 6,
742
+ inject: sessionId => ({
743
+ controller,
744
+ remove: (occurrence: PasteOccurrence) => { controller.remove(String(sessionId), occurrence) },
745
+ }),
746
+ }, PasteImageDock))
747
+ }