@anionex/dsh-vision-toolkit 0.1.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (152) hide show
  1. package/LICENSE +21 -0
  2. package/README.i18n.yaml +6 -0
  3. package/README.md +383 -0
  4. package/README.zh.md +383 -0
  5. package/assets/dsh-conversation-artifact.png +0 -0
  6. package/assets/dsh-conversation-image-qa-top.png +0 -0
  7. package/assets/dsh-conversation-image-qa.png +0 -0
  8. package/assets/dsh-conversation-pixel-diff.png +0 -0
  9. package/assets/dsh-conversation-screenshot-debugging-top.png +0 -0
  10. package/assets/dsh-conversation-screenshot-debugging.png +0 -0
  11. package/assets/dsh-conversation-tool-call.png +0 -0
  12. package/assets/dsh-conversation-vision-trace.png +0 -0
  13. package/assets/hero.png +0 -0
  14. package/assets/social-preview.png +0 -0
  15. package/assets/upstream/README.md +16 -0
  16. package/assets/upstream/image-qa.webp +0 -0
  17. package/assets/upstream/infographic-reference.webp +0 -0
  18. package/assets/upstream/infographic-result.webp +0 -0
  19. package/assets/upstream/screenshot-debugging.webp +0 -0
  20. package/assets/upstream/ui-result.webp +0 -0
  21. package/assets/upstream/ui-sketch.webp +0 -0
  22. package/assets/vision-settings.png +0 -0
  23. package/cordis.patch.yml +6 -0
  24. package/docs/assets/vision-settings.png +0 -0
  25. package/docs/requirements-traceability/README.i18n.yaml +6 -0
  26. package/docs/requirements-traceability/README.md +75 -0
  27. package/docs/requirements-traceability/README.zh.md +75 -0
  28. package/examples/ui-restoration/README.i18n.yaml +6 -0
  29. package/examples/ui-restoration/README.md +70 -0
  30. package/examples/ui-restoration/README.zh.md +70 -0
  31. package/examples/ui-restoration/assets/final-heatmap.png +0 -0
  32. package/examples/ui-restoration/assets/final-report.json +83 -0
  33. package/examples/ui-restoration/assets/implementation.png +0 -0
  34. package/examples/ui-restoration/assets/initial-heatmap.png +0 -0
  35. package/examples/ui-restoration/assets/initial-report.json +83 -0
  36. package/examples/ui-restoration/assets/initial.png +0 -0
  37. package/examples/ui-restoration/assets/metrics.json +12 -0
  38. package/examples/ui-restoration/assets/reference.png +0 -0
  39. package/examples/ui-restoration/implementation.html +94 -0
  40. package/examples/ui-restoration/initial.html +57 -0
  41. package/lib/artifact-access.js +369 -0
  42. package/lib/artifact-access.js.map +1 -0
  43. package/lib/artifacts.js +56 -0
  44. package/lib/artifacts.js.map +1 -0
  45. package/lib/client.js +952 -0
  46. package/lib/client.js.map +1 -0
  47. package/lib/config.js +117 -0
  48. package/lib/config.js.map +1 -0
  49. package/lib/errors.js +56 -0
  50. package/lib/errors.js.map +1 -0
  51. package/lib/exposure.js +213 -0
  52. package/lib/exposure.js.map +1 -0
  53. package/lib/index.js +97 -0
  54. package/lib/index.js.map +1 -0
  55. package/lib/paste-images.js +199 -0
  56. package/lib/paste-images.js.map +1 -0
  57. package/lib/paths.js +325 -0
  58. package/lib/paths.js.map +1 -0
  59. package/lib/runtime-install.js +601 -0
  60. package/lib/runtime-install.js.map +1 -0
  61. package/lib/runtime-manager.js +126 -0
  62. package/lib/runtime-manager.js.map +1 -0
  63. package/lib/runtime.js +1344 -0
  64. package/lib/runtime.js.map +1 -0
  65. package/lib/skill.js +139 -0
  66. package/lib/skill.js.map +1 -0
  67. package/lib/tools.js +528 -0
  68. package/lib/tools.js.map +1 -0
  69. package/lib/types/artifact-access.d.ts +61 -0
  70. package/lib/types/artifact-access.d.ts.map +1 -0
  71. package/lib/types/artifacts.d.ts +42 -0
  72. package/lib/types/artifacts.d.ts.map +1 -0
  73. package/lib/types/client/index.d.ts +179 -0
  74. package/lib/types/client/index.d.ts.map +1 -0
  75. package/lib/types/client/paste-images.d.ts +57 -0
  76. package/lib/types/client/paste-images.d.ts.map +1 -0
  77. package/lib/types/config.d.ts +73 -0
  78. package/lib/types/config.d.ts.map +1 -0
  79. package/lib/types/errors.d.ts +35 -0
  80. package/lib/types/errors.d.ts.map +1 -0
  81. package/lib/types/exposure.d.ts +40 -0
  82. package/lib/types/exposure.d.ts.map +1 -0
  83. package/lib/types/index.d.ts +18 -0
  84. package/lib/types/index.d.ts.map +1 -0
  85. package/lib/types/paste-images.d.ts +21 -0
  86. package/lib/types/paste-images.d.ts.map +1 -0
  87. package/lib/types/paths.d.ts +107 -0
  88. package/lib/types/paths.d.ts.map +1 -0
  89. package/lib/types/runtime-install.d.ts +49 -0
  90. package/lib/types/runtime-install.d.ts.map +1 -0
  91. package/lib/types/runtime-manager.d.ts +60 -0
  92. package/lib/types/runtime-manager.d.ts.map +1 -0
  93. package/lib/types/runtime.d.ts +389 -0
  94. package/lib/types/runtime.d.ts.map +1 -0
  95. package/lib/types/skill.d.ts +15 -0
  96. package/lib/types/skill.d.ts.map +1 -0
  97. package/lib/types/tools.d.ts +22 -0
  98. package/lib/types/tools.d.ts.map +1 -0
  99. package/lib/types/upstream.d.ts +207 -0
  100. package/lib/types/upstream.d.ts.map +1 -0
  101. package/lib/types/version.d.ts +15 -0
  102. package/lib/types/version.d.ts.map +1 -0
  103. package/lib/types/web-request.d.ts +4 -0
  104. package/lib/types/web-request.d.ts.map +1 -0
  105. package/lib/types/web.d.ts +74 -0
  106. package/lib/types/web.d.ts.map +1 -0
  107. package/lib/upstream.js +675 -0
  108. package/lib/upstream.js.map +1 -0
  109. package/lib/version.js +18 -0
  110. package/lib/version.js.map +1 -0
  111. package/lib/web-request.js +20 -0
  112. package/lib/web-request.js.map +1 -0
  113. package/lib/web.js +244 -0
  114. package/lib/web.js.map +1 -0
  115. package/package.json +139 -0
  116. package/runtime/requirements.lock +3 -0
  117. package/src/artifact-access.ts +386 -0
  118. package/src/artifacts.ts +85 -0
  119. package/src/client/index.tsx +866 -0
  120. package/src/client/paste-images.tsx +426 -0
  121. package/src/config.ts +177 -0
  122. package/src/errors.ts +62 -0
  123. package/src/exposure.ts +227 -0
  124. package/src/index.ts +122 -0
  125. package/src/paste-images.ts +234 -0
  126. package/src/paths.ts +348 -0
  127. package/src/runtime-install.ts +723 -0
  128. package/src/runtime-manager.ts +166 -0
  129. package/src/runtime.ts +1783 -0
  130. package/src/skill.ts +143 -0
  131. package/src/tools.ts +668 -0
  132. package/src/upstream.ts +861 -0
  133. package/src/version.ts +37 -0
  134. package/src/web-request.ts +17 -0
  135. package/src/web.ts +329 -0
  136. package/vendor/agent-vision-toolkit/CHANGELOG.md +16 -0
  137. package/vendor/agent-vision-toolkit/LICENSE +21 -0
  138. package/vendor/agent-vision-toolkit/README.md +399 -0
  139. package/vendor/agent-vision-toolkit/UPSTREAM_MANIFEST.json +89 -0
  140. package/vendor/agent-vision-toolkit/bin/crop +90 -0
  141. package/vendor/agent-vision-toolkit/bin/detect +13 -0
  142. package/vendor/agent-vision-toolkit/bin/glance +93 -0
  143. package/vendor/agent-vision-toolkit/bin/ground +13 -0
  144. package/vendor/agent-vision-toolkit/bin/trace +129 -0
  145. package/vendor/agent-vision-toolkit/detect.py +56 -0
  146. package/vendor/agent-vision-toolkit/ground.py +216 -0
  147. package/vendor/agent-vision-toolkit/skills/vision-tools/scripts/dominant_colors.py +224 -0
  148. package/vendor/agent-vision-toolkit/skills/vision-tools/scripts/extract_fg.py +278 -0
  149. package/vendor/agent-vision-toolkit/skills/vision-tools/scripts/html_shot.py +108 -0
  150. package/vendor/agent-vision-toolkit/skills/vision-tools/scripts/long_screenshot_ocr.py +1245 -0
  151. package/vendor/agent-vision-toolkit/skills/vision-tools/scripts/pixel_diff.py +88 -0
  152. package/vendor/agent-vision-toolkit/vision_client.py +156 -0
@@ -0,0 +1,426 @@
1
+ /** Clipboard-only multi-image input for DSH Web. */
2
+
3
+ import { useSyncExternalStore, type ReactNode } from 'react'
4
+ import type { ClientContext } from '@deepseek-ai/dsh-client-runtime/client'
5
+ import type {} from '@deepseek-ai/dsh-client-ui-conversation/client'
6
+ import type { InputTriggerSource } from '@deepseek-ai/dsh-client-ui-input-trigger/client'
7
+ import type { PropsRuntime } from '@deepseek-ai/dsh-client-ui-slots'
8
+
9
+ const SOURCE = 'vision-toolkit-pasted-image'
10
+ export const PASTE_IMAGES_ROUTE = '/_dsh/vision-toolkit/paste-images'
11
+ const MAX_IMAGES = 20
12
+ const MAX_IMAGE_BYTES = 20 * 1024 * 1024
13
+ const MAX_BATCH_BYTES = 80 * 1024 * 1024
14
+
15
+ interface PasteRecord {
16
+ ref: string
17
+ file: File
18
+ batch: PasteBatch
19
+ status: 'ready' | 'copying' | 'copied' | 'error'
20
+ error?: string | undefined
21
+ absolutePath?: string | undefined
22
+ }
23
+
24
+ interface PasteBatch {
25
+ sessionId: string
26
+ records: PasteRecord[]
27
+ inflight?: Promise<void> | undefined
28
+ unsubscribe?: (() => void) | undefined
29
+ }
30
+
31
+ interface PasteResponse {
32
+ ok: boolean
33
+ value?: { absolutePath?: string }
34
+ error?: { message?: string }
35
+ }
36
+
37
+ interface PasteOccurrence {
38
+ occurrenceId: number
39
+ source: string
40
+ ref: string
41
+ offset: number
42
+ label: string
43
+ }
44
+
45
+ type PasteDockProps = PropsRuntime<'conversation.input.dock'> & {
46
+ controller: PasteImageController
47
+ remove: (occurrence: PasteOccurrence) => void
48
+ }
49
+
50
+ interface ReferenceSourceRegistry {
51
+ registerSource: (source: InputTriggerSource) => () => void
52
+ }
53
+
54
+ interface ReferenceSourceRegistration {
55
+ dispose: () => void
56
+ owners: number
57
+ }
58
+
59
+ interface LegacyTriggerContext {
60
+ inputTriggers: ReferenceSourceRegistry
61
+ }
62
+
63
+ interface LegacySlashContext {
64
+ slash: ReferenceSourceRegistry
65
+ }
66
+
67
+ const CORDIS_ORIGINAL = Symbol.for('cordis.original')
68
+
69
+ function registryIdentity(registry: ReferenceSourceRegistry): object {
70
+ let current: object = registry
71
+ while (true) {
72
+ const original = (current as Record<symbol, unknown>)[CORDIS_ORIGINAL]
73
+ if ((typeof original !== 'object' && typeof original !== 'function') || original === null || original === current) {
74
+ return current
75
+ }
76
+ current = original
77
+ }
78
+ }
79
+
80
+ let fallbackId = 0
81
+
82
+ function id(): string {
83
+ if (typeof globalThis.crypto?.randomUUID === 'function') return globalThis.crypto.randomUUID()
84
+ fallbackId += 1
85
+ return `paste-${Date.now()}-${fallbackId}`
86
+ }
87
+
88
+ function humanBytes(bytes: number): string {
89
+ if (bytes < 1024) return `${bytes} B`
90
+ if (bytes < 1024 ** 2) return `${(bytes / 1024).toFixed(1)} KB`
91
+ return `${(bytes / 1024 ** 2).toFixed(1)} MB`
92
+ }
93
+
94
+ function message(error: unknown): string {
95
+ return error instanceof Error ? error.message : String(error)
96
+ }
97
+
98
+ function imageFiles(data: DataTransfer | null): File[] {
99
+ if (data === null) return []
100
+ const itemFiles = Array.from(data.items)
101
+ .filter(item => item.kind === 'file')
102
+ .map(item => item.getAsFile())
103
+ .filter((file): file is File => file !== null)
104
+ const candidates = itemFiles.length > 0 ? itemFiles : Array.from(data.files)
105
+ return candidates.filter(file => file.type.toLowerCase().startsWith('image/'))
106
+ }
107
+
108
+ function validateImages(files: readonly File[]): void {
109
+ if (files.length > MAX_IMAGES) throw new Error(`Paste at most ${MAX_IMAGES} images at a time`)
110
+ let total = 0
111
+ for (const file of files) {
112
+ if (!file.type.toLowerCase().startsWith('image/')) throw new Error(`${file.name || 'clipboard item'} is not an image`)
113
+ if (file.size <= 0) throw new Error(`${file.name || 'clipboard image'} is empty`)
114
+ if (file.size > MAX_IMAGE_BYTES) throw new Error(`${file.name || 'clipboard image'} exceeds ${humanBytes(MAX_IMAGE_BYTES)}`)
115
+ total += file.size
116
+ }
117
+ if (total > MAX_BATCH_BYTES) throw new Error(`Pasted images exceed ${humanBytes(MAX_BATCH_BYTES)} in total`)
118
+ }
119
+
120
+ async function responseJson(response: Response): Promise<PasteResponse> {
121
+ const body = await response.json() as PasteResponse
122
+ if (!response.ok || body.ok !== true) throw new Error(body.error?.message ?? `Image copy failed (${response.status})`)
123
+ return body
124
+ }
125
+
126
+ function pasteLabel(file: File, index: number): string {
127
+ return file.name.trim() || `clipboard-image-${index + 1}`
128
+ }
129
+
130
+ /** Owns browser File objects until DSH serializes the corresponding text references. */
131
+ export class PasteImageController {
132
+ private readonly records = new Map<string, PasteRecord>()
133
+ private readonly listeners = new Set<() => void>()
134
+ private revision = 0
135
+
136
+ constructor(private readonly ctx: ClientContext) {}
137
+
138
+ subscribe = (listener: () => void): (() => void) => {
139
+ this.listeners.add(listener)
140
+ return () => { this.listeners.delete(listener) }
141
+ }
142
+
143
+ snapshot = (): number => this.revision
144
+
145
+ private changed(): void {
146
+ this.revision += 1
147
+ for (const listener of this.listeners) listener()
148
+ }
149
+
150
+ source(): InputTriggerSource {
151
+ return {
152
+ trigger: '@',
153
+ name: SOURCE,
154
+ order: 1000,
155
+ candidates: () => Promise.resolve([]),
156
+ onPick: () => undefined,
157
+ codec: {
158
+ clipboardText: ref => `[pasted image: ${this.records.get(ref)?.file.name ?? ref}]`,
159
+ serialize: (ref, signal) => this.serialize(ref, signal),
160
+ },
161
+ }
162
+ }
163
+
164
+ recordsFor(occurrences: readonly PasteOccurrence[]): PasteRecord[] {
165
+ return occurrences
166
+ .filter(occurrence => occurrence.source === SOURCE)
167
+ .map(occurrence => this.records.get(occurrence.ref))
168
+ .filter((record): record is PasteRecord => record !== undefined)
169
+ }
170
+
171
+ private inputFor(sessionId: string) {
172
+ const actx = this.ctx.sessions.scope(sessionId as never)
173
+ if (actx === undefined) throw new Error('Open a live session before pasting images')
174
+ return this.ctx.conversation.input.for(actx)
175
+ }
176
+
177
+ private insertText(input: ReturnType<PasteImageController['inputFor']>, text: string, start: number, end = start): number {
178
+ if (text === '') return start
179
+ const snapshot = input.state.getSnapshot()
180
+ input.setDraft(snapshot.draft.slice(0, start) + text + snapshot.draft.slice(end))
181
+ return start + text.length
182
+ }
183
+
184
+ private insertRecords(
185
+ sessionId: string,
186
+ input: ReturnType<PasteImageController['inputFor']>,
187
+ files: readonly File[],
188
+ cursor: number,
189
+ ): number {
190
+ const batch: PasteBatch = { sessionId, records: [] }
191
+ const draftBeforeReferences = input.state.getSnapshot().draft
192
+ try {
193
+ const before = input.state.getSnapshot().draft.slice(0, cursor)
194
+ if (before !== '' && !/\s$/u.test(before)) cursor = this.insertText(input, ' ', cursor)
195
+ for (const [index, file] of files.entries()) {
196
+ const ref = id()
197
+ const record: PasteRecord = { ref, file, batch, status: 'ready' }
198
+ batch.records.push(record)
199
+ this.records.set(ref, record)
200
+ const snapshot = input.state.getSnapshot()
201
+ const accepted = input.insertReference({
202
+ source: SOURCE,
203
+ ref,
204
+ label: pasteLabel(file, index),
205
+ clipboardText: `[pasted image: ${pasteLabel(file, index)}]`,
206
+ }, { start: cursor, end: cursor, draftRev: snapshot.draftRev })
207
+ if (!accepted) throw new Error('The composer changed before pasted images could be inserted')
208
+ cursor += 1
209
+ const hasNext = index + 1 < files.length
210
+ const suffix = input.state.getSnapshot().draft.slice(cursor)
211
+ if (hasNext || (suffix !== '' && !/^\s/u.test(suffix))) cursor = this.insertText(input, ' ', cursor)
212
+ }
213
+ batch.unsubscribe = input.state.subscribe(() => {
214
+ const alive = new Set(input.state.getSnapshot().occurrences
215
+ .filter(occurrence => occurrence.source === SOURCE)
216
+ .map(occurrence => occurrence.ref))
217
+ let changed = false
218
+ for (const record of batch.records) {
219
+ if (alive.has(record.ref) || record.batch.inflight !== undefined) continue
220
+ changed = this.records.delete(record.ref) || changed
221
+ }
222
+ if (batch.records.every(record => !this.records.has(record.ref)) && batch.inflight === undefined) {
223
+ batch.unsubscribe?.()
224
+ batch.unsubscribe = undefined
225
+ }
226
+ if (changed) this.changed()
227
+ })
228
+ this.changed()
229
+ return cursor
230
+ } catch (error) {
231
+ input.setDraft(draftBeforeReferences)
232
+ for (const record of batch.records) this.records.delete(record.ref)
233
+ throw error
234
+ }
235
+ }
236
+
237
+ handlePaste(event: ClipboardEvent): boolean {
238
+ const files = imageFiles(event.clipboardData)
239
+ if (files.length === 0) return false
240
+ const target = event.target
241
+ if (!(target instanceof HTMLTextAreaElement) || target.closest('[data-composer-card]') === null) return false
242
+
243
+ event.preventDefault()
244
+ event.stopPropagation()
245
+ event.stopImmediatePropagation()
246
+
247
+ const sessionId = this.ctx.sessions.list.getSnapshot().current
248
+ if (sessionId === undefined) return true
249
+ const input = this.inputFor(sessionId)
250
+ const snapshot = input.state.getSnapshot()
251
+ if (snapshot.phase !== 'plain') return true
252
+
253
+ const start = Math.max(0, Math.min(target.selectionStart ?? snapshot.draft.length, snapshot.draft.length))
254
+ const end = Math.max(start, Math.min(target.selectionEnd ?? start, snapshot.draft.length))
255
+ const text = (event.clipboardData?.getData('text/plain') ?? '').replaceAll('\uFFFC', '')
256
+ try {
257
+ let cursor = this.insertText(input, text, start, end)
258
+ validateImages(files)
259
+ cursor = this.insertRecords(String(sessionId), input, files, cursor)
260
+ requestAnimationFrame(() => {
261
+ target.focus({ preventScroll: true })
262
+ target.setSelectionRange(cursor, cursor)
263
+ })
264
+ } catch (error) {
265
+ input.notify('error', message(error))
266
+ }
267
+ return true
268
+ }
269
+
270
+ remove(sessionId: string, occurrence: PasteOccurrence): void {
271
+ const record = this.records.get(occurrence.ref)
272
+ if (record?.batch.inflight !== undefined) return
273
+ const input = this.inputFor(sessionId)
274
+ const snapshot = input.state.getSnapshot()
275
+ if (snapshot.phase !== 'plain') return
276
+ const current = snapshot.occurrences.find(candidate =>
277
+ candidate.source === SOURCE
278
+ && candidate.occurrenceId === occurrence.occurrenceId
279
+ && candidate.ref === occurrence.ref)
280
+ if (current === undefined) return
281
+ const accepted = (input as typeof input & {
282
+ insertText: (text: string, span: { start: number; end: number; draftRev: number }) => boolean
283
+ }).insertText('', {
284
+ start: current.offset,
285
+ end: current.offset + 1,
286
+ draftRev: snapshot.draftRev,
287
+ })
288
+ if (!accepted) return
289
+ this.records.delete(occurrence.ref)
290
+ this.changed()
291
+ }
292
+
293
+ private async upload(batch: PasteBatch, signal: AbortSignal): Promise<void> {
294
+ if (batch.inflight !== undefined) return batch.inflight
295
+ const active = batch.records.filter(record => this.records.get(record.ref) === record)
296
+ if (active.length === 0) throw new Error('Pasted images were removed before sending')
297
+ const pending = active.filter(record => record.absolutePath === undefined)
298
+ if (pending.length === 0) return
299
+ const task = (async () => {
300
+ for (const record of pending) {
301
+ record.status = 'copying'
302
+ record.error = undefined
303
+ }
304
+ this.changed()
305
+ try {
306
+ const failures = await Promise.all(pending.map(async (record) => {
307
+ try {
308
+ if (signal.aborted) throw signal.reason ?? new DOMException('Aborted', 'AbortError')
309
+ const query = new URLSearchParams({
310
+ sessionId: batch.sessionId,
311
+ name: record.file.name || 'clipboard-image',
312
+ size: String(record.file.size),
313
+ })
314
+ const body = await responseJson(await fetch(`${PASTE_IMAGES_ROUTE}?${query.toString()}`, {
315
+ method: 'POST',
316
+ headers: { 'Content-Type': record.file.type },
317
+ body: record.file,
318
+ signal,
319
+ }))
320
+ const absolutePath = body.value?.absolutePath
321
+ if (typeof absolutePath !== 'string' || absolutePath === '') {
322
+ throw new Error('Image copy response contained an invalid path')
323
+ }
324
+ record.absolutePath = absolutePath
325
+ record.status = 'copied'
326
+ record.error = undefined
327
+ return undefined
328
+ } catch (error) {
329
+ const failure = error instanceof Error ? error : new Error(message(error))
330
+ record.status = 'error'
331
+ record.error = failure.message
332
+ return failure
333
+ }
334
+ }))
335
+ this.changed()
336
+ const failure = failures.find((error): error is Error => error !== undefined)
337
+ if (failure !== undefined) throw failure
338
+ } finally {
339
+ batch.inflight = undefined
340
+ this.changed()
341
+ }
342
+ })()
343
+ batch.inflight = task
344
+ return task
345
+ }
346
+
347
+ private async serialize(ref: string, signal: AbortSignal): Promise<string> {
348
+ const record = this.records.get(ref)
349
+ if (record === undefined) throw new Error('Pasted image is no longer available in this browser tab')
350
+ await this.upload(record.batch, signal)
351
+ if (record.absolutePath === undefined) throw new Error('Pasted image was not copied into the workspace')
352
+ return `[Pasted image available at absolute path: ${JSON.stringify(record.absolutePath)}]`
353
+ }
354
+ }
355
+
356
+ /** Minimal per-image progress, failure, and removal feedback above the composer. */
357
+ export function PasteImageDock(props: PasteDockProps): ReactNode {
358
+ useSyncExternalStore(props.controller.subscribe, props.controller.snapshot)
359
+ const occurrences = props.input.occurrences.filter(occurrence => occurrence.source === SOURCE)
360
+ const records = props.controller.recordsFor(occurrences)
361
+ if (records.length === 0) return null
362
+ return <div className="dvt-paste-dock" role="status" aria-label="Pasted images">
363
+ {occurrences.map((occurrence) => {
364
+ const record = props.controller.recordsFor([occurrence])[0]
365
+ if (record === undefined) return null
366
+ const detail = record.status === 'copying' ? 'copying…'
367
+ : record.status === 'copied' ? 'copied'
368
+ : record.status === 'error' ? record.error ?? 'copy failed'
369
+ : humanBytes(record.file.size)
370
+ return <div className="dvt-paste-chip" data-status={record.status} key={occurrence.occurrenceId}>
371
+ <span className="dvt-paste-name" title={record.file.name}>{record.file.name || 'clipboard image'}</span>
372
+ <span className="dvt-paste-detail" title={record.error}>{detail}</span>
373
+ <button
374
+ type="button"
375
+ aria-label={`Remove ${record.file.name || 'clipboard image'}`}
376
+ disabled={props.input.phase !== 'plain' || record.status === 'copying'}
377
+ onClick={() => { props.remove(occurrence) }}
378
+ >×</button>
379
+ </div>
380
+ })}
381
+ </div>
382
+ }
383
+
384
+ /** Install capture interception, the text-reference codec, and composer feedback. */
385
+ export function installPasteImages(ctx: ClientContext): void {
386
+ const controller = new PasteImageController(ctx)
387
+ const registered = new WeakMap<object, ReferenceSourceRegistration>()
388
+ const register = (scope: ClientContext, registry: ReferenceSourceRegistry): void => {
389
+ scope.effect(() => {
390
+ const identity = registryIdentity(registry)
391
+ let registration = registered.get(identity)
392
+ if (registration === undefined) {
393
+ registration = { dispose: registry.registerSource(controller.source()), owners: 0 }
394
+ registered.set(identity, registration)
395
+ }
396
+ registration.owners += 1
397
+ return () => {
398
+ if (registered.get(identity) !== registration) return
399
+ registration.owners -= 1
400
+ if (registration.owners > 0) return
401
+ registered.delete(identity)
402
+ registration.dispose()
403
+ }
404
+ }, 'dsh-vision-toolkit: pasted image reference codec')
405
+ }
406
+ ctx.inject(['slash'], (scope: ClientContext) => {
407
+ register(scope, (scope as unknown as LegacySlashContext).slash)
408
+ })
409
+ ctx.inject(['inputTriggers'], (scope: ClientContext) => {
410
+ register(scope, (scope as unknown as LegacyTriggerContext).inputTriggers)
411
+ })
412
+ ctx.effect(() => {
413
+ const listener = (event: ClipboardEvent): void => { controller.handlePaste(event) }
414
+ document.addEventListener('paste', listener, true)
415
+ return () => { document.removeEventListener('paste', listener, true) }
416
+ }, 'dsh-vision-toolkit: clipboard image capture')
417
+ ctx.slots.inject('conversation.input.dock', () => ctx.slots.register({
418
+ name: 'conversation.input.dock',
419
+ id: 'vision-toolkit-pasted-images',
420
+ order: 6,
421
+ inject: sessionId => ({
422
+ controller,
423
+ remove: (occurrence: PasteOccurrence) => { controller.remove(String(sessionId), occurrence) },
424
+ }),
425
+ }, PasteImageDock))
426
+ }
package/src/config.ts ADDED
@@ -0,0 +1,177 @@
1
+ /**
2
+ * Plugin configuration: provider endpoint and credential reference, output
3
+ * language, limits, and the external upstream runtime location. Secrets never
4
+ * live here — `provider.credential` is a DSH Credential reference resolved per
5
+ * operation through `ctx.credentials`.
6
+ * @module dsh-vision-toolkit/config
7
+ */
8
+
9
+ import z from '@deepseek-ai/schemastery'
10
+ import type Schema from '@deepseek-ai/schemastery'
11
+ import { credentialRef, type CredentialRef } from '@deepseek-ai/dsh-credentials'
12
+ import { settingsNamespace } from '@deepseek-ai/dsh-settings'
13
+ import { VisionToolkitError } from './errors.ts'
14
+
15
+ /** Settings document namespace owned by this plugin. */
16
+ export const VISION_TOOLKIT_SETTINGS_NAMESPACE = settingsNamespace('vision-toolkit')
17
+
18
+ /** Full user-facing configuration; every field defaults at the schema boundary. */
19
+ export interface VisionToolkitConfig {
20
+ provider?: {
21
+ /** OpenAI-compatible chat/completions base URL. */
22
+ baseUrl?: string
23
+ /** DSH Credential reference holding the API key (an environment-style name). */
24
+ credential?: string
25
+ /** Multimodal model name. */
26
+ model?: string
27
+ }
28
+ /** Vision output language (`zh` or `en`). */
29
+ language?: 'zh' | 'en'
30
+ /** Single remote/upstream call budget in milliseconds. */
31
+ timeoutMs?: number
32
+ /** Maximum accepted input image size in bytes. */
33
+ maxImageBytes?: number
34
+ /** Maximum decoded pixel count per input image. */
35
+ maxImagePixels?: number
36
+ /** In-flight tool execution cap per session. */
37
+ concurrency?: number
38
+ runtime?: {
39
+ /** `managed` uses the packaged snapshot and isolated venv; `external` uses a clean pinned checkout. */
40
+ mode?: 'managed' | 'external'
41
+ /** Required path to the clean pinned checkout when `mode` is `external`. */
42
+ agentVisionToolkitPath?: string
43
+ /** Optional Python 3.11+ bootstrap/interpreter override. */
44
+ python?: string
45
+ }
46
+ /** Extra directories (besides the workspace) inputs may come from. */
47
+ allowedDirs?: string[]
48
+ }
49
+
50
+ /** Configuration schema with the documented P0 defaults. */
51
+ export const Config: Schema<VisionToolkitConfig> = z.object({
52
+ provider: z.object({
53
+ baseUrl: z.string().default('https://api.inferera.com/v1'),
54
+ credential: z.string().default('VISION_API_KEY'),
55
+ model: z.string().default('gemini-3.6-flash'),
56
+ }),
57
+ language: z.union(['zh', 'en'] as const).default('zh'),
58
+ timeoutMs: z.number().default(60000),
59
+ maxImageBytes: z.number().default(10485760),
60
+ maxImagePixels: z.number().default(40000000),
61
+ concurrency: z.number().default(4),
62
+ runtime: z.object({
63
+ mode: z.union(['managed', 'external'] as const).default('managed'),
64
+ agentVisionToolkitPath: z.string(),
65
+ python: z.string(),
66
+ }),
67
+ allowedDirs: z.array(z.string()).default([]),
68
+ })
69
+
70
+ /** Configuration after static validation, with every default materialized. */
71
+ export interface ResolvedVisionToolkitConfig {
72
+ provider: {
73
+ baseUrl: string
74
+ credential: CredentialRef
75
+ model: string
76
+ }
77
+ language: 'zh' | 'en'
78
+ timeoutMs: number
79
+ maxImageBytes: number
80
+ maxImagePixels: number
81
+ concurrency: number
82
+ runtime: {
83
+ mode: 'managed' | 'external'
84
+ agentVisionToolkitPath?: string
85
+ python?: string
86
+ }
87
+ allowedDirs: string[]
88
+ }
89
+
90
+ const MAX_TIMEOUT_MS = 600000
91
+ const MAX_IMAGE_BYTES = 268435456
92
+ const MAX_IMAGE_PIXELS = 268435456
93
+ const MAX_CONCURRENCY = 16
94
+
95
+ /**
96
+ * Validate and normalize a config object (partial inputs receive the same
97
+ * defaults the schemastery schema applies). Configuration mistakes fail loud
98
+ * at plugin load (the earliest resolvable point); runtime availability is a
99
+ * separate, later concern.
100
+ * @param config - parsed config with defaults applied.
101
+ * @returns the fully defaulted, validated configuration.
102
+ */
103
+ export function resolveConfig(config: VisionToolkitConfig = {}): ResolvedVisionToolkitConfig {
104
+ const provider = config.provider ?? {}
105
+ const runtime = config.runtime ?? {}
106
+ const baseUrl = (provider.baseUrl ?? 'https://api.inferera.com/v1').trim().replace(/\/+$/, '')
107
+ if (!/^https?:\/\//i.test(baseUrl) || baseUrl.length <= 'https://'.length) {
108
+ throw new VisionToolkitError('config', 'provider.baseUrl must be an http(s) URL')
109
+ }
110
+ let credential: CredentialRef
111
+ try {
112
+ credential = credentialRef((provider.credential ?? 'VISION_API_KEY').trim())
113
+ } catch (error) {
114
+ throw new VisionToolkitError(
115
+ 'config',
116
+ `provider.credential "${provider.credential ?? 'VISION_API_KEY'}" is not a valid credential reference`,
117
+ { cause: error },
118
+ )
119
+ }
120
+ const model = (provider.model ?? 'gemini-3.6-flash').trim()
121
+ if (model.length === 0) {
122
+ throw new VisionToolkitError('config', 'provider.model must not be empty')
123
+ }
124
+ const language = config.language ?? 'zh'
125
+ if (language !== 'zh' && language !== 'en') {
126
+ throw new VisionToolkitError('config', 'language must be "zh" or "en"')
127
+ }
128
+ const timeoutMs = config.timeoutMs ?? 60000
129
+ if (!Number.isInteger(timeoutMs) || timeoutMs < 1000 || timeoutMs > MAX_TIMEOUT_MS) {
130
+ throw new VisionToolkitError('config', `timeoutMs must be an integer between 1000 and ${MAX_TIMEOUT_MS}`)
131
+ }
132
+ const maxImageBytes = config.maxImageBytes ?? 10485760
133
+ if (!Number.isInteger(maxImageBytes) || maxImageBytes < 1024 || maxImageBytes > MAX_IMAGE_BYTES) {
134
+ throw new VisionToolkitError('config', `maxImageBytes must be an integer between 1024 and ${MAX_IMAGE_BYTES}`)
135
+ }
136
+ const maxImagePixels = config.maxImagePixels ?? 40000000
137
+ if (!Number.isInteger(maxImagePixels) || maxImagePixels < 1 || maxImagePixels > MAX_IMAGE_PIXELS) {
138
+ throw new VisionToolkitError('config', `maxImagePixels must be an integer between 1 and ${MAX_IMAGE_PIXELS}`)
139
+ }
140
+ const concurrency = config.concurrency ?? 4
141
+ if (!Number.isInteger(concurrency) || concurrency < 1 || concurrency > MAX_CONCURRENCY) {
142
+ throw new VisionToolkitError('config', `concurrency must be an integer between 1 and ${MAX_CONCURRENCY}`)
143
+ }
144
+ const mode = runtime.mode ?? 'managed'
145
+ if (mode !== 'managed' && mode !== 'external') {
146
+ throw new VisionToolkitError('config', 'runtime.mode must be "managed" or "external"')
147
+ }
148
+ const toolkitPath = runtime.agentVisionToolkitPath?.trim()
149
+ if (toolkitPath !== undefined && toolkitPath.length === 0) {
150
+ throw new VisionToolkitError('config', 'runtime.agentVisionToolkitPath must not be empty when provided')
151
+ }
152
+ if (mode === 'external' && toolkitPath === undefined) {
153
+ throw new VisionToolkitError('config', 'runtime.agentVisionToolkitPath is required when runtime.mode is external')
154
+ }
155
+ if (mode === 'managed' && toolkitPath !== undefined) {
156
+ throw new VisionToolkitError('config', 'runtime.agentVisionToolkitPath is only valid when runtime.mode is external')
157
+ }
158
+ const python = runtime.python?.trim()
159
+ if (python !== undefined && python.length === 0) {
160
+ throw new VisionToolkitError('config', 'runtime.python must not be empty')
161
+ }
162
+ const allowedDirs = (config.allowedDirs ?? []).map(dir => dir.trim()).filter(dir => dir.length > 0)
163
+ return {
164
+ provider: { baseUrl, credential, model },
165
+ language,
166
+ timeoutMs,
167
+ maxImageBytes,
168
+ maxImagePixels,
169
+ concurrency,
170
+ runtime: {
171
+ mode,
172
+ ...(toolkitPath !== undefined ? { agentVisionToolkitPath: toolkitPath } : {}),
173
+ ...(python !== undefined ? { python } : {}),
174
+ },
175
+ allowedDirs,
176
+ }
177
+ }
package/src/errors.ts ADDED
@@ -0,0 +1,62 @@
1
+ /**
2
+ * Stable error vocabulary shared by the runtime, upstream adapter, and tools.
3
+ * Every failure reaching the model carries one of these codes and a message
4
+ * that never contains credentials or raw upstream stack traces.
5
+ * @module dsh-vision-toolkit/errors
6
+ */
7
+
8
+ /** Discriminant tag for every Vision Toolkit failure. */
9
+ export const VISION_TOOLKIT_ERROR_CODES = [
10
+ 'config',
11
+ 'input',
12
+ 'capacity',
13
+ 'service',
14
+ 'runtime',
15
+ 'output',
16
+ 'timeout',
17
+ 'cancelled',
18
+ 'path',
19
+ ] as const
20
+
21
+ /** Stable machine-readable error category. */
22
+ export type VisionToolkitErrorCode = typeof VISION_TOOLKIT_ERROR_CODES[number]
23
+
24
+ /** Error with a stable category; safe to surface to the model. */
25
+ export class VisionToolkitError extends Error {
26
+ readonly code: VisionToolkitErrorCode
27
+
28
+ constructor(code: VisionToolkitErrorCode, message: string, options?: { cause?: unknown }) {
29
+ super(message, options)
30
+ this.name = 'VisionToolkitError'
31
+ this.code = code
32
+ }
33
+ }
34
+
35
+ /**
36
+ * Replace every known secret occurrence in untrusted text. Used before
37
+ * upstream stderr, exit messages, or trace reports enter logs or results.
38
+ * @param text - text that may embed a secret.
39
+ * @param secrets - values that must never be surfaced.
40
+ * @returns text with each secret replaced by a fixed marker.
41
+ */
42
+ export function redactText(text: string, secrets: readonly string[]): string {
43
+ let result = text
44
+ for (const secret of secrets) {
45
+ if (secret.length === 0) continue
46
+ result = result.split(secret).join('<redacted>')
47
+ }
48
+ return result
49
+ }
50
+
51
+ /**
52
+ * Build a model-safe upstream failure line: the tool prefix plus the
53
+ * redacted stderr tail, never a JavaScript stack.
54
+ * @param tool - upstream CLI name.
55
+ * @param stderr - captured upstream stderr.
56
+ * @param secrets - values to redact.
57
+ * @returns one-line safe message.
58
+ */
59
+ export function upstreamFailureMessage(tool: string, stderr: string, secrets: readonly string[]): string {
60
+ const tail = redactText(stderr, secrets).trim().split(/\r?\n/).filter(Boolean).slice(-2).join(' ')
61
+ return tail.length === 0 ? `${tool}: upstream command failed` : `${tool}: ${tail}`
62
+ }