@anionex/dsh-vision-toolkit 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.i18n.yaml +6 -0
- package/README.md +383 -0
- package/README.zh.md +383 -0
- package/assets/dsh-conversation-artifact.png +0 -0
- package/assets/dsh-conversation-image-qa-top.png +0 -0
- package/assets/dsh-conversation-image-qa.png +0 -0
- package/assets/dsh-conversation-pixel-diff.png +0 -0
- package/assets/dsh-conversation-screenshot-debugging-top.png +0 -0
- package/assets/dsh-conversation-screenshot-debugging.png +0 -0
- package/assets/dsh-conversation-tool-call.png +0 -0
- package/assets/dsh-conversation-vision-trace.png +0 -0
- package/assets/hero.png +0 -0
- package/assets/social-preview.png +0 -0
- package/assets/upstream/README.md +16 -0
- package/assets/upstream/image-qa.webp +0 -0
- package/assets/upstream/infographic-reference.webp +0 -0
- package/assets/upstream/infographic-result.webp +0 -0
- package/assets/upstream/screenshot-debugging.webp +0 -0
- package/assets/upstream/ui-result.webp +0 -0
- package/assets/upstream/ui-sketch.webp +0 -0
- package/assets/vision-settings.png +0 -0
- package/cordis.patch.yml +6 -0
- package/docs/assets/vision-settings.png +0 -0
- package/docs/requirements-traceability/README.i18n.yaml +6 -0
- package/docs/requirements-traceability/README.md +75 -0
- package/docs/requirements-traceability/README.zh.md +75 -0
- package/examples/ui-restoration/README.i18n.yaml +6 -0
- package/examples/ui-restoration/README.md +70 -0
- package/examples/ui-restoration/README.zh.md +70 -0
- package/examples/ui-restoration/assets/final-heatmap.png +0 -0
- package/examples/ui-restoration/assets/final-report.json +83 -0
- package/examples/ui-restoration/assets/implementation.png +0 -0
- package/examples/ui-restoration/assets/initial-heatmap.png +0 -0
- package/examples/ui-restoration/assets/initial-report.json +83 -0
- package/examples/ui-restoration/assets/initial.png +0 -0
- package/examples/ui-restoration/assets/metrics.json +12 -0
- package/examples/ui-restoration/assets/reference.png +0 -0
- package/examples/ui-restoration/implementation.html +94 -0
- package/examples/ui-restoration/initial.html +57 -0
- package/lib/artifact-access.js +369 -0
- package/lib/artifact-access.js.map +1 -0
- package/lib/artifacts.js +56 -0
- package/lib/artifacts.js.map +1 -0
- package/lib/client.js +952 -0
- package/lib/client.js.map +1 -0
- package/lib/config.js +117 -0
- package/lib/config.js.map +1 -0
- package/lib/errors.js +56 -0
- package/lib/errors.js.map +1 -0
- package/lib/exposure.js +213 -0
- package/lib/exposure.js.map +1 -0
- package/lib/index.js +97 -0
- package/lib/index.js.map +1 -0
- package/lib/paste-images.js +199 -0
- package/lib/paste-images.js.map +1 -0
- package/lib/paths.js +325 -0
- package/lib/paths.js.map +1 -0
- package/lib/runtime-install.js +601 -0
- package/lib/runtime-install.js.map +1 -0
- package/lib/runtime-manager.js +126 -0
- package/lib/runtime-manager.js.map +1 -0
- package/lib/runtime.js +1344 -0
- package/lib/runtime.js.map +1 -0
- package/lib/skill.js +139 -0
- package/lib/skill.js.map +1 -0
- package/lib/tools.js +528 -0
- package/lib/tools.js.map +1 -0
- package/lib/types/artifact-access.d.ts +61 -0
- package/lib/types/artifact-access.d.ts.map +1 -0
- package/lib/types/artifacts.d.ts +42 -0
- package/lib/types/artifacts.d.ts.map +1 -0
- package/lib/types/client/index.d.ts +179 -0
- package/lib/types/client/index.d.ts.map +1 -0
- package/lib/types/client/paste-images.d.ts +57 -0
- package/lib/types/client/paste-images.d.ts.map +1 -0
- package/lib/types/config.d.ts +73 -0
- package/lib/types/config.d.ts.map +1 -0
- package/lib/types/errors.d.ts +35 -0
- package/lib/types/errors.d.ts.map +1 -0
- package/lib/types/exposure.d.ts +40 -0
- package/lib/types/exposure.d.ts.map +1 -0
- package/lib/types/index.d.ts +18 -0
- package/lib/types/index.d.ts.map +1 -0
- package/lib/types/paste-images.d.ts +21 -0
- package/lib/types/paste-images.d.ts.map +1 -0
- package/lib/types/paths.d.ts +107 -0
- package/lib/types/paths.d.ts.map +1 -0
- package/lib/types/runtime-install.d.ts +49 -0
- package/lib/types/runtime-install.d.ts.map +1 -0
- package/lib/types/runtime-manager.d.ts +60 -0
- package/lib/types/runtime-manager.d.ts.map +1 -0
- package/lib/types/runtime.d.ts +389 -0
- package/lib/types/runtime.d.ts.map +1 -0
- package/lib/types/skill.d.ts +15 -0
- package/lib/types/skill.d.ts.map +1 -0
- package/lib/types/tools.d.ts +22 -0
- package/lib/types/tools.d.ts.map +1 -0
- package/lib/types/upstream.d.ts +207 -0
- package/lib/types/upstream.d.ts.map +1 -0
- package/lib/types/version.d.ts +15 -0
- package/lib/types/version.d.ts.map +1 -0
- package/lib/types/web-request.d.ts +4 -0
- package/lib/types/web-request.d.ts.map +1 -0
- package/lib/types/web.d.ts +74 -0
- package/lib/types/web.d.ts.map +1 -0
- package/lib/upstream.js +675 -0
- package/lib/upstream.js.map +1 -0
- package/lib/version.js +18 -0
- package/lib/version.js.map +1 -0
- package/lib/web-request.js +20 -0
- package/lib/web-request.js.map +1 -0
- package/lib/web.js +244 -0
- package/lib/web.js.map +1 -0
- package/package.json +139 -0
- package/runtime/requirements.lock +3 -0
- package/src/artifact-access.ts +386 -0
- package/src/artifacts.ts +85 -0
- package/src/client/index.tsx +866 -0
- package/src/client/paste-images.tsx +426 -0
- package/src/config.ts +177 -0
- package/src/errors.ts +62 -0
- package/src/exposure.ts +227 -0
- package/src/index.ts +122 -0
- package/src/paste-images.ts +234 -0
- package/src/paths.ts +348 -0
- package/src/runtime-install.ts +723 -0
- package/src/runtime-manager.ts +166 -0
- package/src/runtime.ts +1783 -0
- package/src/skill.ts +143 -0
- package/src/tools.ts +668 -0
- package/src/upstream.ts +861 -0
- package/src/version.ts +37 -0
- package/src/web-request.ts +17 -0
- package/src/web.ts +329 -0
- package/vendor/agent-vision-toolkit/CHANGELOG.md +16 -0
- package/vendor/agent-vision-toolkit/LICENSE +21 -0
- package/vendor/agent-vision-toolkit/README.md +399 -0
- package/vendor/agent-vision-toolkit/UPSTREAM_MANIFEST.json +89 -0
- package/vendor/agent-vision-toolkit/bin/crop +90 -0
- package/vendor/agent-vision-toolkit/bin/detect +13 -0
- package/vendor/agent-vision-toolkit/bin/glance +93 -0
- package/vendor/agent-vision-toolkit/bin/ground +13 -0
- package/vendor/agent-vision-toolkit/bin/trace +129 -0
- package/vendor/agent-vision-toolkit/detect.py +56 -0
- package/vendor/agent-vision-toolkit/ground.py +216 -0
- package/vendor/agent-vision-toolkit/skills/vision-tools/scripts/dominant_colors.py +224 -0
- package/vendor/agent-vision-toolkit/skills/vision-tools/scripts/extract_fg.py +278 -0
- package/vendor/agent-vision-toolkit/skills/vision-tools/scripts/html_shot.py +108 -0
- package/vendor/agent-vision-toolkit/skills/vision-tools/scripts/long_screenshot_ocr.py +1245 -0
- package/vendor/agent-vision-toolkit/skills/vision-tools/scripts/pixel_diff.py +88 -0
- package/vendor/agent-vision-toolkit/vision_client.py +156 -0
|
@@ -0,0 +1,426 @@
|
|
|
1
|
+
/** Clipboard-only multi-image input for DSH Web. */
|
|
2
|
+
|
|
3
|
+
import { useSyncExternalStore, type ReactNode } from 'react'
|
|
4
|
+
import type { ClientContext } from '@deepseek-ai/dsh-client-runtime/client'
|
|
5
|
+
import type {} from '@deepseek-ai/dsh-client-ui-conversation/client'
|
|
6
|
+
import type { InputTriggerSource } from '@deepseek-ai/dsh-client-ui-input-trigger/client'
|
|
7
|
+
import type { PropsRuntime } from '@deepseek-ai/dsh-client-ui-slots'
|
|
8
|
+
|
|
9
|
+
const SOURCE = 'vision-toolkit-pasted-image'
|
|
10
|
+
export const PASTE_IMAGES_ROUTE = '/_dsh/vision-toolkit/paste-images'
|
|
11
|
+
const MAX_IMAGES = 20
|
|
12
|
+
const MAX_IMAGE_BYTES = 20 * 1024 * 1024
|
|
13
|
+
const MAX_BATCH_BYTES = 80 * 1024 * 1024
|
|
14
|
+
|
|
15
|
+
interface PasteRecord {
|
|
16
|
+
ref: string
|
|
17
|
+
file: File
|
|
18
|
+
batch: PasteBatch
|
|
19
|
+
status: 'ready' | 'copying' | 'copied' | 'error'
|
|
20
|
+
error?: string | undefined
|
|
21
|
+
absolutePath?: string | undefined
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
interface PasteBatch {
|
|
25
|
+
sessionId: string
|
|
26
|
+
records: PasteRecord[]
|
|
27
|
+
inflight?: Promise<void> | undefined
|
|
28
|
+
unsubscribe?: (() => void) | undefined
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
interface PasteResponse {
|
|
32
|
+
ok: boolean
|
|
33
|
+
value?: { absolutePath?: string }
|
|
34
|
+
error?: { message?: string }
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
interface PasteOccurrence {
|
|
38
|
+
occurrenceId: number
|
|
39
|
+
source: string
|
|
40
|
+
ref: string
|
|
41
|
+
offset: number
|
|
42
|
+
label: string
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
type PasteDockProps = PropsRuntime<'conversation.input.dock'> & {
|
|
46
|
+
controller: PasteImageController
|
|
47
|
+
remove: (occurrence: PasteOccurrence) => void
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
interface ReferenceSourceRegistry {
|
|
51
|
+
registerSource: (source: InputTriggerSource) => () => void
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
interface ReferenceSourceRegistration {
|
|
55
|
+
dispose: () => void
|
|
56
|
+
owners: number
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
interface LegacyTriggerContext {
|
|
60
|
+
inputTriggers: ReferenceSourceRegistry
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
interface LegacySlashContext {
|
|
64
|
+
slash: ReferenceSourceRegistry
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
const CORDIS_ORIGINAL = Symbol.for('cordis.original')
|
|
68
|
+
|
|
69
|
+
function registryIdentity(registry: ReferenceSourceRegistry): object {
|
|
70
|
+
let current: object = registry
|
|
71
|
+
while (true) {
|
|
72
|
+
const original = (current as Record<symbol, unknown>)[CORDIS_ORIGINAL]
|
|
73
|
+
if ((typeof original !== 'object' && typeof original !== 'function') || original === null || original === current) {
|
|
74
|
+
return current
|
|
75
|
+
}
|
|
76
|
+
current = original
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
let fallbackId = 0
|
|
81
|
+
|
|
82
|
+
function id(): string {
|
|
83
|
+
if (typeof globalThis.crypto?.randomUUID === 'function') return globalThis.crypto.randomUUID()
|
|
84
|
+
fallbackId += 1
|
|
85
|
+
return `paste-${Date.now()}-${fallbackId}`
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
function humanBytes(bytes: number): string {
|
|
89
|
+
if (bytes < 1024) return `${bytes} B`
|
|
90
|
+
if (bytes < 1024 ** 2) return `${(bytes / 1024).toFixed(1)} KB`
|
|
91
|
+
return `${(bytes / 1024 ** 2).toFixed(1)} MB`
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
function message(error: unknown): string {
|
|
95
|
+
return error instanceof Error ? error.message : String(error)
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
function imageFiles(data: DataTransfer | null): File[] {
|
|
99
|
+
if (data === null) return []
|
|
100
|
+
const itemFiles = Array.from(data.items)
|
|
101
|
+
.filter(item => item.kind === 'file')
|
|
102
|
+
.map(item => item.getAsFile())
|
|
103
|
+
.filter((file): file is File => file !== null)
|
|
104
|
+
const candidates = itemFiles.length > 0 ? itemFiles : Array.from(data.files)
|
|
105
|
+
return candidates.filter(file => file.type.toLowerCase().startsWith('image/'))
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
function validateImages(files: readonly File[]): void {
|
|
109
|
+
if (files.length > MAX_IMAGES) throw new Error(`Paste at most ${MAX_IMAGES} images at a time`)
|
|
110
|
+
let total = 0
|
|
111
|
+
for (const file of files) {
|
|
112
|
+
if (!file.type.toLowerCase().startsWith('image/')) throw new Error(`${file.name || 'clipboard item'} is not an image`)
|
|
113
|
+
if (file.size <= 0) throw new Error(`${file.name || 'clipboard image'} is empty`)
|
|
114
|
+
if (file.size > MAX_IMAGE_BYTES) throw new Error(`${file.name || 'clipboard image'} exceeds ${humanBytes(MAX_IMAGE_BYTES)}`)
|
|
115
|
+
total += file.size
|
|
116
|
+
}
|
|
117
|
+
if (total > MAX_BATCH_BYTES) throw new Error(`Pasted images exceed ${humanBytes(MAX_BATCH_BYTES)} in total`)
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
async function responseJson(response: Response): Promise<PasteResponse> {
|
|
121
|
+
const body = await response.json() as PasteResponse
|
|
122
|
+
if (!response.ok || body.ok !== true) throw new Error(body.error?.message ?? `Image copy failed (${response.status})`)
|
|
123
|
+
return body
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
function pasteLabel(file: File, index: number): string {
|
|
127
|
+
return file.name.trim() || `clipboard-image-${index + 1}`
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/** Owns browser File objects until DSH serializes the corresponding text references. */
|
|
131
|
+
export class PasteImageController {
|
|
132
|
+
private readonly records = new Map<string, PasteRecord>()
|
|
133
|
+
private readonly listeners = new Set<() => void>()
|
|
134
|
+
private revision = 0
|
|
135
|
+
|
|
136
|
+
constructor(private readonly ctx: ClientContext) {}
|
|
137
|
+
|
|
138
|
+
subscribe = (listener: () => void): (() => void) => {
|
|
139
|
+
this.listeners.add(listener)
|
|
140
|
+
return () => { this.listeners.delete(listener) }
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
snapshot = (): number => this.revision
|
|
144
|
+
|
|
145
|
+
private changed(): void {
|
|
146
|
+
this.revision += 1
|
|
147
|
+
for (const listener of this.listeners) listener()
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
source(): InputTriggerSource {
|
|
151
|
+
return {
|
|
152
|
+
trigger: '@',
|
|
153
|
+
name: SOURCE,
|
|
154
|
+
order: 1000,
|
|
155
|
+
candidates: () => Promise.resolve([]),
|
|
156
|
+
onPick: () => undefined,
|
|
157
|
+
codec: {
|
|
158
|
+
clipboardText: ref => `[pasted image: ${this.records.get(ref)?.file.name ?? ref}]`,
|
|
159
|
+
serialize: (ref, signal) => this.serialize(ref, signal),
|
|
160
|
+
},
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
recordsFor(occurrences: readonly PasteOccurrence[]): PasteRecord[] {
|
|
165
|
+
return occurrences
|
|
166
|
+
.filter(occurrence => occurrence.source === SOURCE)
|
|
167
|
+
.map(occurrence => this.records.get(occurrence.ref))
|
|
168
|
+
.filter((record): record is PasteRecord => record !== undefined)
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
private inputFor(sessionId: string) {
|
|
172
|
+
const actx = this.ctx.sessions.scope(sessionId as never)
|
|
173
|
+
if (actx === undefined) throw new Error('Open a live session before pasting images')
|
|
174
|
+
return this.ctx.conversation.input.for(actx)
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
private insertText(input: ReturnType<PasteImageController['inputFor']>, text: string, start: number, end = start): number {
|
|
178
|
+
if (text === '') return start
|
|
179
|
+
const snapshot = input.state.getSnapshot()
|
|
180
|
+
input.setDraft(snapshot.draft.slice(0, start) + text + snapshot.draft.slice(end))
|
|
181
|
+
return start + text.length
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
private insertRecords(
|
|
185
|
+
sessionId: string,
|
|
186
|
+
input: ReturnType<PasteImageController['inputFor']>,
|
|
187
|
+
files: readonly File[],
|
|
188
|
+
cursor: number,
|
|
189
|
+
): number {
|
|
190
|
+
const batch: PasteBatch = { sessionId, records: [] }
|
|
191
|
+
const draftBeforeReferences = input.state.getSnapshot().draft
|
|
192
|
+
try {
|
|
193
|
+
const before = input.state.getSnapshot().draft.slice(0, cursor)
|
|
194
|
+
if (before !== '' && !/\s$/u.test(before)) cursor = this.insertText(input, ' ', cursor)
|
|
195
|
+
for (const [index, file] of files.entries()) {
|
|
196
|
+
const ref = id()
|
|
197
|
+
const record: PasteRecord = { ref, file, batch, status: 'ready' }
|
|
198
|
+
batch.records.push(record)
|
|
199
|
+
this.records.set(ref, record)
|
|
200
|
+
const snapshot = input.state.getSnapshot()
|
|
201
|
+
const accepted = input.insertReference({
|
|
202
|
+
source: SOURCE,
|
|
203
|
+
ref,
|
|
204
|
+
label: pasteLabel(file, index),
|
|
205
|
+
clipboardText: `[pasted image: ${pasteLabel(file, index)}]`,
|
|
206
|
+
}, { start: cursor, end: cursor, draftRev: snapshot.draftRev })
|
|
207
|
+
if (!accepted) throw new Error('The composer changed before pasted images could be inserted')
|
|
208
|
+
cursor += 1
|
|
209
|
+
const hasNext = index + 1 < files.length
|
|
210
|
+
const suffix = input.state.getSnapshot().draft.slice(cursor)
|
|
211
|
+
if (hasNext || (suffix !== '' && !/^\s/u.test(suffix))) cursor = this.insertText(input, ' ', cursor)
|
|
212
|
+
}
|
|
213
|
+
batch.unsubscribe = input.state.subscribe(() => {
|
|
214
|
+
const alive = new Set(input.state.getSnapshot().occurrences
|
|
215
|
+
.filter(occurrence => occurrence.source === SOURCE)
|
|
216
|
+
.map(occurrence => occurrence.ref))
|
|
217
|
+
let changed = false
|
|
218
|
+
for (const record of batch.records) {
|
|
219
|
+
if (alive.has(record.ref) || record.batch.inflight !== undefined) continue
|
|
220
|
+
changed = this.records.delete(record.ref) || changed
|
|
221
|
+
}
|
|
222
|
+
if (batch.records.every(record => !this.records.has(record.ref)) && batch.inflight === undefined) {
|
|
223
|
+
batch.unsubscribe?.()
|
|
224
|
+
batch.unsubscribe = undefined
|
|
225
|
+
}
|
|
226
|
+
if (changed) this.changed()
|
|
227
|
+
})
|
|
228
|
+
this.changed()
|
|
229
|
+
return cursor
|
|
230
|
+
} catch (error) {
|
|
231
|
+
input.setDraft(draftBeforeReferences)
|
|
232
|
+
for (const record of batch.records) this.records.delete(record.ref)
|
|
233
|
+
throw error
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
handlePaste(event: ClipboardEvent): boolean {
|
|
238
|
+
const files = imageFiles(event.clipboardData)
|
|
239
|
+
if (files.length === 0) return false
|
|
240
|
+
const target = event.target
|
|
241
|
+
if (!(target instanceof HTMLTextAreaElement) || target.closest('[data-composer-card]') === null) return false
|
|
242
|
+
|
|
243
|
+
event.preventDefault()
|
|
244
|
+
event.stopPropagation()
|
|
245
|
+
event.stopImmediatePropagation()
|
|
246
|
+
|
|
247
|
+
const sessionId = this.ctx.sessions.list.getSnapshot().current
|
|
248
|
+
if (sessionId === undefined) return true
|
|
249
|
+
const input = this.inputFor(sessionId)
|
|
250
|
+
const snapshot = input.state.getSnapshot()
|
|
251
|
+
if (snapshot.phase !== 'plain') return true
|
|
252
|
+
|
|
253
|
+
const start = Math.max(0, Math.min(target.selectionStart ?? snapshot.draft.length, snapshot.draft.length))
|
|
254
|
+
const end = Math.max(start, Math.min(target.selectionEnd ?? start, snapshot.draft.length))
|
|
255
|
+
const text = (event.clipboardData?.getData('text/plain') ?? '').replaceAll('\uFFFC', '')
|
|
256
|
+
try {
|
|
257
|
+
let cursor = this.insertText(input, text, start, end)
|
|
258
|
+
validateImages(files)
|
|
259
|
+
cursor = this.insertRecords(String(sessionId), input, files, cursor)
|
|
260
|
+
requestAnimationFrame(() => {
|
|
261
|
+
target.focus({ preventScroll: true })
|
|
262
|
+
target.setSelectionRange(cursor, cursor)
|
|
263
|
+
})
|
|
264
|
+
} catch (error) {
|
|
265
|
+
input.notify('error', message(error))
|
|
266
|
+
}
|
|
267
|
+
return true
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
remove(sessionId: string, occurrence: PasteOccurrence): void {
|
|
271
|
+
const record = this.records.get(occurrence.ref)
|
|
272
|
+
if (record?.batch.inflight !== undefined) return
|
|
273
|
+
const input = this.inputFor(sessionId)
|
|
274
|
+
const snapshot = input.state.getSnapshot()
|
|
275
|
+
if (snapshot.phase !== 'plain') return
|
|
276
|
+
const current = snapshot.occurrences.find(candidate =>
|
|
277
|
+
candidate.source === SOURCE
|
|
278
|
+
&& candidate.occurrenceId === occurrence.occurrenceId
|
|
279
|
+
&& candidate.ref === occurrence.ref)
|
|
280
|
+
if (current === undefined) return
|
|
281
|
+
const accepted = (input as typeof input & {
|
|
282
|
+
insertText: (text: string, span: { start: number; end: number; draftRev: number }) => boolean
|
|
283
|
+
}).insertText('', {
|
|
284
|
+
start: current.offset,
|
|
285
|
+
end: current.offset + 1,
|
|
286
|
+
draftRev: snapshot.draftRev,
|
|
287
|
+
})
|
|
288
|
+
if (!accepted) return
|
|
289
|
+
this.records.delete(occurrence.ref)
|
|
290
|
+
this.changed()
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
private async upload(batch: PasteBatch, signal: AbortSignal): Promise<void> {
|
|
294
|
+
if (batch.inflight !== undefined) return batch.inflight
|
|
295
|
+
const active = batch.records.filter(record => this.records.get(record.ref) === record)
|
|
296
|
+
if (active.length === 0) throw new Error('Pasted images were removed before sending')
|
|
297
|
+
const pending = active.filter(record => record.absolutePath === undefined)
|
|
298
|
+
if (pending.length === 0) return
|
|
299
|
+
const task = (async () => {
|
|
300
|
+
for (const record of pending) {
|
|
301
|
+
record.status = 'copying'
|
|
302
|
+
record.error = undefined
|
|
303
|
+
}
|
|
304
|
+
this.changed()
|
|
305
|
+
try {
|
|
306
|
+
const failures = await Promise.all(pending.map(async (record) => {
|
|
307
|
+
try {
|
|
308
|
+
if (signal.aborted) throw signal.reason ?? new DOMException('Aborted', 'AbortError')
|
|
309
|
+
const query = new URLSearchParams({
|
|
310
|
+
sessionId: batch.sessionId,
|
|
311
|
+
name: record.file.name || 'clipboard-image',
|
|
312
|
+
size: String(record.file.size),
|
|
313
|
+
})
|
|
314
|
+
const body = await responseJson(await fetch(`${PASTE_IMAGES_ROUTE}?${query.toString()}`, {
|
|
315
|
+
method: 'POST',
|
|
316
|
+
headers: { 'Content-Type': record.file.type },
|
|
317
|
+
body: record.file,
|
|
318
|
+
signal,
|
|
319
|
+
}))
|
|
320
|
+
const absolutePath = body.value?.absolutePath
|
|
321
|
+
if (typeof absolutePath !== 'string' || absolutePath === '') {
|
|
322
|
+
throw new Error('Image copy response contained an invalid path')
|
|
323
|
+
}
|
|
324
|
+
record.absolutePath = absolutePath
|
|
325
|
+
record.status = 'copied'
|
|
326
|
+
record.error = undefined
|
|
327
|
+
return undefined
|
|
328
|
+
} catch (error) {
|
|
329
|
+
const failure = error instanceof Error ? error : new Error(message(error))
|
|
330
|
+
record.status = 'error'
|
|
331
|
+
record.error = failure.message
|
|
332
|
+
return failure
|
|
333
|
+
}
|
|
334
|
+
}))
|
|
335
|
+
this.changed()
|
|
336
|
+
const failure = failures.find((error): error is Error => error !== undefined)
|
|
337
|
+
if (failure !== undefined) throw failure
|
|
338
|
+
} finally {
|
|
339
|
+
batch.inflight = undefined
|
|
340
|
+
this.changed()
|
|
341
|
+
}
|
|
342
|
+
})()
|
|
343
|
+
batch.inflight = task
|
|
344
|
+
return task
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
private async serialize(ref: string, signal: AbortSignal): Promise<string> {
|
|
348
|
+
const record = this.records.get(ref)
|
|
349
|
+
if (record === undefined) throw new Error('Pasted image is no longer available in this browser tab')
|
|
350
|
+
await this.upload(record.batch, signal)
|
|
351
|
+
if (record.absolutePath === undefined) throw new Error('Pasted image was not copied into the workspace')
|
|
352
|
+
return `[Pasted image available at absolute path: ${JSON.stringify(record.absolutePath)}]`
|
|
353
|
+
}
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
/** Minimal per-image progress, failure, and removal feedback above the composer. */
|
|
357
|
+
export function PasteImageDock(props: PasteDockProps): ReactNode {
|
|
358
|
+
useSyncExternalStore(props.controller.subscribe, props.controller.snapshot)
|
|
359
|
+
const occurrences = props.input.occurrences.filter(occurrence => occurrence.source === SOURCE)
|
|
360
|
+
const records = props.controller.recordsFor(occurrences)
|
|
361
|
+
if (records.length === 0) return null
|
|
362
|
+
return <div className="dvt-paste-dock" role="status" aria-label="Pasted images">
|
|
363
|
+
{occurrences.map((occurrence) => {
|
|
364
|
+
const record = props.controller.recordsFor([occurrence])[0]
|
|
365
|
+
if (record === undefined) return null
|
|
366
|
+
const detail = record.status === 'copying' ? 'copying…'
|
|
367
|
+
: record.status === 'copied' ? 'copied'
|
|
368
|
+
: record.status === 'error' ? record.error ?? 'copy failed'
|
|
369
|
+
: humanBytes(record.file.size)
|
|
370
|
+
return <div className="dvt-paste-chip" data-status={record.status} key={occurrence.occurrenceId}>
|
|
371
|
+
<span className="dvt-paste-name" title={record.file.name}>{record.file.name || 'clipboard image'}</span>
|
|
372
|
+
<span className="dvt-paste-detail" title={record.error}>{detail}</span>
|
|
373
|
+
<button
|
|
374
|
+
type="button"
|
|
375
|
+
aria-label={`Remove ${record.file.name || 'clipboard image'}`}
|
|
376
|
+
disabled={props.input.phase !== 'plain' || record.status === 'copying'}
|
|
377
|
+
onClick={() => { props.remove(occurrence) }}
|
|
378
|
+
>×</button>
|
|
379
|
+
</div>
|
|
380
|
+
})}
|
|
381
|
+
</div>
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
/** Install capture interception, the text-reference codec, and composer feedback. */
|
|
385
|
+
export function installPasteImages(ctx: ClientContext): void {
|
|
386
|
+
const controller = new PasteImageController(ctx)
|
|
387
|
+
const registered = new WeakMap<object, ReferenceSourceRegistration>()
|
|
388
|
+
const register = (scope: ClientContext, registry: ReferenceSourceRegistry): void => {
|
|
389
|
+
scope.effect(() => {
|
|
390
|
+
const identity = registryIdentity(registry)
|
|
391
|
+
let registration = registered.get(identity)
|
|
392
|
+
if (registration === undefined) {
|
|
393
|
+
registration = { dispose: registry.registerSource(controller.source()), owners: 0 }
|
|
394
|
+
registered.set(identity, registration)
|
|
395
|
+
}
|
|
396
|
+
registration.owners += 1
|
|
397
|
+
return () => {
|
|
398
|
+
if (registered.get(identity) !== registration) return
|
|
399
|
+
registration.owners -= 1
|
|
400
|
+
if (registration.owners > 0) return
|
|
401
|
+
registered.delete(identity)
|
|
402
|
+
registration.dispose()
|
|
403
|
+
}
|
|
404
|
+
}, 'dsh-vision-toolkit: pasted image reference codec')
|
|
405
|
+
}
|
|
406
|
+
ctx.inject(['slash'], (scope: ClientContext) => {
|
|
407
|
+
register(scope, (scope as unknown as LegacySlashContext).slash)
|
|
408
|
+
})
|
|
409
|
+
ctx.inject(['inputTriggers'], (scope: ClientContext) => {
|
|
410
|
+
register(scope, (scope as unknown as LegacyTriggerContext).inputTriggers)
|
|
411
|
+
})
|
|
412
|
+
ctx.effect(() => {
|
|
413
|
+
const listener = (event: ClipboardEvent): void => { controller.handlePaste(event) }
|
|
414
|
+
document.addEventListener('paste', listener, true)
|
|
415
|
+
return () => { document.removeEventListener('paste', listener, true) }
|
|
416
|
+
}, 'dsh-vision-toolkit: clipboard image capture')
|
|
417
|
+
ctx.slots.inject('conversation.input.dock', () => ctx.slots.register({
|
|
418
|
+
name: 'conversation.input.dock',
|
|
419
|
+
id: 'vision-toolkit-pasted-images',
|
|
420
|
+
order: 6,
|
|
421
|
+
inject: sessionId => ({
|
|
422
|
+
controller,
|
|
423
|
+
remove: (occurrence: PasteOccurrence) => { controller.remove(String(sessionId), occurrence) },
|
|
424
|
+
}),
|
|
425
|
+
}, PasteImageDock))
|
|
426
|
+
}
|
package/src/config.ts
ADDED
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Plugin configuration: provider endpoint and credential reference, output
|
|
3
|
+
* language, limits, and the external upstream runtime location. Secrets never
|
|
4
|
+
* live here — `provider.credential` is a DSH Credential reference resolved per
|
|
5
|
+
* operation through `ctx.credentials`.
|
|
6
|
+
* @module dsh-vision-toolkit/config
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import z from '@deepseek-ai/schemastery'
|
|
10
|
+
import type Schema from '@deepseek-ai/schemastery'
|
|
11
|
+
import { credentialRef, type CredentialRef } from '@deepseek-ai/dsh-credentials'
|
|
12
|
+
import { settingsNamespace } from '@deepseek-ai/dsh-settings'
|
|
13
|
+
import { VisionToolkitError } from './errors.ts'
|
|
14
|
+
|
|
15
|
+
/** Settings document namespace owned by this plugin. */
|
|
16
|
+
export const VISION_TOOLKIT_SETTINGS_NAMESPACE = settingsNamespace('vision-toolkit')
|
|
17
|
+
|
|
18
|
+
/** Full user-facing configuration; every field defaults at the schema boundary. */
|
|
19
|
+
export interface VisionToolkitConfig {
|
|
20
|
+
provider?: {
|
|
21
|
+
/** OpenAI-compatible chat/completions base URL. */
|
|
22
|
+
baseUrl?: string
|
|
23
|
+
/** DSH Credential reference holding the API key (an environment-style name). */
|
|
24
|
+
credential?: string
|
|
25
|
+
/** Multimodal model name. */
|
|
26
|
+
model?: string
|
|
27
|
+
}
|
|
28
|
+
/** Vision output language (`zh` or `en`). */
|
|
29
|
+
language?: 'zh' | 'en'
|
|
30
|
+
/** Single remote/upstream call budget in milliseconds. */
|
|
31
|
+
timeoutMs?: number
|
|
32
|
+
/** Maximum accepted input image size in bytes. */
|
|
33
|
+
maxImageBytes?: number
|
|
34
|
+
/** Maximum decoded pixel count per input image. */
|
|
35
|
+
maxImagePixels?: number
|
|
36
|
+
/** In-flight tool execution cap per session. */
|
|
37
|
+
concurrency?: number
|
|
38
|
+
runtime?: {
|
|
39
|
+
/** `managed` uses the packaged snapshot and isolated venv; `external` uses a clean pinned checkout. */
|
|
40
|
+
mode?: 'managed' | 'external'
|
|
41
|
+
/** Required path to the clean pinned checkout when `mode` is `external`. */
|
|
42
|
+
agentVisionToolkitPath?: string
|
|
43
|
+
/** Optional Python 3.11+ bootstrap/interpreter override. */
|
|
44
|
+
python?: string
|
|
45
|
+
}
|
|
46
|
+
/** Extra directories (besides the workspace) inputs may come from. */
|
|
47
|
+
allowedDirs?: string[]
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/** Configuration schema with the documented P0 defaults. */
|
|
51
|
+
export const Config: Schema<VisionToolkitConfig> = z.object({
|
|
52
|
+
provider: z.object({
|
|
53
|
+
baseUrl: z.string().default('https://api.inferera.com/v1'),
|
|
54
|
+
credential: z.string().default('VISION_API_KEY'),
|
|
55
|
+
model: z.string().default('gemini-3.6-flash'),
|
|
56
|
+
}),
|
|
57
|
+
language: z.union(['zh', 'en'] as const).default('zh'),
|
|
58
|
+
timeoutMs: z.number().default(60000),
|
|
59
|
+
maxImageBytes: z.number().default(10485760),
|
|
60
|
+
maxImagePixels: z.number().default(40000000),
|
|
61
|
+
concurrency: z.number().default(4),
|
|
62
|
+
runtime: z.object({
|
|
63
|
+
mode: z.union(['managed', 'external'] as const).default('managed'),
|
|
64
|
+
agentVisionToolkitPath: z.string(),
|
|
65
|
+
python: z.string(),
|
|
66
|
+
}),
|
|
67
|
+
allowedDirs: z.array(z.string()).default([]),
|
|
68
|
+
})
|
|
69
|
+
|
|
70
|
+
/** Configuration after static validation, with every default materialized. */
|
|
71
|
+
export interface ResolvedVisionToolkitConfig {
|
|
72
|
+
provider: {
|
|
73
|
+
baseUrl: string
|
|
74
|
+
credential: CredentialRef
|
|
75
|
+
model: string
|
|
76
|
+
}
|
|
77
|
+
language: 'zh' | 'en'
|
|
78
|
+
timeoutMs: number
|
|
79
|
+
maxImageBytes: number
|
|
80
|
+
maxImagePixels: number
|
|
81
|
+
concurrency: number
|
|
82
|
+
runtime: {
|
|
83
|
+
mode: 'managed' | 'external'
|
|
84
|
+
agentVisionToolkitPath?: string
|
|
85
|
+
python?: string
|
|
86
|
+
}
|
|
87
|
+
allowedDirs: string[]
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
const MAX_TIMEOUT_MS = 600000
|
|
91
|
+
const MAX_IMAGE_BYTES = 268435456
|
|
92
|
+
const MAX_IMAGE_PIXELS = 268435456
|
|
93
|
+
const MAX_CONCURRENCY = 16
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* Validate and normalize a config object (partial inputs receive the same
|
|
97
|
+
* defaults the schemastery schema applies). Configuration mistakes fail loud
|
|
98
|
+
* at plugin load (the earliest resolvable point); runtime availability is a
|
|
99
|
+
* separate, later concern.
|
|
100
|
+
* @param config - parsed config with defaults applied.
|
|
101
|
+
* @returns the fully defaulted, validated configuration.
|
|
102
|
+
*/
|
|
103
|
+
export function resolveConfig(config: VisionToolkitConfig = {}): ResolvedVisionToolkitConfig {
|
|
104
|
+
const provider = config.provider ?? {}
|
|
105
|
+
const runtime = config.runtime ?? {}
|
|
106
|
+
const baseUrl = (provider.baseUrl ?? 'https://api.inferera.com/v1').trim().replace(/\/+$/, '')
|
|
107
|
+
if (!/^https?:\/\//i.test(baseUrl) || baseUrl.length <= 'https://'.length) {
|
|
108
|
+
throw new VisionToolkitError('config', 'provider.baseUrl must be an http(s) URL')
|
|
109
|
+
}
|
|
110
|
+
let credential: CredentialRef
|
|
111
|
+
try {
|
|
112
|
+
credential = credentialRef((provider.credential ?? 'VISION_API_KEY').trim())
|
|
113
|
+
} catch (error) {
|
|
114
|
+
throw new VisionToolkitError(
|
|
115
|
+
'config',
|
|
116
|
+
`provider.credential "${provider.credential ?? 'VISION_API_KEY'}" is not a valid credential reference`,
|
|
117
|
+
{ cause: error },
|
|
118
|
+
)
|
|
119
|
+
}
|
|
120
|
+
const model = (provider.model ?? 'gemini-3.6-flash').trim()
|
|
121
|
+
if (model.length === 0) {
|
|
122
|
+
throw new VisionToolkitError('config', 'provider.model must not be empty')
|
|
123
|
+
}
|
|
124
|
+
const language = config.language ?? 'zh'
|
|
125
|
+
if (language !== 'zh' && language !== 'en') {
|
|
126
|
+
throw new VisionToolkitError('config', 'language must be "zh" or "en"')
|
|
127
|
+
}
|
|
128
|
+
const timeoutMs = config.timeoutMs ?? 60000
|
|
129
|
+
if (!Number.isInteger(timeoutMs) || timeoutMs < 1000 || timeoutMs > MAX_TIMEOUT_MS) {
|
|
130
|
+
throw new VisionToolkitError('config', `timeoutMs must be an integer between 1000 and ${MAX_TIMEOUT_MS}`)
|
|
131
|
+
}
|
|
132
|
+
const maxImageBytes = config.maxImageBytes ?? 10485760
|
|
133
|
+
if (!Number.isInteger(maxImageBytes) || maxImageBytes < 1024 || maxImageBytes > MAX_IMAGE_BYTES) {
|
|
134
|
+
throw new VisionToolkitError('config', `maxImageBytes must be an integer between 1024 and ${MAX_IMAGE_BYTES}`)
|
|
135
|
+
}
|
|
136
|
+
const maxImagePixels = config.maxImagePixels ?? 40000000
|
|
137
|
+
if (!Number.isInteger(maxImagePixels) || maxImagePixels < 1 || maxImagePixels > MAX_IMAGE_PIXELS) {
|
|
138
|
+
throw new VisionToolkitError('config', `maxImagePixels must be an integer between 1 and ${MAX_IMAGE_PIXELS}`)
|
|
139
|
+
}
|
|
140
|
+
const concurrency = config.concurrency ?? 4
|
|
141
|
+
if (!Number.isInteger(concurrency) || concurrency < 1 || concurrency > MAX_CONCURRENCY) {
|
|
142
|
+
throw new VisionToolkitError('config', `concurrency must be an integer between 1 and ${MAX_CONCURRENCY}`)
|
|
143
|
+
}
|
|
144
|
+
const mode = runtime.mode ?? 'managed'
|
|
145
|
+
if (mode !== 'managed' && mode !== 'external') {
|
|
146
|
+
throw new VisionToolkitError('config', 'runtime.mode must be "managed" or "external"')
|
|
147
|
+
}
|
|
148
|
+
const toolkitPath = runtime.agentVisionToolkitPath?.trim()
|
|
149
|
+
if (toolkitPath !== undefined && toolkitPath.length === 0) {
|
|
150
|
+
throw new VisionToolkitError('config', 'runtime.agentVisionToolkitPath must not be empty when provided')
|
|
151
|
+
}
|
|
152
|
+
if (mode === 'external' && toolkitPath === undefined) {
|
|
153
|
+
throw new VisionToolkitError('config', 'runtime.agentVisionToolkitPath is required when runtime.mode is external')
|
|
154
|
+
}
|
|
155
|
+
if (mode === 'managed' && toolkitPath !== undefined) {
|
|
156
|
+
throw new VisionToolkitError('config', 'runtime.agentVisionToolkitPath is only valid when runtime.mode is external')
|
|
157
|
+
}
|
|
158
|
+
const python = runtime.python?.trim()
|
|
159
|
+
if (python !== undefined && python.length === 0) {
|
|
160
|
+
throw new VisionToolkitError('config', 'runtime.python must not be empty')
|
|
161
|
+
}
|
|
162
|
+
const allowedDirs = (config.allowedDirs ?? []).map(dir => dir.trim()).filter(dir => dir.length > 0)
|
|
163
|
+
return {
|
|
164
|
+
provider: { baseUrl, credential, model },
|
|
165
|
+
language,
|
|
166
|
+
timeoutMs,
|
|
167
|
+
maxImageBytes,
|
|
168
|
+
maxImagePixels,
|
|
169
|
+
concurrency,
|
|
170
|
+
runtime: {
|
|
171
|
+
mode,
|
|
172
|
+
...(toolkitPath !== undefined ? { agentVisionToolkitPath: toolkitPath } : {}),
|
|
173
|
+
...(python !== undefined ? { python } : {}),
|
|
174
|
+
},
|
|
175
|
+
allowedDirs,
|
|
176
|
+
}
|
|
177
|
+
}
|
package/src/errors.ts
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Stable error vocabulary shared by the runtime, upstream adapter, and tools.
|
|
3
|
+
* Every failure reaching the model carries one of these codes and a message
|
|
4
|
+
* that never contains credentials or raw upstream stack traces.
|
|
5
|
+
* @module dsh-vision-toolkit/errors
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
/** Discriminant tag for every Vision Toolkit failure. */
|
|
9
|
+
export const VISION_TOOLKIT_ERROR_CODES = [
|
|
10
|
+
'config',
|
|
11
|
+
'input',
|
|
12
|
+
'capacity',
|
|
13
|
+
'service',
|
|
14
|
+
'runtime',
|
|
15
|
+
'output',
|
|
16
|
+
'timeout',
|
|
17
|
+
'cancelled',
|
|
18
|
+
'path',
|
|
19
|
+
] as const
|
|
20
|
+
|
|
21
|
+
/** Stable machine-readable error category. */
|
|
22
|
+
export type VisionToolkitErrorCode = typeof VISION_TOOLKIT_ERROR_CODES[number]
|
|
23
|
+
|
|
24
|
+
/** Error with a stable category; safe to surface to the model. */
|
|
25
|
+
export class VisionToolkitError extends Error {
|
|
26
|
+
readonly code: VisionToolkitErrorCode
|
|
27
|
+
|
|
28
|
+
constructor(code: VisionToolkitErrorCode, message: string, options?: { cause?: unknown }) {
|
|
29
|
+
super(message, options)
|
|
30
|
+
this.name = 'VisionToolkitError'
|
|
31
|
+
this.code = code
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* Replace every known secret occurrence in untrusted text. Used before
|
|
37
|
+
* upstream stderr, exit messages, or trace reports enter logs or results.
|
|
38
|
+
* @param text - text that may embed a secret.
|
|
39
|
+
* @param secrets - values that must never be surfaced.
|
|
40
|
+
* @returns text with each secret replaced by a fixed marker.
|
|
41
|
+
*/
|
|
42
|
+
export function redactText(text: string, secrets: readonly string[]): string {
|
|
43
|
+
let result = text
|
|
44
|
+
for (const secret of secrets) {
|
|
45
|
+
if (secret.length === 0) continue
|
|
46
|
+
result = result.split(secret).join('<redacted>')
|
|
47
|
+
}
|
|
48
|
+
return result
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Build a model-safe upstream failure line: the tool prefix plus the
|
|
53
|
+
* redacted stderr tail, never a JavaScript stack.
|
|
54
|
+
* @param tool - upstream CLI name.
|
|
55
|
+
* @param stderr - captured upstream stderr.
|
|
56
|
+
* @param secrets - values to redact.
|
|
57
|
+
* @returns one-line safe message.
|
|
58
|
+
*/
|
|
59
|
+
export function upstreamFailureMessage(tool: string, stderr: string, secrets: readonly string[]): string {
|
|
60
|
+
const tail = redactText(stderr, secrets).trim().split(/\r?\n/).filter(Boolean).slice(-2).join(' ')
|
|
61
|
+
return tail.length === 0 ? `${tool}: upstream command failed` : `${tool}: ${tail}`
|
|
62
|
+
}
|