@mengruo/dsh-vision-toolkit 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.i18n.yaml +6 -0
- package/README.md +356 -0
- package/README.zh.md +354 -0
- package/assets/community-group-qr.png +0 -0
- package/assets/dsh-conversation-artifact.png +0 -0
- package/assets/dsh-conversation-image-qa-top.png +0 -0
- package/assets/dsh-conversation-image-qa.png +0 -0
- package/assets/dsh-conversation-pixel-diff.png +0 -0
- package/assets/dsh-conversation-screenshot-debugging-top.png +0 -0
- package/assets/dsh-conversation-screenshot-debugging.png +0 -0
- package/assets/dsh-conversation-tool-call.png +0 -0
- package/assets/dsh-conversation-vision-trace.png +0 -0
- package/assets/dsh-view-example.png +0 -0
- package/assets/hero-v2.png +0 -0
- package/assets/logo_aihubmix.png +0 -0
- package/assets/python-bootstrap.json +48 -0
- package/assets/skill/SKILL.md +329 -0
- package/assets/skill/UPSTREAM.json +71 -0
- package/assets/skill/references/gui.md +88 -0
- package/assets/skill/references/long-screenshot-ocr.md +77 -0
- package/assets/skill/references/restore-graphic.md +84 -0
- package/assets/skill/references/restore-structure.md +45 -0
- package/assets/skill/references/restore-ui.md +202 -0
- package/assets/social-preview.png +0 -0
- package/assets/upstream/README.md +18 -0
- package/assets/upstream/focus-hint-comparison-1.webp +0 -0
- package/assets/upstream/focus-hint-comparison-2.webp +0 -0
- package/assets/upstream/infographic-reference.webp +0 -0
- package/assets/upstream/infographic-result.webp +0 -0
- package/assets/upstream/ui-fast-restore-reference.webp +0 -0
- package/assets/upstream/ui-fast-restore-result.webp +0 -0
- package/assets/upstream/ui-result.webp +0 -0
- package/assets/upstream/ui-sketch.webp +0 -0
- package/assets/vision-model-test.png +0 -0
- package/assets/vision-settings.png +0 -0
- package/assets/wechat-reward.png +0 -0
- package/cordis.patch.yml +6 -0
- package/docs/aihubmix-gemini-vision.i18n.yaml +6 -0
- package/docs/aihubmix-gemini-vision.md +138 -0
- package/docs/aihubmix-gemini-vision.zh.md +138 -0
- package/docs/assets/aihubmix-api-keys.png +0 -0
- package/docs/assets/aihubmix-create-key.png +0 -0
- package/docs/assets/aihubmix-free-vision-model.png +0 -0
- package/docs/assets/aihubmix-home.png +0 -0
- package/docs/assets/aihubmix-sign-up.png +0 -0
- package/docs/assets/groq-console-home.png +0 -0
- package/docs/assets/groq-console-keys-login.png +0 -0
- package/docs/assets/groq-docs-quickstart.png +0 -0
- package/docs/assets/groq-docs-qwen3.6.png +0 -0
- package/docs/assets/groq-docs-vision.png +0 -0
- package/docs/assets/vision-settings.png +0 -0
- package/docs/dsh-desktop-install.md +67 -0
- package/docs/dsh-desktop-install.zh.md +67 -0
- package/docs/groq-qwen3.6-vision.i18n.yaml +6 -0
- package/docs/groq-qwen3.6-vision.md +236 -0
- package/docs/groq-qwen3.6-vision.zh.md +236 -0
- package/docs/python-runtime.i18n.yaml +6 -0
- package/docs/python-runtime.md +89 -0
- package/docs/python-runtime.zh.md +89 -0
- package/docs/requirements-traceability/README.i18n.yaml +6 -0
- package/docs/requirements-traceability/README.md +75 -0
- package/docs/requirements-traceability/README.zh.md +75 -0
- package/examples/ui-restoration/README.i18n.yaml +6 -0
- package/examples/ui-restoration/README.md +70 -0
- package/examples/ui-restoration/README.zh.md +70 -0
- package/examples/ui-restoration/assets/final-heatmap.png +0 -0
- package/examples/ui-restoration/assets/final-report.json +83 -0
- package/examples/ui-restoration/assets/implementation.png +0 -0
- package/examples/ui-restoration/assets/initial-heatmap.png +0 -0
- package/examples/ui-restoration/assets/initial-report.json +83 -0
- package/examples/ui-restoration/assets/initial.png +0 -0
- package/examples/ui-restoration/assets/metrics.json +12 -0
- package/examples/ui-restoration/assets/reference.png +0 -0
- package/examples/ui-restoration/implementation.html +94 -0
- package/examples/ui-restoration/initial.html +57 -0
- package/lib/artifact-access.js +369 -0
- package/lib/artifact-access.js.map +1 -0
- package/lib/artifacts.js +56 -0
- package/lib/artifacts.js.map +1 -0
- package/lib/client.js +2099 -0
- package/lib/client.js.map +1 -0
- package/lib/config.js +172 -0
- package/lib/config.js.map +1 -0
- package/lib/defaults.js +6 -0
- package/lib/defaults.js.map +1 -0
- package/lib/errors.js +56 -0
- package/lib/errors.js.map +1 -0
- package/lib/evidence-cache.js +350 -0
- package/lib/evidence-cache.js.map +1 -0
- package/lib/exposure.js +247 -0
- package/lib/exposure.js.map +1 -0
- package/lib/image-input-variants.js +882 -0
- package/lib/image-input-variants.js.map +1 -0
- package/lib/index.js +105 -0
- package/lib/index.js.map +1 -0
- package/lib/paste-images.js +217 -0
- package/lib/paste-images.js.map +1 -0
- package/lib/paths.js +354 -0
- package/lib/paths.js.map +1 -0
- package/lib/plugin-update.js +1003 -0
- package/lib/plugin-update.js.map +1 -0
- package/lib/runtime-install.js +964 -0
- package/lib/runtime-install.js.map +1 -0
- package/lib/runtime-manager.js +133 -0
- package/lib/runtime-manager.js.map +1 -0
- package/lib/runtime.js +1678 -0
- package/lib/runtime.js.map +1 -0
- package/lib/skill.js +25 -0
- package/lib/skill.js.map +1 -0
- package/lib/tools.js +549 -0
- package/lib/tools.js.map +1 -0
- package/lib/types/artifact-access.d.ts +61 -0
- package/lib/types/artifact-access.d.ts.map +1 -0
- package/lib/types/artifacts.d.ts +42 -0
- package/lib/types/artifacts.d.ts.map +1 -0
- package/lib/types/client/display-config.d.ts +24 -0
- package/lib/types/client/display-config.d.ts.map +1 -0
- package/lib/types/client/index.d.ts +350 -0
- package/lib/types/client/index.d.ts.map +1 -0
- package/lib/types/client/model-variants-hider.d.ts +40 -0
- package/lib/types/client/model-variants-hider.d.ts.map +1 -0
- package/lib/types/client/paste-images.d.ts +126 -0
- package/lib/types/client/paste-images.d.ts.map +1 -0
- package/lib/types/config.d.ts +123 -0
- package/lib/types/config.d.ts.map +1 -0
- package/lib/types/defaults.d.ts +6 -0
- package/lib/types/defaults.d.ts.map +1 -0
- package/lib/types/errors.d.ts +35 -0
- package/lib/types/errors.d.ts.map +1 -0
- package/lib/types/evidence-cache.d.ts +91 -0
- package/lib/types/evidence-cache.d.ts.map +1 -0
- package/lib/types/exposure.d.ts +50 -0
- package/lib/types/exposure.d.ts.map +1 -0
- package/lib/types/image-input-variants.d.ts +144 -0
- package/lib/types/image-input-variants.d.ts.map +1 -0
- package/lib/types/index.d.ts +19 -0
- package/lib/types/index.d.ts.map +1 -0
- package/lib/types/paste-images.d.ts +73 -0
- package/lib/types/paste-images.d.ts.map +1 -0
- package/lib/types/paths.d.ts +117 -0
- package/lib/types/paths.d.ts.map +1 -0
- package/lib/types/plugin-update.d.ts +119 -0
- package/lib/types/plugin-update.d.ts.map +1 -0
- package/lib/types/runtime-install.d.ts +92 -0
- package/lib/types/runtime-install.d.ts.map +1 -0
- package/lib/types/runtime-manager.d.ts +60 -0
- package/lib/types/runtime-manager.d.ts.map +1 -0
- package/lib/types/runtime.d.ts +412 -0
- package/lib/types/runtime.d.ts.map +1 -0
- package/lib/types/skill.d.ts +14 -0
- package/lib/types/skill.d.ts.map +1 -0
- package/lib/types/tools.d.ts +35 -0
- package/lib/types/tools.d.ts.map +1 -0
- package/lib/types/upstream.d.ts +233 -0
- package/lib/types/upstream.d.ts.map +1 -0
- package/lib/types/version.d.ts +15 -0
- package/lib/types/version.d.ts.map +1 -0
- package/lib/types/web-request.d.ts +11 -0
- package/lib/types/web-request.d.ts.map +1 -0
- package/lib/types/web.d.ts +118 -0
- package/lib/types/web.d.ts.map +1 -0
- package/lib/upstream.js +860 -0
- package/lib/upstream.js.map +1 -0
- package/lib/version.js +18 -0
- package/lib/version.js.map +1 -0
- package/lib/web-request.js +29 -0
- package/lib/web-request.js.map +1 -0
- package/lib/web.js +462 -0
- package/lib/web.js.map +1 -0
- package/package.json +149 -0
- package/patches/vision-tools-dsh.patch +970 -0
- package/runtime/requirements.lock +3 -0
- package/src/artifact-access.ts +386 -0
- package/src/artifacts.ts +85 -0
- package/src/client/display-config.ts +62 -0
- package/src/client/index.tsx +1580 -0
- package/src/client/model-variants-hider.ts +159 -0
- package/src/client/paste-images.tsx +747 -0
- package/src/config.ts +290 -0
- package/src/defaults.ts +5 -0
- package/src/errors.ts +62 -0
- package/src/evidence-cache.ts +418 -0
- package/src/exposure.ts +265 -0
- package/src/image-input-variants.ts +996 -0
- package/src/index.ts +141 -0
- package/src/paste-images.ts +286 -0
- package/src/paths.ts +383 -0
- package/src/plugin-update.ts +1143 -0
- package/src/runtime-install.ts +1161 -0
- package/src/runtime-manager.ts +174 -0
- package/src/runtime.ts +2189 -0
- package/src/skill.ts +35 -0
- package/src/tools.ts +691 -0
- package/src/upstream.ts +1078 -0
- package/src/version.ts +37 -0
- package/src/web-request.ts +27 -0
- package/src/web.ts +594 -0
- package/vendor/agent-vision-toolkit/CHANGELOG.md +30 -0
- package/vendor/agent-vision-toolkit/LICENSE +21 -0
- package/vendor/agent-vision-toolkit/README.md +405 -0
- package/vendor/agent-vision-toolkit/UPSTREAM_MANIFEST.json +94 -0
- package/vendor/agent-vision-toolkit/bin/crop +90 -0
- package/vendor/agent-vision-toolkit/bin/detect +13 -0
- package/vendor/agent-vision-toolkit/bin/glance +93 -0
- package/vendor/agent-vision-toolkit/bin/ground +13 -0
- package/vendor/agent-vision-toolkit/bin/trace +129 -0
- package/vendor/agent-vision-toolkit/detect.py +60 -0
- package/vendor/agent-vision-toolkit/ground.py +267 -0
- package/vendor/agent-vision-toolkit/skills/vision-tools/scripts/dominant_colors.py +224 -0
- package/vendor/agent-vision-toolkit/skills/vision-tools/scripts/extract_fg.py +278 -0
- package/vendor/agent-vision-toolkit/skills/vision-tools/scripts/html_shot.py +420 -0
- package/vendor/agent-vision-toolkit/skills/vision-tools/scripts/long_screenshot_ocr.py +1245 -0
- package/vendor/agent-vision-toolkit/skills/vision-tools/scripts/pixel_diff.py +88 -0
- package/vendor/agent-vision-toolkit/tests/test_vision_client.py +450 -0
- package/vendor/agent-vision-toolkit/vision_client.py +310 -0
|
@@ -0,0 +1,747 @@
|
|
|
1
|
+
/** Clipboard-only multi-image input for DSH Web. */
|
|
2
|
+
|
|
3
|
+
import { useSyncExternalStore, type ReactNode } from 'react'
|
|
4
|
+
import type { ClientContext } from '@deepseek-ai/dsh-client-runtime/client'
|
|
5
|
+
import type {} from '@deepseek-ai/dsh-client-ui-conversation/client'
|
|
6
|
+
import type { InputTriggerSource } from '@deepseek-ai/dsh-client-ui-input-trigger/client'
|
|
7
|
+
import type { PropsRuntime } from '@deepseek-ai/dsh-client-ui-slots'
|
|
8
|
+
import { readDisplayConfig } from './display-config.ts'
|
|
9
|
+
|
|
10
|
+
const SOURCE = 'vision-toolkit-pasted-image'
|
|
11
|
+
export const PASTE_IMAGES_ROUTE = '/_dsh/vision-toolkit/paste-images'
|
|
12
|
+
export const PASTE_POLICY_ROUTE = '/_dsh/vision-toolkit/paste-policy'
|
|
13
|
+
const MAX_IMAGES = 20
|
|
14
|
+
/** Hard per-image paste ceiling; must match MAX_PASTE_IMAGE_BYTES on the server. */
|
|
15
|
+
const MAX_IMAGE_BYTES = 20 * 1024 * 1024
|
|
16
|
+
const MAX_BATCH_BYTES = 80 * 1024 * 1024
|
|
17
|
+
/** A confirmed paste verdict older than this is unknown again, even while a refresh is in flight. */
|
|
18
|
+
const VERDICT_MAX_AGE_MS = 15000
|
|
19
|
+
|
|
20
|
+
interface PasteRecord {
|
|
21
|
+
ref: string
|
|
22
|
+
file: File
|
|
23
|
+
batch: PasteBatch
|
|
24
|
+
status: 'ready' | 'copying' | 'copied' | 'error'
|
|
25
|
+
error?: string | undefined
|
|
26
|
+
absolutePath?: string | undefined
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
interface PasteBatch {
|
|
30
|
+
sessionId: string
|
|
31
|
+
records: PasteRecord[]
|
|
32
|
+
inflight?: Promise<void> | undefined
|
|
33
|
+
unsubscribe?: (() => void) | undefined
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/** One model route the host asks the browser to switch to before the native paste flow. */
|
|
37
|
+
interface PasteSwitchRoute {
|
|
38
|
+
provider: string
|
|
39
|
+
model: string
|
|
40
|
+
label: string
|
|
41
|
+
reasoningEffort?: string
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/** A fresh host verdict for one Session and model label. */
|
|
45
|
+
interface PasteVerdictValue {
|
|
46
|
+
takeOver: boolean
|
|
47
|
+
autoSwitch?: PasteSwitchRoute
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
interface PasteResponse {
|
|
51
|
+
ok: boolean
|
|
52
|
+
value?: { absolutePath?: string }
|
|
53
|
+
error?: { message?: string }
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
interface PasteOccurrence {
|
|
57
|
+
occurrenceId: number
|
|
58
|
+
source: string
|
|
59
|
+
ref: string
|
|
60
|
+
offset: number
|
|
61
|
+
label: string
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
type PasteDockProps = PropsRuntime<'conversation.input.dock'> & {
|
|
65
|
+
controller: PasteImageController
|
|
66
|
+
remove: (occurrence: PasteOccurrence) => void
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
interface ReferenceSourceRegistry {
|
|
70
|
+
registerSource: (source: InputTriggerSource) => () => void
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
interface ReferenceSourceRegistration {
|
|
74
|
+
dispose: () => void
|
|
75
|
+
owners: number
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
interface LegacyTriggerContext {
|
|
79
|
+
inputTriggers: ReferenceSourceRegistry
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
interface LegacySlashContext {
|
|
83
|
+
slash: ReferenceSourceRegistry
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
const CORDIS_ORIGINAL = Symbol.for('cordis.original')
|
|
87
|
+
|
|
88
|
+
function registryIdentity(registry: ReferenceSourceRegistry): object {
|
|
89
|
+
let current: object = registry
|
|
90
|
+
while (true) {
|
|
91
|
+
const original = (current as Record<symbol, unknown>)[CORDIS_ORIGINAL]
|
|
92
|
+
if ((typeof original !== 'object' && typeof original !== 'function') || original === null || original === current) {
|
|
93
|
+
return current
|
|
94
|
+
}
|
|
95
|
+
current = original
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
let fallbackId = 0
|
|
100
|
+
|
|
101
|
+
function id(): string {
|
|
102
|
+
if (typeof globalThis.crypto?.randomUUID === 'function') return globalThis.crypto.randomUUID()
|
|
103
|
+
fallbackId += 1
|
|
104
|
+
return `paste-${Date.now()}-${fallbackId}`
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
function humanBytes(bytes: number): string {
|
|
108
|
+
if (bytes < 1024) return `${bytes} B`
|
|
109
|
+
if (bytes < 1024 ** 2) return `${(bytes / 1024).toFixed(1)} KB`
|
|
110
|
+
return `${(bytes / 1024 ** 2).toFixed(1)} MB`
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
function message(error: unknown): string {
|
|
114
|
+
return error instanceof Error ? error.message : String(error)
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
function imageFiles(data: DataTransfer | null): File[] {
|
|
118
|
+
if (data === null) return []
|
|
119
|
+
const itemFiles = Array.from(data.items)
|
|
120
|
+
.filter(item => item.kind === 'file')
|
|
121
|
+
.map(item => item.getAsFile())
|
|
122
|
+
.filter((file): file is File => file !== null)
|
|
123
|
+
const candidates = itemFiles.length > 0 ? itemFiles : Array.from(data.files)
|
|
124
|
+
return candidates.filter(file => file.type.toLowerCase().startsWith('image/'))
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/**
|
|
128
|
+
* The selector label the model picker currently shows, or '' when none is
|
|
129
|
+
* readable. Matches the host ModelSelect trigger aria-labels ("Select model,
|
|
130
|
+
* current …" / "选择模型,当前 …"); any other label wording falls back to the
|
|
131
|
+
* session-header verdict, which is stale until the next request.
|
|
132
|
+
*/
|
|
133
|
+
function currentModelLabel(): string {
|
|
134
|
+
const buttons = document.querySelectorAll('button[aria-label]')
|
|
135
|
+
for (const button of buttons) {
|
|
136
|
+
const label = button.getAttribute('aria-label') ?? ''
|
|
137
|
+
if (/select model|current model|选择模型/iu.test(label)) return label
|
|
138
|
+
}
|
|
139
|
+
return ''
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
/** Verdict cache key: the model label is part of the answer, so a switch invalidates it. */
|
|
143
|
+
function verdictKey(sessionId: string, modelLabel: string): string {
|
|
144
|
+
return `${sessionId}|${modelLabel}`
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
function validateImages(files: readonly File[]): void {
|
|
148
|
+
if (files.length > MAX_IMAGES) throw new Error(`Paste at most ${MAX_IMAGES} images at a time`)
|
|
149
|
+
let total = 0
|
|
150
|
+
for (const file of files) {
|
|
151
|
+
if (!file.type.toLowerCase().startsWith('image/')) throw new Error(`${file.name || 'clipboard item'} is not an image`)
|
|
152
|
+
if (file.size <= 0) throw new Error(`${file.name || 'clipboard image'} is empty`)
|
|
153
|
+
if (file.size > MAX_IMAGE_BYTES) throw new Error(`${file.name || 'clipboard image'} exceeds ${humanBytes(MAX_IMAGE_BYTES)}`)
|
|
154
|
+
total += file.size
|
|
155
|
+
}
|
|
156
|
+
if (total > MAX_BATCH_BYTES) throw new Error(`Pasted images exceed ${humanBytes(MAX_BATCH_BYTES)} in total`)
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
async function responseJson(response: Response): Promise<PasteResponse> {
|
|
160
|
+
const body = await response.json() as PasteResponse
|
|
161
|
+
if (!response.ok || body.ok !== true) throw new Error(body.error?.message ?? `Image copy failed (${response.status})`)
|
|
162
|
+
return body
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
function pasteLabel(file: File, index: number): string {
|
|
166
|
+
return file.name.trim() || `clipboard-image-${index + 1}`
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
/** Owns browser File objects until DSH serializes the corresponding text references. */
|
|
170
|
+
export class PasteImageController {
|
|
171
|
+
private readonly records = new Map<string, PasteRecord>()
|
|
172
|
+
private readonly listeners = new Set<() => void>()
|
|
173
|
+
private revision = 0
|
|
174
|
+
private readonly verdicts = new Map<string, {
|
|
175
|
+
takeOver: boolean
|
|
176
|
+
autoSwitch?: PasteSwitchRoute
|
|
177
|
+
at: number
|
|
178
|
+
pending: boolean
|
|
179
|
+
}>()
|
|
180
|
+
/** Guards the synthetic replay paste from re-entering capture interception. */
|
|
181
|
+
private replaying = false
|
|
182
|
+
|
|
183
|
+
constructor(private readonly ctx: ClientContext) {}
|
|
184
|
+
|
|
185
|
+
subscribe = (listener: () => void): (() => void) => {
|
|
186
|
+
this.listeners.add(listener)
|
|
187
|
+
return () => { this.listeners.delete(listener) }
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
snapshot = (): number => this.revision
|
|
191
|
+
|
|
192
|
+
private changed(): void {
|
|
193
|
+
this.revision += 1
|
|
194
|
+
for (const listener of this.listeners) listener()
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
source(): InputTriggerSource {
|
|
198
|
+
return {
|
|
199
|
+
trigger: '@',
|
|
200
|
+
name: SOURCE,
|
|
201
|
+
order: 1000,
|
|
202
|
+
candidates: () => Promise.resolve([]),
|
|
203
|
+
onPick: () => undefined,
|
|
204
|
+
codec: {
|
|
205
|
+
clipboardText: ref => `[pasted image: ${this.records.get(ref)?.file.name ?? ref}]`,
|
|
206
|
+
serialize: (ref, signal) => this.serialize(ref, signal),
|
|
207
|
+
},
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
recordsFor(occurrences: readonly PasteOccurrence[]): PasteRecord[] {
|
|
212
|
+
return occurrences
|
|
213
|
+
.filter(occurrence => occurrence.source === SOURCE)
|
|
214
|
+
.map(occurrence => this.records.get(occurrence.ref))
|
|
215
|
+
.filter((record): record is PasteRecord => record !== undefined)
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
private inputFor(sessionId: string) {
|
|
219
|
+
const actx = this.ctx.sessions.scope(sessionId as never)
|
|
220
|
+
if (actx === undefined) throw new Error('Open a live session before pasting images')
|
|
221
|
+
return this.ctx.conversation.input.for(actx)
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
private insertText(input: ReturnType<PasteImageController['inputFor']>, text: string, start: number, end = start): number {
|
|
225
|
+
if (text === '') return start
|
|
226
|
+
const snapshot = input.state.getSnapshot()
|
|
227
|
+
input.setDraft(snapshot.draft.slice(0, start) + text + snapshot.draft.slice(end))
|
|
228
|
+
return start + text.length
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
private insertRecords(
|
|
232
|
+
sessionId: string,
|
|
233
|
+
input: ReturnType<PasteImageController['inputFor']>,
|
|
234
|
+
files: readonly File[],
|
|
235
|
+
cursor: number,
|
|
236
|
+
): number {
|
|
237
|
+
const batch: PasteBatch = { sessionId, records: [] }
|
|
238
|
+
const draftBeforeReferences = input.state.getSnapshot().draft
|
|
239
|
+
try {
|
|
240
|
+
const before = input.state.getSnapshot().draft.slice(0, cursor)
|
|
241
|
+
if (before !== '' && !/\s$/u.test(before)) cursor = this.insertText(input, ' ', cursor)
|
|
242
|
+
for (const [index, file] of files.entries()) {
|
|
243
|
+
const ref = id()
|
|
244
|
+
const record: PasteRecord = { ref, file, batch, status: 'ready' }
|
|
245
|
+
batch.records.push(record)
|
|
246
|
+
this.records.set(ref, record)
|
|
247
|
+
const snapshot = input.state.getSnapshot()
|
|
248
|
+
const accepted = input.insertReference({
|
|
249
|
+
source: SOURCE,
|
|
250
|
+
ref,
|
|
251
|
+
label: pasteLabel(file, index),
|
|
252
|
+
clipboardText: `[pasted image: ${pasteLabel(file, index)}]`,
|
|
253
|
+
}, { start: cursor, end: cursor, draftRev: snapshot.draftRev })
|
|
254
|
+
if (!accepted) throw new Error('The composer changed before pasted images could be inserted')
|
|
255
|
+
cursor += 1
|
|
256
|
+
const hasNext = index + 1 < files.length
|
|
257
|
+
const suffix = input.state.getSnapshot().draft.slice(cursor)
|
|
258
|
+
if (hasNext || (suffix !== '' && !/^\s/u.test(suffix))) cursor = this.insertText(input, ' ', cursor)
|
|
259
|
+
}
|
|
260
|
+
batch.unsubscribe = input.state.subscribe(() => {
|
|
261
|
+
const alive = new Set(input.state.getSnapshot().occurrences
|
|
262
|
+
.filter(occurrence => occurrence.source === SOURCE)
|
|
263
|
+
.map(occurrence => occurrence.ref))
|
|
264
|
+
let changed = false
|
|
265
|
+
for (const record of batch.records) {
|
|
266
|
+
if (alive.has(record.ref) || record.batch.inflight !== undefined) continue
|
|
267
|
+
changed = this.records.delete(record.ref) || changed
|
|
268
|
+
}
|
|
269
|
+
if (batch.records.every(record => !this.records.has(record.ref)) && batch.inflight === undefined) {
|
|
270
|
+
batch.unsubscribe?.()
|
|
271
|
+
batch.unsubscribe = undefined
|
|
272
|
+
}
|
|
273
|
+
if (changed) this.changed()
|
|
274
|
+
})
|
|
275
|
+
this.changed()
|
|
276
|
+
return cursor
|
|
277
|
+
} catch (error) {
|
|
278
|
+
input.setDraft(draftBeforeReferences)
|
|
279
|
+
for (const record of batch.records) this.records.delete(record.ref)
|
|
280
|
+
throw error
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
/**
|
|
285
|
+
* The host's verdict for one Session and selector label, when fresh. The
|
|
286
|
+
* last CONFIRMED answer is authoritative while a background refresh is in
|
|
287
|
+
* flight (the paste acts on what the host last said; the refresh only
|
|
288
|
+
* covers the next paste). A label that changed since the confirmation
|
|
289
|
+
* answers undefined, so the native attachment flow stays the default.
|
|
290
|
+
* @param sessionId - the live Session the paste belongs to.
|
|
291
|
+
* @param modelLabel - the model-selector label currently shown.
|
|
292
|
+
* @returns the fresh confirmed verdict, or undefined when unconfirmed.
|
|
293
|
+
*/
|
|
294
|
+
private verdictFor(sessionId: string, modelLabel: string): PasteVerdictValue | undefined {
|
|
295
|
+
const entry = this.verdicts.get(verdictKey(sessionId, modelLabel))
|
|
296
|
+
if (entry === undefined || entry.at === 0) return undefined
|
|
297
|
+
if (Date.now() - entry.at > VERDICT_MAX_AGE_MS) return undefined
|
|
298
|
+
return { takeOver: entry.takeOver, ...(entry.autoSwitch === undefined ? {} : { autoSwitch: entry.autoSwitch }) }
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
/**
|
|
302
|
+
* The exact model route the live model catalog reports for one Session.
|
|
303
|
+
* Unreadable routes answer undefined, so the verdict falls back to the
|
|
304
|
+
* selector label alone.
|
|
305
|
+
* @param sessionId - the live Session id.
|
|
306
|
+
* @returns the current provider/model selection, when readable.
|
|
307
|
+
*/
|
|
308
|
+
private async readSelection(sessionId: string): Promise<{ provider: string; model: string; reasoningEffort?: string } | undefined> {
|
|
309
|
+
const connection = this.ctx.get('connection') as { api: { sessions: {
|
|
310
|
+
models(request: { sessionId: string }): Promise<{ result: {
|
|
311
|
+
ok: true
|
|
312
|
+
value: { current?: { provider: string; model: string; reasoningEffort?: string } | null }
|
|
313
|
+
} | { ok: false; error: { code: string; message: string } } }>
|
|
314
|
+
} } } | undefined
|
|
315
|
+
if (connection === undefined) return undefined
|
|
316
|
+
try {
|
|
317
|
+
const { result } = await connection.api.sessions.models({ sessionId })
|
|
318
|
+
if (!result.ok) return undefined
|
|
319
|
+
const current = result.value.current
|
|
320
|
+
if (current === undefined || current === null || current.provider === '' || current.model === '') return undefined
|
|
321
|
+
return {
|
|
322
|
+
provider: current.provider,
|
|
323
|
+
model: current.model,
|
|
324
|
+
...(current.reasoningEffort === undefined ? {} : { reasoningEffort: current.reasoningEffort }),
|
|
325
|
+
}
|
|
326
|
+
} catch {
|
|
327
|
+
return undefined
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
/**
|
|
332
|
+
* Ask the host what to do with a paste for the current model, and cache the
|
|
333
|
+
* answer per Session and selector label. A model switch changes the label,
|
|
334
|
+
* which changes the cache key, so a stale verdict never outlives the model
|
|
335
|
+
* it described. The exact selection rides along when the live model catalog
|
|
336
|
+
* is readable, so the host can answer with an auto-switch route; a 404
|
|
337
|
+
* simply leaves the verdict unconfirmed; the next focus or paste retries.
|
|
338
|
+
* @param sessionId - the live Session to ask about.
|
|
339
|
+
* @param modelLabel - the model-selector label currently shown.
|
|
340
|
+
*/
|
|
341
|
+
refreshVerdict(sessionId: string, modelLabel: string): void {
|
|
342
|
+
const key = verdictKey(sessionId, modelLabel)
|
|
343
|
+
const cached = this.verdicts.get(key)
|
|
344
|
+
// Dedupe only on an in-flight request, never on freshness: the host's
|
|
345
|
+
// model route can change under an unchanged Session id.
|
|
346
|
+
if (cached?.pending) return
|
|
347
|
+
const entry = {
|
|
348
|
+
pending: true,
|
|
349
|
+
takeOver: cached ? cached.takeOver : false,
|
|
350
|
+
at: cached ? cached.at : 0,
|
|
351
|
+
...(cached?.autoSwitch === undefined ? {} : { autoSwitch: cached.autoSwitch }),
|
|
352
|
+
}
|
|
353
|
+
this.verdicts.set(key, entry)
|
|
354
|
+
void (async () => {
|
|
355
|
+
const selection = await this.readSelection(sessionId)
|
|
356
|
+
const query = new URLSearchParams({ sessionId })
|
|
357
|
+
if (modelLabel !== '') query.set('model', modelLabel)
|
|
358
|
+
if (selection !== undefined) {
|
|
359
|
+
query.set('provider', selection.provider)
|
|
360
|
+
query.set('modelId', selection.model)
|
|
361
|
+
if (selection.reasoningEffort !== undefined) query.set('reasoningEffort', selection.reasoningEffort)
|
|
362
|
+
}
|
|
363
|
+
let request: Promise<Response>
|
|
364
|
+
try {
|
|
365
|
+
request = fetch(`${PASTE_POLICY_ROUTE}?${query.toString()}`)
|
|
366
|
+
} catch {
|
|
367
|
+
// No fetch surface (test runtime, pre-fetch bootstrap): leave the
|
|
368
|
+
// verdict unconfirmed rather than letting the paste listener die.
|
|
369
|
+
entry.pending = false
|
|
370
|
+
return
|
|
371
|
+
}
|
|
372
|
+
request
|
|
373
|
+
.then((response) => {
|
|
374
|
+
if (response.status === 404) {
|
|
375
|
+
// Route not mounted yet (plugin load race, hot reload): forget every
|
|
376
|
+
// verdict and retry on the next focus or paste instead of standing
|
|
377
|
+
// down for the page lifetime.
|
|
378
|
+
this.verdicts.clear()
|
|
379
|
+
return null
|
|
380
|
+
}
|
|
381
|
+
if (!response.ok) throw new Error(`paste policy ${response.status}`)
|
|
382
|
+
return response.json() as Promise<{ ok: true; value: PasteVerdictValue }>
|
|
383
|
+
})
|
|
384
|
+
.then((body) => {
|
|
385
|
+
entry.pending = false
|
|
386
|
+
if (body !== null) {
|
|
387
|
+
entry.takeOver = body.value.takeOver === true
|
|
388
|
+
if (body.value.autoSwitch !== undefined) entry.autoSwitch = body.value.autoSwitch
|
|
389
|
+
else delete entry.autoSwitch
|
|
390
|
+
entry.at = Date.now()
|
|
391
|
+
}
|
|
392
|
+
})
|
|
393
|
+
.catch(() => {
|
|
394
|
+
entry.pending = false
|
|
395
|
+
})
|
|
396
|
+
})()
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
/**
|
|
400
|
+
* Switch one Session to the route the host validated, through the same
|
|
401
|
+
* model-directory seat the selector uses when present (so the shared UI
|
|
402
|
+
* state moves with the session), falling back to the raw RPC.
|
|
403
|
+
* @param sessionId - the live Session id.
|
|
404
|
+
* @param route - the validated variant route.
|
|
405
|
+
*/
|
|
406
|
+
private async switchModel(sessionId: string, route: PasteSwitchRoute): Promise<void> {
|
|
407
|
+
const directories = this.ctx.get('modelDirectories') as {
|
|
408
|
+
directoryFor(id: string): { select(selection: { provider: string; model: string; reasoningEffort?: string }): Promise<void> }
|
|
409
|
+
} | undefined
|
|
410
|
+
if (directories !== undefined) {
|
|
411
|
+
// The label is a display hint; the seat only needs the exact route.
|
|
412
|
+
await directories.directoryFor(sessionId).select({
|
|
413
|
+
provider: route.provider,
|
|
414
|
+
model: route.model,
|
|
415
|
+
...(route.reasoningEffort === undefined ? {} : { reasoningEffort: route.reasoningEffort }),
|
|
416
|
+
})
|
|
417
|
+
return
|
|
418
|
+
}
|
|
419
|
+
const connection = this.ctx.get('connection') as { api: { sessions: {
|
|
420
|
+
selectModel(request: {
|
|
421
|
+
sessionId: string
|
|
422
|
+
provider: string
|
|
423
|
+
model: string
|
|
424
|
+
reasoningEffort?: string
|
|
425
|
+
}): Promise<{ result: { ok: boolean; error?: { code: string; message: string } } }>
|
|
426
|
+
} } } | undefined
|
|
427
|
+
if (connection === undefined) throw new Error('No model switch channel is available in this Web application')
|
|
428
|
+
const { result } = await connection.api.sessions.selectModel({
|
|
429
|
+
sessionId,
|
|
430
|
+
provider: route.provider,
|
|
431
|
+
model: route.model,
|
|
432
|
+
...(route.reasoningEffort === undefined ? {} : { reasoningEffort: route.reasoningEffort }),
|
|
433
|
+
})
|
|
434
|
+
if (!result.ok) throw new Error(`${result.error?.code ?? 'select-model-failed'}: ${result.error?.message ?? 'unknown error'}`)
|
|
435
|
+
}
|
|
436
|
+
|
|
437
|
+
/**
|
|
438
|
+
* Replay a swallowed paste as a synthetic clipboard event so the composer's
|
|
439
|
+
* own intake (limits, thumbnails, keyboard) runs with the captured files.
|
|
440
|
+
* @returns false when the environment cannot construct a clipboard payload.
|
|
441
|
+
*/
|
|
442
|
+
private replayPaste(target: HTMLTextAreaElement, files: readonly File[], text: string): boolean {
|
|
443
|
+
let data: DataTransfer
|
|
444
|
+
try {
|
|
445
|
+
data = new DataTransfer()
|
|
446
|
+
for (const file of files) data.items.add(file)
|
|
447
|
+
if (text !== '') data.setData('text/plain', text)
|
|
448
|
+
} catch {
|
|
449
|
+
return false
|
|
450
|
+
}
|
|
451
|
+
let event: ClipboardEvent
|
|
452
|
+
try {
|
|
453
|
+
event = new ClipboardEvent('paste', {
|
|
454
|
+
clipboardData: data,
|
|
455
|
+
bubbles: true,
|
|
456
|
+
cancelable: true,
|
|
457
|
+
} as ClipboardEventInit)
|
|
458
|
+
} catch {
|
|
459
|
+
return false
|
|
460
|
+
}
|
|
461
|
+
if (event.clipboardData === null || event.clipboardData.files.length === 0) return false
|
|
462
|
+
this.replaying = true
|
|
463
|
+
try {
|
|
464
|
+
target.dispatchEvent(event)
|
|
465
|
+
} finally {
|
|
466
|
+
this.replaying = false
|
|
467
|
+
}
|
|
468
|
+
return true
|
|
469
|
+
}
|
|
470
|
+
|
|
471
|
+
/**
|
|
472
|
+
* Auto-switch flow: switch the Session to the image-input variant, announce
|
|
473
|
+
* it, then replay the paste into the composer's native intake. A failed
|
|
474
|
+
* switch, or an environment that cannot replay clipboard bytes, degrades to
|
|
475
|
+
* the path takeover with the same files.
|
|
476
|
+
* @param sessionId - the live Session id.
|
|
477
|
+
* @param target - the composer textarea the paste landed on.
|
|
478
|
+
* @param files - the captured image files.
|
|
479
|
+
* @param text - same-paste text, replayed alongside the files.
|
|
480
|
+
* @param route - the validated variant route to switch to.
|
|
481
|
+
*/
|
|
482
|
+
private async autoSwitchPaste(
|
|
483
|
+
sessionId: string,
|
|
484
|
+
target: HTMLTextAreaElement,
|
|
485
|
+
files: readonly File[],
|
|
486
|
+
text: string,
|
|
487
|
+
route: PasteSwitchRoute,
|
|
488
|
+
): Promise<void> {
|
|
489
|
+
const input = this.inputFor(sessionId)
|
|
490
|
+
try {
|
|
491
|
+
await this.switchModel(sessionId, route)
|
|
492
|
+
const { hidden } = await readDisplayConfig()
|
|
493
|
+
input.notify('info', hidden
|
|
494
|
+
? 'Visual enhancement active: pasted images keep the native attachment flow'
|
|
495
|
+
: `Switched to ${route.label || `${route.model} (Vision Toolkit)`}; pasted images now keep the native attachment flow`)
|
|
496
|
+
} catch (error) {
|
|
497
|
+
input.notify('error', `Model switch failed; images will be sent as workspace paths: ${message(error)}`)
|
|
498
|
+
this.takeoverPaste(sessionId, target, files, text)
|
|
499
|
+
return
|
|
500
|
+
}
|
|
501
|
+
// Replaying lets the composer's own intake run (thumbnail, limits,
|
|
502
|
+
// keyboard); if the environment cannot replay clipboard bytes, the
|
|
503
|
+
// images still land as workspace paths.
|
|
504
|
+
const before = input.state.getSnapshot().imageIds.length
|
|
505
|
+
const replayed = this.replayPaste(target, files, text)
|
|
506
|
+
const after = input.state.getSnapshot().imageIds.length
|
|
507
|
+
if (!replayed || after <= before) {
|
|
508
|
+
this.takeoverPaste(sessionId, target, files, text)
|
|
509
|
+
}
|
|
510
|
+
}
|
|
511
|
+
|
|
512
|
+
/**
|
|
513
|
+
* Path-takeover flow: insert the same-paste text and every image as a text
|
|
514
|
+
* reference that serializes to the image's workspace path on send.
|
|
515
|
+
* @param sessionId - the live Session id.
|
|
516
|
+
* @param target - the composer textarea the paste landed on.
|
|
517
|
+
* @param files - the captured image files.
|
|
518
|
+
* @param text - same-paste text.
|
|
519
|
+
*/
|
|
520
|
+
private takeoverPaste(
|
|
521
|
+
sessionId: string,
|
|
522
|
+
target: HTMLTextAreaElement,
|
|
523
|
+
files: readonly File[],
|
|
524
|
+
text: string,
|
|
525
|
+
): void {
|
|
526
|
+
const input = this.inputFor(sessionId)
|
|
527
|
+
const snapshot = input.state.getSnapshot()
|
|
528
|
+
if (snapshot.phase !== 'plain') return
|
|
529
|
+
const start = Math.max(0, Math.min(target.selectionStart ?? snapshot.draft.length, snapshot.draft.length))
|
|
530
|
+
const end = Math.max(start, Math.min(target.selectionEnd ?? start, snapshot.draft.length))
|
|
531
|
+
try {
|
|
532
|
+
let cursor = this.insertText(input, text, start, end)
|
|
533
|
+
validateImages(files)
|
|
534
|
+
cursor = this.insertRecords(sessionId, input, files, cursor)
|
|
535
|
+
requestAnimationFrame(() => {
|
|
536
|
+
target.focus({ preventScroll: true })
|
|
537
|
+
target.setSelectionRange(cursor, cursor)
|
|
538
|
+
})
|
|
539
|
+
} catch (error) {
|
|
540
|
+
input.notify('error', message(error))
|
|
541
|
+
}
|
|
542
|
+
}
|
|
543
|
+
|
|
544
|
+
handlePaste(event: ClipboardEvent): boolean {
|
|
545
|
+
if (this.replaying) return false
|
|
546
|
+
const files = imageFiles(event.clipboardData)
|
|
547
|
+
if (files.length === 0) return false
|
|
548
|
+
const target = event.target
|
|
549
|
+
if (!(target instanceof HTMLTextAreaElement) || target.closest('[data-composer-card]') === null) return false
|
|
550
|
+
|
|
551
|
+
const sessionId = this.ctx.sessions.list.getSnapshot().current
|
|
552
|
+
if (sessionId === undefined) return false
|
|
553
|
+
const modelLabel = currentModelLabel()
|
|
554
|
+
this.refreshVerdict(sessionId, modelLabel)
|
|
555
|
+
// Only a fresh host verdict acts; the native attachment flow stays the
|
|
556
|
+
// default while the host is unconfirmed.
|
|
557
|
+
const verdict = this.verdictFor(sessionId, modelLabel)
|
|
558
|
+
if (verdict === undefined) return false
|
|
559
|
+
// An image-capable model (the variant routes included) keeps its native
|
|
560
|
+
// paste: no switch, no takeover.
|
|
561
|
+
if (verdict.takeOver === false && verdict.autoSwitch === undefined) return false
|
|
562
|
+
|
|
563
|
+
event.preventDefault()
|
|
564
|
+
event.stopPropagation()
|
|
565
|
+
event.stopImmediatePropagation()
|
|
566
|
+
|
|
567
|
+
const input = this.inputFor(sessionId)
|
|
568
|
+
if (input.state.getSnapshot().phase !== 'plain') return true
|
|
569
|
+
|
|
570
|
+
const text = (event.clipboardData?.getData('text/plain') ?? '').replaceAll('\uFFFC', '')
|
|
571
|
+
if (verdict.autoSwitch !== undefined) {
|
|
572
|
+
// The text-only model has an image-input variant: switch first, then
|
|
573
|
+
// let the paste flow natively so the thumbnail and durable session
|
|
574
|
+
// image are preserved.
|
|
575
|
+
void this.autoSwitchPaste(sessionId, target, files, text, verdict.autoSwitch)
|
|
576
|
+
return true
|
|
577
|
+
}
|
|
578
|
+
this.takeoverPaste(sessionId, target, files, text)
|
|
579
|
+
return true
|
|
580
|
+
}
|
|
581
|
+
|
|
582
|
+
remove(sessionId: string, occurrence: PasteOccurrence): void {
|
|
583
|
+
const record = this.records.get(occurrence.ref)
|
|
584
|
+
if (record?.batch.inflight !== undefined) return
|
|
585
|
+
const input = this.inputFor(sessionId)
|
|
586
|
+
const snapshot = input.state.getSnapshot()
|
|
587
|
+
if (snapshot.phase !== 'plain') return
|
|
588
|
+
const current = snapshot.occurrences.find(candidate =>
|
|
589
|
+
candidate.source === SOURCE
|
|
590
|
+
&& candidate.occurrenceId === occurrence.occurrenceId
|
|
591
|
+
&& candidate.ref === occurrence.ref)
|
|
592
|
+
if (current === undefined) return
|
|
593
|
+
const accepted = (input as typeof input & {
|
|
594
|
+
insertText: (text: string, span: { start: number; end: number; draftRev: number }) => boolean
|
|
595
|
+
}).insertText('', {
|
|
596
|
+
start: current.offset,
|
|
597
|
+
end: current.offset + 1,
|
|
598
|
+
draftRev: snapshot.draftRev,
|
|
599
|
+
})
|
|
600
|
+
if (!accepted) return
|
|
601
|
+
this.records.delete(occurrence.ref)
|
|
602
|
+
this.changed()
|
|
603
|
+
}
|
|
604
|
+
|
|
605
|
+
private async upload(batch: PasteBatch, signal: AbortSignal): Promise<void> {
|
|
606
|
+
if (batch.inflight !== undefined) return batch.inflight
|
|
607
|
+
const active = batch.records.filter(record => this.records.get(record.ref) === record)
|
|
608
|
+
if (active.length === 0) throw new Error('Pasted images were removed before sending')
|
|
609
|
+
const pending = active.filter(record => record.absolutePath === undefined)
|
|
610
|
+
if (pending.length === 0) return
|
|
611
|
+
const task = (async () => {
|
|
612
|
+
for (const record of pending) {
|
|
613
|
+
record.status = 'copying'
|
|
614
|
+
record.error = undefined
|
|
615
|
+
}
|
|
616
|
+
this.changed()
|
|
617
|
+
try {
|
|
618
|
+
const failures = await Promise.all(pending.map(async (record) => {
|
|
619
|
+
try {
|
|
620
|
+
if (signal.aborted) throw signal.reason ?? new DOMException('Aborted', 'AbortError')
|
|
621
|
+
const query = new URLSearchParams({
|
|
622
|
+
sessionId: batch.sessionId,
|
|
623
|
+
name: record.file.name || 'clipboard-image',
|
|
624
|
+
size: String(record.file.size),
|
|
625
|
+
})
|
|
626
|
+
const body = await responseJson(await fetch(`${PASTE_IMAGES_ROUTE}?${query.toString()}`, {
|
|
627
|
+
method: 'POST',
|
|
628
|
+
headers: { 'Content-Type': record.file.type },
|
|
629
|
+
body: record.file,
|
|
630
|
+
signal,
|
|
631
|
+
}))
|
|
632
|
+
const absolutePath = body.value?.absolutePath
|
|
633
|
+
if (typeof absolutePath !== 'string' || absolutePath === '') {
|
|
634
|
+
throw new Error('Image copy response contained an invalid path')
|
|
635
|
+
}
|
|
636
|
+
record.absolutePath = absolutePath
|
|
637
|
+
record.status = 'copied'
|
|
638
|
+
record.error = undefined
|
|
639
|
+
return undefined
|
|
640
|
+
} catch (error) {
|
|
641
|
+
const failure = error instanceof Error ? error : new Error(message(error))
|
|
642
|
+
record.status = 'error'
|
|
643
|
+
record.error = failure.message
|
|
644
|
+
return failure
|
|
645
|
+
}
|
|
646
|
+
}))
|
|
647
|
+
this.changed()
|
|
648
|
+
const failure = failures.find((error): error is Error => error !== undefined)
|
|
649
|
+
if (failure !== undefined) throw failure
|
|
650
|
+
} finally {
|
|
651
|
+
batch.inflight = undefined
|
|
652
|
+
this.changed()
|
|
653
|
+
}
|
|
654
|
+
})()
|
|
655
|
+
batch.inflight = task
|
|
656
|
+
return task
|
|
657
|
+
}
|
|
658
|
+
|
|
659
|
+
private async serialize(ref: string, signal: AbortSignal): Promise<string> {
|
|
660
|
+
const record = this.records.get(ref)
|
|
661
|
+
if (record === undefined) throw new Error('Pasted image is no longer available in this browser tab')
|
|
662
|
+
await this.upload(record.batch, signal)
|
|
663
|
+
if (record.absolutePath === undefined) throw new Error('Pasted image was not copied into the workspace')
|
|
664
|
+
return `[Pasted image available at absolute path: ${JSON.stringify(record.absolutePath)}]`
|
|
665
|
+
}
|
|
666
|
+
}
|
|
667
|
+
|
|
668
|
+
/** Minimal per-image progress, failure, and removal feedback above the composer. */
|
|
669
|
+
export function PasteImageDock(props: PasteDockProps): ReactNode {
|
|
670
|
+
useSyncExternalStore(props.controller.subscribe, props.controller.snapshot)
|
|
671
|
+
const occurrences = props.input.occurrences.filter(occurrence => occurrence.source === SOURCE)
|
|
672
|
+
const records = props.controller.recordsFor(occurrences)
|
|
673
|
+
if (records.length === 0) return null
|
|
674
|
+
return <div className="dvt-paste-dock" role="status" aria-label="Pasted images">
|
|
675
|
+
{occurrences.map((occurrence) => {
|
|
676
|
+
const record = props.controller.recordsFor([occurrence])[0]
|
|
677
|
+
if (record === undefined) return null
|
|
678
|
+
const detail = record.status === 'copying' ? 'copying…'
|
|
679
|
+
: record.status === 'copied' ? 'copied'
|
|
680
|
+
: record.status === 'error' ? record.error ?? 'copy failed'
|
|
681
|
+
: humanBytes(record.file.size)
|
|
682
|
+
return <div className="dvt-paste-chip" data-status={record.status} key={occurrence.occurrenceId}>
|
|
683
|
+
<span className="dvt-paste-name" title={record.file.name}>{record.file.name || 'clipboard image'}</span>
|
|
684
|
+
<span className="dvt-paste-detail" title={record.error}>{detail}</span>
|
|
685
|
+
<button
|
|
686
|
+
type="button"
|
|
687
|
+
aria-label={`Remove ${record.file.name || 'clipboard image'}`}
|
|
688
|
+
disabled={props.input.phase !== 'plain' || record.status === 'copying'}
|
|
689
|
+
onClick={() => { props.remove(occurrence) }}
|
|
690
|
+
>×</button>
|
|
691
|
+
</div>
|
|
692
|
+
})}
|
|
693
|
+
</div>
|
|
694
|
+
}
|
|
695
|
+
|
|
696
|
+
/** Install capture interception, the text-reference codec, and composer feedback. */
|
|
697
|
+
export function installPasteImages(ctx: ClientContext): void {
|
|
698
|
+
const controller = new PasteImageController(ctx)
|
|
699
|
+
const registered = new WeakMap<object, ReferenceSourceRegistration>()
|
|
700
|
+
const register = (scope: ClientContext, registry: ReferenceSourceRegistry): void => {
|
|
701
|
+
scope.effect(() => {
|
|
702
|
+
const identity = registryIdentity(registry)
|
|
703
|
+
let registration = registered.get(identity)
|
|
704
|
+
if (registration === undefined) {
|
|
705
|
+
registration = { dispose: registry.registerSource(controller.source()), owners: 0 }
|
|
706
|
+
registered.set(identity, registration)
|
|
707
|
+
}
|
|
708
|
+
registration.owners += 1
|
|
709
|
+
return () => {
|
|
710
|
+
if (registered.get(identity) !== registration) return
|
|
711
|
+
registration.owners -= 1
|
|
712
|
+
if (registration.owners > 0) return
|
|
713
|
+
registered.delete(identity)
|
|
714
|
+
registration.dispose()
|
|
715
|
+
}
|
|
716
|
+
}, 'dsh-vision-toolkit: pasted image reference codec')
|
|
717
|
+
}
|
|
718
|
+
ctx.inject(['slash'], (scope: ClientContext) => {
|
|
719
|
+
register(scope, (scope as unknown as LegacySlashContext).slash)
|
|
720
|
+
})
|
|
721
|
+
ctx.inject(['inputTriggers'], (scope: ClientContext) => {
|
|
722
|
+
register(scope, (scope as unknown as LegacyTriggerContext).inputTriggers)
|
|
723
|
+
})
|
|
724
|
+
ctx.effect(() => {
|
|
725
|
+
const listener = (event: ClipboardEvent): void => { controller.handlePaste(event) }
|
|
726
|
+
// A focus-time prefetch has the verdict ready before the first paste can land.
|
|
727
|
+
const onFocusIn = (): void => {
|
|
728
|
+
const sessionId = ctx.sessions.list.getSnapshot().current
|
|
729
|
+
if (sessionId !== undefined) controller.refreshVerdict(String(sessionId), currentModelLabel())
|
|
730
|
+
}
|
|
731
|
+
document.addEventListener('paste', listener, true)
|
|
732
|
+
document.addEventListener('focusin', onFocusIn, true)
|
|
733
|
+
return () => {
|
|
734
|
+
document.removeEventListener('paste', listener, true)
|
|
735
|
+
document.removeEventListener('focusin', onFocusIn, true)
|
|
736
|
+
}
|
|
737
|
+
}, 'dsh-vision-toolkit: clipboard image capture')
|
|
738
|
+
ctx.slots.inject('conversation.input.dock', () => ctx.slots.register({
|
|
739
|
+
name: 'conversation.input.dock',
|
|
740
|
+
id: 'vision-toolkit-pasted-images',
|
|
741
|
+
order: 6,
|
|
742
|
+
inject: sessionId => ({
|
|
743
|
+
controller,
|
|
744
|
+
remove: (occurrence: PasteOccurrence) => { controller.remove(String(sessionId), occurrence) },
|
|
745
|
+
}),
|
|
746
|
+
}, PasteImageDock))
|
|
747
|
+
}
|