@mengruo/dsh-vision-toolkit 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (216) hide show
  1. package/LICENSE +21 -0
  2. package/README.i18n.yaml +6 -0
  3. package/README.md +356 -0
  4. package/README.zh.md +354 -0
  5. package/assets/community-group-qr.png +0 -0
  6. package/assets/dsh-conversation-artifact.png +0 -0
  7. package/assets/dsh-conversation-image-qa-top.png +0 -0
  8. package/assets/dsh-conversation-image-qa.png +0 -0
  9. package/assets/dsh-conversation-pixel-diff.png +0 -0
  10. package/assets/dsh-conversation-screenshot-debugging-top.png +0 -0
  11. package/assets/dsh-conversation-screenshot-debugging.png +0 -0
  12. package/assets/dsh-conversation-tool-call.png +0 -0
  13. package/assets/dsh-conversation-vision-trace.png +0 -0
  14. package/assets/dsh-view-example.png +0 -0
  15. package/assets/hero-v2.png +0 -0
  16. package/assets/logo_aihubmix.png +0 -0
  17. package/assets/python-bootstrap.json +48 -0
  18. package/assets/skill/SKILL.md +329 -0
  19. package/assets/skill/UPSTREAM.json +71 -0
  20. package/assets/skill/references/gui.md +88 -0
  21. package/assets/skill/references/long-screenshot-ocr.md +77 -0
  22. package/assets/skill/references/restore-graphic.md +84 -0
  23. package/assets/skill/references/restore-structure.md +45 -0
  24. package/assets/skill/references/restore-ui.md +202 -0
  25. package/assets/social-preview.png +0 -0
  26. package/assets/upstream/README.md +18 -0
  27. package/assets/upstream/focus-hint-comparison-1.webp +0 -0
  28. package/assets/upstream/focus-hint-comparison-2.webp +0 -0
  29. package/assets/upstream/infographic-reference.webp +0 -0
  30. package/assets/upstream/infographic-result.webp +0 -0
  31. package/assets/upstream/ui-fast-restore-reference.webp +0 -0
  32. package/assets/upstream/ui-fast-restore-result.webp +0 -0
  33. package/assets/upstream/ui-result.webp +0 -0
  34. package/assets/upstream/ui-sketch.webp +0 -0
  35. package/assets/vision-model-test.png +0 -0
  36. package/assets/vision-settings.png +0 -0
  37. package/assets/wechat-reward.png +0 -0
  38. package/cordis.patch.yml +6 -0
  39. package/docs/aihubmix-gemini-vision.i18n.yaml +6 -0
  40. package/docs/aihubmix-gemini-vision.md +138 -0
  41. package/docs/aihubmix-gemini-vision.zh.md +138 -0
  42. package/docs/assets/aihubmix-api-keys.png +0 -0
  43. package/docs/assets/aihubmix-create-key.png +0 -0
  44. package/docs/assets/aihubmix-free-vision-model.png +0 -0
  45. package/docs/assets/aihubmix-home.png +0 -0
  46. package/docs/assets/aihubmix-sign-up.png +0 -0
  47. package/docs/assets/groq-console-home.png +0 -0
  48. package/docs/assets/groq-console-keys-login.png +0 -0
  49. package/docs/assets/groq-docs-quickstart.png +0 -0
  50. package/docs/assets/groq-docs-qwen3.6.png +0 -0
  51. package/docs/assets/groq-docs-vision.png +0 -0
  52. package/docs/assets/vision-settings.png +0 -0
  53. package/docs/dsh-desktop-install.md +67 -0
  54. package/docs/dsh-desktop-install.zh.md +67 -0
  55. package/docs/groq-qwen3.6-vision.i18n.yaml +6 -0
  56. package/docs/groq-qwen3.6-vision.md +236 -0
  57. package/docs/groq-qwen3.6-vision.zh.md +236 -0
  58. package/docs/python-runtime.i18n.yaml +6 -0
  59. package/docs/python-runtime.md +89 -0
  60. package/docs/python-runtime.zh.md +89 -0
  61. package/docs/requirements-traceability/README.i18n.yaml +6 -0
  62. package/docs/requirements-traceability/README.md +75 -0
  63. package/docs/requirements-traceability/README.zh.md +75 -0
  64. package/examples/ui-restoration/README.i18n.yaml +6 -0
  65. package/examples/ui-restoration/README.md +70 -0
  66. package/examples/ui-restoration/README.zh.md +70 -0
  67. package/examples/ui-restoration/assets/final-heatmap.png +0 -0
  68. package/examples/ui-restoration/assets/final-report.json +83 -0
  69. package/examples/ui-restoration/assets/implementation.png +0 -0
  70. package/examples/ui-restoration/assets/initial-heatmap.png +0 -0
  71. package/examples/ui-restoration/assets/initial-report.json +83 -0
  72. package/examples/ui-restoration/assets/initial.png +0 -0
  73. package/examples/ui-restoration/assets/metrics.json +12 -0
  74. package/examples/ui-restoration/assets/reference.png +0 -0
  75. package/examples/ui-restoration/implementation.html +94 -0
  76. package/examples/ui-restoration/initial.html +57 -0
  77. package/lib/artifact-access.js +369 -0
  78. package/lib/artifact-access.js.map +1 -0
  79. package/lib/artifacts.js +56 -0
  80. package/lib/artifacts.js.map +1 -0
  81. package/lib/client.js +2099 -0
  82. package/lib/client.js.map +1 -0
  83. package/lib/config.js +172 -0
  84. package/lib/config.js.map +1 -0
  85. package/lib/defaults.js +6 -0
  86. package/lib/defaults.js.map +1 -0
  87. package/lib/errors.js +56 -0
  88. package/lib/errors.js.map +1 -0
  89. package/lib/evidence-cache.js +350 -0
  90. package/lib/evidence-cache.js.map +1 -0
  91. package/lib/exposure.js +247 -0
  92. package/lib/exposure.js.map +1 -0
  93. package/lib/image-input-variants.js +882 -0
  94. package/lib/image-input-variants.js.map +1 -0
  95. package/lib/index.js +105 -0
  96. package/lib/index.js.map +1 -0
  97. package/lib/paste-images.js +217 -0
  98. package/lib/paste-images.js.map +1 -0
  99. package/lib/paths.js +354 -0
  100. package/lib/paths.js.map +1 -0
  101. package/lib/plugin-update.js +1003 -0
  102. package/lib/plugin-update.js.map +1 -0
  103. package/lib/runtime-install.js +964 -0
  104. package/lib/runtime-install.js.map +1 -0
  105. package/lib/runtime-manager.js +133 -0
  106. package/lib/runtime-manager.js.map +1 -0
  107. package/lib/runtime.js +1678 -0
  108. package/lib/runtime.js.map +1 -0
  109. package/lib/skill.js +25 -0
  110. package/lib/skill.js.map +1 -0
  111. package/lib/tools.js +549 -0
  112. package/lib/tools.js.map +1 -0
  113. package/lib/types/artifact-access.d.ts +61 -0
  114. package/lib/types/artifact-access.d.ts.map +1 -0
  115. package/lib/types/artifacts.d.ts +42 -0
  116. package/lib/types/artifacts.d.ts.map +1 -0
  117. package/lib/types/client/display-config.d.ts +24 -0
  118. package/lib/types/client/display-config.d.ts.map +1 -0
  119. package/lib/types/client/index.d.ts +350 -0
  120. package/lib/types/client/index.d.ts.map +1 -0
  121. package/lib/types/client/model-variants-hider.d.ts +40 -0
  122. package/lib/types/client/model-variants-hider.d.ts.map +1 -0
  123. package/lib/types/client/paste-images.d.ts +126 -0
  124. package/lib/types/client/paste-images.d.ts.map +1 -0
  125. package/lib/types/config.d.ts +123 -0
  126. package/lib/types/config.d.ts.map +1 -0
  127. package/lib/types/defaults.d.ts +6 -0
  128. package/lib/types/defaults.d.ts.map +1 -0
  129. package/lib/types/errors.d.ts +35 -0
  130. package/lib/types/errors.d.ts.map +1 -0
  131. package/lib/types/evidence-cache.d.ts +91 -0
  132. package/lib/types/evidence-cache.d.ts.map +1 -0
  133. package/lib/types/exposure.d.ts +50 -0
  134. package/lib/types/exposure.d.ts.map +1 -0
  135. package/lib/types/image-input-variants.d.ts +144 -0
  136. package/lib/types/image-input-variants.d.ts.map +1 -0
  137. package/lib/types/index.d.ts +19 -0
  138. package/lib/types/index.d.ts.map +1 -0
  139. package/lib/types/paste-images.d.ts +73 -0
  140. package/lib/types/paste-images.d.ts.map +1 -0
  141. package/lib/types/paths.d.ts +117 -0
  142. package/lib/types/paths.d.ts.map +1 -0
  143. package/lib/types/plugin-update.d.ts +119 -0
  144. package/lib/types/plugin-update.d.ts.map +1 -0
  145. package/lib/types/runtime-install.d.ts +92 -0
  146. package/lib/types/runtime-install.d.ts.map +1 -0
  147. package/lib/types/runtime-manager.d.ts +60 -0
  148. package/lib/types/runtime-manager.d.ts.map +1 -0
  149. package/lib/types/runtime.d.ts +412 -0
  150. package/lib/types/runtime.d.ts.map +1 -0
  151. package/lib/types/skill.d.ts +14 -0
  152. package/lib/types/skill.d.ts.map +1 -0
  153. package/lib/types/tools.d.ts +35 -0
  154. package/lib/types/tools.d.ts.map +1 -0
  155. package/lib/types/upstream.d.ts +233 -0
  156. package/lib/types/upstream.d.ts.map +1 -0
  157. package/lib/types/version.d.ts +15 -0
  158. package/lib/types/version.d.ts.map +1 -0
  159. package/lib/types/web-request.d.ts +11 -0
  160. package/lib/types/web-request.d.ts.map +1 -0
  161. package/lib/types/web.d.ts +118 -0
  162. package/lib/types/web.d.ts.map +1 -0
  163. package/lib/upstream.js +860 -0
  164. package/lib/upstream.js.map +1 -0
  165. package/lib/version.js +18 -0
  166. package/lib/version.js.map +1 -0
  167. package/lib/web-request.js +29 -0
  168. package/lib/web-request.js.map +1 -0
  169. package/lib/web.js +462 -0
  170. package/lib/web.js.map +1 -0
  171. package/package.json +149 -0
  172. package/patches/vision-tools-dsh.patch +970 -0
  173. package/runtime/requirements.lock +3 -0
  174. package/src/artifact-access.ts +386 -0
  175. package/src/artifacts.ts +85 -0
  176. package/src/client/display-config.ts +62 -0
  177. package/src/client/index.tsx +1580 -0
  178. package/src/client/model-variants-hider.ts +159 -0
  179. package/src/client/paste-images.tsx +747 -0
  180. package/src/config.ts +290 -0
  181. package/src/defaults.ts +5 -0
  182. package/src/errors.ts +62 -0
  183. package/src/evidence-cache.ts +418 -0
  184. package/src/exposure.ts +265 -0
  185. package/src/image-input-variants.ts +996 -0
  186. package/src/index.ts +141 -0
  187. package/src/paste-images.ts +286 -0
  188. package/src/paths.ts +383 -0
  189. package/src/plugin-update.ts +1143 -0
  190. package/src/runtime-install.ts +1161 -0
  191. package/src/runtime-manager.ts +174 -0
  192. package/src/runtime.ts +2189 -0
  193. package/src/skill.ts +35 -0
  194. package/src/tools.ts +691 -0
  195. package/src/upstream.ts +1078 -0
  196. package/src/version.ts +37 -0
  197. package/src/web-request.ts +27 -0
  198. package/src/web.ts +594 -0
  199. package/vendor/agent-vision-toolkit/CHANGELOG.md +30 -0
  200. package/vendor/agent-vision-toolkit/LICENSE +21 -0
  201. package/vendor/agent-vision-toolkit/README.md +405 -0
  202. package/vendor/agent-vision-toolkit/UPSTREAM_MANIFEST.json +94 -0
  203. package/vendor/agent-vision-toolkit/bin/crop +90 -0
  204. package/vendor/agent-vision-toolkit/bin/detect +13 -0
  205. package/vendor/agent-vision-toolkit/bin/glance +93 -0
  206. package/vendor/agent-vision-toolkit/bin/ground +13 -0
  207. package/vendor/agent-vision-toolkit/bin/trace +129 -0
  208. package/vendor/agent-vision-toolkit/detect.py +60 -0
  209. package/vendor/agent-vision-toolkit/ground.py +267 -0
  210. package/vendor/agent-vision-toolkit/skills/vision-tools/scripts/dominant_colors.py +224 -0
  211. package/vendor/agent-vision-toolkit/skills/vision-tools/scripts/extract_fg.py +278 -0
  212. package/vendor/agent-vision-toolkit/skills/vision-tools/scripts/html_shot.py +420 -0
  213. package/vendor/agent-vision-toolkit/skills/vision-tools/scripts/long_screenshot_ocr.py +1245 -0
  214. package/vendor/agent-vision-toolkit/skills/vision-tools/scripts/pixel_diff.py +88 -0
  215. package/vendor/agent-vision-toolkit/tests/test_vision_client.py +450 -0
  216. package/vendor/agent-vision-toolkit/vision_client.py +310 -0
@@ -0,0 +1,418 @@
1
+ /** Durable, Session-scoped cache for image descriptions used by model variants. */
2
+
3
+ import { createHash } from 'node:crypto'
4
+ import type { Context, Fiber } from '@deepseek-ai/cordis'
5
+ import type { ContentBlock } from '@deepseek-ai/dsh-llm'
6
+ import type { SessionHeader, SessionId } from '@deepseek-ai/dsh-session'
7
+ import { defineDomain, domainTable } from '@deepseek-ai/dsh-storage-domain'
8
+ import type { KvTable } from '@deepseek-ai/dsh-storage-domain'
9
+ import { z } from 'zod'
10
+ import type { ResolvedVisionToolkitConfig } from './config.ts'
11
+ import { UPSTREAM_COMMIT } from './version.ts'
12
+
13
+ /** Bump only when the model-visible evidence contract changes incompatibly. */
14
+ export const EVIDENCE_CONTRACT_VERSION = 1
15
+
16
+ /** Persistent cache bounds keep the Profile storage proportional and predictable. */
17
+ const DEFAULT_PERSISTED_ENTRY_LIMIT = 512
18
+ const DEFAULT_PERSISTED_BYTE_LIMIT = 8 * 1024 * 1024
19
+ const DEFAULT_PERSISTED_ENTRY_BYTE_LIMIT = 64 * 1024
20
+ const MAX_SCHEMA_TEXT_CHARS = 256 * 1024
21
+ const MAX_SCHEMA_RECORD_CHARS = 512 * 1024
22
+
23
+ const hexDigestSchema = z.string().regex(/^[0-9a-f]{64}$/u)
24
+ const sessionIdentitySchema = z.object({
25
+ createdAt: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER),
26
+ cwd: z.string().optional(),
27
+ })
28
+ const evidenceRecordSchema = z.object({
29
+ contractVersion: z.number().int().nonnegative(),
30
+ sessionId: z.string().min(1),
31
+ session: sessionIdentitySchema,
32
+ attachmentId: z.string().min(1),
33
+ promptHash: hexDigestSchema,
34
+ runtimeHash: hexDigestSchema,
35
+ text: z.string().min(1).max(MAX_SCHEMA_TEXT_CHARS),
36
+ storedAt: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER),
37
+ })
38
+
39
+ type EvidenceRecord = z.infer<typeof evidenceRecordSchema>
40
+ type EvidenceRecordKey = string & { readonly __evidenceRecordKey: unique symbol }
41
+
42
+ /** Plugin-owned sidecar domain; the host backend handles atomicity and file safety. */
43
+ export const evidenceCacheDomainSpec = defineDomain({
44
+ name: 'vision_toolkit_evidence',
45
+ version: 0,
46
+ tables: {
47
+ // Keep the durable boundary tolerant of one damaged cache payload: each
48
+ // record is parsed and validated independently below, so corruption causes
49
+ // a miss instead of preventing the entire optional domain from opening.
50
+ evidence: domainTable<EvidenceRecordKey, string>(z.string().max(MAX_SCHEMA_RECORD_CHARS)),
51
+ },
52
+ })
53
+
54
+ /** Stable metadata for one description lookup. Raw focus prompts are never persisted. */
55
+ export interface EvidenceCacheKey {
56
+ readonly digest: string
57
+ readonly contractVersion: number
58
+ readonly sessionId?: string
59
+ readonly sessionCreatedAt?: number
60
+ readonly sessionCwd?: string
61
+ readonly attachmentId: string
62
+ readonly promptHash: string
63
+ readonly runtimeHash: string
64
+ }
65
+
66
+ /** Optional durable layer behind the process-local promise/LRU cache. */
67
+ export interface EvidencePersistence {
68
+ read(key: EvidenceCacheKey): Promise<ContentBlock | undefined>
69
+ write(key: EvidenceCacheKey, block: ContentBlock): Promise<void>
70
+ }
71
+
72
+ function hash(value: string): string {
73
+ return createHash('sha256').update(value).digest('hex')
74
+ }
75
+
76
+ /** Fingerprint every runtime setting that can change the generated description. */
77
+ export function evidenceRuntimeFingerprint(
78
+ config: ResolvedVisionToolkitConfig,
79
+ credentialSha256?: string,
80
+ sslVerify?: string,
81
+ ): string {
82
+ return hash(JSON.stringify({
83
+ upstreamCommit: UPSTREAM_COMMIT,
84
+ provider: {
85
+ baseUrl: config.provider.baseUrl,
86
+ credential: {
87
+ ref: String(config.provider.credential),
88
+ sha256: credentialSha256 ?? null,
89
+ },
90
+ model: config.provider.model,
91
+ protocol: config.provider.protocol,
92
+ anthropicThinking: config.provider.anthropicThinking,
93
+ sslVerify: sslVerify ?? null,
94
+ userAgent: config.provider.userAgent,
95
+ },
96
+ language: config.language,
97
+ timeoutMs: config.timeoutMs,
98
+ concurrency: config.concurrency,
99
+ maxImageBytes: config.maxImageBytes,
100
+ maxImagePixels: config.maxImagePixels,
101
+ runtime: config.runtime,
102
+ }))
103
+ }
104
+
105
+ /** Build a non-secret cache key from the Session, attachment, focus, and runtime contract. */
106
+ export function createEvidenceCacheKey(input: {
107
+ sessionId?: string
108
+ sessionIdentity?: SessionIdentity
109
+ attachmentId: string
110
+ prompt: string
111
+ runtimeHash: string
112
+ }): EvidenceCacheKey {
113
+ const promptHash = hash(input.prompt)
114
+ const digest = hash(JSON.stringify([
115
+ EVIDENCE_CONTRACT_VERSION,
116
+ input.sessionId ?? '',
117
+ input.sessionIdentity?.createdAt ?? '',
118
+ input.sessionIdentity?.cwd ?? '',
119
+ input.attachmentId,
120
+ promptHash,
121
+ input.runtimeHash,
122
+ ]))
123
+ return Object.freeze({
124
+ digest,
125
+ contractVersion: EVIDENCE_CONTRACT_VERSION,
126
+ ...(input.sessionId === undefined ? {} : { sessionId: input.sessionId }),
127
+ ...(input.sessionIdentity === undefined ? {} : { sessionCreatedAt: input.sessionIdentity.createdAt }),
128
+ ...(input.sessionIdentity?.cwd === undefined ? {} : { sessionCwd: input.sessionIdentity.cwd }),
129
+ attachmentId: input.attachmentId,
130
+ promptHash,
131
+ runtimeHash: input.runtimeHash,
132
+ })
133
+ }
134
+
135
+ /** Bounded promise cache; concurrent readers join one load and rejected loads are evicted. */
136
+ export class EvidenceCache {
137
+ private readonly entries = new Map<string, Promise<ContentBlock>>()
138
+
139
+ constructor(
140
+ private readonly limit: number,
141
+ private readonly persistence?: EvidencePersistence,
142
+ ) {}
143
+
144
+ /** Read a memory/durable hit or compute and persist one model-visible result. */
145
+ read(key: string | EvidenceCacheKey, load: () => Promise<ContentBlock>): Promise<ContentBlock> {
146
+ const memoryKey = typeof key === 'string' ? key : key.digest
147
+ const existing = this.entries.get(memoryKey)
148
+ if (existing !== undefined) {
149
+ // Refresh recency: Map iteration order is insertion order.
150
+ this.entries.delete(memoryKey)
151
+ this.entries.set(memoryKey, existing)
152
+ return existing
153
+ }
154
+
155
+ const pending = (async (): Promise<ContentBlock> => {
156
+ if (typeof key !== 'string' && this.persistence !== undefined) {
157
+ try {
158
+ const persisted = await this.persistence.read(key)
159
+ if (persisted !== undefined) return persisted
160
+ } catch {
161
+ // Persistence is an optimization; a damaged/unavailable sidecar must
162
+ // never block the model request from recomputing evidence.
163
+ }
164
+ }
165
+
166
+ const block = await load()
167
+ if (typeof key !== 'string' && this.persistence !== undefined) {
168
+ try {
169
+ await this.persistence.write(key, block)
170
+ } catch {
171
+ // Keep the process-local result even when durability fails.
172
+ }
173
+ }
174
+ return block
175
+ })().then(
176
+ block => block,
177
+ (error: unknown) => {
178
+ // Only evict our own entry: this promise may have been LRU-evicted and
179
+ // the key re-populated by a newer read meanwhile.
180
+ if (this.entries.get(memoryKey) === pending) {
181
+ this.entries.delete(memoryKey)
182
+ }
183
+ throw error
184
+ },
185
+ )
186
+
187
+ this.entries.set(memoryKey, pending)
188
+ while (this.entries.size > this.limit) {
189
+ const oldest = this.entries.keys().next().value
190
+ if (oldest === undefined) break
191
+ this.entries.delete(oldest)
192
+ }
193
+ return pending
194
+ }
195
+
196
+ /** Drop process-local descriptions; durable rows stay versioned by their runtime fingerprint. */
197
+ clear(): void {
198
+ this.entries.clear()
199
+ }
200
+ }
201
+
202
+ interface StorageBinding {
203
+ table: KvTable<EvidenceRecordKey, string>
204
+ }
205
+
206
+ interface SessionIdentity {
207
+ createdAt: number
208
+ cwd?: string
209
+ }
210
+
211
+ export interface SessionEvidenceStoreOptions {
212
+ maxEntries?: number
213
+ maxBytes?: number
214
+ maxEntryBytes?: number
215
+ now?: () => number
216
+ }
217
+
218
+ function identityOf(header: SessionHeader): SessionIdentity {
219
+ return Object.freeze({
220
+ createdAt: header.createdAt,
221
+ ...(header.cwd === undefined ? {} : { cwd: header.cwd }),
222
+ })
223
+ }
224
+
225
+ function sameIdentity(record: EvidenceRecord, header: SessionHeader): boolean {
226
+ return record.session.createdAt === header.createdAt && record.session.cwd === header.cwd
227
+ }
228
+
229
+ function matchesKey(record: EvidenceRecord, key: EvidenceCacheKey): boolean {
230
+ return record.contractVersion === key.contractVersion
231
+ && record.sessionId === key.sessionId
232
+ && record.attachmentId === key.attachmentId
233
+ && record.promptHash === key.promptHash
234
+ && record.runtimeHash === key.runtimeHash
235
+ }
236
+
237
+ function byteLength(text: string): number {
238
+ return Buffer.byteLength(text, 'utf8')
239
+ }
240
+
241
+ function messageOf(error: unknown): string {
242
+ return error instanceof Error ? error.message : String(error)
243
+ }
244
+
245
+ function parseRecord(value: string): EvidenceRecord | undefined {
246
+ try {
247
+ const parsed: unknown = JSON.parse(value)
248
+ const result = evidenceRecordSchema.safeParse(parsed)
249
+ return result.success ? result.data : undefined
250
+ } catch {
251
+ return undefined
252
+ }
253
+ }
254
+
255
+ /** Official DSH storage-domain sidecar used to survive Profile restarts. */
256
+ export class SessionEvidenceStore implements EvidencePersistence {
257
+ private storage: StorageBinding | undefined
258
+ private storageFiber: (Fiber & PromiseLike<Fiber>) | undefined
259
+ private storageReady: Promise<void> | undefined
260
+ private mutationTail: Promise<void> = Promise.resolve()
261
+ private readonly flushes = new Map<string, Promise<boolean>>()
262
+ private warned = false
263
+ private readonly maxEntries: number
264
+ private readonly maxBytes: number
265
+ private readonly maxEntryBytes: number
266
+ private readonly now: () => number
267
+
268
+ constructor(
269
+ private readonly ctx: Context,
270
+ options: SessionEvidenceStoreOptions = {},
271
+ ) {
272
+ this.maxEntries = options.maxEntries ?? DEFAULT_PERSISTED_ENTRY_LIMIT
273
+ this.maxBytes = options.maxBytes ?? DEFAULT_PERSISTED_BYTE_LIMIT
274
+ this.maxEntryBytes = options.maxEntryBytes ?? DEFAULT_PERSISTED_ENTRY_BYTE_LIMIT
275
+ this.now = options.now ?? Date.now
276
+
277
+ if (typeof ctx.inject !== 'function') return
278
+ this.storageFiber = ctx.inject(['storageDomain'], async (storageCtx: Context) => {
279
+ const domain = await storageCtx.storageDomain.open(evidenceCacheDomainSpec)
280
+ const binding: StorageBinding = { table: domain.table('evidence') }
281
+ try {
282
+ await this.enqueueMutation(async () => { await this.prune(binding.table) })
283
+ this.storage = binding
284
+ } catch (error) {
285
+ await domain.close()
286
+ throw error
287
+ }
288
+ return async () => {
289
+ if (this.storage === binding) this.storage = undefined
290
+ await this.mutationTail
291
+ await domain.close()
292
+ }
293
+ })
294
+ this.storageReady = Promise.resolve(this.storageFiber).then(
295
+ () => undefined,
296
+ (error: unknown) => { this.warnOnce(error) },
297
+ )
298
+ }
299
+
300
+ /** Release the optional storage binding with the owning variant lifecycle. */
301
+ dispose(): void {
302
+ const fiber = this.storageFiber
303
+ this.storageFiber = undefined
304
+ this.storageReady = undefined
305
+ if (fiber !== undefined) void fiber.dispose().catch(error => { this.warnOnce(error) })
306
+ }
307
+
308
+ async read(key: EvidenceCacheKey): Promise<ContentBlock | undefined> {
309
+ const session = this.sessionFor(key)
310
+ if (session === undefined) return undefined
311
+ const binding = await this.prepareStorage()
312
+ if (binding === undefined) return undefined
313
+ const stored = binding.table.get(key.digest as EvidenceRecordKey)
314
+ const record = stored === undefined ? undefined : parseRecord(stored)
315
+ if (record === undefined || !matchesKey(record, key) || !sameIdentity(record, session.header)) {
316
+ return undefined
317
+ }
318
+ if (byteLength(record.text) > this.maxEntryBytes) return undefined
319
+ return { type: 'text', text: record.text }
320
+ }
321
+
322
+ async write(key: EvidenceCacheKey, block: ContentBlock): Promise<void> {
323
+ if (block.type !== 'text' || block.text.length === 0 || byteLength(block.text) > this.maxEntryBytes) return
324
+ const session = this.sessionFor(key)
325
+ if (session === undefined) return
326
+ const binding = await this.prepareStorage()
327
+ if (binding === undefined) return
328
+
329
+ try {
330
+ const participated = await this.flushSession(session)
331
+ if (!participated) return
332
+ const record: EvidenceRecord = Object.freeze({
333
+ contractVersion: key.contractVersion,
334
+ sessionId: key.sessionId as string,
335
+ session: identityOf(session.header),
336
+ attachmentId: key.attachmentId,
337
+ promptHash: key.promptHash,
338
+ runtimeHash: key.runtimeHash,
339
+ text: block.text,
340
+ storedAt: this.now(),
341
+ })
342
+ await this.enqueueMutation(async () => {
343
+ await binding.table.put(key.digest as EvidenceRecordKey, JSON.stringify(record))
344
+ await this.prune(binding.table)
345
+ })
346
+ } catch (error) {
347
+ this.warnOnce(error)
348
+ }
349
+ }
350
+
351
+ private sessionFor(key: EvidenceCacheKey): ReturnType<Context['sessions']['get']> {
352
+ if (key.sessionId === undefined) return undefined
353
+ const session = this.ctx.sessions.get(key.sessionId as SessionId)
354
+ if (session === undefined) return undefined
355
+ if (key.sessionCreatedAt !== undefined
356
+ && (session.header.createdAt !== key.sessionCreatedAt || session.header.cwd !== key.sessionCwd)) {
357
+ return undefined
358
+ }
359
+ return session
360
+ }
361
+
362
+ private async prepareStorage(): Promise<StorageBinding | undefined> {
363
+ if (this.storage !== undefined) return this.storage
364
+ if (this.ctx.get('storageDomain') === undefined) return undefined
365
+ await this.storageReady
366
+ return this.storage
367
+ }
368
+
369
+ private enqueueMutation(operation: () => Promise<void>): Promise<void> {
370
+ const result = this.mutationTail.then(operation)
371
+ this.mutationTail = result.then(() => undefined, () => undefined)
372
+ return result
373
+ }
374
+
375
+ private flushSession(session: NonNullable<ReturnType<Context['sessions']['get']>>): Promise<boolean> {
376
+ const key = `${session.id}\u0000${session.header.createdAt}\u0000${session.header.cwd ?? ''}`
377
+ const existing = this.flushes.get(key)
378
+ if (existing !== undefined) return existing
379
+ const pending = this.ctx.sessions.flush(session).finally(() => {
380
+ if (this.flushes.get(key) === pending) this.flushes.delete(key)
381
+ })
382
+ this.flushes.set(key, pending)
383
+ return pending
384
+ }
385
+
386
+ private async prune(table: KvTable<EvidenceRecordKey, string>): Promise<void> {
387
+ const records = [...table.entries()].map(([key, stored]) => {
388
+ const record = parseRecord(stored)
389
+ return {
390
+ key,
391
+ bytes: byteLength(stored),
392
+ storedAt: record?.storedAt ?? -1,
393
+ valid: record !== undefined && byteLength(record.text) <= this.maxEntryBytes,
394
+ }
395
+ })
396
+ let totalBytes = records.reduce((sum, record) => sum + record.bytes, 0)
397
+ let totalEntries = records.length
398
+ records.sort((left, right) => Number(left.valid) - Number(right.valid)
399
+ || left.storedAt - right.storedAt
400
+ || left.key.localeCompare(right.key))
401
+ for (const record of records) {
402
+ if (record.valid && totalEntries <= this.maxEntries && totalBytes <= this.maxBytes) break
403
+ if (await table.delete(record.key)) {
404
+ totalEntries -= 1
405
+ totalBytes -= record.bytes
406
+ }
407
+ }
408
+ }
409
+
410
+ private warnOnce(error: unknown): void {
411
+ if (this.warned) return
412
+ this.warned = true
413
+ this.ctx.logger?.warn(
414
+ 'dsh-vision-toolkit: persistent image evidence cache is unavailable; using the process cache only. %s',
415
+ messageOf(error).slice(0, 500),
416
+ )
417
+ }
418
+ }
@@ -0,0 +1,265 @@
1
+ /**
2
+ * Agent-scoped progressive exposure for the model-facing visual tools.
3
+ * Runtime readiness is global, while tool schemas enter only an Agent through
4
+ * the matching Skill or its bootstrap tool; administrative diagnostics stay on
5
+ * the Web seam.
6
+ * @module dsh-vision-toolkit/exposure
7
+ */
8
+
9
+ import type { Agent } from '@deepseek-ai/dsh-agent'
10
+ import type { ContentBlock } from '@deepseek-ai/dsh-llm'
11
+ import type { Session } from '@deepseek-ai/dsh-session'
12
+ import { defineTool, type ToolDefinition } from '@deepseek-ai/dsh-tools'
13
+ import type { Context } from '@deepseek-ai/cordis'
14
+ import { VISION_SKILLS_CONTENT, VISION_SKILLS_NAME } from './skill.ts'
15
+ import { VISION_TOOL_NAMES } from './tools.ts'
16
+
17
+ /** Small bootstrap tool retained only until the current Agent gains visual tools. */
18
+ export const VISION_TOOLKIT_ACTIVATE = 'vision_toolkit_activate'
19
+
20
+ /** Skill name used by releases before the rename to vision-skills. */
21
+ export const LEGACY_VISION_TOOLS_SKILL_NAME = 'vision-tools'
22
+
23
+ /** Unique pre-rename line in bundled instructions, kept for Session restore. */
24
+ export const LEGACY_VISION_TOOLS_SKILL_MARKER = 'If this content arrived through a direct `/vision-tools` invocation and the'
25
+
26
+ interface AgentExposure {
27
+ active: boolean
28
+ liftRestriction?: () => void
29
+ toolDisposers: Array<() => void>
30
+ toolNames: string[]
31
+ }
32
+
33
+ /** Result returned by the one-shot activation transport. */
34
+ export interface VisionToolkitActivationResult {
35
+ activated: boolean
36
+ tools: string[]
37
+ }
38
+
39
+ function renderJson(_args: unknown, value: unknown): ContentBlock[] {
40
+ return [{ type: 'text', text: JSON.stringify(value, null, 2) }]
41
+ }
42
+
43
+ function isRecord(value: unknown): value is Record<string, unknown> {
44
+ return value !== null && typeof value === 'object' && !Array.isArray(value)
45
+ }
46
+
47
+ function isBundledSkillName(name: unknown): boolean {
48
+ return name === VISION_SKILLS_NAME || name === LEGACY_VISION_TOOLS_SKILL_NAME
49
+ }
50
+
51
+ function isBundledSkillContent(text: string): boolean {
52
+ return text.includes(VISION_SKILLS_CONTENT) || text.includes(LEGACY_VISION_TOOLS_SKILL_MARKER)
53
+ }
54
+
55
+ function isVisionSkillArguments(value: unknown): boolean {
56
+ return isRecord(value) && isBundledSkillName(value.name)
57
+ }
58
+
59
+ function nativeSkillCall(raw: string): boolean {
60
+ try {
61
+ return isVisionSkillArguments(JSON.parse(raw))
62
+ } catch {
63
+ return false
64
+ }
65
+ }
66
+
67
+ function containsBundledSkillContent(blocks: readonly unknown[]): boolean {
68
+ return blocks.some(block => isRecord(block)
69
+ && block.type === 'text'
70
+ && typeof block.text === 'string'
71
+ && isBundledSkillContent(block.text))
72
+ }
73
+
74
+ function isBundledSkillResult(value: unknown): boolean {
75
+ return isRecord(value)
76
+ && isBundledSkillName(value.name)
77
+ && typeof value.content === 'string'
78
+ && isBundledSkillContent(value.content)
79
+ }
80
+
81
+ /** Whether durable history proves that this Session loaded the bundled Skill. */
82
+ function hasLoadedVisionSkill(session: Session): boolean {
83
+ const nativeCalls = new Set<string>()
84
+ for (const event of session.events) {
85
+ if (event.type === 'user/message') {
86
+ const source = event.data.source
87
+ if (source.kind === 'skill-invocation'
88
+ && isBundledSkillName(source.name)
89
+ && containsBundledSkillContent(event.data.content)) return true
90
+ continue
91
+ }
92
+ if (event.type === 'tool/call') {
93
+ if (event.data.name === 'skill' && nativeSkillCall(event.data.arguments)) {
94
+ nativeCalls.add(String(event.data.callId))
95
+ }
96
+ continue
97
+ }
98
+ if (event.type === 'tool/result') {
99
+ const [block] = event.data.message.content
100
+ if (block?.type === 'tool-result'
101
+ && block.isError !== true
102
+ && nativeCalls.has(String(block.toolCallId))
103
+ && containsBundledSkillContent(block.content)) return true
104
+ continue
105
+ }
106
+ if (event.type === 'tool/code-dispatch'
107
+ && event.data.name === 'skill'
108
+ && event.data.isError === false
109
+ && isVisionSkillArguments(event.data.arguments)
110
+ && containsBundledSkillContent(event.data.content)) return true
111
+ }
112
+ return false
113
+ }
114
+
115
+ /**
116
+ * Owns one progressive-exposure generation for a ready Vision Toolkit runtime.
117
+ * The bootstrap tool is global; visual definitions are created and registered
118
+ * in an Agent scope after the Skill load is durable, just succeeded, or the
119
+ * model explicitly invokes the bootstrap fallback.
120
+ */
121
+ export class VisionToolExposure {
122
+ readonly activationTool: ToolDefinition
123
+ private readonly states = new Map<Agent, AgentExposure>()
124
+ private installed = false
125
+
126
+ /**
127
+ * @param ctx - Plugin context with Tool and Agent registries.
128
+ * @param createTools - Fresh definitions bound to the current runtime generation.
129
+ */
130
+ constructor(
131
+ private readonly ctx: Context,
132
+ private readonly createTools: () => ToolDefinition[],
133
+ ) {
134
+ this.activationTool = defineTool({
135
+ name: VISION_TOOLKIT_ACTIVATE,
136
+ description: `Activate the independent Vision Toolkit execution tools for this Agent: ${Object.values(VISION_TOOL_NAMES).join(', ')}. `
137
+ + `Loading the ${VISION_SKILLS_NAME} Skill normally activates them automatically; call this once when the visual tools are still absent, then use them for image understanding, OCR, UI detection, and related tasks. `
138
+ + 'It is safe to call before the Skill is loaded, and this activation tool disappears after success.',
139
+ parameters: {},
140
+ output: {
141
+ schema: {
142
+ type: 'object',
143
+ additionalProperties: false,
144
+ properties: {
145
+ activated: { type: 'boolean', required: true },
146
+ tools: { type: 'array', items: { type: 'string' }, required: true },
147
+ },
148
+ },
149
+ render: renderJson,
150
+ },
151
+ execute: (_args, exec): Promise<VisionToolkitActivationResult> => {
152
+ if (exec.agent === undefined) {
153
+ throw new Error(`${VISION_TOOLKIT_ACTIVATE}: an Agent Session is required`)
154
+ }
155
+ return Promise.resolve(this.activate(exec.agent))
156
+ },
157
+ presentCall: () => ({ card: 'generic', title: 'Activate vision tools', kind: 'execute' }),
158
+ })
159
+ }
160
+
161
+ /** Install lifecycle listeners and adopt Agents that already exist. */
162
+ install(): () => void {
163
+ if (this.installed) throw new Error('dsh-vision-toolkit: progressive exposure is already installed')
164
+ this.installed = true
165
+ const listeners = [
166
+ this.ctx.on('agent/created', ({ agent }) => { this.attach(agent) }),
167
+ this.ctx.on('agent/disposed', ({ agent }) => { this.detach(agent) }),
168
+ this.ctx.on('session/event', (session, event) => {
169
+ if (event.type === 'step/end') this.applyHideActivationForSession(session)
170
+ }),
171
+ this.ctx.on('tools/result', (exec, result) => {
172
+ if (result.isError === false
173
+ && exec.name === 'skill'
174
+ && exec.agent !== undefined
175
+ && isVisionSkillArguments(exec.arguments)
176
+ && isBundledSkillResult(result.value)) {
177
+ this.activate(exec.agent)
178
+ }
179
+ return undefined
180
+ }),
181
+ ]
182
+ try {
183
+ for (const agent of this.ctx.agents.list()) this.attach(agent)
184
+ } catch (error) {
185
+ for (const dispose of listeners.reverse()) dispose()
186
+ this.disposeStates()
187
+ this.installed = false
188
+ throw error
189
+ }
190
+ return () => {
191
+ if (!this.installed) return
192
+ this.installed = false
193
+ for (const dispose of listeners.reverse()) dispose()
194
+ this.disposeStates()
195
+ }
196
+ }
197
+
198
+ private attach(agent: Agent): void {
199
+ if (this.states.has(agent)) return
200
+ this.states.set(agent, { active: false, toolDisposers: [], toolNames: [] })
201
+ if (hasLoadedVisionSkill(agent.session)) this.activate(agent)
202
+ }
203
+
204
+ private activate(agent: Agent): VisionToolkitActivationResult {
205
+ this.attach(agent)
206
+ const state = this.states.get(agent)
207
+ /* v8 ignore next -- attach() synchronously creates this exact entry. */
208
+ if (state === undefined) throw new Error(`dsh-vision-toolkit: Agent ${String(agent.id)} has no exposure state`)
209
+ if (state.active) return { activated: false, tools: [...state.toolNames] }
210
+
211
+ const definitions = this.createTools()
212
+ const toolDisposers: Array<() => void> = []
213
+ try {
214
+ for (const definition of definitions) toolDisposers.push(agent.ctx.tools.register(definition))
215
+ // A Skill call and the bootstrap can be issued in the same model step.
216
+ // Restricting immediately would turn the still-in-flight bootstrap call
217
+ // into an UNKNOWN_TOOL error, so live sessions hide at step/end.
218
+ if (!this.isLiveSession(agent.session)) this.applyHideActivation(agent)
219
+ } catch (error) {
220
+ for (const dispose of toolDisposers.reverse()) dispose()
221
+ throw error
222
+ }
223
+
224
+ state.active = true
225
+ state.toolDisposers = toolDisposers
226
+ state.toolNames = definitions.map(definition => definition.name)
227
+ return { activated: true, tools: [...state.toolNames] }
228
+ }
229
+
230
+ /** Whether the session is attached to the live SessionStore (production). */
231
+ private isLiveSession(session: Session): boolean {
232
+ return this.ctx.sessions.get(session.id) === session
233
+ }
234
+
235
+ private applyHideActivationForSession(session: Session): void {
236
+ for (const agent of this.ctx.agents.list()) {
237
+ if (agent.session === session && this.states.get(agent)?.active === true) {
238
+ this.applyHideActivation(agent)
239
+ }
240
+ }
241
+ }
242
+
243
+ private applyHideActivation(agent: Agent): void {
244
+ const state = this.states.get(agent)
245
+ if (state === undefined || state.liftRestriction !== undefined) return
246
+ state.liftRestriction = agent.ctx.tools.restrict({ deny: [VISION_TOOLKIT_ACTIVATE] })
247
+ }
248
+
249
+ private detach(agent: Agent): void {
250
+ const state = this.states.get(agent)
251
+ if (state === undefined) return
252
+ this.states.delete(agent)
253
+ this.disposeState(state)
254
+ }
255
+
256
+ private disposeStates(): void {
257
+ for (const state of this.states.values()) this.disposeState(state)
258
+ this.states.clear()
259
+ }
260
+
261
+ private disposeState(state: AgentExposure): void {
262
+ state.liftRestriction?.()
263
+ for (const dispose of state.toolDisposers.reverse()) dispose()
264
+ }
265
+ }