@anionex/dsh-vision-toolkit 0.1.30 → 0.1.32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -20
- package/README.zh.md +16 -14
- package/assets/skill/SKILL.md +2 -2
- package/assets/skill/UPSTREAM.json +5 -5
- package/assets/skill/references/restore-ui.md +6 -4
- package/docs/requirements-traceability/README.md +1 -1
- package/docs/requirements-traceability/README.zh.md +1 -1
- package/lib/exposure.js +18 -7
- package/lib/exposure.js.map +1 -1
- package/lib/image-input-variants.js +72 -22
- package/lib/image-input-variants.js.map +1 -1
- package/lib/index.js +4 -4
- package/lib/index.js.map +1 -1
- package/lib/plugin-update.js +8 -1
- package/lib/plugin-update.js.map +1 -1
- package/lib/runtime.js +12 -9
- package/lib/runtime.js.map +1 -1
- package/lib/skill.js +7 -7
- package/lib/skill.js.map +1 -1
- package/lib/types/exposure.d.ts +4 -0
- package/lib/types/exposure.d.ts.map +1 -1
- package/lib/types/image-input-variants.d.ts.map +1 -1
- package/lib/types/index.d.ts +1 -1
- package/lib/types/plugin-update.d.ts.map +1 -1
- package/lib/types/runtime.d.ts.map +1 -1
- package/lib/types/skill.d.ts +4 -4
- package/lib/types/skill.d.ts.map +1 -1
- package/package.json +2 -1
- package/patches/vision-tools-dsh.patch +29 -13
- package/src/exposure.ts +22 -7
- package/src/image-input-variants.ts +74 -21
- package/src/index.ts +4 -4
- package/src/plugin-update.ts +8 -1
- package/src/runtime.ts +12 -9
- package/src/skill.ts +7 -7
package/README.md
CHANGED
|
@@ -6,9 +6,12 @@
|
|
|
6
6
|
|
|
7
7
|
# DSH Vision Toolkit
|
|
8
8
|
|
|
9
|
+
<a href="https://trendshift.io/repositories/149708?utm_source=trendshift-badge&utm_medium=badge&utm_campaign=badge-trendshift-149708" target="_blank" rel="noopener noreferrer"><img src="https://trendshift.io/api/badge/trendshift/repositories/149708/daily?language=TypeScript" alt="Anionex%2Fdsh-vision-toolkit | Trendshift" width="250" height="55"/></a>
|
|
10
|
+
|
|
9
11
|
[](https://dshfind.com/en/plugins/Anionex/dsh-vision-toolkit)
|
|
10
12
|
[](https://dshfind.com/en/plugins/Anionex/dsh-vision-toolkit)
|
|
11
13
|
[](https://www.theagenticleaderboard.com)
|
|
14
|
+
|
|
12
15
|
[](https://www.npmjs.com/package/@anionex/dsh-vision-toolkit)
|
|
13
16
|
[](LICENSE)
|
|
14
17
|
[](cordis.patch.yml)
|
|
@@ -18,6 +21,10 @@
|
|
|
18
21
|
|
|
19
22
|
🚀 Paste an image and ask directly | Install with one command | Built-in free vision | Broad use cases
|
|
20
23
|
|
|
24
|
+
<p align="center">
|
|
25
|
+
<a href="#highlights">Highlights</a> | <a href="#quick-start-three-steps">Quick start</a> | <a href="#common-workflows">Common workflows</a> | <a href="#toolbox">Toolbox</a> | <a href="#configuration-and-limits">Configuration</a> | <a href="#troubleshooting">Troubleshooting</a> | <a href="#development-and-community">Community</a>
|
|
26
|
+
</p>
|
|
27
|
+
|
|
21
28
|
🌐 **English** | [中文](README.zh.md)
|
|
22
29
|
|
|
23
30
|
</div>
|
|
@@ -26,14 +33,14 @@ If you use DeepSeek or another text-only model in DeepSeek Harness (DSH), you ma
|
|
|
26
33
|
|
|
27
34
|
🏆 This project is the first comprehensive vision-tool plugin in the DeepSeek Harness ecosystem: it was initiated before internal beta and built during the beta with reference to [`agent-vision-toolkit`](https://github.com/Anionex/agent-vision-toolkit).
|
|
28
35
|
|
|
29
|
-
> **Original work:** The system and division of responsibilities behind these visual tools, together with the `vision-
|
|
36
|
+
> **Original work:** The system and division of responsibilities behind these visual tools, together with the `vision-skills` Skill, were personally created and continuously refined by the author through long-term real-world use and repeated iteration.
|
|
30
37
|
|
|
31
38
|
## Highlights
|
|
32
39
|
|
|
33
40
|
- **Paste and use it immediately.** Paste an image in DSH Web and the text-only route switches to its `(Vision Toolkit)` variant without manual path copying or model changes.
|
|
34
41
|
- **A seamless image workflow.** Native thumbnails, session history, and workspace paths stay intact; Web can preview artifacts and Headless can continue using the same structured results.
|
|
35
42
|
- **One command to install.** The built-in free Gemini 3.7 Flash vision service is ready after installation, with no API key required.
|
|
36
|
-
- **Built-in free vision.** The shared service works immediately after installation
|
|
43
|
+
- **Built-in free vision.** The shared service works immediately after installation with a quota of **300 images per machine per day**.
|
|
37
44
|
- **Vision guided by intent.** The agent extracts evidence for the task at hand, such as “Where is the error?” or “Where is the button?”, instead of returning a generic caption.
|
|
38
45
|
- **A complete screenshot-to-verification loop.** Reference images, HTML screenshots, difference regions, and pixel comparison work together for UI restoration.
|
|
39
46
|
|
|
@@ -147,7 +154,7 @@ The first start prepares an isolated runtime: the plugin prefers a system Python
|
|
|
147
154
|
|
|
148
155
|
### 3. Paste an image and describe the outcome you want
|
|
149
156
|
|
|
150
|
-
Paste a screenshot into the conversation or place an image in the session workspace, then invoke `/vision-
|
|
157
|
+
Paste a screenshot into the conversation or place an image in the session workspace, then invoke `/vision-skills`. For example:
|
|
151
158
|
|
|
152
159
|
```text
|
|
153
160
|
Inspect this screenshot. Explain the error and tell me what to fix first.
|
|
@@ -197,7 +204,7 @@ The plugin keeps image understanding and deterministic local image processing in
|
|
|
197
204
|
|
|
198
205
|
```mermaid
|
|
199
206
|
flowchart LR
|
|
200
|
-
Image["Screenshot or local HTML"] --> Skill["vision-
|
|
207
|
+
Image["Screenshot or local HTML"] --> Skill["vision-skills Skill"]
|
|
201
208
|
Skill --> Agent["Text agent selects a task"]
|
|
202
209
|
Agent --> Vision["Use a vision model when image understanding is needed"]
|
|
203
210
|
Agent --> Local["Run crop, SVG, and pixel work locally"]
|
|
@@ -209,13 +216,13 @@ flowchart LR
|
|
|
209
216
|
|
|
210
217
|
The visual capabilities come from a packaged, pinned `agent-vision-toolkit` snapshot. The DSH plugin handles installation, session-scoped tool exposure, Credentials, path checks, cancellation, timeouts, result files, and Web presentation. The runtime never fetches upstream `main` in the background.
|
|
211
218
|
|
|
212
|
-
The `vision-
|
|
213
|
-
`SKILL.md` plus all five upstream playbooks. Tool
|
|
214
|
-
Artifact delivery, progressive exposure, and DSH
|
|
215
|
-
adapted; the upstream tool-selection rules,
|
|
216
|
-
remain intact. The exact upstream Skill
|
|
217
|
-
and reviewable adapter patch are
|
|
218
|
-
`patches/vision-tools-dsh.patch`.
|
|
219
|
+
The bundled `vision-skills` Skill is the DSH adapter of the upstream
|
|
220
|
+
`vision-tools` Skill: its `SKILL.md` plus all five upstream playbooks. Tool
|
|
221
|
+
names, argument syntax, Artifact delivery, progressive exposure, and DSH
|
|
222
|
+
path/lifecycle boundaries are adapted; the upstream tool-selection rules,
|
|
223
|
+
coarse-to-fine method, and task SOPs remain intact. The exact upstream Skill
|
|
224
|
+
commit, source hashes, adapted hashes, and reviewable adapter patch are
|
|
225
|
+
recorded in `assets/skill/UPSTREAM.json` and `patches/vision-tools-dsh.patch`.
|
|
219
226
|
|
|
220
227
|
For routes that DSH positively identifies as text-only, the plugin registers a sibling `<model> (Vision Toolkit)` variant. By default, pasting an image in DSH Web switches to that variant and gives the model both a reusable workspace path and a visual description focused on the current task.
|
|
221
228
|
|
|
@@ -239,6 +246,7 @@ This is a shared zero-configuration entry point, not an unlimited private endpoi
|
|
|
239
246
|
|
|
240
247
|
| Limit | Current value |
|
|
241
248
|
|---|---:|
|
|
249
|
+
| Daily quota | 300 images per machine per day |
|
|
242
250
|
| Images per request | Up to 5 |
|
|
243
251
|
| Image size | 4 MiB per image |
|
|
244
252
|
| Decoded pixels | 20,000,000 per image |
|
|
@@ -383,7 +391,7 @@ The updater revalidates the Profile before mutation, snapshots the original mani
|
|
|
383
391
|
|
|
384
392
|
| Problem | What to do |
|
|
385
393
|
|---|---|
|
|
386
|
-
| Pasting an image still says the model does not support image input | Restart the Web Profile, refresh the page, and confirm the selected route has the `(Vision Toolkit)` suffix. You can also place the image in the session workspace and invoke `/vision-
|
|
394
|
+
| Pasting an image still says the model does not support image input | Restart the Web Profile, refresh the page, and confirm the selected route has the `(Vision Toolkit)` suffix. You can also place the image in the session workspace and invoke `/vision-skills` |
|
|
387
395
|
| The free service returns 429 | Wait for the `Retry-After` interval, or switch to your own endpoint when you need stable higher volume |
|
|
388
396
|
| The image exceeds a size or pixel limit | Crop or resize it first; the error identifies whether bytes or decoded pixels caused the rejection |
|
|
389
397
|
| A custom Credential is missing | Enter the API key in **Settings → Vision Toolkit** and confirm the Credential name matches the provider configuration |
|
|
@@ -397,14 +405,6 @@ The current release focuses on screenshot understanding, visual grounding, OCR,
|
|
|
397
405
|
|
|
398
406
|
## Development and community
|
|
399
407
|
|
|
400
|
-
```sh
|
|
401
|
-
pnpm install --frozen-lockfile --trust-lockfile
|
|
402
|
-
pnpm run verify:portable
|
|
403
|
-
pnpm run build
|
|
404
|
-
pnpm test
|
|
405
|
-
TSX_TSCONFIG_PATH=tsconfig.json pnpm dlx tsx scripts/ui-restoration-example.ts --check
|
|
406
|
-
```
|
|
407
|
-
|
|
408
408
|
- Read [CONTRIBUTING.md](CONTRIBUTING.md) before contributing.
|
|
409
409
|
- Use [GitHub Issues](https://github.com/Anionex/dsh-vision-toolkit/issues) for bugs, focused feature requests, and usage questions; see [SUPPORT.md](SUPPORT.md) for channel guidance.
|
|
410
410
|
- Report vulnerabilities privately through [SECURITY.md](SECURITY.md).
|
|
@@ -415,6 +415,8 @@ TSX_TSCONFIG_PATH=tsconfig.json pnpm dlx tsx scripts/ui-restoration-example.ts -
|
|
|
415
415
|
<img src="assets/community-group-qr.png" alt="QR code for the agent-vision-toolkit community group" width="240" />
|
|
416
416
|
</p>
|
|
417
417
|
|
|
418
|
+
I'm <a href="https://anionex.me/">anionex</a>, an AI-native developer who once ranked <strong>No. 3</strong> on GitHub's global developer trending list, with more than 16k stars across my projects. If you would like to follow my future work, <a href="https://github.com/Anionex">follow me on GitHub</a>.
|
|
419
|
+
|
|
418
420
|
[`agent-vision-toolkit`](https://github.com/Anionex/agent-vision-toolkit) was created by [Anionex](https://anionex.me/). This repository maintains its native DeepSeek Harness integration.
|
|
419
421
|
|
|
420
422
|
## License
|
package/README.zh.md
CHANGED
|
@@ -6,9 +6,12 @@
|
|
|
6
6
|
|
|
7
7
|
# DSH Vision Toolkit
|
|
8
8
|
|
|
9
|
+
<a href="https://trendshift.io/repositories/149708?utm_source=trendshift-badge&utm_medium=badge&utm_campaign=badge-trendshift-149708" target="_blank" rel="noopener noreferrer"><img src="https://trendshift.io/api/badge/trendshift/repositories/149708/daily?language=TypeScript" alt="Anionex%2Fdsh-vision-toolkit | Trendshift" width="250" height="55"/></a>
|
|
10
|
+
|
|
9
11
|
[](https://dshfind.com/zh/plugins/Anionex/dsh-vision-toolkit)
|
|
10
12
|
[](https://dshfind.com/zh/plugins/Anionex/dsh-vision-toolkit)
|
|
11
13
|
[](https://www.npmjs.com/package/@anionex/dsh-vision-toolkit)
|
|
14
|
+
|
|
12
15
|
[](LICENSE)
|
|
13
16
|
[](cordis.patch.yml)
|
|
14
17
|
|
|
@@ -16,6 +19,10 @@
|
|
|
16
19
|
|
|
17
20
|
🚀 粘贴图片，直接提问 ｜ 一行命令安装即用 ｜ 内置免费视觉 ｜ 场景丰富
|
|
18
21
|
|
|
22
|
+
<p align="center">
|
|
23
|
+
<a href="#亮点">亮点</a> ｜ <a href="#快速开始三步完成">快速开始</a> ｜ <a href="#常见任务">常见任务</a> ｜ <a href="#工具一览">工具一览</a> ｜ <a href="#配置与限制">配置与限制</a> ｜ <a href="#常见问题">常见问题</a> ｜ <a href="#开发与社区">交流群</a>
|
|
24
|
+
</p>
|
|
25
|
+
|
|
19
26
|
🌐 [English](README.md) ｜ **中文**
|
|
20
27
|
|
|
21
28
|
</div>
|
|
@@ -28,14 +35,14 @@
|
|
|
28
35
|
|
|
29
36
|
🏆 本项目为deepseek harness生态首个综合性视觉工具插件：内测前已立项，并在内测期间参考本人的[`agent-vision-toolkit`](https://github.com/Anionex/agent-vision-toolkit)做出
|
|
30
37
|
|
|
31
|
-
> **原创声明：** 这套视觉工具的体系和划分方式，以及 `vision-
|
|
38
|
+
> **原创声明：** 这套视觉工具的体系和划分方式，以及 `vision-skills` Skill，均由作者个人原创并持续打磨，相关工具、方法和工作流来自长期的真实使用与反复迭代。
|
|
32
39
|
|
|
33
40
|
## 亮点
|
|
34
41
|
|
|
35
42
|
- **粘贴即可使用。** 在 DSH Web 里粘贴图片，文本模型会自动切换到看图模式变体，不需要手动复制路径或更换模型。
|
|
36
43
|
- **无缝体验。** 图片保留原生缩略图、会话记录和工作区路径；Web 可以预览产物，Headless 也能继续使用同一份结构化结果。
|
|
37
44
|
- **一行命令安装即用。** 安装插件后默认使用内置免费 Gemini 3.7 Flash 视觉服务，不需要申请 API Key。
|
|
38
|
-
- **内置免费视觉。**
|
|
45
|
+
- **内置免费视觉。** 安装后即可直接使用共享服务，每台机器每天有 **300 张图** 的免费额度。
|
|
39
46
|
- **带着意图去看图。** Agent 不只生成通用描述，而是围绕“报错在哪里”“按钮在哪”等当前任务提取证据。
|
|
40
47
|
- **从截图到可验证结果。** 参考图、HTML 截图、差异定位和像素对比组成一条完整 UI 还原闭环。
|
|
41
48
|
|
|
@@ -150,7 +157,7 @@ dsh plugin --profile headless add @anionex/dsh-vision-toolkit
|
|
|
150
157
|
|
|
151
158
|
### 3. 粘贴图片，直接说你要做什么
|
|
152
159
|
|
|
153
|
-
在会话中粘贴截图，或把图片放进会话工作区，然后调用 `/vision-
|
|
160
|
+
在会话中粘贴截图，或把图片放进会话工作区，然后调用 `/vision-skills`。例如：
|
|
154
161
|
|
|
155
162
|
```text
|
|
156
163
|
看看这张截图，告诉我报错原因和最值得先修的地方。
|
|
@@ -200,7 +207,7 @@ dsh plugin --profile headless add @anionex/dsh-vision-toolkit
|
|
|
200
207
|
|
|
201
208
|
```mermaid
|
|
202
209
|
flowchart LR
|
|
203
|
-
Image["截图或本地 HTML"] --> Skill["vision-
|
|
210
|
+
Image["截图或本地 HTML"] --> Skill["vision-skills Skill"]
|
|
204
211
|
Skill --> Agent["文本 Agent 选择任务"]
|
|
205
212
|
Agent --> Vision["需要理解图片时调用视觉模型"]
|
|
206
213
|
Agent --> Local["裁剪、SVG、像素等任务在本地处理"]
|
|
@@ -212,7 +219,7 @@ flowchart LR
|
|
|
212
219
|
|
|
213
220
|
视觉能力来自打包的固定版本 `agent-vision-toolkit`。DSH 插件负责安装、会话级工具暴露、Credential、路径校验、取消、超时、结果文件和 Web 展示。运行时不会在后台拉取上游 `main`。
|
|
214
221
|
|
|
215
|
-
`vision-
|
|
222
|
+
`vision-skills` Skill（上游原名 `vision-tools`）现在以上游 `SKILL.md` 和全部 5 篇上游 SOP 为明确底稿：
|
|
216
223
|
只适配工具名、结构化参数、Artifact 交付、渐进式暴露，以及 DSH 的路径和生命周期边界；
|
|
217
224
|
上游的工具选择规则、由粗到细方法和任务流程保持不变。精确的上游 Skill commit、
|
|
218
225
|
源文件哈希、适配后哈希和可审查补丁分别记录在 `assets/skill/UPSTREAM.json` 与
|
|
@@ -240,6 +247,7 @@ API Key: https://agent-vision.anionex.me（自动填写）
|
|
|
240
247
|
|
|
241
248
|
| 限制 | 当前值 |
|
|
242
249
|
|---|---:|
|
|
250
|
+
| 每日额度 | 每台机器每天 300 张图 |
|
|
243
251
|
| 单次请求图片数 | 最多 5 张 |
|
|
244
252
|
| 单张图片大小 | 4 MiB |
|
|
245
253
|
| 单张图片像素 | 20,000,000 |
|
|
@@ -384,7 +392,7 @@ dsh plugin --profile web remove @anionex/dsh-vision-toolkit
|
|
|
384
392
|
|
|
385
393
|
| 问题 | 处理方式 |
|
|
386
394
|
|---|---|
|
|
387
|
-
| 粘贴图片后仍提示模型不支持图片 | 重启 Web Profile 并刷新页面，确认当前模型已切换到带 `(Vision Toolkit)` 的变体；也可以把图片先放进会话工作区，再调用 `/vision-
|
|
395
|
+
| 粘贴图片后仍提示模型不支持图片 | 重启 Web Profile 并刷新页面，确认当前模型已切换到带 `(Vision Toolkit)` 的变体；也可以把图片先放进会话工作区，再调用 `/vision-skills` |
|
|
388
396
|
| 免费服务提示 429 | 按错误中的 `Retry-After` 等待后重试；如果需要稳定高额度，切换到自己的视觉端点 |
|
|
389
397
|
| 图片过大或像素超限 | 先裁剪或缩放图片；错误会明确显示是字节还是像素限制 |
|
|
390
398
|
| 自定义 Credential 缺失 | 在 **设置 → 视觉工具** 填写 API Key，并确认 Credential 名称与配置一致 |
|
|
@@ -398,14 +406,6 @@ dsh plugin --profile web remove @anionex/dsh-vision-toolkit
|
|
|
398
406
|
|
|
399
407
|
## 开发与社区
|
|
400
408
|
|
|
401
|
-
```sh
|
|
402
|
-
pnpm install --frozen-lockfile --trust-lockfile
|
|
403
|
-
pnpm run verify:portable
|
|
404
|
-
pnpm run build
|
|
405
|
-
pnpm test
|
|
406
|
-
TSX_TSCONFIG_PATH=tsconfig.json pnpm dlx tsx scripts/ui-restoration-example.ts --check
|
|
407
|
-
```
|
|
408
|
-
|
|
409
409
|
- 贡献前请阅读 [CONTRIBUTING.md](CONTRIBUTING.md)。
|
|
410
410
|
- Bug、功能建议和使用问题请提交到 [GitHub Issues](https://github.com/Anionex/dsh-vision-toolkit/issues)；渠道说明见 [SUPPORT.md](SUPPORT.md)。
|
|
411
411
|
- 安全漏洞请按 [SECURITY.md](SECURITY.md) 私下报告。
|
|
@@ -416,6 +416,8 @@ TSX_TSCONFIG_PATH=tsconfig.json pnpm dlx tsx scripts/ui-restoration-example.ts -
|
|
|
416
416
|
<img src="assets/community-group-qr.png" alt="agent-vision-toolkit 项目交流群二维码" width="240" />
|
|
417
417
|
</p>
|
|
418
418
|
|
|
419
|
+
我是 <a href="https://anionex.me/">anionex</a>，一位 AI 原生开发者，曾位列 GitHub 全球开发者趋势榜第 <strong>3</strong> 名，项目累计超过 16k stars。想了解我后续的工作，欢迎在 <a href="https://github.com/Anionex">GitHub</a> 关注我。
|
|
420
|
+
|
|
419
421
|
[`agent-vision-toolkit`](https://github.com/Anionex/agent-vision-toolkit) 由 [Anionex](https://anionex.me/) 创建。本仓库维护它面向 DeepSeek Harness 的原生集成。
|
|
420
422
|
|
|
421
423
|
## 许可证
|
package/assets/skill/SKILL.md
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# vision-
|
|
1
|
+
# vision-skills
|
|
2
2
|
|
|
3
3
|
Ten native DSH tools give a text-only agent eyes. Use these structured tools
|
|
4
4
|
directly; do not shell out to the bundled Python scripts or reproduce their
|
|
@@ -7,7 +7,7 @@ plugin, so tool calls do not receive credentials.
|
|
|
7
7
|
|
|
8
8
|
The visual execution schemas are mounted only for the current Agent after this
|
|
9
9
|
Skill is loaded. A normal `skill` call activates them for the next model step.
|
|
10
|
-
If this content arrived through a direct `/vision-
|
|
10
|
+
If this content arrived through a direct `/vision-skills` invocation and the
|
|
11
11
|
visual tools are still absent, call `vision_toolkit_activate` once. Do not call
|
|
12
12
|
that bootstrap when the visual tools are already present.
|
|
13
13
|
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
"repository": "https://github.com/Anionex/agent-vision-toolkit",
|
|
4
4
|
"commit": "77c24ad5b5d7a123119862893129f939307f1d3f",
|
|
5
5
|
"patch": "patches/vision-tools-dsh.patch",
|
|
6
|
-
"patchSha256": "
|
|
6
|
+
"patchSha256": "ebbd49035945f094be9a582c5095d3e1689879d2501a7c82d0ca4517e38abf1b",
|
|
7
7
|
"sourceFiles": [
|
|
8
8
|
{
|
|
9
9
|
"path": "SKILL.md",
|
|
@@ -39,8 +39,8 @@
|
|
|
39
39
|
"adaptedFiles": [
|
|
40
40
|
{
|
|
41
41
|
"path": "SKILL.md",
|
|
42
|
-
"bytes":
|
|
43
|
-
"sha256": "
|
|
42
|
+
"bytes": 16070,
|
|
43
|
+
"sha256": "5e66863f14e85beaba46a6d148ab82304aa00d63ce2ae90cb25e3aad975f1b22"
|
|
44
44
|
},
|
|
45
45
|
{
|
|
46
46
|
"path": "references/gui.md",
|
|
@@ -64,8 +64,8 @@
|
|
|
64
64
|
},
|
|
65
65
|
{
|
|
66
66
|
"path": "references/restore-ui.md",
|
|
67
|
-
"bytes":
|
|
68
|
-
"sha256": "
|
|
67
|
+
"bytes": 10197,
|
|
68
|
+
"sha256": "8cd479a195c526098733ac518cb9928ffdafdbb3ad93e08864efe4fd7bd312b3"
|
|
69
69
|
}
|
|
70
70
|
]
|
|
71
71
|
}
|
|
@@ -37,8 +37,9 @@ typography, shadows, decorative details, and icon geometry.
|
|
|
37
37
|
or questions into the same round instead of waiting for each result before
|
|
38
38
|
starting the next call. Normally one or two rounds are enough.
|
|
39
39
|
4. Do not use `vision_trace`, foreground extraction, repeated color sampling,
|
|
40
|
-
|
|
41
|
-
consume the delivery window.
|
|
40
|
+
iterative `vision_pixel_diff` work, or hand-written SVG in fast mode. Those
|
|
41
|
+
are fidelity tools and will consume the delivery window. Icons and decorative
|
|
42
|
+
marks stay library-based or screenshot-backed; never hand-write SVG code.
|
|
42
43
|
|
|
43
44
|
### Build the approximation
|
|
44
45
|
|
|
@@ -47,8 +48,9 @@ typography, shadows, decorative details, and icon geometry.
|
|
|
47
48
|
when zoomed in.
|
|
48
49
|
2. Reuse the project's existing components and CSS tokens. If its frontend or
|
|
49
50
|
icon library contains a reasonably similar component or icon, use it
|
|
50
|
-
directly instead of recreating the reference.
|
|
51
|
-
|
|
51
|
+
directly instead of recreating the reference. Never hand-write SVG in fast
|
|
52
|
+
mode: use an approximate library icon, or extract the original pixels as a
|
|
53
|
+
screenshot-backed asset.
|
|
52
54
|
3. Use nearby existing palette tokens or visually similar CSS values. Exact
|
|
53
55
|
sampled hex values, gradients, subtle borders, and shadow opacity are out of
|
|
54
56
|
scope unless one of them defines the whole composition.
|
|
@@ -36,7 +36,7 @@ This reference maps the DSH Vision Toolkit product brief's committed P0/P1 requi
|
|
|
36
36
|
| Portability | Runtime and browser calls use argv vectors and Node filesystem/process APIs rather than POSIX shell composition. Managed preparation has `uv` and `venv`/pip paths; Python and Chrome discovery include platform-specific candidates. Package tests avoid machine-local dependency specifiers, and the checked-in fixture/profile flow is keyless. |
|
|
37
37
|
| Performance and cancellation | [`src/runtime.ts`](../../src/runtime.ts) applies one hard deadline, propagates `AbortSignal`, bounds per-session concurrency, rejects oversized decoded images before remote I/O, deduplicates repeated inputs within one glance operation, and keeps a one-entry content/configuration-keyed cache for the last successful identical glance in each live Session. [`src/index.ts`](../../src/index.ts) aborts plugin-owned calls before unregistering tools. Timeout, caller/plugin cancellation, cache-hit/miss, semaphore, and independent-session behavior are automated. |
|
|
38
38
|
| Observability | [`src/runtime.ts`](../../src/runtime.ts) logs bounded tool name, outcome, total/upstream duration, image count/bytes/pixels, cache hits, model, and error category while excluding base64, credentials, headers, and unbounded upstream output. |
|
|
39
|
-
| Model-context economy | The profile-global runtime publishes one small bootstrap, while [`src/exposure.ts`](../../src/exposure.ts) mounts the ten execution schemas only in an Agent that loads `vision-
|
|
39
|
+
| Model-context economy | The profile-global runtime publishes one small bootstrap, while [`src/exposure.ts`](../../src/exposure.ts) mounts the ten execution schemas only in an Agent that loads `vision-skills` and hides the bootstrap afterward. Other Agents remain unchanged. Health, connection, and version diagnostics stay exclusively on the Web Settings seam and never become model tools. Unit tests cover Agent isolation and restoration; every real-profile tool flow asserts the initial and activated schema sets. |
|
|
40
40
|
| Maintainability | Every tool calls one [`VisionToolkitRuntime`](../../src/runtime.ts); DSH-specific adaptation remains in [`src/tools.ts`](../../src/tools.ts), [`src/exposure.ts`](../../src/exposure.ts), [`src/index.ts`](../../src/index.ts), [`src/web.ts`](../../src/web.ts), and [`src/client/index.tsx`](../../src/client/index.tsx), while the pinned upstream remains the sole algorithm source. Runtime readiness, Skill/bootstrap publication, and Agent-scoped schemas belong to one lifecycle generation. |
|
|
41
41
|
| HTML screenshot isolation | The pinned screenshot guard uses a disposable Chrome profile, `--use-mock-keychain`, and `--incognito`, removes the profile after each call, and keeps network disabled. [`tests/html-screenshot-guard.spec.ts`](../../tests/html-screenshot-guard.spec.ts) captures the actual argv and cleanup behavior. |
|
|
42
42
|
| UI restoration acceptance | [`scripts/ui-restoration-example.ts`](../../scripts/ui-restoration-example.ts) performs reference → HTML render → pixel diff through the real runtime. [`examples/ui-restoration/README.md`](../../examples/ui-restoration/README.md) owns the reproducible procedure and checked-in evidence; [`tests/ui-restoration-example.spec.ts`](../../tests/ui-restoration-example.spec.ts) enforces it. |
|
|
@@ -36,7 +36,7 @@
|
|
|
36
36
|
| 可移植性 | 运行时和浏览器调用使用 argv 向量与 Node 文件系统/进程 API，不拼接 POSIX Shell。Managed 准备同时提供 `uv` 和 `venv`/pip 路径；Python 与 Chrome 探测包含平台候选。软件包测试拒绝机器本地依赖声明，已提交 fixture/Profile 流程无需真实 Key。 |
|
|
37
37
|
| 性能与取消 | [`src/runtime.ts`](../../src/runtime.ts) 应用一个硬截止时间、传递 `AbortSignal`、限制每个会话的并发数、在远程 I/O 前拒绝解码后过大的图片、在一次 glance 操作中对重复输入去重，并为每个活动会话保留一条按内容/配置键控的最近成功相同 glance 缓存。[`src/index.ts`](../../src/index.ts) 会在注销工具前中止插件拥有的调用。超时、调用方/插件取消、缓存命中/未命中、信号量和独立会话行为均有自动化覆盖。 |
|
|
38
38
|
| 可观察性 | [`src/runtime.ts`](../../src/runtime.ts) 记录有界的工具名、结果、总耗时/上游耗时、图片数量/字节/像素、缓存命中、模型和错误类别，同时排除 base64、Credential、鉴权头和无界上游输出。 |
|
|
39
|
-
| 模型上下文经济性 | Profile 级运行时只发布一个很小的引导工具，[`src/exposure.ts`](../../src/exposure.ts) 仅为加载 `vision-
|
|
39
|
+
| 模型上下文经济性 | Profile 级运行时只发布一个很小的引导工具，[`src/exposure.ts`](../../src/exposure.ts) 仅为加载 `vision-skills` 的 Agent 挂载 10 个执行 schema，并在成功后隐藏引导工具。其他 Agent 不受影响。健康检查、连接测试和版本诊断只存在于 Web Settings 边界，永远不会成为模型工具。单元测试覆盖 Agent 隔离与恢复；每条真实 Profile 工具流程都会断言初始和激活后的 schema 集合。 |
|
|
40
40
|
| 可维护性 | 每个工具都调用同一个 [`VisionToolkitRuntime`](../../src/runtime.ts)；DSH 专用适配保留在 [`src/tools.ts`](../../src/tools.ts)、[`src/exposure.ts`](../../src/exposure.ts)、[`src/index.ts`](../../src/index.ts)、[`src/web.ts`](../../src/web.ts) 和 [`src/client/index.tsx`](../../src/client/index.tsx)，固定上游继续作为算法唯一来源。运行时就绪、skill/引导工具发布和 Agent 级 schema 属于同一个生命周期 generation。 |
|
|
41
41
|
| HTML 截图隔离 | 固定截图 guard 使用一次性 Chrome Profile、`--use-mock-keychain` 和 `--incognito`，每次调用后删除 Profile，并保持网络禁用。[`tests/html-screenshot-guard.spec.ts`](../../tests/html-screenshot-guard.spec.ts) 捕获实际 argv 与清理行为。 |
|
|
42
42
|
| UI 还原验收 | [`scripts/ui-restoration-example.ts`](../../scripts/ui-restoration-example.ts) 通过真实运行时执行参考页面 → HTML 渲染 → 像素差异。[`examples/ui-restoration/README.md`](../../examples/ui-restoration/README.md) 负责可复现流程与已提交证据；[`tests/ui-restoration-example.spec.ts`](../../tests/ui-restoration-example.spec.ts) 强制执行该契约。 |
|
package/lib/exposure.js
CHANGED
|
@@ -6,18 +6,28 @@
|
|
|
6
6
|
* @module dsh-vision-toolkit/exposure
|
|
7
7
|
*/
|
|
8
8
|
import { defineTool } from '@deepseek-ai/dsh-tools';
|
|
9
|
-
import {
|
|
9
|
+
import { VISION_SKILLS_CONTENT, VISION_SKILLS_NAME } from "./skill.js";
|
|
10
10
|
import { VISION_TOOL_NAMES } from "./tools.js";
|
|
11
11
|
/** Small bootstrap tool retained only until the current Agent gains visual tools. */
|
|
12
12
|
export const VISION_TOOLKIT_ACTIVATE = 'vision_toolkit_activate';
|
|
13
|
+
/** Skill name used by releases before the rename to vision-skills. */
|
|
14
|
+
export const LEGACY_VISION_TOOLS_SKILL_NAME = 'vision-tools';
|
|
15
|
+
/** Unique pre-rename line in bundled instructions, kept for Session restore. */
|
|
16
|
+
export const LEGACY_VISION_TOOLS_SKILL_MARKER = 'If this content arrived through a direct `/vision-tools` invocation and the';
|
|
13
17
|
function renderJson(_args, value) {
|
|
14
18
|
return [{ type: 'text', text: JSON.stringify(value, null, 2) }];
|
|
15
19
|
}
|
|
16
20
|
function isRecord(value) {
|
|
17
21
|
return value !== null && typeof value === 'object' && !Array.isArray(value);
|
|
18
22
|
}
|
|
23
|
+
function isBundledSkillName(name) {
|
|
24
|
+
return name === VISION_SKILLS_NAME || name === LEGACY_VISION_TOOLS_SKILL_NAME;
|
|
25
|
+
}
|
|
26
|
+
function isBundledSkillContent(text) {
|
|
27
|
+
return text.includes(VISION_SKILLS_CONTENT) || text.includes(LEGACY_VISION_TOOLS_SKILL_MARKER);
|
|
28
|
+
}
|
|
19
29
|
function isVisionSkillArguments(value) {
|
|
20
|
-
return isRecord(value) && value.name
|
|
30
|
+
return isRecord(value) && isBundledSkillName(value.name);
|
|
21
31
|
}
|
|
22
32
|
function nativeSkillCall(raw) {
|
|
23
33
|
try {
|
|
@@ -31,12 +41,13 @@ function containsBundledSkillContent(blocks) {
|
|
|
31
41
|
return blocks.some(block => isRecord(block)
|
|
32
42
|
&& block.type === 'text'
|
|
33
43
|
&& typeof block.text === 'string'
|
|
34
|
-
&& block.text
|
|
44
|
+
&& isBundledSkillContent(block.text));
|
|
35
45
|
}
|
|
36
46
|
function isBundledSkillResult(value) {
|
|
37
47
|
return isRecord(value)
|
|
38
|
-
&& value.name
|
|
39
|
-
&& value.content ===
|
|
48
|
+
&& isBundledSkillName(value.name)
|
|
49
|
+
&& typeof value.content === 'string'
|
|
50
|
+
&& isBundledSkillContent(value.content);
|
|
40
51
|
}
|
|
41
52
|
/** Whether durable history proves that this Session loaded the bundled Skill. */
|
|
42
53
|
function hasLoadedVisionSkill(session) {
|
|
@@ -45,7 +56,7 @@ function hasLoadedVisionSkill(session) {
|
|
|
45
56
|
if (event.type === 'user/message') {
|
|
46
57
|
const source = event.data.source;
|
|
47
58
|
if (source.kind === 'skill-invocation'
|
|
48
|
-
&& source.name
|
|
59
|
+
&& isBundledSkillName(source.name)
|
|
49
60
|
&& containsBundledSkillContent(event.data.content))
|
|
50
61
|
return true;
|
|
51
62
|
continue;
|
|
@@ -96,7 +107,7 @@ export class VisionToolExposure {
|
|
|
96
107
|
this.activationTool = defineTool({
|
|
97
108
|
name: VISION_TOOLKIT_ACTIVATE,
|
|
98
109
|
description: `Activate the independent Vision Toolkit execution tools for this Agent: ${Object.values(VISION_TOOL_NAMES).join(', ')}. `
|
|
99
|
-
+ `Loading the ${
|
|
110
|
+
+ `Loading the ${VISION_SKILLS_NAME} Skill normally activates them automatically; call this once when the visual tools are still absent, then use them for image understanding, OCR, UI detection, and related tasks. `
|
|
100
111
|
+ 'It is safe to call before the Skill is loaded, and this activation tool disappears after success.',
|
|
101
112
|
parameters: {},
|
|
102
113
|
output: {
|
package/lib/exposure.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"exposure.js","sourceRoot":"","sources":["../src/exposure.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAKH,OAAO,EAAE,UAAU,EAAuB,MAAM,wBAAwB,CAAA;AAExE,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"exposure.js","sourceRoot":"","sources":["../src/exposure.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAKH,OAAO,EAAE,UAAU,EAAuB,MAAM,wBAAwB,CAAA;AAExE,OAAO,EAAE,qBAAqB,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAA;AACtE,OAAO,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAA;AAE9C,qFAAqF;AACrF,MAAM,CAAC,MAAM,uBAAuB,GAAG,yBAAyB,CAAA;AAEhE,sEAAsE;AACtE,MAAM,CAAC,MAAM,8BAA8B,GAAG,cAAc,CAAA;AAE5D,gFAAgF;AAChF,MAAM,CAAC,MAAM,gCAAgC,GAAG,6EAA6E,CAAA;AAe7H,SAAS,UAAU,CAAC,KAAc,EAAE,KAAc;IAChD,OAAO,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC,CAAA;AACjE,CAAC;AAED,SAAS,QAAQ,CAAC,KAAc;IAC9B,OAAO,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;AAC7E,CAAC;AAED,SAAS,kBAAkB,CAAC,IAAa;IACvC,OAAO,IAAI,KAAK,kBAAkB,IAAI,IAAI,KAAK,8BAA8B,CAAA;AAC/E,CAAC;AAED,SAAS,qBAAqB,CAAC,IAAY;IACzC,OAAO,IAAI,CAAC,QAAQ,CAAC,qBAAqB,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,gCAAgC,CAAC,CAAA;AAChG,CAAC;AAED,SAAS,sBAAsB,CAAC,KAAc;IAC5C,OAAO,QAAQ,CAAC,KAAK,CAAC,IAAI,kBAAkB,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;AAC1D,CAAC;AAED,SAAS,eAAe,CAAC,GAAW;IAClC,IAAI,CAAC;QACH,OAAO,sBAAsB,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAA;IAChD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAA;IACd,CAAC;AACH,CAAC;AAED,SAAS,2BAA2B,CAAC,MAA0B;IAC7D,OAAO,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC;WACtC,KAAK,CAAC,IAAI,KAAK,MAAM;WACrB,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ;WAC9B,qBAAqB,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAA;AACzC,CAAC;AAED,SAAS,oBAAoB,CAAC,KAAc;IAC1C,OAAO,QAAQ,CAAC,KAAK,CAAC;WACjB,kBAAkB,CAAC,KAAK,CAAC,IAAI,CAAC;WAC9B,OAAO,KAAK,CAAC,OAAO,KAAK,QAAQ;WACjC,qBAAqB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAA;AAC3C,CAAC;AAED,iFAAiF;AACjF,SAAS,oBAAoB,CAAC,OAAgB;IAC5C,MAAM,WAAW,GAAG,IAAI,GAAG,EAAU,CAAA;IACrC,KAAK,MAAM,KAAK,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACnC,IAAI,KAAK,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;YAClC,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAA;YAChC,IAAI,MAAM,CAAC,IAAI,KAAK,kBAAkB;mBACjC,kBAAkB,CAAC,MAAM,CAAC,IAAI,CAAC;mBAC/B,2BAA2B,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC;gBAAE,OAAO,IAAI,CAAA;YACjE,SAAQ;QACV,CAAC;QACD,IAAI,KAAK,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;YAC/B,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,KAAK,OAAO,IAAI,eAAe,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;gBACzE,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAA;YAC5C,CAAC;YACD,SAAQ;QACV,CAAC;QACD,IAAI,KAAK,CAAC,IAAI,KAAK,aAAa,EAAE,CAAC;YACjC,MAAM,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAA;YAC1C,IAAI,KAAK,EAAE,IAAI,KAAK,aAAa;mBAC5B,KAAK,CAAC,OAAO,KAAK,IAAI;mBACtB,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;mBACzC,2BAA2B,CAAC,KAAK,CAAC,OAAO,CAAC;gBAAE,OAAO,IAAI,CAAA;YAC5D,SAAQ;QACV,CAAC;QACD,IAAI,KAAK,CAAC,IAAI,KAAK,oBAAoB;eAClC,KAAK,CAAC,IAAI,CAAC,IAAI,KAAK,OAAO;eAC3B,KAAK,CAAC,IAAI,CAAC,OAAO,KAAK,KAAK;eAC5B,sBAAsB,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC;eAC5C,2BAA2B,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC;YAAE,OAAO,IAAI,CAAA;IACnE,CAAC;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAED;;;;;GAKG;AACH,MAAM,OAAO,kBAAkB;IAUV;IACA;IAVV,cAAc,CAAgB;IACtB,MAAM,GAAG,IAAI,GAAG,EAAwB,CAAA;IACjD,SAAS,GAAG,KAAK,CAAA;IAEzB;;;OAGG;IACH,YACmB,GAAY,EACZ,WAAmC;QADnC,QAAG,GAAH,GAAG,CAAS;QACZ,gBAAW,GAAX,WAAW,CAAwB;QAEpD,IAAI,CAAC,cAAc,GAAG,UAAU,CAAC;YAC/B,IAAI,EAAE,uBAAuB;YAC7B,WAAW,EAAE,2EAA2E,MAAM,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI;kBACnI,eAAe,kBAAkB,oLAAoL;kBACrN,mGAAmG;YACvG,UAAU,EAAE,EAAE;YACd,MAAM,EAAE;gBACN,MAAM,EAAE;oBACN,IAAI,EAAE,QAAQ;oBACd,oBAAoB,EAAE,KAAK;oBAC3B,UAAU,EAAE;wBACV,SAAS,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,IAAI,EAAE;wBAC9C,KAAK,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE;qBACpE;iBACF;gBACD,MAAM,EAAE,UAAU;aACnB;YACD,OAAO,EAAE,CAAC,KAAK,EAAE,IAAI,EAA0C,EAAE;gBAC/D,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;oBAC7B,MAAM,IAAI,KAAK,CAAC,GAAG,uBAAuB,gCAAgC,CAAC,CAAA;gBAC7E,CAAC;gBACD,OAAO,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAA;YACnD,CAAC;YACD,WAAW,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,uBAAuB,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC;SAC1F,CAAC,CAAA;IACJ,CAAC;IAED,uEAAuE;IACvE,OAAO;QACL,IAAI,IAAI,CAAC,SAAS;YAAE,MAAM,IAAI,KAAK,CAAC,+DAA+D,CAAC,CAAA;QACpG,IAAI,CAAC,SAAS,GAAG,IAAI,CAAA;QACrB,MAAM,SAAS,GAAG;YAChB,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,eAAe,EAAE,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA,CAAC,CAAC,CAAC;YACnE,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,gBAAgB,EAAE,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA,CAAC,CAAC,CAAC;YACpE,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,eAAe,EAAE,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE;gBAC9C,IAAI,KAAK,CAAC,IAAI,KAAK,UAAU;oBAAE,IAAI,CAAC,6BAA6B,CAAC,OAAO,CAAC,CAAA;YAC5E,CAAC,CAAC;YACF,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,cAAc,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE;gBAC3C,IAAI,MAAM,CAAC,OAAO,KAAK,KAAK;uBACvB,IAAI,CAAC,IAAI,KAAK,OAAO;uBACrB,IAAI,CAAC,KAAK,KAAK,SAAS;uBACxB,sBAAsB,CAAC,IAAI,CAAC,SAAS,CAAC;uBACtC,oBAAoB,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;oBACxC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;gBAC3B,CAAC;gBACD,OAAO,SAAS,CAAA;YAClB,CAAC,CAAC;SACH,CAAA;QACD,IAAI,CAAC;YACH,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE;gBAAE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QAChE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,KAAK,MAAM,OAAO,IAAI,SAAS,CAAC,OAAO,EAAE;gBAAE,OAAO,EAAE,CAAA;YACpD,IAAI,CAAC,aAAa,EAAE,CAAA;YACpB,IAAI,CAAC,SAAS,GAAG,KAAK,CAAA;YACtB,MAAM,KAAK,CAAA;QACb,CAAC;QACD,OAAO,GAAG,EAAE;YACV,IAAI,CAAC,IAAI,CAAC,SAAS;gBAAE,OAAM;YAC3B,IAAI,CAAC,SAAS,GAAG,KAAK,CAAA;YACtB,KAAK,MAAM,OAAO,IAAI,SAAS,CAAC,OAAO,EAAE;gBAAE,OAAO,EAAE,CAAA;YACpD,IAAI,CAAC,aAAa,EAAE,CAAA;QACtB,CAAC,CAAA;IACH,CAAC;IAEO,MAAM,CAAC,KAAY;QACzB,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC;YAAE,OAAM;QAClC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,aAAa,EAAE,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC,CAAA;QAC3E,IAAI,oBAAoB,CAAC,KAAK,CAAC,OAAO,CAAC;YAAE,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAA;IAC/D,CAAC;IAEO,QAAQ,CAAC,KAAY;QAC3B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QAClB,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;QACpC,wEAAwE;QACxE,IAAI,KAAK,KAAK,SAAS;YAAE,MAAM,IAAI,KAAK,CAAC,6BAA6B,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,wBAAwB,CAAC,CAAA;QAC/G,IAAI,KAAK,CAAC,MAAM;YAAE,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,GAAG,KAAK,CAAC,SAAS,CAAC,EAAE,CAAA;QAE1E,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,EAAE,CAAA;QACtC,MAAM,aAAa,GAAsB,EAAE,CAAA;QAC3C,IAAI,CAAC;YACH,KAAK,MAAM,UAAU,IAAI,WAAW;gBAAE,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAA;YAC9F,uEAAuE;YACvE,wEAAwE;YACxE,iEAAiE;YACjE,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,OAAO,CAAC;gBAAE,IAAI,CAAC,mBAAmB,CAAC,KAAK,CAAC,CAAA;QACzE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,KAAK,MAAM,OAAO,IAAI,aAAa,CAAC,OAAO,EAAE;gBAAE,OAAO,EAAE,CAAA;YACxD,MAAM,KAAK,CAAA;QACb,CAAC;QAED,KAAK,CAAC,MAAM,GAAG,IAAI,CAAA;QACnB,KAAK,CAAC,aAAa,GAAG,aAAa,CAAA;QACnC,KAAK,CAAC,SAAS,GAAG,WAAW,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,CAAA;QAChE,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,GAAG,KAAK,CAAC,SAAS,CAAC,EAAE,CAAA;IACzD,CAAC;IAED,6EAA6E;IACrE,aAAa,CAAC,OAAgB;QACpC,OAAO,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,KAAK,OAAO,CAAA;IACtD,CAAC;IAEO,6BAA6B,CAAC,OAAgB;QACpD,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC;YAC3C,IAAI,KAAK,CAAC,OAAO,KAAK,OAAO,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,KAAK,IAAI,EAAE,CAAC;gBACzE,IAAI,CAAC,mBAAmB,CAAC,KAAK,CAAC,CAAA;YACjC,CAAC;QACH,CAAC;IACH,CAAC;IAEO,mBAAmB,CAAC,KAAY;QACtC,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;QACpC,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,CAAC,eAAe,KAAK,SAAS;YAAE,OAAM;QACtE,KAAK,CAAC,eAAe,GAAG,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,CAAC,uBAAuB,CAAC,EAAE,CAAC,CAAA;IACvF,CAAC;IAEO,MAAM,CAAC,KAAY;QACzB,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;QACpC,IAAI,KAAK,KAAK,SAAS;YAAE,OAAM;QAC/B,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QACzB,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAA;IAC1B,CAAC;IAEO,aAAa;QACnB,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE;YAAE,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAA;QAClE,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAA;IACrB,CAAC;IAEO,YAAY,CAAC,KAAoB;QACvC,KAAK,CAAC,eAAe,EAAE,EAAE,CAAA;QACzB,KAAK,MAAM,OAAO,IAAI,KAAK,CAAC,aAAa,CAAC,OAAO,EAAE;YAAE,OAAO,EAAE,CAAA;IAChE,CAAC;CACF"}
|
|
@@ -20,6 +20,10 @@ export const VARIANT_PROVIDER_PREFIX = 'vision-toolkit-';
|
|
|
20
20
|
export const VARIANT_SUFFIX = ' (Vision Toolkit)';
|
|
21
21
|
/** Promise-cache bound for image descriptions, so a long-lived Web profile cannot hoard evidence text. */
|
|
22
22
|
const EVIDENCE_CACHE_LIMIT = 64;
|
|
23
|
+
/** Periodic self-heal sweep for image-input variant routes. */
|
|
24
|
+
const VARIANT_SWEEP_INTERVAL_MS = 10_000;
|
|
25
|
+
/** How long an upstream must stay absent before its wrapper is released. */
|
|
26
|
+
const VARIANT_RELEASE_GRACE_MS = 10_000;
|
|
23
27
|
/**
|
|
24
28
|
* Media types the Vision Toolkit glance pipeline accepts, by declared media
|
|
25
29
|
* type. Narrower than the paste-to-workspace route (which stores any image
|
|
@@ -731,6 +735,12 @@ export function createPasteTakeoverResolver(ctx, getConfig) {
|
|
|
731
735
|
*/
|
|
732
736
|
export function installImageInputVariants(ctx, getConfig, getRuntime) {
|
|
733
737
|
const registrations = new Map();
|
|
738
|
+
// Upstream ids observed missing and the timestamp of the first missing
|
|
739
|
+
// observation. A wrapper is only released after its upstream stays absent
|
|
740
|
+
// for the full grace period, so a transient registry gap (adapter
|
|
741
|
+
// re-registration, HMR reload) cannot drop a wrapper that a live session is
|
|
742
|
+
// still pointed at, no matter how many sweeps fire in between.
|
|
743
|
+
const stale = new Map();
|
|
734
744
|
let disposed = false;
|
|
735
745
|
// Serialize sweeps: a registration itself announces llm/adapters-updated,
|
|
736
746
|
// and two interleaved sweeps must never probe the same route concurrently.
|
|
@@ -738,13 +748,26 @@ export function installImageInputVariants(ctx, getConfig, getRuntime) {
|
|
|
738
748
|
// Coalesce bursts: a sweep triggered while one is pending is one extra pass,
|
|
739
749
|
// not one per notification (a single registration emits a notification).
|
|
740
750
|
let sweepQueued = false;
|
|
741
|
-
const
|
|
742
|
-
|
|
751
|
+
const release = (upstream) => {
|
|
752
|
+
const dispose = registrations.get(upstream);
|
|
753
|
+
if (dispose === undefined)
|
|
754
|
+
return;
|
|
755
|
+
registrations.delete(upstream);
|
|
756
|
+
stale.delete(upstream);
|
|
757
|
+
try {
|
|
743
758
|
dispose();
|
|
744
|
-
|
|
759
|
+
}
|
|
760
|
+
catch (error) {
|
|
761
|
+
ctx.logger.warn('dsh-vision-toolkit: image-input variant release failed for "%s": %s', upstream, messageOf(error));
|
|
762
|
+
}
|
|
763
|
+
};
|
|
764
|
+
const releaseAll = () => {
|
|
765
|
+
stale.clear();
|
|
766
|
+
for (const upstream of [...registrations.keys()])
|
|
767
|
+
release(upstream);
|
|
745
768
|
};
|
|
746
769
|
const sweep = () => {
|
|
747
|
-
if (sweepQueued)
|
|
770
|
+
if (disposed || sweepQueued)
|
|
748
771
|
return;
|
|
749
772
|
sweepQueued = true;
|
|
750
773
|
queueMicrotask(() => {
|
|
@@ -772,6 +795,27 @@ export function installImageInputVariants(ctx, getConfig, getRuntime) {
|
|
|
772
795
|
catch {
|
|
773
796
|
return;
|
|
774
797
|
}
|
|
798
|
+
const live = new Set(providers.map(provider => provider.id));
|
|
799
|
+
// Release restrict-narrowed routes immediately (a configuration change is
|
|
800
|
+
// authoritative). For routes merely missing from the live registry, keep
|
|
801
|
+
// the wrapper until the absence outlives the grace period.
|
|
802
|
+
for (const upstream of [...registrations.keys()]) {
|
|
803
|
+
if (restrict.size > 0 && !restrict.has(upstream)) {
|
|
804
|
+
release(upstream);
|
|
805
|
+
continue;
|
|
806
|
+
}
|
|
807
|
+
if (live.has(upstream)) {
|
|
808
|
+
stale.delete(upstream);
|
|
809
|
+
continue;
|
|
810
|
+
}
|
|
811
|
+
const missedAt = stale.get(upstream);
|
|
812
|
+
if (missedAt !== undefined && Date.now() - missedAt >= VARIANT_RELEASE_GRACE_MS) {
|
|
813
|
+
release(upstream);
|
|
814
|
+
}
|
|
815
|
+
else if (missedAt === undefined) {
|
|
816
|
+
stale.set(upstream, Date.now());
|
|
817
|
+
}
|
|
818
|
+
}
|
|
775
819
|
for (const provider of providers) {
|
|
776
820
|
const upstream = provider.id;
|
|
777
821
|
if (restrict.size > 0 && !restrict.has(upstream))
|
|
@@ -780,6 +824,7 @@ export function installImageInputVariants(ctx, getConfig, getRuntime) {
|
|
|
780
824
|
// probing them would just re-probe their upstream route.
|
|
781
825
|
if (upstream.startsWith(VARIANT_PROVIDER_PREFIX))
|
|
782
826
|
continue;
|
|
827
|
+
const variantId = variantProviderId(upstream);
|
|
783
828
|
let models;
|
|
784
829
|
try {
|
|
785
830
|
models = await llm.listModels(upstream);
|
|
@@ -789,35 +834,35 @@ export function installImageInputVariants(ctx, getConfig, getRuntime) {
|
|
|
789
834
|
}
|
|
790
835
|
const eligible = models.some(shouldWrapModel);
|
|
791
836
|
const registered = registrations.has(upstream);
|
|
792
|
-
if (
|
|
793
|
-
//
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
837
|
+
if (registered && live.has(variantId)) {
|
|
838
|
+
// Re-read the live registry after the await: a rebuild inside the
|
|
839
|
+
// probe window can drop our wrapper without touching our map.
|
|
840
|
+
const liveNow = new Set(llm.listProviders().map(provider => provider.id));
|
|
841
|
+
if (liveNow.has(variantId)) {
|
|
842
|
+
// Healthy handle: only react when the route lost every eligible model.
|
|
843
|
+
if (!eligible)
|
|
844
|
+
release(upstream);
|
|
845
|
+
continue;
|
|
846
|
+
}
|
|
847
|
+
}
|
|
848
|
+
if (registered) {
|
|
849
|
+
// Dead handle from a registry rebuild/reset: drop it and register a
|
|
850
|
+
// fresh wrapper below.
|
|
851
|
+
release(upstream);
|
|
798
852
|
}
|
|
799
|
-
if (!eligible
|
|
853
|
+
if (!eligible)
|
|
800
854
|
continue;
|
|
801
855
|
if (disposed)
|
|
802
856
|
return;
|
|
803
857
|
try {
|
|
804
|
-
const dispose = llm.registerAdapter([
|
|
858
|
+
const dispose = llm.registerAdapter([variantId], new ImageInputVariantAdapter(ctx, llm, upstream, provider.name, getRuntime, new EvidenceCache(EVIDENCE_CACHE_LIMIT)));
|
|
805
859
|
registrations.set(upstream, dispose);
|
|
860
|
+
stale.delete(upstream);
|
|
806
861
|
}
|
|
807
862
|
catch (error) {
|
|
808
863
|
ctx.logger.warn('dsh-vision-toolkit: image-input variant registration skipped for "%s": %s', upstream, messageOf(error));
|
|
809
864
|
}
|
|
810
865
|
}
|
|
811
|
-
const live = new Set(providers.map(provider => provider.id));
|
|
812
|
-
for (const [upstream, dispose] of [...registrations]) {
|
|
813
|
-
// A wrapper is released when its upstream route vanished OR the
|
|
814
|
-
// current configuration no longer allows that route (restrict
|
|
815
|
-
// narrowing must not leave stale variants behind).
|
|
816
|
-
if (!live.has(upstream) || (restrict.size > 0 && !restrict.has(upstream))) {
|
|
817
|
-
dispose();
|
|
818
|
-
registrations.delete(upstream);
|
|
819
|
-
}
|
|
820
|
-
}
|
|
821
866
|
}
|
|
822
867
|
catch (error) {
|
|
823
868
|
ctx.logger.warn('dsh-vision-toolkit: image-input variant sweep failed: %s', messageOf(error));
|
|
@@ -827,9 +872,14 @@ export function installImageInputVariants(ctx, getConfig, getRuntime) {
|
|
|
827
872
|
ctx.on('llm/adapters-updated', () => { sweep(); });
|
|
828
873
|
}
|
|
829
874
|
sweep();
|
|
875
|
+
// Self-heal safety net: re-verify periodically so a wrapper dropped by an
|
|
876
|
+
// unobserved cause (registry rebuild without an event we can see) is restored
|
|
877
|
+
// within one interval.
|
|
878
|
+
const timer = setInterval(() => { sweep(); }, VARIANT_SWEEP_INTERVAL_MS);
|
|
830
879
|
return {
|
|
831
880
|
dispose: () => {
|
|
832
881
|
disposed = true;
|
|
882
|
+
clearInterval(timer);
|
|
833
883
|
releaseAll();
|
|
834
884
|
},
|
|
835
885
|
reconcile: () => { sweep(); },
|