@anionex/dsh-vision-toolkit 0.1.7 → 0.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.i18n.yaml +2 -2
- package/README.md +17 -1
- package/README.zh.md +17 -1
- package/lib/client.js +306 -16
- package/lib/client.js.map +1 -1
- package/lib/config.js +14 -0
- package/lib/config.js.map +1 -1
- package/lib/image-input-variants.js +615 -0
- package/lib/image-input-variants.js.map +1 -0
- package/lib/index.js +8 -1
- package/lib/index.js.map +1 -1
- package/lib/paste-images.js +6 -0
- package/lib/paste-images.js.map +1 -1
- package/lib/types/client/paste-images.d.ts +69 -0
- package/lib/types/client/paste-images.d.ts.map +1 -1
- package/lib/types/config.d.ts +26 -0
- package/lib/types/config.d.ts.map +1 -1
- package/lib/types/image-input-variants.d.ts +153 -0
- package/lib/types/image-input-variants.d.ts.map +1 -0
- package/lib/types/index.d.ts.map +1 -1
- package/lib/types/paste-images.d.ts +35 -0
- package/lib/types/paste-images.d.ts.map +1 -1
- package/lib/types/web-request.d.ts +7 -0
- package/lib/types/web-request.d.ts.map +1 -1
- package/lib/types/web.d.ts +17 -2
- package/lib/types/web.d.ts.map +1 -1
- package/lib/web-request.js +11 -2
- package/lib/web-request.js.map +1 -1
- package/lib/web.js +81 -3
- package/lib/web.js.map +1 -1
- package/package.json +2 -1
- package/src/client/paste-images.tsx +331 -15
- package/src/config.ts +40 -0
- package/src/image-input-variants.ts +688 -0
- package/src/index.ts +18 -1
- package/src/paste-images.ts +39 -0
- package/src/web-request.ts +12 -2
- package/src/web.ts +84 -2
package/README.i18n.yaml
CHANGED
|
@@ -2,5 +2,5 @@
|
|
|
2
2
|
# side as of the last confirmed-consistent state. Both languages carry equal authority;
|
|
3
3
|
# after editing either side, bring the other along and re-record with:
|
|
4
4
|
# pnpm run verify-translation-pairing --write dsh-vision-dark-theme/README.md
|
|
5
|
-
README.md:
|
|
6
|
-
README.zh.md:
|
|
5
|
+
README.md: 73215f08bb8e697ddc9c49cc4e1a5491a0f4c735
|
|
6
|
+
README.zh.md: 8a8979e5b3ea6e03e58b51dea8ce49b0d1e218ea
|
package/README.md
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
# DSH Vision Toolkit
|
|
4
4
|
|
|
5
|
+
[](https://dshfind.com/)
|
|
5
6
|
[](https://x.com/anion_ex)
|
|
6
7
|
[](https://github.com/Anionex/dsh-vision-toolkit/releases/tag/v0.1.7)
|
|
7
8
|
[](tests)
|
|
@@ -145,6 +146,14 @@ Runtime readiness is profile-wide, but the ten visual execution schemas are Agen
|
|
|
145
146
|
|
|
146
147
|
Health checks, connection testing, and plugin/upstream version inspection are administrative Web Settings operations. `vision_toolkit_health` and `vision_toolkit_version` are not model tools and never enter an Agent's schema, including after visual-tool activation.
|
|
147
148
|
|
|
149
|
+
## Image-input variants for text-only models
|
|
150
|
+
|
|
151
|
+
Text-only model routes get sibling model-selector entries named `<model> (Vision Toolkit)` under a matching provider group. A variant declares image input, so pasted images keep the native attachment flow — composer thumbnail, durable session image, and history rendering — and the plugin rewrites every image block into a Vision Toolkit description only on the wire to the model, before the request reaches the upstream route. The session log is untouched; replay and the UI keep the real image.
|
|
152
|
+
|
|
153
|
+
A variant is registered automatically for every model the host positively declares text-only (for example the DeepSeek chat family). Paste handling is automatic: when the current model is confirmed text-only and its variant exists, the browser integration switches the session to the variant by itself (a short notice names the new model) and the paste then keeps the native flow; no manual model change is needed. The host's verdict uses the exact model route the browser read from the live model catalog, with the selector label as fallback; unconfirmed or image-capable routes always keep the native flow, and a text-only model without a variant (for example when variants are disabled) keeps the paste-to-path takeover, which copies the image into the session workspace and inserts its path as text.
|
|
154
|
+
|
|
155
|
+
Description conversion needs the configured vision provider and its credential; when the runtime is not ready or a read fails, the wire block degrades to an explanatory note instead of failing the turn. Disable variants with `imageInputVariants.enabled: false`, restrict the wrapped routes with `imageInputVariants.providers`, or keep the paste-to-path behavior for text-only models with `imageInputVariants.autoSwitch: false`.
|
|
156
|
+
|
|
148
157
|
## Requirements
|
|
149
158
|
|
|
150
159
|
- DeepSeek Harness with a Web or Headless profile and `pnpm` available to `dsh plugin`.
|
|
@@ -233,6 +242,10 @@ The bundle defaults to the managed runtime. A profile patch can override the pro
|
|
|
233
242
|
runtime:
|
|
234
243
|
mode: managed
|
|
235
244
|
allowedDirs: []
|
|
245
|
+
imageInputVariants:
|
|
246
|
+
enabled: true
|
|
247
|
+
providers: []
|
|
248
|
+
autoSwitch: true
|
|
236
249
|
```
|
|
237
250
|
|
|
238
251
|
### Configuration fields
|
|
@@ -254,6 +267,9 @@ The bundle defaults to the managed runtime. A profile patch can override the pro
|
|
|
254
267
|
| `runtime.agentVisionToolkitPath` | unset | Required in `external` mode; exported exact snapshot or clean pinned Git checkout |
|
|
255
268
|
| `runtime.python` | unset | Optional Python 3.11+ bootstrap/interpreter override |
|
|
256
269
|
| `allowedDirs` | `[]` | Additional realpath-resolved input roots; the session workspace is always allowed |
|
|
270
|
+
| `imageInputVariants.enabled` | `true` | Register image-input variant entries for text-only model routes in the model selector |
|
|
271
|
+
| `imageInputVariants.providers` | `[]` | Restrict wrapped upstream routes by provider id; empty wraps every eligible route |
|
|
272
|
+
| `imageInputVariants.autoSwitch` | `true` | Automatically switch a text-only session to its image-input variant on paste, so the image keeps the native flow; `false` keeps the paste-to-path takeover for text-only models |
|
|
257
273
|
|
|
258
274
|
### Credentials
|
|
259
275
|
|
|
@@ -341,7 +357,7 @@ The committed evidence records an initial `6.04%` difference across six non-zero
|
|
|
341
357
|
|
|
342
358
|
| Symptom | Resolution |
|
|
343
359
|
|---|---|
|
|
344
|
-
| `Model "..." does not support image input. (attachment-error)` | The image used DSH's native model-attachment channel, so a text-only model rejected the turn before the Skill or Vision Toolkit could run.
|
|
360
|
+
| `Model "..." does not support image input. (attachment-error)` | The image used DSH's native model-attachment channel, so a text-only model rejected the turn before the Skill or Vision Toolkit could run. With image-input variants enabled this is rare: pasting normally auto-switches the session to the `<model> (Vision Toolkit)` variant. If variants are disabled or auto-switch is off, use DSH Paste Input's attachment button, paste, or drop flow so the file is copied into the session workspace and represented by a path, then invoke `/vision-tools`. Restart the Web profile and reload the page after installing or upgrading either browser plugin. |
|
|
345
361
|
| Credential reported missing | Paste the key into Web Settings **API key**, keep the advanced **Credential name** aligned with `provider.credential`, save, then rerun health. Headless deployments can provision the same reference in `$DSH_HOME/.credentials.yaml`. Local-only tools do not need it. |
|
|
346
362
|
| Runtime preparation fails | Read the Settings runtime error, verify Python 3.11+, package-cache/network access, disk permissions, and the exact external pin. Save only after correcting the candidate; the active generation remains intact. |
|
|
347
363
|
| Chrome is not found | Install Chrome, Chromium, or Edge or configure an environment where one is discoverable. Only `vision_html_screenshot` is unavailable. |
|
package/README.zh.md
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
# DSH Vision Toolkit
|
|
4
4
|
|
|
5
|
+
[](https://dshfind.com/)
|
|
5
6
|
[](https://x.com/anion_ex)
|
|
6
7
|
[](https://github.com/Anionex/dsh-vision-toolkit/releases/tag/v0.1.7)
|
|
7
8
|
[](tests)
|
|
@@ -145,6 +146,14 @@ flowchart LR
|
|
|
145
146
|
|
|
146
147
|
健康检查、连接测试以及插件/上游版本检查属于 Web Settings 管理操作。`vision_toolkit_health` 和 `vision_toolkit_version` 不是模型工具,即使视觉执行工具已经激活,也永远不会进入 Agent schema。
|
|
147
148
|
|
|
149
|
+
## 纯文本模型的图片输入变体
|
|
150
|
+
|
|
151
|
+
纯文本模型路由会获得同名的兄弟模型条目:`<模型名> (Vision Toolkit)`,挂在对应的提供方分组下。变体声明支持图片输入,因此粘贴的图片走原生附件流程——输入框缩略图、会话持久化图片与历史渲染全部保留——插件只在发往模型的请求链路上把每个图片块改写成 Vision Toolkit 描述文本,再转交上游路由。会话日志不被改动;回放与 UI 看到的始终是真实图片。
|
|
152
|
+
|
|
153
|
+
插件会自动为宿主明确声明为纯文本的每个模型注册变体(例如 DeepSeek 对话家族)。粘贴处理是全自动的:当当前模型被确认为纯文本、且它的变体已注册时,浏览器端集成会自动把会话切换到变体(会有一条简短提示说明新模型名),随后粘贴走原生流程,无需手动切换模型。宿主依据浏览器从实时模型目录读到的精确模型路由来裁决,模型选择器标签作为兜底;无法确认或支持图片的路由一律保持原生流程,而"纯文本但没有变体"的模型(例如变体被关闭时)继续走"粘贴转路径":图片被复制进会话工作区,输入框里插入的是它的路径文本。
|
|
154
|
+
|
|
155
|
+
描述转换需要已配置的视觉提供方及其 Credential;当运行时未就绪或读取失败时,请求链路上的图片块降级为说明文本,而不是让整轮失败。用 `imageInputVariants.enabled: false` 关闭变体,用 `imageInputVariants.providers` 限制被包装的路由,或用 `imageInputVariants.autoSwitch: false` 让纯文本模型继续走"粘贴转路径"。
|
|
156
|
+
|
|
148
157
|
## 运行要求
|
|
149
158
|
|
|
150
159
|
- 启用 Web 或 Headless Profile 的 DeepSeek Harness,并确保 `dsh plugin` 可以使用 `pnpm`。
|
|
@@ -233,6 +242,10 @@ Bundle 默认使用 managed 运行时。Profile patch 可以覆盖提供方与
|
|
|
233
242
|
runtime:
|
|
234
243
|
mode: managed
|
|
235
244
|
allowedDirs: []
|
|
245
|
+
imageInputVariants:
|
|
246
|
+
enabled: true
|
|
247
|
+
providers: []
|
|
248
|
+
autoSwitch: true
|
|
236
249
|
```
|
|
237
250
|
|
|
238
251
|
### 配置字段
|
|
@@ -254,6 +267,9 @@ Bundle 默认使用 managed 运行时。Profile patch 可以覆盖提供方与
|
|
|
254
267
|
| `runtime.agentVisionToolkitPath` | 未设置 | `external` 模式必填;必须是精确导出快照或固定 commit 的干净 Git checkout |
|
|
255
268
|
| `runtime.python` | 未设置 | 可选的 Python 3.11+ 引导程序/解释器覆盖值 |
|
|
256
269
|
| `allowedDirs` | `[]` | 额外的 realpath 解析输入根目录;会话工作区始终允许 |
|
|
270
|
+
| `imageInputVariants.enabled` | `true` | 为纯文本模型路由在模型选择器中注册图片输入变体条目 |
|
|
271
|
+
| `imageInputVariants.providers` | `[]` | 按提供方 id 限制被包装的上游路由;为空时包装所有符合条件的路由 |
|
|
272
|
+
| `imageInputVariants.autoSwitch` | `true` | 粘贴时自动把纯文本会话切换到其图片输入变体,让图片保持原生流程;`false` 时纯文本模型继续走"粘贴转路径" |
|
|
257
273
|
|
|
258
274
|
### Credential
|
|
259
275
|
|
|
@@ -341,7 +357,7 @@ npm run example:ui-restoration:write
|
|
|
341
357
|
|
|
342
358
|
| 症状 | 解决方法 |
|
|
343
359
|
|---|---|
|
|
344
|
-
| `Model "..." does not support image input. (attachment-error)` | 图片走了 DSH 的模型原生附件通道,纯文本模型会在 Skill 或 Vision Toolkit
|
|
360
|
+
| `Model "..." does not support image input. (attachment-error)` | 图片走了 DSH 的模型原生附件通道,纯文本模型会在 Skill 或 Vision Toolkit 运行前拒绝该轮。启用图片输入变体时这很少发生:粘贴会自动把会话切换到 `<模型名> (Vision Toolkit)` 变体。若变体被关闭或自动切换被禁用,请使用 DSH Paste Input 的附件按钮、粘贴或拖放流程,让文件先复制到会话工作区并以路径形式进入消息,再调用 `/vision-tools`。安装或升级任一浏览器插件后,需要重启 Web Profile 并刷新页面。 |
|
|
345
361
|
| Credential 显示缺失 | 在 Web 设置页的 **API 密钥** 中粘贴密钥,确认高级设置中的 **凭据名称** 与 `provider.credential` 一致,保存后重新运行健康检查。Headless 部署可以在 `$DSH_HOME/.credentials.yaml` 中预置同名引用。本地工具不需要它。 |
|
|
346
362
|
| 运行时准备失败 | 查看 Settings 中的运行时错误,检查 Python 3.11+、软件包缓存/网络、磁盘权限和精确 external 固定版本。修正候选后再保存;当前 generation 不受影响。 |
|
|
347
363
|
| 找不到 Chrome | 安装 Chrome、Chromium 或 Edge,或让其中一个可被运行环境发现。只有 `vision_html_screenshot` 不可用。 |
|
package/lib/client.js
CHANGED
|
@@ -895,7 +895,7 @@ function apply(ctx) {
|
|
|
895
895
|
__modules["./paste-images.js"] = function(module, exports, require, __load_) {
|
|
896
896
|
"use strict";
|
|
897
897
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
898
|
-
exports.PasteImageController = exports.PASTE_IMAGES_ROUTE = void 0;
|
|
898
|
+
exports.PasteImageController = exports.PASTE_POLICY_ROUTE = exports.PASTE_IMAGES_ROUTE = void 0;
|
|
899
899
|
exports.PasteImageDock = PasteImageDock;
|
|
900
900
|
exports.installPasteImages = installPasteImages;
|
|
901
901
|
const jsx_runtime_1 = require("react/jsx-runtime");
|
|
@@ -903,9 +903,12 @@ const jsx_runtime_1 = require("react/jsx-runtime");
|
|
|
903
903
|
const react_1 = require("react");
|
|
904
904
|
const SOURCE = 'vision-toolkit-pasted-image';
|
|
905
905
|
exports.PASTE_IMAGES_ROUTE = '/_dsh/vision-toolkit/paste-images';
|
|
906
|
+
exports.PASTE_POLICY_ROUTE = '/_dsh/vision-toolkit/paste-policy';
|
|
906
907
|
const MAX_IMAGES = 20;
|
|
907
908
|
const MAX_IMAGE_BYTES = 20 * 1024 * 1024;
|
|
908
909
|
const MAX_BATCH_BYTES = 80 * 1024 * 1024;
|
|
910
|
+
/** A confirmed paste verdict older than this is unknown again, even while a refresh is in flight. */
|
|
911
|
+
const VERDICT_MAX_AGE_MS = 15000;
|
|
909
912
|
const CORDIS_ORIGINAL = Symbol.for('cordis.original');
|
|
910
913
|
function registryIdentity(registry) {
|
|
911
914
|
let current = registry;
|
|
@@ -944,6 +947,25 @@ function imageFiles(data) {
|
|
|
944
947
|
const candidates = itemFiles.length > 0 ? itemFiles : Array.from(data.files);
|
|
945
948
|
return candidates.filter(file => file.type.toLowerCase().startsWith('image/'));
|
|
946
949
|
}
|
|
950
|
+
/**
|
|
951
|
+
* The selector label the model picker currently shows, or '' when none is
|
|
952
|
+
* readable. Matches the host ModelSelect trigger aria-labels ("Select model,
|
|
953
|
+
* current …" / "选择模型,当前 …"); any other label wording falls back to the
|
|
954
|
+
* session-header verdict, which is stale until the next request.
|
|
955
|
+
*/
|
|
956
|
+
function currentModelLabel() {
|
|
957
|
+
const buttons = document.querySelectorAll('button[aria-label]');
|
|
958
|
+
for (const button of buttons) {
|
|
959
|
+
const label = button.getAttribute('aria-label') ?? '';
|
|
960
|
+
if (/select model|current model|选择模型/iu.test(label))
|
|
961
|
+
return label;
|
|
962
|
+
}
|
|
963
|
+
return '';
|
|
964
|
+
}
|
|
965
|
+
/** Verdict cache key: the model label is part of the answer, so a switch invalidates it. */
|
|
966
|
+
function verdictKey(sessionId, modelLabel) {
|
|
967
|
+
return `${sessionId}|${modelLabel}`;
|
|
968
|
+
}
|
|
947
969
|
function validateImages(files) {
|
|
948
970
|
if (files.length > MAX_IMAGES)
|
|
949
971
|
throw new Error(`Paste at most ${MAX_IMAGES} images at a time`);
|
|
@@ -975,6 +997,9 @@ class PasteImageController {
|
|
|
975
997
|
records = new Map();
|
|
976
998
|
listeners = new Set();
|
|
977
999
|
revision = 0;
|
|
1000
|
+
verdicts = new Map();
|
|
1001
|
+
/** Guards the synthetic replay paste from re-entering capture interception. */
|
|
1002
|
+
replaying = false;
|
|
978
1003
|
constructor(ctx) {
|
|
979
1004
|
this.ctx = ctx;
|
|
980
1005
|
}
|
|
@@ -1074,30 +1099,246 @@ class PasteImageController {
|
|
|
1074
1099
|
throw error;
|
|
1075
1100
|
}
|
|
1076
1101
|
}
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1102
|
+
/**
|
|
1103
|
+
* The host's verdict for one Session and selector label, when fresh. The
|
|
1104
|
+
* last CONFIRMED answer is authoritative while a background refresh is in
|
|
1105
|
+
* flight (the paste acts on what the host last said; the refresh only
|
|
1106
|
+
* covers the next paste). A label that changed since the confirmation
|
|
1107
|
+
* answers undefined, so the native attachment flow stays the default.
|
|
1108
|
+
* @param sessionId - the live Session the paste belongs to.
|
|
1109
|
+
* @param modelLabel - the model-selector label currently shown.
|
|
1110
|
+
* @returns the fresh confirmed verdict, or undefined when unconfirmed.
|
|
1111
|
+
*/
|
|
1112
|
+
verdictFor(sessionId, modelLabel) {
|
|
1113
|
+
const entry = this.verdicts.get(verdictKey(sessionId, modelLabel));
|
|
1114
|
+
if (entry === undefined || entry.at === 0)
|
|
1115
|
+
return undefined;
|
|
1116
|
+
if (Date.now() - entry.at > VERDICT_MAX_AGE_MS)
|
|
1117
|
+
return undefined;
|
|
1118
|
+
return { takeOver: entry.takeOver, ...(entry.autoSwitch === undefined ? {} : { autoSwitch: entry.autoSwitch }) };
|
|
1119
|
+
}
|
|
1120
|
+
/**
|
|
1121
|
+
* The exact model route the live model catalog reports for one Session.
|
|
1122
|
+
* Unreadable routes answer undefined, so the verdict falls back to the
|
|
1123
|
+
* selector label alone.
|
|
1124
|
+
* @param sessionId - the live Session id.
|
|
1125
|
+
* @returns the current provider/model selection, when readable.
|
|
1126
|
+
*/
|
|
1127
|
+
async readSelection(sessionId) {
|
|
1128
|
+
const connection = this.ctx.get('connection');
|
|
1129
|
+
if (connection === undefined)
|
|
1130
|
+
return undefined;
|
|
1131
|
+
try {
|
|
1132
|
+
const { result } = await connection.api.sessions.models({ sessionId });
|
|
1133
|
+
if (!result.ok)
|
|
1134
|
+
return undefined;
|
|
1135
|
+
const current = result.value.current;
|
|
1136
|
+
if (current === undefined || current === null || current.provider === '' || current.model === '')
|
|
1137
|
+
return undefined;
|
|
1138
|
+
return {
|
|
1139
|
+
provider: current.provider,
|
|
1140
|
+
model: current.model,
|
|
1141
|
+
...(current.reasoningEffort === undefined ? {} : { reasoningEffort: current.reasoningEffort }),
|
|
1142
|
+
};
|
|
1143
|
+
}
|
|
1144
|
+
catch {
|
|
1145
|
+
return undefined;
|
|
1146
|
+
}
|
|
1147
|
+
}
|
|
1148
|
+
/**
|
|
1149
|
+
* Ask the host what to do with a paste for the current model, and cache the
|
|
1150
|
+
* answer per Session and selector label. A model switch changes the label,
|
|
1151
|
+
* which changes the cache key, so a stale verdict never outlives the model
|
|
1152
|
+
* it described. The exact selection rides along when the live model catalog
|
|
1153
|
+
* is readable, so the host can answer with an auto-switch route; a 404
|
|
1154
|
+
* simply leaves the verdict unconfirmed; the next focus or paste retries.
|
|
1155
|
+
* @param sessionId - the live Session to ask about.
|
|
1156
|
+
* @param modelLabel - the model-selector label currently shown.
|
|
1157
|
+
*/
|
|
1158
|
+
refreshVerdict(sessionId, modelLabel) {
|
|
1159
|
+
const key = verdictKey(sessionId, modelLabel);
|
|
1160
|
+
const cached = this.verdicts.get(key);
|
|
1161
|
+
// Dedupe only on an in-flight request, never on freshness: the host's
|
|
1162
|
+
// model route can change under an unchanged Session id.
|
|
1163
|
+
if (cached?.pending)
|
|
1164
|
+
return;
|
|
1165
|
+
const entry = {
|
|
1166
|
+
pending: true,
|
|
1167
|
+
takeOver: cached ? cached.takeOver : false,
|
|
1168
|
+
at: cached ? cached.at : 0,
|
|
1169
|
+
...(cached?.autoSwitch === undefined ? {} : { autoSwitch: cached.autoSwitch }),
|
|
1170
|
+
};
|
|
1171
|
+
this.verdicts.set(key, entry);
|
|
1172
|
+
void (async () => {
|
|
1173
|
+
const selection = await this.readSelection(sessionId);
|
|
1174
|
+
const query = new URLSearchParams({ sessionId });
|
|
1175
|
+
if (modelLabel !== '')
|
|
1176
|
+
query.set('model', modelLabel);
|
|
1177
|
+
if (selection !== undefined) {
|
|
1178
|
+
query.set('provider', selection.provider);
|
|
1179
|
+
query.set('modelId', selection.model);
|
|
1180
|
+
if (selection.reasoningEffort !== undefined)
|
|
1181
|
+
query.set('reasoningEffort', selection.reasoningEffort);
|
|
1182
|
+
}
|
|
1183
|
+
let request;
|
|
1184
|
+
try {
|
|
1185
|
+
request = fetch(`${exports.PASTE_POLICY_ROUTE}?${query.toString()}`);
|
|
1186
|
+
}
|
|
1187
|
+
catch {
|
|
1188
|
+
// No fetch surface (test runtime, pre-fetch bootstrap): leave the
|
|
1189
|
+
// verdict unconfirmed rather than letting the paste listener die.
|
|
1190
|
+
entry.pending = false;
|
|
1191
|
+
return;
|
|
1192
|
+
}
|
|
1193
|
+
request
|
|
1194
|
+
.then((response) => {
|
|
1195
|
+
if (response.status === 404) {
|
|
1196
|
+
// Route not mounted yet (plugin load race, hot reload): forget every
|
|
1197
|
+
// verdict and retry on the next focus or paste instead of standing
|
|
1198
|
+
// down for the page lifetime.
|
|
1199
|
+
this.verdicts.clear();
|
|
1200
|
+
return null;
|
|
1201
|
+
}
|
|
1202
|
+
if (!response.ok)
|
|
1203
|
+
throw new Error(`paste policy ${response.status}`);
|
|
1204
|
+
return response.json();
|
|
1205
|
+
})
|
|
1206
|
+
.then((body) => {
|
|
1207
|
+
entry.pending = false;
|
|
1208
|
+
if (body !== null) {
|
|
1209
|
+
entry.takeOver = body.value.takeOver === true;
|
|
1210
|
+
if (body.value.autoSwitch !== undefined)
|
|
1211
|
+
entry.autoSwitch = body.value.autoSwitch;
|
|
1212
|
+
else
|
|
1213
|
+
delete entry.autoSwitch;
|
|
1214
|
+
entry.at = Date.now();
|
|
1215
|
+
}
|
|
1216
|
+
})
|
|
1217
|
+
.catch(() => {
|
|
1218
|
+
entry.pending = false;
|
|
1219
|
+
});
|
|
1220
|
+
})();
|
|
1221
|
+
}
|
|
1222
|
+
/**
|
|
1223
|
+
* Switch one Session to the route the host validated, through the same
|
|
1224
|
+
* model-directory seat the selector uses when present (so the shared UI
|
|
1225
|
+
* state moves with the session), falling back to the raw RPC.
|
|
1226
|
+
* @param sessionId - the live Session id.
|
|
1227
|
+
* @param route - the validated variant route.
|
|
1228
|
+
*/
|
|
1229
|
+
async switchModel(sessionId, route) {
|
|
1230
|
+
const directories = this.ctx.get('modelDirectories');
|
|
1231
|
+
if (directories !== undefined) {
|
|
1232
|
+
// The label is a display hint; the seat only needs the exact route.
|
|
1233
|
+
await directories.directoryFor(sessionId).select({
|
|
1234
|
+
provider: route.provider,
|
|
1235
|
+
model: route.model,
|
|
1236
|
+
...(route.reasoningEffort === undefined ? {} : { reasoningEffort: route.reasoningEffort }),
|
|
1237
|
+
});
|
|
1238
|
+
return;
|
|
1239
|
+
}
|
|
1240
|
+
const connection = this.ctx.get('connection');
|
|
1241
|
+
if (connection === undefined)
|
|
1242
|
+
throw new Error('No model switch channel is available in this Web application');
|
|
1243
|
+
const { result } = await connection.api.sessions.selectModel({
|
|
1244
|
+
sessionId,
|
|
1245
|
+
provider: route.provider,
|
|
1246
|
+
model: route.model,
|
|
1247
|
+
...(route.reasoningEffort === undefined ? {} : { reasoningEffort: route.reasoningEffort }),
|
|
1248
|
+
});
|
|
1249
|
+
if (!result.ok)
|
|
1250
|
+
throw new Error(`${result.error?.code ?? 'select-model-failed'}: ${result.error?.message ?? 'unknown error'}`);
|
|
1251
|
+
}
|
|
1252
|
+
/**
|
|
1253
|
+
* Replay a swallowed paste as a synthetic clipboard event so the composer's
|
|
1254
|
+
* own intake (limits, thumbnails, keyboard) runs with the captured files.
|
|
1255
|
+
* @returns false when the environment cannot construct a clipboard payload.
|
|
1256
|
+
*/
|
|
1257
|
+
replayPaste(target, files, text) {
|
|
1258
|
+
let data;
|
|
1259
|
+
try {
|
|
1260
|
+
data = new DataTransfer();
|
|
1261
|
+
for (const file of files)
|
|
1262
|
+
data.items.add(file);
|
|
1263
|
+
if (text !== '')
|
|
1264
|
+
data.setData('text/plain', text);
|
|
1265
|
+
}
|
|
1266
|
+
catch {
|
|
1080
1267
|
return false;
|
|
1081
|
-
|
|
1082
|
-
|
|
1268
|
+
}
|
|
1269
|
+
let event;
|
|
1270
|
+
try {
|
|
1271
|
+
event = new ClipboardEvent('paste', {
|
|
1272
|
+
clipboardData: data,
|
|
1273
|
+
bubbles: true,
|
|
1274
|
+
cancelable: true,
|
|
1275
|
+
});
|
|
1276
|
+
}
|
|
1277
|
+
catch {
|
|
1083
1278
|
return false;
|
|
1084
|
-
|
|
1085
|
-
event.
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1279
|
+
}
|
|
1280
|
+
if (event.clipboardData === null || event.clipboardData.files.length === 0)
|
|
1281
|
+
return false;
|
|
1282
|
+
this.replaying = true;
|
|
1283
|
+
try {
|
|
1284
|
+
target.dispatchEvent(event);
|
|
1285
|
+
}
|
|
1286
|
+
finally {
|
|
1287
|
+
this.replaying = false;
|
|
1288
|
+
}
|
|
1289
|
+
return true;
|
|
1290
|
+
}
|
|
1291
|
+
/**
|
|
1292
|
+
* Auto-switch flow: switch the Session to the image-input variant, announce
|
|
1293
|
+
* it, then replay the paste into the composer's native intake. A failed
|
|
1294
|
+
* switch, or an environment that cannot replay clipboard bytes, degrades to
|
|
1295
|
+
* the path takeover with the same files.
|
|
1296
|
+
* @param sessionId - the live Session id.
|
|
1297
|
+
* @param target - the composer textarea the paste landed on.
|
|
1298
|
+
* @param files - the captured image files.
|
|
1299
|
+
* @param text - same-paste text, replayed alongside the files.
|
|
1300
|
+
* @param route - the validated variant route to switch to.
|
|
1301
|
+
*/
|
|
1302
|
+
async autoSwitchPaste(sessionId, target, files, text, route) {
|
|
1303
|
+
const input = this.inputFor(sessionId);
|
|
1304
|
+
try {
|
|
1305
|
+
await this.switchModel(sessionId, route);
|
|
1306
|
+
input.notify('info', `Switched to ${route.label || `${route.model} (Vision Toolkit)`}; pasted images now keep the native attachment flow`);
|
|
1307
|
+
}
|
|
1308
|
+
catch (error) {
|
|
1309
|
+
input.notify('error', `Model switch failed; images will be sent as workspace paths: ${message(error)}`);
|
|
1310
|
+
this.takeoverPaste(sessionId, target, files, text);
|
|
1311
|
+
return;
|
|
1312
|
+
}
|
|
1313
|
+
// Replaying lets the composer's own intake run (thumbnail, limits,
|
|
1314
|
+
// keyboard); if the environment cannot replay clipboard bytes, the
|
|
1315
|
+
// images still land as workspace paths.
|
|
1316
|
+
const before = input.state.getSnapshot().imageIds.length;
|
|
1317
|
+
const replayed = this.replayPaste(target, files, text);
|
|
1318
|
+
const after = input.state.getSnapshot().imageIds.length;
|
|
1319
|
+
if (!replayed || after <= before) {
|
|
1320
|
+
this.takeoverPaste(sessionId, target, files, text);
|
|
1321
|
+
}
|
|
1322
|
+
}
|
|
1323
|
+
/**
|
|
1324
|
+
* Path-takeover flow: insert the same-paste text and every image as a text
|
|
1325
|
+
* reference that serializes to the image's workspace path on send.
|
|
1326
|
+
* @param sessionId - the live Session id.
|
|
1327
|
+
* @param target - the composer textarea the paste landed on.
|
|
1328
|
+
* @param files - the captured image files.
|
|
1329
|
+
* @param text - same-paste text.
|
|
1330
|
+
*/
|
|
1331
|
+
takeoverPaste(sessionId, target, files, text) {
|
|
1090
1332
|
const input = this.inputFor(sessionId);
|
|
1091
1333
|
const snapshot = input.state.getSnapshot();
|
|
1092
1334
|
if (snapshot.phase !== 'plain')
|
|
1093
|
-
return
|
|
1335
|
+
return;
|
|
1094
1336
|
const start = Math.max(0, Math.min(target.selectionStart ?? snapshot.draft.length, snapshot.draft.length));
|
|
1095
1337
|
const end = Math.max(start, Math.min(target.selectionEnd ?? start, snapshot.draft.length));
|
|
1096
|
-
const text = (event.clipboardData?.getData('text/plain') ?? '').replaceAll('\uFFFC', '');
|
|
1097
1338
|
try {
|
|
1098
1339
|
let cursor = this.insertText(input, text, start, end);
|
|
1099
1340
|
validateImages(files);
|
|
1100
|
-
cursor = this.insertRecords(
|
|
1341
|
+
cursor = this.insertRecords(sessionId, input, files, cursor);
|
|
1101
1342
|
requestAnimationFrame(() => {
|
|
1102
1343
|
target.focus({ preventScroll: true });
|
|
1103
1344
|
target.setSelectionRange(cursor, cursor);
|
|
@@ -1106,6 +1347,45 @@ class PasteImageController {
|
|
|
1106
1347
|
catch (error) {
|
|
1107
1348
|
input.notify('error', message(error));
|
|
1108
1349
|
}
|
|
1350
|
+
}
|
|
1351
|
+
handlePaste(event) {
|
|
1352
|
+
if (this.replaying)
|
|
1353
|
+
return false;
|
|
1354
|
+
const files = imageFiles(event.clipboardData);
|
|
1355
|
+
if (files.length === 0)
|
|
1356
|
+
return false;
|
|
1357
|
+
const target = event.target;
|
|
1358
|
+
if (!(target instanceof HTMLTextAreaElement) || target.closest('[data-composer-card]') === null)
|
|
1359
|
+
return false;
|
|
1360
|
+
const sessionId = this.ctx.sessions.list.getSnapshot().current;
|
|
1361
|
+
if (sessionId === undefined)
|
|
1362
|
+
return false;
|
|
1363
|
+
const modelLabel = currentModelLabel();
|
|
1364
|
+
this.refreshVerdict(sessionId, modelLabel);
|
|
1365
|
+
// Only a fresh host verdict acts; the native attachment flow stays the
|
|
1366
|
+
// default while the host is unconfirmed.
|
|
1367
|
+
const verdict = this.verdictFor(sessionId, modelLabel);
|
|
1368
|
+
if (verdict === undefined)
|
|
1369
|
+
return false;
|
|
1370
|
+
// An image-capable model (the variant routes included) keeps its native
|
|
1371
|
+
// paste: no switch, no takeover.
|
|
1372
|
+
if (verdict.takeOver === false && verdict.autoSwitch === undefined)
|
|
1373
|
+
return false;
|
|
1374
|
+
event.preventDefault();
|
|
1375
|
+
event.stopPropagation();
|
|
1376
|
+
event.stopImmediatePropagation();
|
|
1377
|
+
const input = this.inputFor(sessionId);
|
|
1378
|
+
if (input.state.getSnapshot().phase !== 'plain')
|
|
1379
|
+
return true;
|
|
1380
|
+
const text = (event.clipboardData?.getData('text/plain') ?? '').replaceAll('\uFFFC', '');
|
|
1381
|
+
if (verdict.autoSwitch !== undefined) {
|
|
1382
|
+
// The text-only model has an image-input variant: switch first, then
|
|
1383
|
+
// let the paste flow natively so the thumbnail and durable session
|
|
1384
|
+
// image are preserved.
|
|
1385
|
+
void this.autoSwitchPaste(sessionId, target, files, text, verdict.autoSwitch);
|
|
1386
|
+
return true;
|
|
1387
|
+
}
|
|
1388
|
+
this.takeoverPaste(sessionId, target, files, text);
|
|
1109
1389
|
return true;
|
|
1110
1390
|
}
|
|
1111
1391
|
remove(sessionId, occurrence) {
|
|
@@ -1252,8 +1532,18 @@ function installPasteImages(ctx) {
|
|
|
1252
1532
|
});
|
|
1253
1533
|
ctx.effect(() => {
|
|
1254
1534
|
const listener = (event) => { controller.handlePaste(event); };
|
|
1535
|
+
// A focus-time prefetch has the verdict ready before the first paste can land.
|
|
1536
|
+
const onFocusIn = () => {
|
|
1537
|
+
const sessionId = ctx.sessions.list.getSnapshot().current;
|
|
1538
|
+
if (sessionId !== undefined)
|
|
1539
|
+
controller.refreshVerdict(String(sessionId), currentModelLabel());
|
|
1540
|
+
};
|
|
1255
1541
|
document.addEventListener('paste', listener, true);
|
|
1256
|
-
|
|
1542
|
+
document.addEventListener('focusin', onFocusIn, true);
|
|
1543
|
+
return () => {
|
|
1544
|
+
document.removeEventListener('paste', listener, true);
|
|
1545
|
+
document.removeEventListener('focusin', onFocusIn, true);
|
|
1546
|
+
};
|
|
1257
1547
|
}, 'dsh-vision-toolkit: clipboard image capture');
|
|
1258
1548
|
ctx.slots.inject('conversation.input.dock', () => ctx.slots.register({
|
|
1259
1549
|
name: 'conversation.input.dock',
|