dsh-vision-router 3.0.3 → 3.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -10
- package/README.zh.md +11 -8
- package/cordis.patch.yml +14 -10
- package/docs/architecture/dsh-0.2.1-alpha.2-compat.md +41 -0
- package/docs/development/dsh-alpha3-compatibility.md +36 -0
- package/docs/development/dsh-alpha4-adaptation.md +12 -0
- package/docs/development/dsh-alpha4-validation-evidence.md +29 -0
- package/docs/development/dsh-alpha5-adaptation.md +31 -0
- package/docs/development/web-live-reload.md +157 -0
- package/docs/releases/v3.0.4.md +18 -0
- package/docs/remote-settings.md +4 -0
- package/docs/v2-capability-routing.md +25 -9
- package/lib/adapter-update-coalescer.js +41 -8
- package/lib/client-presentation-boundary-main.js +26 -3
- package/lib/client.js +20 -19
- package/lib/core-primitives.js +4 -0
- package/lib/maintenance-routes.js +7 -1
- package/lib/vision-backend-runtime-policy.js +3 -3
- package/lib/vision-background-benchmark.js +33 -5
- package/lib/vision-capability-benchmark-service.js +61 -23
- package/lib/vision-exact-check-client.js +24 -6
- package/lib/vision-image-input-verdict.js +5 -3
- package/lib/vision-toggle-root-hardening.js +20 -3
- package/lib/web-capability-boundary.js +164 -31
- package/package.json +6 -4
- package/presets/web-connection-ready/index.mjs +48 -0
- package/presets/web-connection-ready/package.json +6 -0
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
const wrappedContexts = new WeakMap()
|
|
2
2
|
const wrappedLlmServices = new WeakMap()
|
|
3
|
-
const
|
|
4
|
-
const fullAdapterContractCompat = new WeakMap()
|
|
3
|
+
const adapterContractCompat = Array.from({ length: 4 }, () => new WeakMap())
|
|
5
4
|
|
|
6
5
|
export const DEFAULT_ADAPTER_RECONCILE_MAX_PASSES = 32
|
|
7
6
|
|
|
@@ -37,13 +36,18 @@ function noImageRequestPricing() {
|
|
|
37
36
|
return undefined
|
|
38
37
|
}
|
|
39
38
|
|
|
39
|
+
function normalizedImageRequestTarget(_provider, _model, image) {
|
|
40
|
+
return { width: image.width, height: image.height, maxBytes: image.bytes }
|
|
41
|
+
}
|
|
42
|
+
|
|
40
43
|
/**
|
|
41
44
|
* Normalize Vision Router-owned duck-typed adapters to the Host adapter
|
|
42
45
|
* contracts that have base-class defaults in newer DSH releases.
|
|
43
46
|
*
|
|
44
47
|
* DSH 0.1.1 started dispatching through adapter.prepareCall(). DSH 0.1.2 adds
|
|
45
48
|
* adapter.imageRequestPricing() and the token meter resolves it synchronously
|
|
46
|
-
* on every measurement.
|
|
49
|
+
* on every measurement. DSH 0.2.1 adds adapter.imageRequestTarget() for
|
|
50
|
+
* screenshot consumers, with a base-class identity projection default. Vision Router intentionally registers plain adapter
|
|
47
51
|
* objects, so neither inherited default exists unless we materialize it here.
|
|
48
52
|
*
|
|
49
53
|
* Request-image pricing is deliberately NOT delegated to some other adapter.
|
|
@@ -51,6 +55,9 @@ function noImageRequestPricing() {
|
|
|
51
55
|
* only that exact route can truthfully declare its projection. `undefined` is
|
|
52
56
|
* LlmAdapter's default and keeps the Host's provider-neutral heuristic. An
|
|
53
57
|
* adapter that explicitly implements imageRequestPricing remains authoritative.
|
|
58
|
+
* Image request targets likewise default to the Host's normalized-image
|
|
59
|
+
* identity projection, never a guessed provider-specific resize. An explicit
|
|
60
|
+
* route-owned imageRequestTarget remains authoritative.
|
|
54
61
|
*
|
|
55
62
|
* The compatibility method is installed non-enumerably on extensible adapters
|
|
56
63
|
* to preserve registration identity. Frozen/sealed adapters fall back to one
|
|
@@ -61,17 +68,20 @@ export function ensureAdapterContracts(adapter, options = {}) {
|
|
|
61
68
|
if (!isAdapter(adapter)) return adapter
|
|
62
69
|
|
|
63
70
|
const includeImageRequestPricing = options.imageRequestPricing === true
|
|
71
|
+
const includeImageRequestTarget = options.imageRequestTarget === true
|
|
64
72
|
const needsPrepareCall = !hasFunction(adapter, 'prepareCall')
|
|
65
73
|
const needsImageRequestPricing =
|
|
66
74
|
includeImageRequestPricing && !hasFunction(adapter, 'imageRequestPricing')
|
|
67
|
-
|
|
75
|
+
const needsImageRequestTarget = includeImageRequestTarget && !hasFunction(adapter, 'imageRequestTarget')
|
|
76
|
+
if (!needsPrepareCall && !needsImageRequestPricing && !needsImageRequestTarget) return adapter
|
|
68
77
|
|
|
69
|
-
const cache = includeImageRequestPricing
|
|
78
|
+
const cache = adapterContractCompat[Number(includeImageRequestPricing) + 2 * Number(includeImageRequestTarget)]
|
|
70
79
|
const cached = cache.get(adapter)
|
|
71
80
|
if (cached) return cached
|
|
72
81
|
|
|
73
82
|
const prepareCall = needsPrepareCall ? createPrepareCallFallback(adapter) : undefined
|
|
74
83
|
const imageRequestPricing = needsImageRequestPricing ? noImageRequestPricing : undefined
|
|
84
|
+
const imageRequestTarget = needsImageRequestTarget ? normalizedImageRequestTarget : undefined
|
|
75
85
|
const descriptors = {}
|
|
76
86
|
if (prepareCall !== undefined) {
|
|
77
87
|
descriptors.prepareCall = {
|
|
@@ -87,6 +97,13 @@ export function ensureAdapterContracts(adapter, options = {}) {
|
|
|
87
97
|
value: imageRequestPricing,
|
|
88
98
|
}
|
|
89
99
|
}
|
|
100
|
+
if (imageRequestTarget !== undefined) {
|
|
101
|
+
descriptors.imageRequestTarget = {
|
|
102
|
+
configurable: true,
|
|
103
|
+
writable: true,
|
|
104
|
+
value: imageRequestTarget,
|
|
105
|
+
}
|
|
106
|
+
}
|
|
90
107
|
|
|
91
108
|
// Preserve the exact adapter object whenever possible. Some runtime paths
|
|
92
109
|
// retain registration.adapter and compare/reuse it later, so mutation is less
|
|
@@ -103,6 +120,9 @@ export function ensureAdapterContracts(adapter, options = {}) {
|
|
|
103
120
|
if (property === 'imageRequestPricing' && imageRequestPricing !== undefined) {
|
|
104
121
|
return imageRequestPricing
|
|
105
122
|
}
|
|
123
|
+
if (property === 'imageRequestTarget' && imageRequestTarget !== undefined) {
|
|
124
|
+
return imageRequestTarget
|
|
125
|
+
}
|
|
106
126
|
const value = Reflect.get(target, property, target)
|
|
107
127
|
if (typeof value !== 'function') return value
|
|
108
128
|
let bound = boundFunctions.get(value)
|
|
@@ -131,6 +151,14 @@ function llmSupportsImageRequestPricing(llm) {
|
|
|
131
151
|
}
|
|
132
152
|
}
|
|
133
153
|
|
|
154
|
+
function llmSupportsImageRequestTarget(llm) {
|
|
155
|
+
try {
|
|
156
|
+
return typeof Reflect.get(llm, 'imageRequestTarget', llm) === 'function'
|
|
157
|
+
} catch {
|
|
158
|
+
return false
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
|
|
134
162
|
function wrapLlmService(llm) {
|
|
135
163
|
if (!isAdapter(llm)) return llm
|
|
136
164
|
const cached = wrappedLlmServices.get(llm)
|
|
@@ -140,6 +168,7 @@ function wrapLlmService(llm) {
|
|
|
140
168
|
// historical adapter shape, while Hosts exposing the new route-pricing seam
|
|
141
169
|
// receive the base-class-compatible default on our duck-typed adapters.
|
|
142
170
|
const includeImageRequestPricing = llmSupportsImageRequestPricing(llm)
|
|
171
|
+
const includeImageRequestTarget = llmSupportsImageRequestTarget(llm)
|
|
143
172
|
const wrapped = new Proxy(llm, {
|
|
144
173
|
get(target, property) {
|
|
145
174
|
if (property === 'registerAdapter') {
|
|
@@ -149,7 +178,10 @@ function wrapLlmService(llm) {
|
|
|
149
178
|
registerAdapter.call(
|
|
150
179
|
target,
|
|
151
180
|
providers,
|
|
152
|
-
ensureAdapterContracts(adapter, {
|
|
181
|
+
ensureAdapterContracts(adapter, {
|
|
182
|
+
imageRequestPricing: includeImageRequestPricing,
|
|
183
|
+
imageRequestTarget: includeImageRequestTarget,
|
|
184
|
+
}),
|
|
153
185
|
...rest,
|
|
154
186
|
)
|
|
155
187
|
}
|
|
@@ -238,8 +270,9 @@ export function createCoalescingRunner(run, options = {}) {
|
|
|
238
270
|
* their native event semantics.
|
|
239
271
|
*
|
|
240
272
|
* The same deepest registration boundary also normalizes Vision Router-owned
|
|
241
|
-
* adapter contracts: prepareCall for DSH 0.1.1+,
|
|
242
|
-
* the Host exposes the DSH 0.1.2 route-pricing seam
|
|
273
|
+
* adapter contracts: prepareCall for DSH 0.1.1+, imageRequestPricing when
|
|
274
|
+
* the Host exposes the DSH 0.1.2 route-pricing seam, and imageRequestTarget
|
|
275
|
+
* when the Host exposes the DSH 0.2.1 screenshot projection seam. This stays scoped to the
|
|
243
276
|
* wrapped context instead of monkey-patching the Host LLM service process-wide.
|
|
244
277
|
*/
|
|
245
278
|
export function contextWithCoalescedAdapterUpdates(ctx) {
|
|
@@ -1170,6 +1170,13 @@ export const CLIENT_PRESENTATION_PRELUDE = String.raw`(function(){
|
|
|
1170
1170
|
var complete = usable && livePair.mode !== 'unavailable';
|
|
1171
1171
|
var settledMode = React.useRef(null);
|
|
1172
1172
|
var toggleTransition = React.useRef(null);
|
|
1173
|
+
// The Host may render a ready directory while the RPC that selected
|
|
1174
|
+
// it is still in flight. Keep only an in-flight transaction fence here,
|
|
1175
|
+
// never a second source of truth for Vision ON/OFF.
|
|
1176
|
+
var selectionTransaction = React.useRef(null);
|
|
1177
|
+
var refreshSelection = React.useState(0)[1];
|
|
1178
|
+
var selectionInFlight = !!(selectionTransaction.current &&
|
|
1179
|
+
selectionTransaction.current.directory === directory);
|
|
1173
1180
|
if (settledMode.current && settledMode.current.directory !== directory) settledMode.current = null;
|
|
1174
1181
|
if (toggleTransition.current && toggleTransition.current.directory !== directory) toggleTransition.current = null;
|
|
1175
1182
|
if (complete) settledMode.current = { directory: directory, mode: livePair.mode };
|
|
@@ -1194,7 +1201,7 @@ export const CLIENT_PRESENTATION_PRELUDE = String.raw`(function(){
|
|
|
1194
1201
|
var toastState = React.useState(null);
|
|
1195
1202
|
var toast = toastState[0];
|
|
1196
1203
|
var setToast = toastState[1];
|
|
1197
|
-
var busy = state.status === 'selecting';
|
|
1204
|
+
var busy = state.status === 'selecting' || selectionInFlight;
|
|
1198
1205
|
var loading = settling;
|
|
1199
1206
|
var removed = props.session && props.session.removed === true;
|
|
1200
1207
|
var disabled = props.available !== true || removed || busy || loading || livePair.mode === 'unavailable';
|
|
@@ -1236,15 +1243,23 @@ export const CLIENT_PRESENTATION_PRELUDE = String.raw`(function(){
|
|
|
1236
1243
|
'data-active': active ? 'true' : 'false',
|
|
1237
1244
|
'data-dimmed': disabled && !ownSettling && (livePair.mode === 'unavailable' || loading) ? 'true' : 'false',
|
|
1238
1245
|
onClick: function() {
|
|
1239
|
-
if (disabled || !pair.target || typeof props.select !== 'function'
|
|
1246
|
+
if (disabled || !pair.target || typeof props.select !== 'function' ||
|
|
1247
|
+
(selectionTransaction.current && selectionTransaction.current.directory === directory)) return;
|
|
1240
1248
|
var target = pair.target;
|
|
1249
|
+
var ticket = { directory: directory, target: target };
|
|
1250
|
+
selectionTransaction.current = ticket;
|
|
1251
|
+
refreshSelection(function(revision){ return revision + 1; });
|
|
1241
1252
|
toggleTransition.current = {
|
|
1242
1253
|
directory: directory,
|
|
1243
1254
|
target: target,
|
|
1244
1255
|
seenSelecting: false
|
|
1245
1256
|
};
|
|
1246
1257
|
setToast(null);
|
|
1247
|
-
|
|
1258
|
+
var pending;
|
|
1259
|
+
try { pending = props.select(target); }
|
|
1260
|
+
catch (error) { pending = Promise.reject(error); }
|
|
1261
|
+
void Promise.resolve(pending).then(function(accepted){
|
|
1262
|
+
if (selectionTransaction.current !== ticket) return;
|
|
1248
1263
|
if (!accepted) {
|
|
1249
1264
|
toggleTransition.current = null;
|
|
1250
1265
|
announceRejectedSelection();
|
|
@@ -1255,6 +1270,14 @@ export const CLIENT_PRESENTATION_PRELUDE = String.raw`(function(){
|
|
|
1255
1270
|
if (!latest || (latest.status !== 'selecting' && latest.status !== 'idle' && latest.status !== 'loading')) {
|
|
1256
1271
|
toggleTransition.current = null;
|
|
1257
1272
|
}
|
|
1273
|
+
}, function(error){
|
|
1274
|
+
if (selectionTransaction.current !== ticket) return;
|
|
1275
|
+
toggleTransition.current = null;
|
|
1276
|
+
announceRejectedSelection();
|
|
1277
|
+
}).finally(function(){
|
|
1278
|
+
if (selectionTransaction.current !== ticket) return;
|
|
1279
|
+
selectionTransaction.current = null;
|
|
1280
|
+
refreshSelection(function(revision){ return revision + 1; });
|
|
1258
1281
|
});
|
|
1259
1282
|
}
|
|
1260
1283
|
},
|
package/lib/client.js
CHANGED
|
@@ -268,18 +268,18 @@ openLogFolder,
|
|
|
268
268
|
catalogFallback: '),模型字段已退回手动输入。',
|
|
269
269
|
catalogPartialFailure: '部分已配置供应商的模型目录加载失败:{detail}。这些供应商可能仍会显示在「设置 → 模型」中,但暂时不会出现在这里。',
|
|
270
270
|
visionCapsLoading: '正在读取模型图片能力声明(仅用于提示,不影响可选模型)…',
|
|
271
|
-
visionCapsError: '
|
|
272
|
-
visionCapsFiltered: '
|
|
273
|
-
visionCapabilityUndeclaredWarning: '
|
|
274
|
-
visionCapabilityTextOnlyWarning: '
|
|
275
|
-
visionCapabilityUnknownWarning: '
|
|
271
|
+
visionCapsError: '能力数据不可用,仍可选择模型;实际图片投递取决于Host声明及传输支持。',
|
|
272
|
+
visionCapsFiltered: '能力声明不限制后端选择;Host标为仅文本可能导致图片无法送达。',
|
|
273
|
+
visionCapabilityUndeclaredWarning: '图片能力未知:调用时尝试原生adapter,失败后尝试下一后端。',
|
|
274
|
+
visionCapabilityTextOnlyWarning: 'Host 将此模型标为仅文本,adapter 调用前会剥离图片;仅兼容的 HTTP Chat Completions 可桥接。若支持图片,在 DSH「设置 → 模型 → 输入类型」勾选图片;旧 Host 设 llm-pi-ai 的 input: [text, image] 或 llm-deepseek 的 inputModalities: [text, image] 后重载配置,否则跳过此后端。',
|
|
275
|
+
visionCapabilityUnknownWarning: '图片能力待验证;调用失败会尝试下一后端。',
|
|
276
276
|
visionCapsEmptyTitle: '没有可选的用户视觉模型',
|
|
277
277
|
visionCapsEmptyBody: '检测到 {count} 个用户模型,但它们都没有被 DSH 明确标记为支持图片,因此已被安全隐藏。',
|
|
278
278
|
visionCapsHiddenPrefix: '被隐藏的模型:',
|
|
279
279
|
visionCapsReasonMissingImage: '未声明 image',
|
|
280
280
|
visionCapsReasonUnverified: '无法验证图片能力',
|
|
281
281
|
visionCapsHiddenMore: '另有 {count} 个模型未显示',
|
|
282
|
-
visionCapsMissingImageHint: '
|
|
282
|
+
visionCapsMissingImageHint: '若模型确实支持图片,请到 DSH「设置 → 模型 → 输入类型」启用图片并保存;旧 Host 可在 profile 的 cordis.patch.yml 中声明正确的模型输入类型。仅添加额外视觉模型标记不能改变 Host 图片投递。',
|
|
283
283
|
visionCapsRetry: '重新检测模型',
|
|
284
284
|
chainInvalidCurrent: '当前保存的后端已不在可调用模型目录中,或属于非生成式/递归路由,运行时会跳过:',
|
|
285
285
|
retryCatalog: '重试加载目录',
|
|
@@ -301,7 +301,7 @@ openLogFolder,
|
|
|
301
301
|
extraVisionModelsHint:
|
|
302
302
|
'这个设置不再用于“解锁”下拉或允许调用;所有可调用生成式模型本来就能被选择。' +
|
|
303
303
|
'只有当你希望把某个未声明图片能力的模型明确标记为视觉模型时才需要填写。' +
|
|
304
|
-
'
|
|
304
|
+
'此标记不会更改 Host 输入声明。仅经验证的 HTTP Chat Completions 渠道可直接桥接;其他纯文本声明会被安全拒绝。',
|
|
305
305
|
groupRoutes: '路由名称',
|
|
306
306
|
groupProxy: '网络',
|
|
307
307
|
networkAuthorityHint: '默认不接管网络出口:Vision Router 沿用 DSH/Host 当前网络路径。下方代理只用于需要给视觉请求单独指定出口的高级覆盖。',
|
|
@@ -554,18 +554,18 @@ openLogFolder,
|
|
|
554
554
|
catalogFallback: '); model fields fell back to free-text input.',
|
|
555
555
|
catalogPartialFailure: 'Some configured providers failed to load their model catalog: {detail}. They may still appear in Settings → Models, but are temporarily unavailable here.',
|
|
556
556
|
visionCapsLoading: 'Reading model image-capability metadata (advisory only; it does not hide selectable models)…',
|
|
557
|
-
visionCapsError: '
|
|
558
|
-
visionCapsFiltered: '
|
|
559
|
-
visionCapabilityUndeclaredWarning: '
|
|
560
|
-
visionCapabilityTextOnlyWarning: '
|
|
561
|
-
visionCapabilityUnknownWarning: 'Image capability is
|
|
557
|
+
visionCapsError: 'Capabilities unavailable; models remain selectable, but image delivery depends on Host input declaration.',
|
|
558
|
+
visionCapsFiltered: 'Model capability labels do not restrict selection; a Host text-only label may block image delivery.',
|
|
559
|
+
visionCapabilityUndeclaredWarning: 'Image capability unknown; Vision Router tries the adapter and falls through on failure.',
|
|
560
|
+
visionCapabilityTextOnlyWarning: 'Host text-only strips images before adapter; only verified HTTP Chat Completions can bridge. If supported, enable Image in DSH Settings → Models → Input types. Older Hosts: llm-pi-ai input: [text, image]; llm-deepseek inputModalities: [text, image]. Otherwise skip this backend.',
|
|
561
|
+
visionCapabilityUnknownWarning: 'Image capability is checked at call time; failures fall through.',
|
|
562
562
|
visionCapsEmptyTitle: 'No selectable user vision models',
|
|
563
563
|
visionCapsEmptyBody: 'DSH reported {count} user models, but none are explicitly marked as accepting images, so Vision Router hid them safely.',
|
|
564
564
|
visionCapsHiddenPrefix: 'Hidden models:',
|
|
565
565
|
visionCapsReasonMissingImage: 'image input not declared',
|
|
566
566
|
visionCapsReasonUnverified: 'image capability could not be verified',
|
|
567
567
|
visionCapsHiddenMore: '{count} more models not shown',
|
|
568
|
-
visionCapsMissingImageHint: '
|
|
568
|
+
visionCapsMissingImageHint: 'For a model that truly accepts images, enable Image in DSH Settings → Models → Input types. Older Hosts: set the correct input field in your profile cordis.patch.yml. Extra vision labels cannot change Host pixel delivery.',
|
|
569
569
|
visionCapsRetry: 'Re-detect models',
|
|
570
570
|
chainInvalidCurrent: 'This saved backend is no longer callable, or is a non-generative/recursive route, so runtime will skip it: ',
|
|
571
571
|
retryCatalog: 'Retry catalog',
|
|
@@ -587,7 +587,7 @@ openLogFolder,
|
|
|
587
587
|
extraVisionModelsHint:
|
|
588
588
|
'This setting no longer unlocks the picker or admission: every callable generative model is selectable already. ' +
|
|
589
589
|
'Use it only when you want to explicitly label an undeclared model as visual. ' +
|
|
590
|
-
'
|
|
590
|
+
'This label does not change Host input declarations. Verified HTTP Chat Completions can bridge; other explicit text-only routes fail safely.',
|
|
591
591
|
groupRoutes: 'Route names',
|
|
592
592
|
groupProxy: 'Network',
|
|
593
593
|
networkAuthorityHint: 'Vision Router does not own egress by default: it follows the current DSH/Host network path. The proxy below is only an advanced vision-request override.',
|
|
@@ -845,11 +845,12 @@ openLogFolder,
|
|
|
845
845
|
function visionCapabilityWarningKey(capability, status) {
|
|
846
846
|
if (status === 'loading' || status === 'idle') return undefined
|
|
847
847
|
if (status === 'error' || !capability) return 'visionCapabilityUnknownWarning'
|
|
848
|
-
if (capability.attemptable === false
|
|
849
|
-
const
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
848
|
+
if (capability.attemptable === false) return undefined
|
|
849
|
+
const declared = Array.isArray(capability.hostInputModalities)
|
|
850
|
+
? capability.hostInputModalities : capability.inputModalities
|
|
851
|
+
const modalities = Array.isArray(declared) ? declared : []
|
|
852
|
+
if (modalities.length > 0 && !modalities.includes('image')) return 'visionCapabilityTextOnlyWarning'
|
|
853
|
+
return capability.image === true ? undefined : 'visionCapabilityUndeclaredWarning'
|
|
853
854
|
}
|
|
854
855
|
|
|
855
856
|
// ── field specs ──────────────────────────────────────────────────────────
|
package/lib/core-primitives.js
CHANGED
|
@@ -2768,6 +2768,8 @@ export function decideVisionBackendCapability(info, provider, model, extraVision
|
|
|
2768
2768
|
image: true,
|
|
2769
2769
|
attemptable: true,
|
|
2770
2770
|
inputModalities: [...new Set([...inputModalities, 'image'])],
|
|
2771
|
+
...(inputModalities.length > 0 && !inputModalities.includes('image')
|
|
2772
|
+
? { hostInputModalities: [...inputModalities] } : {}),
|
|
2771
2773
|
inferred: 'override',
|
|
2772
2774
|
reason: undefined,
|
|
2773
2775
|
}
|
|
@@ -2789,6 +2791,8 @@ export function decideVisionBackendCapability(info, provider, model, extraVision
|
|
|
2789
2791
|
image: true,
|
|
2790
2792
|
attemptable: true,
|
|
2791
2793
|
inputModalities: [...new Set([...inputModalities, 'image'])],
|
|
2794
|
+
...(inputModalities.length > 0 && !inputModalities.includes('image')
|
|
2795
|
+
? { hostInputModalities: [...inputModalities] } : {}),
|
|
2792
2796
|
inferred: 'name',
|
|
2793
2797
|
reason: undefined,
|
|
2794
2798
|
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { randomBytes } from 'node:crypto'
|
|
2
2
|
import { createCachedUpdateChecker } from './update-check.js'
|
|
3
|
+
import { isLocalUiRequest } from './web-capability-boundary.js'
|
|
3
4
|
import { detectDshSelfUpdatePlan, runDshPluginUpdate } from './self-update.js'
|
|
4
5
|
|
|
5
6
|
function json(res, status, body, headers = {}) {
|
|
@@ -66,7 +67,12 @@ export function installVisionMaintenanceRoutes(ctx, options = {}) {
|
|
|
66
67
|
res.end()
|
|
67
68
|
return
|
|
68
69
|
}
|
|
69
|
-
|
|
70
|
+
// Forced refresh causes outbound Host network traffic. Remote Web may
|
|
71
|
+
// read already-cached update status, but must never bypass the cache
|
|
72
|
+
// with ?force=1. Reuse the existing local-UI provenance boundary;
|
|
73
|
+
// do not trust the query string or caller-supplied proxy headers.
|
|
74
|
+
const requestedForce = /(?:[?&])force=1(?:&|$)/.test(String(req.url ?? ''))
|
|
75
|
+
const force = requestedForce && isLocalUiRequest(req)
|
|
70
76
|
const result = await updateChecker.check(force)
|
|
71
77
|
json(res, 200, updateResultForClient(result), { 'cache-control': 'no-store' })
|
|
72
78
|
},
|
|
@@ -174,7 +174,7 @@ async function toOpenAIMessages(ctx, messages) {
|
|
|
174
174
|
return out
|
|
175
175
|
}
|
|
176
176
|
|
|
177
|
-
async function resolvedHostInfo(llm, call) {
|
|
177
|
+
export async function resolvedHostInfo(llm, call) {
|
|
178
178
|
if (typeof llm?.prepareCall === 'function') {
|
|
179
179
|
try {
|
|
180
180
|
const prepared = await llm.prepareCall(
|
|
@@ -195,7 +195,7 @@ async function resolvedHostInfo(llm, call) {
|
|
|
195
195
|
return undefined
|
|
196
196
|
}
|
|
197
197
|
|
|
198
|
-
function localOnlyVisionCallAllowed(core, config, provider, model) {
|
|
198
|
+
export function localOnlyVisionCallAllowed(core, config, provider, model) {
|
|
199
199
|
if (config?.localOnlyVision !== true) return true
|
|
200
200
|
if (provider !== 'vision-http') return false
|
|
201
201
|
const allowed = []
|
|
@@ -346,7 +346,7 @@ function llmWithRuntimePolicy(ctx, llm, options) {
|
|
|
346
346
|
plan.reason,
|
|
347
347
|
)
|
|
348
348
|
yield* failureStream(
|
|
349
|
-
`vision backend ${provider}/${model} is declared text-only by the Host
|
|
349
|
+
`vision backend ${provider}/${model} is declared text-only by the Host, which strips image pixels before the registered adapter runs; no safe HTTP image bridge is available. If this endpoint really supports images, enable Image under DSH Settings > Models > provider > Customized settings > Model options > Input types (when available); on older Hosts set the provider-specific model field (llm-pi-ai: input: [text, image]; llm-deepseek: inputModalities: [text, image]), then reload model configuration. Otherwise choose a declared image-capable backend. Direct bridging is supported only for verified HTTP OpenAI Chat Completions endpoints.`,
|
|
350
350
|
)
|
|
351
351
|
return
|
|
352
352
|
}
|
|
@@ -10,6 +10,7 @@ import { resolveVisionRoutingAuthority } from './vision-routing-authority.js'
|
|
|
10
10
|
import { redactDiagnosticText } from './diagnostic-redaction.js'
|
|
11
11
|
import { installVisionExactCheckClient } from './vision-exact-check-client.js'
|
|
12
12
|
import { createImageInputVerdictStore } from './vision-image-input-verdict.js'
|
|
13
|
+
import { hostImageDeliveryFromInfo, localOnlyVisionCallAllowed, resolvedHostInfo } from './vision-backend-runtime-policy.js'
|
|
13
14
|
import { createBackgroundBenchmarkStopStore } from './vision-background-stop-store.js'
|
|
14
15
|
import { backgroundFailurePolicy } from './vision-background-failure-policy.js'
|
|
15
16
|
import { isLocalUiRequest } from './web-capability-boundary.js'
|
|
@@ -288,9 +289,18 @@ function candidateBlockedByFailure(backoff, candidate, at) {
|
|
|
288
289
|
return false
|
|
289
290
|
}
|
|
290
291
|
|
|
291
|
-
async function
|
|
292
|
+
async function currentHostImageDelivery(ctx, provider, model) {
|
|
293
|
+
if (provider === 'vision-http') return 'native-image'
|
|
294
|
+
return hostImageDeliveryFromInfo(await resolvedHostInfo(ctx?.llm, { provider, model }))
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
async function imageVerdictFor(imageVerdictStore, candidate, ctx) {
|
|
292
298
|
if (!candidate?.endpointFingerprint || typeof imageVerdictStore?.get !== 'function') return undefined
|
|
293
|
-
try {
|
|
299
|
+
try {
|
|
300
|
+
const verdict = await imageVerdictStore.get(candidate.endpointFingerprint)
|
|
301
|
+
if (!verdict || verdict.hostMode !== 'native-image') return undefined
|
|
302
|
+
return await currentHostImageDelivery(ctx, candidate.provider, candidate.model) === 'native-image' ? verdict : undefined
|
|
303
|
+
} catch { return undefined }
|
|
294
304
|
}
|
|
295
305
|
|
|
296
306
|
async function chooseNextWork({ ctx, config, core, store, now, backoff, excluded, imageVerdictStore, backgroundStopStore }) {
|
|
@@ -302,7 +312,7 @@ async function chooseNextWork({ ctx, config, core, store, now, backoff, excluded
|
|
|
302
312
|
await restorePersistentStops(backoff, collected, backgroundStopStore, ctx, now)
|
|
303
313
|
const candidates = []
|
|
304
314
|
for (const candidate of collected) {
|
|
305
|
-
const verdict = await imageVerdictFor(imageVerdictStore, candidate)
|
|
315
|
+
const verdict = await imageVerdictFor(imageVerdictStore, candidate, ctx)
|
|
306
316
|
if (verdict?.state === 'unsupported') {
|
|
307
317
|
excluded.set(candidate.key, { key: candidate.key, reason: 'measured-text-only' })
|
|
308
318
|
continue
|
|
@@ -470,6 +480,13 @@ export async function runExactVisionCheck({ ctx, config, core, store, provider,
|
|
|
470
480
|
const wantedModel = typeof model === 'string' ? model.trim() : ''
|
|
471
481
|
if (!wantedProvider || !wantedModel) throw Object.assign(new Error('provider and model are required'), { code: 'VISION_CHECK_BACKEND_REQUIRED' })
|
|
472
482
|
const current = activeSettings(ctx, config)
|
|
483
|
+
if (!localOnlyVisionCallAllowed(core, current, wantedProvider, wantedModel)) {
|
|
484
|
+
const error = new Error('local-only vision policy prevents testing a nonlocal backend')
|
|
485
|
+
error.code = 'VISION_LOCAL_ONLY_POLICY'
|
|
486
|
+
error.benchmarkClass = 'host-configuration'
|
|
487
|
+
throw error
|
|
488
|
+
}
|
|
489
|
+
const declaredBefore = await currentHostImageDelivery(ctx, wantedProvider, wantedModel)
|
|
473
490
|
const candidates = await collectVisionRoutingCandidates(ctx, current, core, store)
|
|
474
491
|
const candidate = candidates.find((entry) => entry?.provider === wantedProvider && entry?.model === wantedModel)
|
|
475
492
|
?? exactAdapterCandidate(ctx, core, wantedProvider, wantedModel)
|
|
@@ -513,6 +530,12 @@ export async function runExactVisionCheck({ ctx, config, core, store, provider,
|
|
|
513
530
|
remaining(),
|
|
514
531
|
'exact image check timed out',
|
|
515
532
|
)
|
|
533
|
+
if (declaredBefore !== await currentHostImageDelivery(ctx, wantedProvider, wantedModel)) {
|
|
534
|
+
const error = new Error('Host image-input declaration changed during the test; retest the model')
|
|
535
|
+
error.code = 'VISION_CHECK_STALE_CONFIG'
|
|
536
|
+
error.benchmarkClass = 'host-configuration'
|
|
537
|
+
throw error
|
|
538
|
+
}
|
|
516
539
|
return {
|
|
517
540
|
ok: true,
|
|
518
541
|
key: candidate.key,
|
|
@@ -602,13 +625,15 @@ export function createBackgroundCapabilityProfiler({
|
|
|
602
625
|
excluded.set(candidate.key, { key: candidate.key, reason: 'measured-text-only' })
|
|
603
626
|
clearDeferredForCandidate(candidate)
|
|
604
627
|
await clearPersistentStopsForCandidate(candidate)
|
|
605
|
-
if (candidate.endpointFingerprint && typeof imageVerdictStore?.markUnsupported === 'function'
|
|
628
|
+
if (candidate.endpointFingerprint && typeof imageVerdictStore?.markUnsupported === 'function'
|
|
629
|
+
&& await currentHostImageDelivery(ctx, candidate.provider, candidate.model) === 'native-image') {
|
|
606
630
|
try {
|
|
607
631
|
await imageVerdictStore.markUnsupported({
|
|
608
632
|
fingerprint: candidate.endpointFingerprint,
|
|
609
633
|
key: candidate.key,
|
|
610
634
|
provider: candidate.provider,
|
|
611
635
|
model: candidate.model,
|
|
636
|
+
hostMode: 'native-image',
|
|
612
637
|
measuredAt: Number(now()),
|
|
613
638
|
})
|
|
614
639
|
} catch (error) {
|
|
@@ -1140,7 +1165,9 @@ export function installBackgroundCapabilityProfiling(ctx, config, core, store, o
|
|
|
1140
1165
|
const timedOut = error?.name === 'TimeoutError' || error?.name === 'AbortError' || /timed out|timeout|deadline/i.test(String(error?.message ?? ''))
|
|
1141
1166
|
const failureClass = error?.benchmarkClass ?? classifyCapabilityBenchmarkFailure(error)
|
|
1142
1167
|
const unsupportedImage = !timedOut && failureClass === 'unsupported-image'
|
|
1143
|
-
if (unsupportedImage
|
|
1168
|
+
if (unsupportedImage && await currentHostImageDelivery(ctx, body.provider, body.model) === 'native-image') {
|
|
1169
|
+
await profiler.recordImageUnsupported(body.provider, body.model)
|
|
1170
|
+
}
|
|
1144
1171
|
const code = timedOut
|
|
1145
1172
|
? 'VISION_CHECK_TIMEOUT'
|
|
1146
1173
|
: unsupportedImage
|
|
@@ -1151,6 +1178,7 @@ export function installBackgroundCapabilityProfiling(ctx, config, core, store, o
|
|
|
1151
1178
|
'VISION_CHECK_INVALID_JSON',
|
|
1152
1179
|
'VISION_CHECK_BACKEND_REQUIRED',
|
|
1153
1180
|
'VISION_CHECK_BACKEND_STALE',
|
|
1181
|
+
'VISION_CHECK_STALE_CONFIG',
|
|
1154
1182
|
]).has(code)
|
|
1155
1183
|
sendJson(res, unsupportedImage ? 422 : clientError ? 400 : 502, {
|
|
1156
1184
|
ok: false,
|
|
@@ -14,6 +14,8 @@ import { resolveVisionCredential } from './vision-capability-identity.js'
|
|
|
14
14
|
import { redactDiagnosticText } from './diagnostic-redaction.js'
|
|
15
15
|
import { wireSessionAffinityId } from './session-affinity.js'
|
|
16
16
|
import { streamWithVisionSessionAffinity } from './session-affinity-runtime.js'
|
|
17
|
+
import { hostImageDeliveryFromInfo, resolvedHostInfo } from './vision-backend-runtime-policy.js'
|
|
18
|
+
import { isLocalImageCapabilityAdmissionFailure } from './vision-execution-policy.js'
|
|
17
19
|
import { streamWithLegacyGlobalProxyScope } from './legacy-global-proxy-boundary.js'
|
|
18
20
|
import {
|
|
19
21
|
hardenCapabilityBenchmarkFixture,
|
|
@@ -205,13 +207,28 @@ function failureError(failure, fallback) {
|
|
|
205
207
|
async function collectStreamText(streamLike) {
|
|
206
208
|
const stream = await streamLike
|
|
207
209
|
let text = ''
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
210
|
+
let observedOutput = false
|
|
211
|
+
try {
|
|
212
|
+
for await (const chunk of stream) {
|
|
213
|
+
// Any non-terminal output proves that the adapter began responding.
|
|
214
|
+
// Retrying through a second transport could duplicate a billed call.
|
|
215
|
+
if (chunk && chunk.type !== 'finish' && chunk.type !== 'error' && chunk.type !== 'aborted') {
|
|
216
|
+
observedOutput = true
|
|
217
|
+
}
|
|
218
|
+
if (chunk && typeof chunk.text === 'string') text += chunk.text
|
|
219
|
+
if (chunk?.type === 'finish') {
|
|
220
|
+
const kind = chunk.reason?.kind
|
|
221
|
+
if (kind === 'error' || kind === 'aborted') throw failureError(chunk.reason?.failure, kind)
|
|
222
|
+
}
|
|
223
|
+
if (chunk?.type === 'error' || chunk?.type === 'aborted') throw failureError(chunk.failure, chunk.type)
|
|
224
|
+
}
|
|
225
|
+
} catch (error) {
|
|
226
|
+
if (observedOutput) {
|
|
227
|
+
const failure = failureError(error)
|
|
228
|
+
failure.visionAdapterProducedOutput = true
|
|
229
|
+
throw failure
|
|
213
230
|
}
|
|
214
|
-
|
|
231
|
+
throw error
|
|
215
232
|
}
|
|
216
233
|
return text
|
|
217
234
|
}
|
|
@@ -223,20 +240,22 @@ function tagInvocationError(value) {
|
|
|
223
240
|
return error
|
|
224
241
|
}
|
|
225
242
|
|
|
226
|
-
function mayUseExactAdapterBridge(
|
|
227
|
-
if (directProvider === undefined) return false
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
243
|
+
function mayUseExactAdapterBridge(_core, error, directProvider) {
|
|
244
|
+
if (directProvider === undefined || error?.visionAdapterProducedOutput === true) return false
|
|
245
|
+
// Share the runtime's exact pre-wire pi-ai admission predicate. Provider
|
|
246
|
+
// HTTP 400s, ambiguous image errors and network failures must never replay.
|
|
247
|
+
return isLocalImageCapabilityAdmissionFailure(error)
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
export function exactHostImageDeliveryPlan(info, candidate, core) {
|
|
251
|
+
const delivery = hostImageDeliveryFromInfo(info)
|
|
252
|
+
if (delivery !== 'text-projected') return { delivery, path: 'adapter' }
|
|
253
|
+
const verified = candidate?.evidenceScope === 'endpoint'
|
|
254
|
+
&& candidate?.endpointConfig?.api === 'openai-completions'
|
|
255
|
+
&& typeof candidate?.endpoint === 'string'
|
|
256
|
+
&& typeof core?.isOpenAIHttpBridgeTransport === 'function'
|
|
257
|
+
&& core.isOpenAIHttpBridgeTransport({ api: 'openai-completions', baseURL: candidate.endpoint })
|
|
258
|
+
return { delivery, path: verified ? 'http-direct' : 'refuse' }
|
|
240
259
|
}
|
|
241
260
|
|
|
242
261
|
function benchmarkMaxTokensForProvider(provider) {
|
|
@@ -295,6 +314,22 @@ export function createExactCapabilityInvoker(ctx, core, candidate, config, optio
|
|
|
295
314
|
)
|
|
296
315
|
const callSignal = mergedSignal(signal, AbortSignal.timeout(fixtureTimeoutMs))
|
|
297
316
|
const exactHttpProvider = directProviderFor(candidate, config, core)
|
|
317
|
+
const hostInfo = candidate.provider === 'vision-http' ? undefined :
|
|
318
|
+
await resolvedHostInfo(ctx?.llm, { provider: candidate.provider, model: candidate.model, signal: callSignal })
|
|
319
|
+
const imagePlan = candidate.provider === 'vision-http'
|
|
320
|
+
? { path: 'http-direct' }
|
|
321
|
+
: exactHostImageDeliveryPlan(hostInfo, candidate, core)
|
|
322
|
+
if (imagePlan.path === 'refuse') {
|
|
323
|
+
const error = new Error('Host declares this model text-only and removes image pixels before adapter dispatch; enable Image in DSH model input types, then retest')
|
|
324
|
+
error.code = 'VISION_IMAGE_DELIVERY_UNAVAILABLE'
|
|
325
|
+
error.benchmarkClass = 'host-configuration'
|
|
326
|
+
throw error
|
|
327
|
+
}
|
|
328
|
+
if (imagePlan.path === 'http-direct' && exactHttpProvider === undefined) {
|
|
329
|
+
const error = new Error('verified HTTP image bridge unavailable for this model')
|
|
330
|
+
error.code = 'CAPABILITY_BENCHMARK_PROTOCOL'
|
|
331
|
+
throw error
|
|
332
|
+
}
|
|
298
333
|
const callExactHttp = async () => {
|
|
299
334
|
if (exactHttpProvider === undefined) {
|
|
300
335
|
const error = new Error('exact HTTP bridge is unavailable for this benchmark backend')
|
|
@@ -323,7 +358,7 @@ export function createExactCapabilityInvoker(ctx, core, candidate, config, optio
|
|
|
323
358
|
let output
|
|
324
359
|
let transport
|
|
325
360
|
let latencyMs
|
|
326
|
-
if (candidate.provider === 'vision-http') {
|
|
361
|
+
if (candidate.provider === 'vision-http' || imagePlan.path === 'http-direct') {
|
|
327
362
|
const direct = await callExactHttp()
|
|
328
363
|
output = direct.output
|
|
329
364
|
latencyMs = direct.latencyMs
|
|
@@ -404,8 +439,11 @@ async function declaredImageState(ctx, core, config, candidate) {
|
|
|
404
439
|
candidate.model,
|
|
405
440
|
config?.extraVisionModels,
|
|
406
441
|
)
|
|
407
|
-
|
|
408
|
-
if (
|
|
442
|
+
const hostMode = hostImageDeliveryFromInfo(info)
|
|
443
|
+
if (hostMode === 'native-image') return 'declared'
|
|
444
|
+
if (hostMode === 'text-projected') return 'text-only'
|
|
445
|
+
// Name-based guesses and user labels do not rewrite Host input metadata.
|
|
446
|
+
if (decision?.image === true) return 'unknown'
|
|
409
447
|
} catch {
|
|
410
448
|
// Metadata is advisory. Exact configured routes remain force-testable.
|
|
411
449
|
}
|