dsh-vision-router 2.1.1 → 2.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/docs/releases/v2.1.2.md +30 -0
- package/lib/agent-request-route-authority.js +94 -0
- package/lib/delegated-call-config.js +27 -0
- package/lib/grounding-coordinate-runtime.js +10 -8
- package/lib/legacy-core-vision-policy-bridge.js +17 -146
- package/lib/runtime-composition.js +30 -8
- package/lib/session-surface-policy.js +59 -32
- package/lib/session-vision-mode-authority.js +254 -0
- package/lib/session-vision-mode-boundary.js +408 -0
- package/lib/twin-image-capability-fallback.js +428 -0
- package/lib/vision-attachment-handle-runtime.js +212 -0
- package/lib/vision-tool-runtime-boundary.js +224 -5
- package/package.json +5 -5
|
@@ -0,0 +1,428 @@
|
|
|
1
|
+
import { AsyncLocalStorage } from 'node:async_hooks'
|
|
2
|
+
|
|
3
|
+
const TWIN_SUFFIX = '-vision'
|
|
4
|
+
const DEFAULT_MAIN_WRAPPER_ROUTE = 'deepseek-vision'
|
|
5
|
+
const DEFAULT_CHAIN_ROUTE = 'vision-chain'
|
|
6
|
+
const forcedTextBridgeScope = new AsyncLocalStorage()
|
|
7
|
+
|
|
8
|
+
export const TWIN_IMAGE_CAPABILITY_NEGATIVE_TTL_MS = 60_000
|
|
9
|
+
export const MAX_TWIN_IMAGE_CAPABILITY_NEGATIVE_MODELS = 64
|
|
10
|
+
|
|
11
|
+
function isObject(value) {
|
|
12
|
+
return value !== null && (typeof value === 'object' || typeof value === 'function')
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
function nonEmptyString(value) {
|
|
16
|
+
return typeof value === 'string' && value.length > 0
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
function blocksHaveImage(content) {
|
|
20
|
+
if (!Array.isArray(content)) return false
|
|
21
|
+
for (const block of content) {
|
|
22
|
+
if (!block) continue
|
|
23
|
+
if (block.type === 'image') return true
|
|
24
|
+
if (Array.isArray(block.content) && blocksHaveImage(block.content)) return true
|
|
25
|
+
}
|
|
26
|
+
return false
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export function twinRequestHasImage(messages) {
|
|
30
|
+
return (messages ?? []).some(
|
|
31
|
+
(message) => message && Array.isArray(message.content) && blocksHaveImage(message.content),
|
|
32
|
+
)
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
function failureText(failure, seen = new Set()) {
|
|
36
|
+
if (failure === undefined || failure === null) return ''
|
|
37
|
+
if (typeof failure === 'string') return failure
|
|
38
|
+
if (!isObject(failure) || seen.has(failure)) return ''
|
|
39
|
+
seen.add(failure)
|
|
40
|
+
const pieces = []
|
|
41
|
+
for (const key of ['message', 'detail', 'reason', 'error']) {
|
|
42
|
+
const value = failure[key]
|
|
43
|
+
if (typeof value === 'string') pieces.push(value)
|
|
44
|
+
else if (isObject(value)) {
|
|
45
|
+
const nested = failureText(value, seen)
|
|
46
|
+
if (nested !== '') pieces.push(nested)
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
if (isObject(failure.cause)) {
|
|
50
|
+
const nested = failureText(failure.cause, seen)
|
|
51
|
+
if (nested !== '') pieces.push(nested)
|
|
52
|
+
}
|
|
53
|
+
return pieces.join(' | ')
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
function failureCodes(failure, seen = new Set(), out = new Set()) {
|
|
57
|
+
if (!isObject(failure) || seen.has(failure)) return out
|
|
58
|
+
seen.add(failure)
|
|
59
|
+
if (typeof failure.code === 'string' && failure.code.trim() !== '') {
|
|
60
|
+
out.add(failure.code.trim().toUpperCase())
|
|
61
|
+
}
|
|
62
|
+
for (const key of ['error', 'cause', 'detail', 'reason']) {
|
|
63
|
+
failureCodes(failure[key], seen, out)
|
|
64
|
+
}
|
|
65
|
+
return out
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* True only for an explicit runtime rejection of image input. Generic 400s,
|
|
70
|
+
* max-token errors, auth failures, rate limits and transport errors must never
|
|
71
|
+
* trigger a second model call. Capability prose is deliberately end-bounded:
|
|
72
|
+
* "does not support image inputs larger than ..." describes a size/format
|
|
73
|
+
* restriction, not proof that the model is text-only.
|
|
74
|
+
*/
|
|
75
|
+
export function isImageInputUnsupportedFailure(failure) {
|
|
76
|
+
const text = failureText(failure)
|
|
77
|
+
const terminal = String.raw`(?:\s*(?:[.!?。!?]|$)|\s*\|)`
|
|
78
|
+
const explicitImageRejection = [
|
|
79
|
+
new RegExp(`does not support (?:images?|image inputs?|image content)${terminal}`, 'i'),
|
|
80
|
+
new RegExp(
|
|
81
|
+
`(?:images?|image inputs?|image content) (?:is|are) not supported(?:\\s+(?:by|for) (?:this|the) (?:model|endpoint))?${terminal}`,
|
|
82
|
+
'i',
|
|
83
|
+
),
|
|
84
|
+
new RegExp(
|
|
85
|
+
`unsupported (?:images?|image inputs?|image content)(?:\\s+(?:by|for) (?:this|the) (?:model|endpoint))?${terminal}`,
|
|
86
|
+
'i',
|
|
87
|
+
),
|
|
88
|
+
new RegExp(`cannot (?:accept|process|handle) (?:images?|image inputs?)${terminal}`, 'i'),
|
|
89
|
+
new RegExp(`(?:model|adapter).*text[- ]only.*(?:images?|image inputs?)${terminal}`, 'i'),
|
|
90
|
+
new RegExp(`(?:模型|适配器).*不支持(?:图片|图像)(?:输入)?${terminal}`),
|
|
91
|
+
new RegExp(`(?:图片|图像)输入(?:不被支持|暂不支持|不支持)${terminal}`),
|
|
92
|
+
].some((pattern) => pattern.test(text))
|
|
93
|
+
if (explicitImageRejection) return true
|
|
94
|
+
|
|
95
|
+
const codes = failureCodes(failure)
|
|
96
|
+
return codes.has('MODEL_DOES_NOT_SUPPORT_IMAGES') || codes.has('UNSUPPORTED_IMAGE_INPUT')
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
function chunkFailure(chunk) {
|
|
100
|
+
if (!chunk || typeof chunk !== 'object') return undefined
|
|
101
|
+
if (chunk.type === 'finish' && chunk.reason && chunk.reason.kind === 'error') {
|
|
102
|
+
return chunk.reason.failure ?? chunk.reason
|
|
103
|
+
}
|
|
104
|
+
if (chunk.type === 'error') return chunk.failure ?? chunk.error ?? chunk
|
|
105
|
+
return undefined
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
function stripImageCapability(info) {
|
|
109
|
+
if (!info || typeof info !== 'object' || !Array.isArray(info.inputModalities)) return info
|
|
110
|
+
if (!info.inputModalities.includes('image')) return info
|
|
111
|
+
return {
|
|
112
|
+
...info,
|
|
113
|
+
inputModalities: info.inputModalities.filter((item) => item !== 'image'),
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
function facadeFor(source, overrides = {}) {
|
|
118
|
+
const shell = Object.create(Reflect.getPrototypeOf(source))
|
|
119
|
+
const hasOverride = (property) => Object.prototype.hasOwnProperty.call(overrides, property)
|
|
120
|
+
const read = (target, property) => {
|
|
121
|
+
if (Object.prototype.hasOwnProperty.call(target, property)) {
|
|
122
|
+
return Reflect.get(target, property, target)
|
|
123
|
+
}
|
|
124
|
+
if (hasOverride(property)) return overrides[property]
|
|
125
|
+
const value = Reflect.get(source, property, source)
|
|
126
|
+
return typeof value === 'function' ? value.bind(source) : value
|
|
127
|
+
}
|
|
128
|
+
return new Proxy(shell, {
|
|
129
|
+
get(target, property) {
|
|
130
|
+
return read(target, property)
|
|
131
|
+
},
|
|
132
|
+
has(target, property) {
|
|
133
|
+
return Reflect.has(target, property) || hasOverride(property) || Reflect.has(source, property)
|
|
134
|
+
},
|
|
135
|
+
ownKeys(target) {
|
|
136
|
+
return [...new Set([...Reflect.ownKeys(target), ...Reflect.ownKeys(source), ...Reflect.ownKeys(overrides)])]
|
|
137
|
+
},
|
|
138
|
+
getOwnPropertyDescriptor(target, property) {
|
|
139
|
+
const own = Reflect.getOwnPropertyDescriptor(target, property)
|
|
140
|
+
if (own !== undefined) return own
|
|
141
|
+
if (!hasOverride(property) && !Reflect.has(source, property)) return undefined
|
|
142
|
+
const sourceDescriptor = Reflect.getOwnPropertyDescriptor(source, property)
|
|
143
|
+
return {
|
|
144
|
+
configurable: true,
|
|
145
|
+
enumerable: sourceDescriptor?.enumerable ?? true,
|
|
146
|
+
writable: false,
|
|
147
|
+
value: read(target, property),
|
|
148
|
+
}
|
|
149
|
+
},
|
|
150
|
+
})
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
function proxiedRegistrationWithoutImage(registration, provider, model) {
|
|
154
|
+
if (!registration || !isObject(registration)) return registration
|
|
155
|
+
const adapter = registration.adapter
|
|
156
|
+
if (!adapter || !isObject(adapter) || typeof adapter.resolveModel !== 'function') return registration
|
|
157
|
+
const adapterView = facadeFor(adapter, {
|
|
158
|
+
async resolveModel(requestedProvider, requestedModel, ...rest) {
|
|
159
|
+
const info = await adapter.resolveModel(requestedProvider, requestedModel, ...rest)
|
|
160
|
+
if (requestedProvider !== provider || requestedModel !== model) return info
|
|
161
|
+
return stripImageCapability(info)
|
|
162
|
+
},
|
|
163
|
+
})
|
|
164
|
+
return facadeFor(registration, { adapter: adapterView })
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
async function closeIterator(iterator) {
|
|
168
|
+
if (iterator && typeof iterator.return === 'function') {
|
|
169
|
+
try {
|
|
170
|
+
await iterator.return()
|
|
171
|
+
} catch {
|
|
172
|
+
// The failed attempt is already being abandoned. Cleanup errors must not
|
|
173
|
+
// replace the capability rejection or block the safe bridge retry.
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
/**
|
|
179
|
+
* Private Core-only boundary for generated `<provider>-vision` twins.
|
|
180
|
+
*
|
|
181
|
+
* Catalog metadata remains the optimistic fast path: a genuinely multimodal
|
|
182
|
+
* source still receives raw image blocks exactly as before. If that source
|
|
183
|
+
* explicitly rejects image input before producing any model-visible output,
|
|
184
|
+
* runtime evidence overrides the stale metadata for this request and the same
|
|
185
|
+
* twin is re-entered once under a scoped metadata view with `image` removed.
|
|
186
|
+
* The existing Core wrapper then performs its canonical image -> tool-marker /
|
|
187
|
+
* cached-description projection; this layer never duplicates that logic.
|
|
188
|
+
*/
|
|
189
|
+
export function contextWithTwinImageCapabilityFallback(ctx, options = {}) {
|
|
190
|
+
if (!ctx || typeof ctx !== 'object' || !ctx.llm) return ctx
|
|
191
|
+
const llm = ctx.llm
|
|
192
|
+
const configuredMainWrapperRoute = nonEmptyString(options.wrapperRoute)
|
|
193
|
+
? options.wrapperRoute
|
|
194
|
+
: DEFAULT_MAIN_WRAPPER_ROUTE
|
|
195
|
+
const configuredChainRoute = nonEmptyString(options.chainRoute)
|
|
196
|
+
? options.chainRoute
|
|
197
|
+
: DEFAULT_CHAIN_ROUTE
|
|
198
|
+
const currentOwnedRoutes = () => {
|
|
199
|
+
try {
|
|
200
|
+
const settings = ctx?.get?.('settings')
|
|
201
|
+
const current = settings?.get?.('vision-router')
|
|
202
|
+
if (current && typeof current === 'object' && !Array.isArray(current)) {
|
|
203
|
+
return new Set([
|
|
204
|
+
nonEmptyString(current.wrapperRoute) ? current.wrapperRoute : configuredMainWrapperRoute,
|
|
205
|
+
nonEmptyString(current.chainRoute) ? current.chainRoute : configuredChainRoute,
|
|
206
|
+
])
|
|
207
|
+
}
|
|
208
|
+
} catch {
|
|
209
|
+
// Settings can be between generations; composition-time routes remain
|
|
210
|
+
// authoritative until the live namespace becomes readable again.
|
|
211
|
+
}
|
|
212
|
+
return new Set([configuredMainWrapperRoute, configuredChainRoute])
|
|
213
|
+
}
|
|
214
|
+
const logger = options.logger ?? ctx.logger
|
|
215
|
+
const ttlMs = Number.isFinite(Number(options.negativeTtlMs)) && Number(options.negativeTtlMs) > 0
|
|
216
|
+
? Number(options.negativeTtlMs)
|
|
217
|
+
: TWIN_IMAGE_CAPABILITY_NEGATIVE_TTL_MS
|
|
218
|
+
|
|
219
|
+
// Runtime rejection is scoped to the exact source adapter + provider/model.
|
|
220
|
+
// One adapter instance may legitimately serve multiple provider routes, so
|
|
221
|
+
// adapter + model alone is not sufficient isolation. A provider reload or
|
|
222
|
+
// adapter replacement clears stale evidence by identity; the short TTL also
|
|
223
|
+
// self-heals an adapter whose capability changes in place.
|
|
224
|
+
const scopeToken = Object.freeze({})
|
|
225
|
+
const rejectedByAdapter = new WeakMap() // source adapter -> Map(provider\0model -> expiresAt)
|
|
226
|
+
const memoFor = (adapter) => {
|
|
227
|
+
if (!isObject(adapter)) return undefined
|
|
228
|
+
let memo = rejectedByAdapter.get(adapter)
|
|
229
|
+
if (memo === undefined) {
|
|
230
|
+
memo = new Map()
|
|
231
|
+
rejectedByAdapter.set(adapter, memo)
|
|
232
|
+
}
|
|
233
|
+
return memo
|
|
234
|
+
}
|
|
235
|
+
const rejectionKey = (provider, model) => `${provider}\u0000${model}`
|
|
236
|
+
const rejectionCached = (adapter, provider, model) => {
|
|
237
|
+
if (!isObject(adapter) || !nonEmptyString(provider) || !nonEmptyString(model)) return false
|
|
238
|
+
const memo = rejectedByAdapter.get(adapter)
|
|
239
|
+
if (memo === undefined) return false
|
|
240
|
+
const now = Date.now()
|
|
241
|
+
for (const [key, expiresAt] of memo) {
|
|
242
|
+
if (expiresAt <= now) memo.delete(key)
|
|
243
|
+
}
|
|
244
|
+
const key = rejectionKey(provider, model)
|
|
245
|
+
const expiresAt = memo.get(key)
|
|
246
|
+
if (expiresAt === undefined) return false
|
|
247
|
+
if (expiresAt <= now) {
|
|
248
|
+
memo.delete(key)
|
|
249
|
+
return false
|
|
250
|
+
}
|
|
251
|
+
// Refresh recency without extending the proof lifetime.
|
|
252
|
+
memo.delete(key)
|
|
253
|
+
memo.set(key, expiresAt)
|
|
254
|
+
return true
|
|
255
|
+
}
|
|
256
|
+
const rememberRejection = (adapter, provider, model) => {
|
|
257
|
+
if (!isObject(adapter) || !nonEmptyString(provider) || !nonEmptyString(model)) return
|
|
258
|
+
const memo = memoFor(adapter)
|
|
259
|
+
if (memo === undefined) return
|
|
260
|
+
const key = rejectionKey(provider, model)
|
|
261
|
+
if (!memo.has(key) && memo.size >= MAX_TWIN_IMAGE_CAPABILITY_NEGATIVE_MODELS) {
|
|
262
|
+
memo.delete(memo.keys().next().value)
|
|
263
|
+
}
|
|
264
|
+
memo.delete(key)
|
|
265
|
+
memo.set(key, Date.now() + ttlMs)
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
const sourceRegistration = (sourceProvider) => {
|
|
269
|
+
try {
|
|
270
|
+
return llm.registration(sourceProvider)
|
|
271
|
+
} catch {
|
|
272
|
+
return undefined
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
const wrapTwinAdapter = (adapter, sourceProvider) => {
|
|
277
|
+
if (!adapter || !isObject(adapter) || typeof adapter.stream !== 'function') return adapter
|
|
278
|
+
const stream = adapter.stream
|
|
279
|
+
return facadeFor(adapter, {
|
|
280
|
+
async *stream(options = {}) {
|
|
281
|
+
const messages = options.messages ?? []
|
|
282
|
+
if (!twinRequestHasImage(messages)) {
|
|
283
|
+
yield* stream.call(adapter, options)
|
|
284
|
+
return
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
const model = nonEmptyString(options.model) ? options.model : ''
|
|
288
|
+
const sourceAdapter = sourceRegistration(sourceProvider)?.adapter
|
|
289
|
+
const forceBridge = async function* () {
|
|
290
|
+
const store = { token: scopeToken, provider: sourceProvider, model }
|
|
291
|
+
const scopedIterator = stream.call(adapter, options)[Symbol.asyncIterator]()
|
|
292
|
+
let scopedFinished = false
|
|
293
|
+
try {
|
|
294
|
+
while (true) {
|
|
295
|
+
const step = await forcedTextBridgeScope.run(store, () => scopedIterator.next())
|
|
296
|
+
if (step.done) {
|
|
297
|
+
scopedFinished = true
|
|
298
|
+
return step.value
|
|
299
|
+
}
|
|
300
|
+
yield step.value
|
|
301
|
+
}
|
|
302
|
+
} finally {
|
|
303
|
+
if (!scopedFinished && typeof scopedIterator.return === 'function') {
|
|
304
|
+
await forcedTextBridgeScope.run(store, () => scopedIterator.return())
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
// Once this exact provider/model on this concrete source adapter has
|
|
310
|
+
// disproved its metadata, skip the known-bad raw-image attempt for the
|
|
311
|
+
// short proof lifetime without poisoning sibling routes on that adapter.
|
|
312
|
+
if (rejectionCached(sourceAdapter, sourceProvider, model)) {
|
|
313
|
+
yield* forceBridge()
|
|
314
|
+
return
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
const iterator = stream.call(adapter, options)[Symbol.asyncIterator]()
|
|
318
|
+
const bufferedUsage = []
|
|
319
|
+
let committed = false
|
|
320
|
+
let finished = false
|
|
321
|
+
try {
|
|
322
|
+
while (true) {
|
|
323
|
+
let step
|
|
324
|
+
try {
|
|
325
|
+
step = await iterator.next()
|
|
326
|
+
} catch (error) {
|
|
327
|
+
if (
|
|
328
|
+
!committed &&
|
|
329
|
+
!(options.signal && options.signal.aborted) &&
|
|
330
|
+
isImageInputUnsupportedFailure(error)
|
|
331
|
+
) {
|
|
332
|
+
await closeIterator(iterator)
|
|
333
|
+
finished = true
|
|
334
|
+
rememberRejection(sourceAdapter, sourceProvider, model)
|
|
335
|
+
logger?.warn?.(
|
|
336
|
+
'vision-router: source %s/%s rejected raw image input despite image-capable metadata; retrying once through the canonical text bridge',
|
|
337
|
+
sourceProvider,
|
|
338
|
+
model,
|
|
339
|
+
)
|
|
340
|
+
yield* forceBridge()
|
|
341
|
+
return
|
|
342
|
+
}
|
|
343
|
+
throw error
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
if (step.done) {
|
|
347
|
+
finished = true
|
|
348
|
+
for (const usage of bufferedUsage) yield usage
|
|
349
|
+
return step.value
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
const chunk = step.value
|
|
353
|
+
const failure = chunkFailure(chunk)
|
|
354
|
+
if (
|
|
355
|
+
!committed &&
|
|
356
|
+
failure !== undefined &&
|
|
357
|
+
!(options.signal && options.signal.aborted) &&
|
|
358
|
+
isImageInputUnsupportedFailure(failure)
|
|
359
|
+
) {
|
|
360
|
+
await closeIterator(iterator)
|
|
361
|
+
finished = true
|
|
362
|
+
rememberRejection(sourceAdapter, sourceProvider, model)
|
|
363
|
+
logger?.warn?.(
|
|
364
|
+
'vision-router: source %s/%s rejected raw image input despite image-capable metadata; retrying once through the canonical text bridge',
|
|
365
|
+
sourceProvider,
|
|
366
|
+
model,
|
|
367
|
+
)
|
|
368
|
+
yield* forceBridge()
|
|
369
|
+
return
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
// Usage before any model-visible block is provisional. Do not
|
|
373
|
+
// publish accounting from a failed pre-validation attempt; flush
|
|
374
|
+
// it unchanged if that attempt actually commits.
|
|
375
|
+
if (!committed && chunk && chunk.type === 'usage') {
|
|
376
|
+
bufferedUsage.push(chunk)
|
|
377
|
+
continue
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
if (!committed) {
|
|
381
|
+
committed = true
|
|
382
|
+
for (const usage of bufferedUsage) yield usage
|
|
383
|
+
bufferedUsage.length = 0
|
|
384
|
+
}
|
|
385
|
+
yield chunk
|
|
386
|
+
}
|
|
387
|
+
} finally {
|
|
388
|
+
if (!finished) await closeIterator(iterator)
|
|
389
|
+
}
|
|
390
|
+
},
|
|
391
|
+
})
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
const llmProxy = facadeFor(llm, {
|
|
395
|
+
registration(provider, ...rest) {
|
|
396
|
+
const hit = llm.registration(provider, ...rest)
|
|
397
|
+
const scope = forcedTextBridgeScope.getStore()
|
|
398
|
+
if (!scope || scope.token !== scopeToken || provider !== scope.provider) return hit
|
|
399
|
+
return proxiedRegistrationWithoutImage(hit, scope.provider, scope.model)
|
|
400
|
+
},
|
|
401
|
+
registerAdapter(providers, adapter, ...rest) {
|
|
402
|
+
const route = Array.isArray(providers) && providers.length === 1 ? providers[0] : undefined
|
|
403
|
+
const looksLikeGeneratedTwin =
|
|
404
|
+
nonEmptyString(route) &&
|
|
405
|
+
!currentOwnedRoutes().has(route) &&
|
|
406
|
+
route.endsWith(TWIN_SUFFIX) &&
|
|
407
|
+
route.length > TWIN_SUFFIX.length
|
|
408
|
+
if (!looksLikeGeneratedTwin) return llm.registerAdapter(providers, adapter, ...rest)
|
|
409
|
+
const sourceProvider = route.slice(0, -TWIN_SUFFIX.length)
|
|
410
|
+
// This context is private to Core.apply, and Core creates its generated
|
|
411
|
+
// twins as single `<source>-vision` registrations. Do not require the
|
|
412
|
+
// source route to exist *yet*: explicit wrappedProviders are allowed to
|
|
413
|
+
// register before a settings-backed source adapter appears, then delegate
|
|
414
|
+
// lazily once that source is mounted. Main/custom wrapper and chain routes
|
|
415
|
+
// are excluded above, so preserving this lazy lifecycle does not broaden
|
|
416
|
+
// the boundary to other Core-owned registrations.
|
|
417
|
+
return llm.registerAdapter(providers, wrapTwinAdapter(adapter, sourceProvider), ...rest)
|
|
418
|
+
},
|
|
419
|
+
})
|
|
420
|
+
|
|
421
|
+
return new Proxy(ctx, {
|
|
422
|
+
get(target, property) {
|
|
423
|
+
if (property === 'llm') return llmProxy
|
|
424
|
+
const value = Reflect.get(target, property, target)
|
|
425
|
+
return typeof value === 'function' ? value.bind(target) : value
|
|
426
|
+
},
|
|
427
|
+
})
|
|
428
|
+
}
|
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
const SHA256_HANDLE_PREFIX = /^sha256:/i
|
|
2
|
+
const CANONICAL_SHA256_ID = /^sha256:[0-9a-f]{64}$/i
|
|
3
|
+
const PROJECTED_SHA256_HANDLE = /^sha256:[0-9a-f]{8,63}$/i
|
|
4
|
+
|
|
5
|
+
const TOOL_FIELDS = Object.freeze({
|
|
6
|
+
vision_describe: { arrays: ['attachmentIds', 'paths'] },
|
|
7
|
+
vision_bootstrap: { arrays: ['attachmentIds', 'paths'] },
|
|
8
|
+
vision_materialize: { scalars: ['image'] },
|
|
9
|
+
vision_ground: { scalars: ['image'] },
|
|
10
|
+
vision_detect: { scalars: ['image'] },
|
|
11
|
+
vision_crop: { scalars: ['image'] },
|
|
12
|
+
vision_present: { scalars: ['image'] },
|
|
13
|
+
vision_pixel_diff: { scalars: ['original', 'rebuilt'] },
|
|
14
|
+
vision_colors: { scalars: ['image'] },
|
|
15
|
+
vision_ocr: { scalars: ['image'] },
|
|
16
|
+
vision_trace: { scalars: ['image'] },
|
|
17
|
+
vision_extract_foreground: { scalars: ['image'] },
|
|
18
|
+
})
|
|
19
|
+
|
|
20
|
+
function isObject(value) {
|
|
21
|
+
return value !== null && typeof value === 'object'
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
function attachmentIdOf(ref) {
|
|
25
|
+
if (!ref || typeof ref !== 'object') return undefined
|
|
26
|
+
const value = ref.attachmentId ?? ref.id
|
|
27
|
+
if (value === undefined || value === null) return undefined
|
|
28
|
+
const id = String(value).trim()
|
|
29
|
+
return id === '' ? undefined : id
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
function sessionEvents(session) {
|
|
33
|
+
try {
|
|
34
|
+
if (session && typeof session.snapshotEvents === 'function') {
|
|
35
|
+
const events = session.snapshotEvents()
|
|
36
|
+
return Array.isArray(events) ? events : []
|
|
37
|
+
}
|
|
38
|
+
return Array.isArray(session?.events) ? session.events : []
|
|
39
|
+
} catch {
|
|
40
|
+
return []
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
function collectImageRefsFromBlocks(blocks, out) {
|
|
45
|
+
if (!Array.isArray(blocks)) return
|
|
46
|
+
for (const block of blocks) {
|
|
47
|
+
if (!block || typeof block !== 'object') continue
|
|
48
|
+
if (block.type === 'image' && block.attachment) out.push(block.attachment)
|
|
49
|
+
if (Array.isArray(block.content)) collectImageRefsFromBlocks(block.content, out)
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
function collectImageRefsFromMessages(messages, out) {
|
|
54
|
+
if (!Array.isArray(messages)) return
|
|
55
|
+
for (const message of messages) collectImageRefsFromBlocks(message?.content, out)
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
function authorizedRefs(core, agent) {
|
|
59
|
+
const refs = []
|
|
60
|
+
const session = agent?.session
|
|
61
|
+
const events = sessionEvents(session)
|
|
62
|
+
if (typeof core?.collectEventAttachmentRefs === 'function') {
|
|
63
|
+
try {
|
|
64
|
+
const durable = core.collectEventAttachmentRefs(events)
|
|
65
|
+
if (Array.isArray(durable)) refs.push(...durable)
|
|
66
|
+
} catch {
|
|
67
|
+
// The resolver remains fail-closed; pending refs below may still suffice.
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
collectImageRefsFromMessages(agent?.inbox?.nextTurn, refs)
|
|
71
|
+
collectImageRefsFromMessages(agent?.inbox?.nextStep, refs)
|
|
72
|
+
|
|
73
|
+
const unique = new Map()
|
|
74
|
+
for (const ref of refs) {
|
|
75
|
+
const id = attachmentIdOf(ref)
|
|
76
|
+
if (id !== undefined && !unique.has(id)) unique.set(id, ref)
|
|
77
|
+
}
|
|
78
|
+
return [...unique.values()]
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
export function isProjectedAttachmentHandle(value) {
|
|
82
|
+
return typeof value === 'string' && PROJECTED_SHA256_HANDLE.test(value.trim())
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
function isSha256HandleLike(value) {
|
|
86
|
+
return typeof value === 'string' && SHA256_HANDLE_PREFIX.test(value.trim())
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
function isCanonicalSha256Id(value) {
|
|
90
|
+
return typeof value === 'string' && CANONICAL_SHA256_ID.test(value.trim())
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* Resolve DSH's model-facing text-only image alias back to one canonical
|
|
95
|
+
* attachment ref without changing durable attachment identity semantics.
|
|
96
|
+
*
|
|
97
|
+
* DSH currently projects `sha256:<full digest>` to a short `sha256:<prefix>`
|
|
98
|
+
* for text-only requests. The alias is not globally addressable: it is valid
|
|
99
|
+
* only when exactly one image authorized by the current Session has that
|
|
100
|
+
* prefix. Zero or multiple matches fail closed.
|
|
101
|
+
*/
|
|
102
|
+
export function resolveProjectedAttachmentHandle(handle, { core, sessionVisionIndex, agent } = {}) {
|
|
103
|
+
const token = typeof handle === 'string' ? handle.trim() : ''
|
|
104
|
+
if (!isProjectedAttachmentHandle(token)) return { kind: 'not-projected', value: handle }
|
|
105
|
+
|
|
106
|
+
const session = agent?.session
|
|
107
|
+
const refs = authorizedRefs(core, agent)
|
|
108
|
+
const wanted = token.toLowerCase()
|
|
109
|
+
const matches = refs.filter((ref) => {
|
|
110
|
+
const id = attachmentIdOf(ref)
|
|
111
|
+
return id !== undefined && id.toLowerCase().startsWith(wanted)
|
|
112
|
+
})
|
|
113
|
+
|
|
114
|
+
if (matches.length === 0) return { kind: 'unknown', handle: token }
|
|
115
|
+
if (matches.length > 1) return { kind: 'ambiguous', handle: token }
|
|
116
|
+
|
|
117
|
+
const candidate = matches[0]
|
|
118
|
+
const canonicalId = attachmentIdOf(candidate)
|
|
119
|
+
if (canonicalId === undefined) return { kind: 'unknown', handle: token }
|
|
120
|
+
|
|
121
|
+
// The Session event/inbox proves authorization; warm the canonical bounded
|
|
122
|
+
// index with that exact Host-owned ref, then require the index to hand it
|
|
123
|
+
// back. This keeps all downstream tools on the same recovery/identity seam.
|
|
124
|
+
try {
|
|
125
|
+
sessionVisionIndex?.recordAttachments?.(session, [candidate])
|
|
126
|
+
} catch {
|
|
127
|
+
// A durable event can still be recovered by lookupAttachment below.
|
|
128
|
+
}
|
|
129
|
+
if (!sessionVisionIndex || typeof sessionVisionIndex.lookupAttachment !== 'function') {
|
|
130
|
+
return { kind: 'unknown', handle: token }
|
|
131
|
+
}
|
|
132
|
+
const ref = sessionVisionIndex.lookupAttachment(session, canonicalId)
|
|
133
|
+
if (ref === undefined) return { kind: 'unknown', handle: token }
|
|
134
|
+
return { kind: 'resolved', handle: token, canonicalId, ref }
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
function canonicalizeValue(value, context) {
|
|
138
|
+
if (!isSha256HandleLike(value)) return value
|
|
139
|
+
if (isCanonicalSha256Id(value)) return value
|
|
140
|
+
if (!isProjectedAttachmentHandle(value)) {
|
|
141
|
+
throw new Error(
|
|
142
|
+
`${context.toolName}: invalid attachment handle "${String(value).trim()}" ` +
|
|
143
|
+
'(sha256 attachment handles must be a canonical id or a DSH-projected 8+ hex prefix)',
|
|
144
|
+
)
|
|
145
|
+
}
|
|
146
|
+
const result = resolveProjectedAttachmentHandle(value, context)
|
|
147
|
+
if (result.kind === 'resolved') return result.canonicalId
|
|
148
|
+
if (result.kind === 'ambiguous') {
|
|
149
|
+
throw new Error(
|
|
150
|
+
`${context.toolName}: ambiguous attachment handle "${result.handle}" ` +
|
|
151
|
+
'(multiple images in this conversation share that prefix; use a full attachment id or attach the image again)',
|
|
152
|
+
)
|
|
153
|
+
}
|
|
154
|
+
throw new Error(
|
|
155
|
+
`${context.toolName}: unknown attachment handle "${result.handle}" ` +
|
|
156
|
+
'(it is not authorized by an image in this conversation; attach the image again if needed)',
|
|
157
|
+
)
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
function canonicalizeArgs(args, context, fields) {
|
|
161
|
+
if (!isObject(args) || Array.isArray(args)) return args
|
|
162
|
+
let next
|
|
163
|
+
for (const field of fields.scalars ?? []) {
|
|
164
|
+
if (!Object.hasOwn(args, field)) continue
|
|
165
|
+
const value = canonicalizeValue(args[field], context)
|
|
166
|
+
if (value !== args[field]) {
|
|
167
|
+
next ??= { ...args }
|
|
168
|
+
next[field] = value
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
for (const field of fields.arrays ?? []) {
|
|
172
|
+
const values = args[field]
|
|
173
|
+
if (!Array.isArray(values)) continue
|
|
174
|
+
let changed = false
|
|
175
|
+
const mapped = values.map((value) => {
|
|
176
|
+
const canonical = canonicalizeValue(value, context)
|
|
177
|
+
if (canonical !== value) changed = true
|
|
178
|
+
return canonical
|
|
179
|
+
})
|
|
180
|
+
if (changed) {
|
|
181
|
+
next ??= { ...args }
|
|
182
|
+
next[field] = mapped
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
return next ?? args
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
/**
|
|
189
|
+
* Canonicalize only image-source arguments of Vision Router tools. Ordinary
|
|
190
|
+
* prose fields are never scanned, so a user mentioning `sha256:deadbeef` in a
|
|
191
|
+
* question cannot be rewritten accidentally. Local paths remain untouched.
|
|
192
|
+
* Any sha256-shaped source is reserved for attachment identity and therefore
|
|
193
|
+
* fails closed rather than falling through to cwd-relative filesystem lookup.
|
|
194
|
+
*/
|
|
195
|
+
export function wrapVisionAttachmentHandleDefinition(def, options = {}) {
|
|
196
|
+
if (!def || typeof def !== 'object' || typeof def.execute !== 'function') return def
|
|
197
|
+
const fields = TOOL_FIELDS[def.name]
|
|
198
|
+
if (!fields || !options.sessionVisionIndex) return def
|
|
199
|
+
const execute = def.execute
|
|
200
|
+
return {
|
|
201
|
+
...def,
|
|
202
|
+
execute(args, exec) {
|
|
203
|
+
const nextArgs = canonicalizeArgs(args, {
|
|
204
|
+
core: options.core,
|
|
205
|
+
sessionVisionIndex: options.sessionVisionIndex,
|
|
206
|
+
agent: exec?.agent,
|
|
207
|
+
toolName: def.name,
|
|
208
|
+
}, fields)
|
|
209
|
+
return execute.call(def, nextArgs, exec)
|
|
210
|
+
},
|
|
211
|
+
}
|
|
212
|
+
}
|