dsh-vision-router 2.1.7 → 2.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -14
- package/README.zh.md +8 -14
- package/docs/architecture/compat-inventory.md +1 -1
- package/docs/architecture/dsh-compatibility-matrix.md +4 -1
- package/docs/architecture/dsh-support-window.md +1 -2
- package/docs/releases/v2.2.0.md +25 -0
- package/docs/releases/v2.2.1.md +22 -0
- package/docs/remote-settings.md +2 -0
- package/entry.js +2 -0
- package/index.js +355 -64
- package/lib/artifact-boundary.js +11 -0
- package/lib/artifact-io.js +60 -7
- package/lib/catalog-corrections.js +5 -0
- package/lib/client.js +22 -17
- package/lib/core-primitives.js +126 -5
- package/lib/degraded-local-evidence.js +39 -0
- package/lib/dsh-contract-compat.js +225 -0
- package/lib/dsh-support-window.js +1 -1
- package/lib/fetch-wrapper-lifecycle.js +56 -0
- package/lib/image-offload-compat.js +38 -0
- package/lib/legacy-global-proxy-boundary.js +4 -3
- package/lib/local-vision-stabilizer.js +15 -6
- package/lib/official-deepseek-catalog.js +120 -0
- package/lib/ollama-cold-start.js +3 -15
- package/lib/pi-ai-bridge-wire-compat.js +4 -3
- package/lib/pixel-diff-stream.js +12 -6
- package/lib/remote-settings-bridge.js +19 -1
- package/lib/replay-delegation.js +41 -21
- package/lib/runtime-composition.js +15 -2
- package/lib/runtime-i18n-boundary.js +4 -13
- package/lib/runtime-i18n.js +2 -2
- package/lib/session-surface-compat.js +40 -0
- package/lib/session-turn-resolver.js +57 -0
- package/lib/session-vision-index.js +438 -97
- package/lib/session-vision-mode-boundary.js +83 -6
- package/lib/session-vision-runtime.js +8 -2
- package/lib/session-vision-state.js +0 -13
- package/lib/tesseract-exec-compat.js +16 -2
- package/lib/twin-image-capability-fallback.js +2 -11
- package/lib/vision-artifact-store.js +16 -3
- package/lib/vision-attachment-handle-runtime.js +9 -21
- package/lib/vision-backend-runtime-policy.js +7 -6
- package/lib/vision-breaker-shadow-health.js +26 -6
- package/lib/vision-evidence-guidance.js +39 -0
- package/lib/vision-resilience.js +73 -1
- package/package.json +12 -7
|
@@ -9,6 +9,7 @@ import {
|
|
|
9
9
|
} from './session-vision-mode-authority.js'
|
|
10
10
|
|
|
11
11
|
const MODE_SYNC_VARIABLE = 'vision_router_mode_sync'
|
|
12
|
+
const VISION_ROUTER_TOOL_OWNER = Symbol.for('dsh-vision-router.tool-owner')
|
|
12
13
|
|
|
13
14
|
function isObject(value) {
|
|
14
15
|
return value !== null && typeof value === 'object'
|
|
@@ -59,6 +60,58 @@ function ownedVisionToolNames(owned) {
|
|
|
59
60
|
return [...owned.keys()].sort()
|
|
60
61
|
}
|
|
61
62
|
|
|
63
|
+
function restrictableOwnedVisionToolNames(tools, owned) {
|
|
64
|
+
const names = ownedVisionToolNames(owned)
|
|
65
|
+
const get = tools?.get
|
|
66
|
+
if (typeof get !== 'function') return names
|
|
67
|
+
|
|
68
|
+
const active = []
|
|
69
|
+
for (const name of names) {
|
|
70
|
+
try {
|
|
71
|
+
// DSH restrict() accepts only names that exist in the current global
|
|
72
|
+
// registry. Logical DVR ownership is broader: an intermediate boundary
|
|
73
|
+
// may capture a conditional definition without mounting it yet
|
|
74
|
+
// (vision_screenshot is the canonical example).
|
|
75
|
+
const definition = get.call(tools, name)
|
|
76
|
+
if (definition?.[VISION_ROUTER_TOOL_OWNER] === true) active.push(name)
|
|
77
|
+
} catch {
|
|
78
|
+
// Older/partial Hosts may not offer a safely readable registry view.
|
|
79
|
+
// Preserve the historical fail-closed path there; per-name fallback
|
|
80
|
+
// below still prevents one bad entry from exposing every other tool.
|
|
81
|
+
return names
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
return active
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
function assemblyOwnedVisionToolNames(registryTools, scopedTools, owned) {
|
|
88
|
+
const active = restrictableOwnedVisionToolNames(registryTools, owned)
|
|
89
|
+
const scopedGet = scopedTools?.get
|
|
90
|
+
if (typeof scopedGet !== 'function') return new Set(active)
|
|
91
|
+
|
|
92
|
+
const projected = new Set()
|
|
93
|
+
for (const name of active) {
|
|
94
|
+
try {
|
|
95
|
+
const visible = scopedGet.call(scopedTools, name)
|
|
96
|
+
// A foreign Agent-scoped definition is allowed to shadow DVR's global
|
|
97
|
+
// name. DSH restrictions deliberately leave scoped registrations visible,
|
|
98
|
+
// so the final assembly projection must preserve the same boundary.
|
|
99
|
+
if (
|
|
100
|
+
visible === undefined
|
|
101
|
+
|| visible?.[VISION_ROUTER_TOOL_OWNER] === true
|
|
102
|
+
) {
|
|
103
|
+
projected.add(name)
|
|
104
|
+
}
|
|
105
|
+
} catch {
|
|
106
|
+
// If the scoped view cannot be read, keep the conservative global DVR
|
|
107
|
+
// projection. This support-window fallback hides only a name whose global
|
|
108
|
+
// registration is proven DVR-owned.
|
|
109
|
+
projected.add(name)
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
return projected
|
|
113
|
+
}
|
|
114
|
+
|
|
62
115
|
function rememberOwnedTool(owned, name) {
|
|
63
116
|
owned.set(name, (owned.get(name) ?? 0) + 1)
|
|
64
117
|
}
|
|
@@ -89,18 +142,30 @@ function releaseAgentToolRestriction(agent, restrictions) {
|
|
|
89
142
|
restrictions.delete(agent)
|
|
90
143
|
}
|
|
91
144
|
|
|
145
|
+
function restrictionFailureLabel(error) {
|
|
146
|
+
const name =
|
|
147
|
+
typeof error?.name === 'string' && error.name !== ''
|
|
148
|
+
? error.name
|
|
149
|
+
: 'Error'
|
|
150
|
+
const code =
|
|
151
|
+
typeof error?.code === 'string' && error.code !== ''
|
|
152
|
+
? `/${error.code}`
|
|
153
|
+
: ''
|
|
154
|
+
return `${name}${code}`
|
|
155
|
+
}
|
|
156
|
+
|
|
92
157
|
function warnRestrictionFailure(agent, error, deny, failed = deny) {
|
|
93
158
|
try {
|
|
94
159
|
agent?.ctx?.logger?.warn?.(
|
|
95
|
-
'vision-router: failed to restrict Session vision tools while Vision mode is off: %s (
|
|
96
|
-
|
|
97
|
-
deny.
|
|
160
|
+
'vision-router: failed to restrict Session vision tools while Vision mode is off: %s (attempted=%s failed=%s)',
|
|
161
|
+
restrictionFailureLabel(error),
|
|
162
|
+
String(deny.length),
|
|
98
163
|
failed.join(','),
|
|
99
164
|
)
|
|
100
165
|
} catch {}
|
|
101
166
|
}
|
|
102
167
|
|
|
103
|
-
function syncAgentToolRestriction(agent, enabled, restrictions, owned) {
|
|
168
|
+
function syncAgentToolRestriction(agent, enabled, restrictions, owned, registryTools) {
|
|
104
169
|
if (!isObject(agent)) return
|
|
105
170
|
if (enabled) {
|
|
106
171
|
releaseAgentToolRestriction(agent, restrictions)
|
|
@@ -109,7 +174,11 @@ function syncAgentToolRestriction(agent, enabled, restrictions, owned) {
|
|
|
109
174
|
|
|
110
175
|
const tools = agent?.ctx?.tools
|
|
111
176
|
const restrict = tools?.restrict
|
|
112
|
-
|
|
177
|
+
// Registration truth must come from the unscoped/global registry view.
|
|
178
|
+
// agent.ctx.tools.get() is restriction-aware: after this boundary denies a
|
|
179
|
+
// DVR tool it reads as absent for that Agent, so using it here would make a
|
|
180
|
+
// later sync tear down its own restriction and re-expose the capability.
|
|
181
|
+
const deny = restrictableOwnedVisionToolNames(registryTools, owned)
|
|
113
182
|
if (typeof restrict !== 'function') return
|
|
114
183
|
if (deny.length === 0) {
|
|
115
184
|
releaseAgentToolRestriction(agent, restrictions)
|
|
@@ -185,6 +254,7 @@ function wrapTools(tools, ctx, config, owned) {
|
|
|
185
254
|
const execute = definition.execute
|
|
186
255
|
const registered = register.call(target, {
|
|
187
256
|
...definition,
|
|
257
|
+
[VISION_ROUTER_TOOL_OWNER]: true,
|
|
188
258
|
async execute(args, exec) {
|
|
189
259
|
if (
|
|
190
260
|
exec?.agent &&
|
|
@@ -264,6 +334,7 @@ export function installSessionVisionModeBoundary(ctx, config = {}) {
|
|
|
264
334
|
visionToolSurfaceEnabled(ctx, config, authority),
|
|
265
335
|
restrictions,
|
|
266
336
|
ownedVisionTools,
|
|
337
|
+
ctx?.tools,
|
|
267
338
|
)
|
|
268
339
|
}
|
|
269
340
|
|
|
@@ -282,6 +353,7 @@ export function installSessionVisionModeBoundary(ctx, config = {}) {
|
|
|
282
353
|
visionToolSurfaceEnabled(ctx, config, authority),
|
|
283
354
|
restrictions,
|
|
284
355
|
ownedVisionTools,
|
|
356
|
+
ctx?.tools,
|
|
285
357
|
)
|
|
286
358
|
}
|
|
287
359
|
return authority
|
|
@@ -314,10 +386,14 @@ export function installSessionVisionModeBoundary(ctx, config = {}) {
|
|
|
314
386
|
if (!authority && isObject(context)) authority = captureForAssembly(context)
|
|
315
387
|
if (!authority && isObject(agent)) authority = resolveForAgent(agent)
|
|
316
388
|
const transformed = typeof next === 'function' ? await next() : assembly
|
|
389
|
+
const projectedOwned =
|
|
390
|
+
ctx?.tools
|
|
391
|
+
? assemblyOwnedVisionToolNames(ctx.tools, agent?.ctx?.tools, ownedVisionTools)
|
|
392
|
+
: ownedVisionTools
|
|
317
393
|
return projectAssemblyTools(
|
|
318
394
|
transformed,
|
|
319
395
|
visionToolSurfaceEnabled(ctx, config, authority),
|
|
320
|
-
|
|
396
|
+
projectedOwned,
|
|
321
397
|
)
|
|
322
398
|
})
|
|
323
399
|
} catch {
|
|
@@ -390,6 +466,7 @@ export function installSessionVisionModeBoundary(ctx, config = {}) {
|
|
|
390
466
|
visionToolSurfaceEnabled(target, config, authority),
|
|
391
467
|
restrictions,
|
|
392
468
|
ownedVisionTools,
|
|
469
|
+
target?.tools,
|
|
393
470
|
)
|
|
394
471
|
return runWithSessionVisionModeAuthority(
|
|
395
472
|
authority,
|
|
@@ -16,13 +16,17 @@ export const DEFAULT_SESSION_VISION_STATE_OPTIONS = Object.freeze({
|
|
|
16
16
|
* It is not a general runtime service locator. Production composition creates
|
|
17
17
|
* one instance and gives the same owner to the Session boundary and Core.
|
|
18
18
|
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
19
|
+
* Cache lookup stays synchronous through index.lookupAttachment(); cold durable
|
|
20
|
+
* recovery is explicit through index.resolveAttachment(). The index never
|
|
21
|
+
* monkey-patches stateStore.lookupAttachment(), and receives only narrow Host
|
|
22
|
+
* Session readers rather than the full Context.
|
|
21
23
|
*/
|
|
22
24
|
export function createSessionVisionRuntime({
|
|
23
25
|
core,
|
|
24
26
|
config = {},
|
|
25
27
|
logger,
|
|
28
|
+
readSessionEvent,
|
|
29
|
+
readSessionLog,
|
|
26
30
|
stateStore,
|
|
27
31
|
stateOptions = {},
|
|
28
32
|
} = {}) {
|
|
@@ -39,6 +43,8 @@ export function createSessionVisionRuntime({
|
|
|
39
43
|
core,
|
|
40
44
|
config,
|
|
41
45
|
logger,
|
|
46
|
+
readSessionEvent,
|
|
47
|
+
readSessionLog,
|
|
42
48
|
})
|
|
43
49
|
|
|
44
50
|
return Object.freeze({
|
|
@@ -123,7 +123,6 @@ function createState(key, stable, options, now) {
|
|
|
123
123
|
attachments: new WeightedLruMap({
|
|
124
124
|
maxEntries: options.attachmentMaxEntries,
|
|
125
125
|
}),
|
|
126
|
-
scannedEventSeq: 0,
|
|
127
126
|
}
|
|
128
127
|
}
|
|
129
128
|
|
|
@@ -356,17 +355,6 @@ export function createSessionVisionStateStore(config = {}) {
|
|
|
356
355
|
return stateFor(session, false)?.attachments.get(id)
|
|
357
356
|
},
|
|
358
357
|
|
|
359
|
-
getScannedEventSeq(session) {
|
|
360
|
-
return stateFor(session, false)?.scannedEventSeq ?? 0
|
|
361
|
-
},
|
|
362
|
-
|
|
363
|
-
setScannedEventSeq(session, seq) {
|
|
364
|
-
const state = stateFor(session, true)
|
|
365
|
-
if (state === undefined) return
|
|
366
|
-
const next = Number.isFinite(Number(seq)) && Number(seq) >= 0 ? Math.floor(Number(seq)) : 0
|
|
367
|
-
state.scannedEventSeq = next
|
|
368
|
-
touchState(state)
|
|
369
|
-
},
|
|
370
358
|
|
|
371
359
|
forgetSession(sessionOrId) {
|
|
372
360
|
const id =
|
|
@@ -389,7 +377,6 @@ export function createSessionVisionStateStore(config = {}) {
|
|
|
389
377
|
descriptions: state.descriptions.size,
|
|
390
378
|
descriptionChars: state.descriptions.weight,
|
|
391
379
|
attachments: state.attachments.size,
|
|
392
|
-
scannedEventSeq: state.scannedEventSeq,
|
|
393
380
|
}
|
|
394
381
|
},
|
|
395
382
|
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { mkdtemp, rm, writeFile } from 'node:fs/promises'
|
|
1
|
+
import { mkdtemp, realpath, rm, writeFile } from 'node:fs/promises'
|
|
2
2
|
import path from 'node:path'
|
|
3
3
|
import { tmpdir } from 'node:os'
|
|
4
4
|
import { createRequire, syncBuiltinESMExports } from 'node:module'
|
|
@@ -80,9 +80,23 @@ export function createTesseractPromisifyCompat(execFileImpl, originalCustom, opt
|
|
|
80
80
|
const makeTempDir = options.mkdtemp ?? mkdtemp
|
|
81
81
|
const writeTempFile = options.writeFile ?? writeFile
|
|
82
82
|
const removeTempDir = options.rm ?? rm
|
|
83
|
+
const resolveRealPath = options.realpath ?? realpath
|
|
83
84
|
const tempDir = options.tempDir ?? tmpdir()
|
|
85
|
+
let resolvedTempDir
|
|
84
86
|
let active = true
|
|
85
87
|
|
|
88
|
+
const stagingRoot = async () => {
|
|
89
|
+
if (resolvedTempDir !== undefined) return resolvedTempDir
|
|
90
|
+
try {
|
|
91
|
+
resolvedTempDir = await resolveRealPath(tempDir)
|
|
92
|
+
} catch {
|
|
93
|
+
// Some tests/embedders provide a virtual temp root. Preserve the old
|
|
94
|
+
// path when canonicalization is unavailable instead of disabling OCR.
|
|
95
|
+
resolvedTempDir = tempDir
|
|
96
|
+
}
|
|
97
|
+
return resolvedTempDir
|
|
98
|
+
}
|
|
99
|
+
|
|
86
100
|
async function tesseractPromisifyCompat(file, args, execOptions) {
|
|
87
101
|
if (!active || !isTesseractCompatCall(file, args, execOptions)) {
|
|
88
102
|
return delegate(file, args, execOptions)
|
|
@@ -91,7 +105,7 @@ export function createTesseractPromisifyCompat(execFileImpl, originalCustom, opt
|
|
|
91
105
|
const bytes = bytesView(execOptions.input)
|
|
92
106
|
let dir
|
|
93
107
|
try {
|
|
94
|
-
dir = await makeTempDir(path.join(
|
|
108
|
+
dir = await makeTempDir(path.join(await stagingRoot(), 'dsh-vision-router-ocr-'))
|
|
95
109
|
const inputPath = path.join(dir, `input${extensionForBytes(bytes)}`)
|
|
96
110
|
await writeTempFile(inputPath, bytes)
|
|
97
111
|
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { AsyncLocalStorage } from 'node:async_hooks'
|
|
2
|
+
import { blocksHaveRetainedImage } from './image-offload-compat.js'
|
|
2
3
|
|
|
3
4
|
const TWIN_SUFFIX = '-vision'
|
|
4
5
|
const DEFAULT_MAIN_WRAPPER_ROUTE = 'deepseek-vision'
|
|
@@ -16,19 +17,9 @@ function nonEmptyString(value) {
|
|
|
16
17
|
return typeof value === 'string' && value.length > 0
|
|
17
18
|
}
|
|
18
19
|
|
|
19
|
-
function blocksHaveImage(content) {
|
|
20
|
-
if (!Array.isArray(content)) return false
|
|
21
|
-
for (const block of content) {
|
|
22
|
-
if (!block) continue
|
|
23
|
-
if (block.type === 'image') return true
|
|
24
|
-
if (Array.isArray(block.content) && blocksHaveImage(block.content)) return true
|
|
25
|
-
}
|
|
26
|
-
return false
|
|
27
|
-
}
|
|
28
|
-
|
|
29
20
|
export function twinRequestHasImage(messages) {
|
|
30
21
|
return (messages ?? []).some(
|
|
31
|
-
(message) => message && Array.isArray(message.content) &&
|
|
22
|
+
(message) => message && Array.isArray(message.content) && blocksHaveRetainedImage(message.content),
|
|
32
23
|
)
|
|
33
24
|
}
|
|
34
25
|
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import {
|
|
2
2
|
resolveArtifactTarget,
|
|
3
3
|
writeArtifactFile,
|
|
4
|
+
writePersistentArtifactFile,
|
|
4
5
|
} from './artifact-io.js'
|
|
5
6
|
import {
|
|
6
7
|
retainArtifactRun,
|
|
@@ -46,6 +47,17 @@ export function createVisionArtifactStore({ workspace, artifactsDir, deps = {} }
|
|
|
46
47
|
)
|
|
47
48
|
}
|
|
48
49
|
|
|
50
|
+
const publishPersistent = (relativePath, data, options = {}) => {
|
|
51
|
+
const current = location()
|
|
52
|
+
return writePersistentArtifactFile(
|
|
53
|
+
current.workspace,
|
|
54
|
+
current.artifactsDir,
|
|
55
|
+
relativePath,
|
|
56
|
+
data,
|
|
57
|
+
callDeps(deps, options),
|
|
58
|
+
)
|
|
59
|
+
}
|
|
60
|
+
|
|
49
61
|
return Object.freeze({
|
|
50
62
|
resolve(relativePath, options = {}) {
|
|
51
63
|
const current = location()
|
|
@@ -58,10 +70,11 @@ export function createVisionArtifactStore({ workspace, artifactsDir, deps = {} }
|
|
|
58
70
|
},
|
|
59
71
|
|
|
60
72
|
publish,
|
|
73
|
+
publishPersistent,
|
|
61
74
|
|
|
62
|
-
// Temporary
|
|
63
|
-
//
|
|
64
|
-
//
|
|
75
|
+
// Temporary placement remains authority/lifetime driven by the ambient
|
|
76
|
+
// random managed artifactRunId. publishPersistent is the narrow handoff
|
|
77
|
+
// exception and uses one stable reserved run that retention still owns.
|
|
65
78
|
publishTemporary(relativePath, data, options = {}) {
|
|
66
79
|
return publish(relativePath, data, options)
|
|
67
80
|
},
|
|
@@ -29,13 +29,11 @@ function attachmentIdOf(ref) {
|
|
|
29
29
|
return id === '' ? undefined : id
|
|
30
30
|
}
|
|
31
31
|
|
|
32
|
-
function
|
|
32
|
+
function sessionMessages(session) {
|
|
33
33
|
try {
|
|
34
|
-
if (session
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
}
|
|
38
|
-
return Array.isArray(session?.events) ? session.events : []
|
|
34
|
+
if (!session || typeof session.deriveMessages !== 'function') return []
|
|
35
|
+
const messages = session.deriveMessages()
|
|
36
|
+
return Array.isArray(messages) ? messages : []
|
|
39
37
|
} catch {
|
|
40
38
|
return []
|
|
41
39
|
}
|
|
@@ -55,18 +53,9 @@ function collectImageRefsFromMessages(messages, out) {
|
|
|
55
53
|
for (const message of messages) collectImageRefsFromBlocks(message?.content, out)
|
|
56
54
|
}
|
|
57
55
|
|
|
58
|
-
function authorizedRefs(
|
|
56
|
+
function authorizedRefs(agent) {
|
|
59
57
|
const refs = []
|
|
60
|
-
|
|
61
|
-
const events = sessionEvents(session)
|
|
62
|
-
if (typeof core?.collectEventAttachmentRefs === 'function') {
|
|
63
|
-
try {
|
|
64
|
-
const durable = core.collectEventAttachmentRefs(events)
|
|
65
|
-
if (Array.isArray(durable)) refs.push(...durable)
|
|
66
|
-
} catch {
|
|
67
|
-
// The resolver remains fail-closed; pending refs below may still suffice.
|
|
68
|
-
}
|
|
69
|
-
}
|
|
58
|
+
collectImageRefsFromMessages(sessionMessages(agent?.session), refs)
|
|
70
59
|
collectImageRefsFromMessages(agent?.inbox?.nextTurn, refs)
|
|
71
60
|
collectImageRefsFromMessages(agent?.inbox?.nextStep, refs)
|
|
72
61
|
|
|
@@ -99,12 +88,12 @@ function isCanonicalSha256Id(value) {
|
|
|
99
88
|
* only when exactly one image authorized by the current Session has that
|
|
100
89
|
* prefix. Zero or multiple matches fail closed.
|
|
101
90
|
*/
|
|
102
|
-
export function resolveProjectedAttachmentHandle(handle, {
|
|
91
|
+
export function resolveProjectedAttachmentHandle(handle, { sessionVisionIndex, agent } = {}) {
|
|
103
92
|
const token = typeof handle === 'string' ? handle.trim() : ''
|
|
104
93
|
if (!isProjectedAttachmentHandle(token)) return { kind: 'not-projected', value: handle }
|
|
105
94
|
|
|
106
95
|
const session = agent?.session
|
|
107
|
-
const refs = authorizedRefs(
|
|
96
|
+
const refs = authorizedRefs(agent)
|
|
108
97
|
const wanted = token.toLowerCase()
|
|
109
98
|
const matches = refs.filter((ref) => {
|
|
110
99
|
const id = attachmentIdOf(ref)
|
|
@@ -124,7 +113,7 @@ export function resolveProjectedAttachmentHandle(handle, { core, sessionVisionIn
|
|
|
124
113
|
try {
|
|
125
114
|
sessionVisionIndex?.recordAttachments?.(session, [candidate])
|
|
126
115
|
} catch {
|
|
127
|
-
//
|
|
116
|
+
// Authorization is still fail-closed if the bounded cache cannot be warmed.
|
|
128
117
|
}
|
|
129
118
|
if (!sessionVisionIndex || typeof sessionVisionIndex.lookupAttachment !== 'function') {
|
|
130
119
|
return { kind: 'unknown', handle: token }
|
|
@@ -201,7 +190,6 @@ export function wrapVisionAttachmentHandleDefinition(def, options = {}) {
|
|
|
201
190
|
...def,
|
|
202
191
|
execute(args, exec) {
|
|
203
192
|
const nextArgs = canonicalizeArgs(args, {
|
|
204
|
-
core: options.core,
|
|
205
193
|
sessionVisionIndex: options.sessionVisionIndex,
|
|
206
194
|
agent: exec?.agent,
|
|
207
195
|
toolName: def.name,
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { AsyncLocalStorage } from 'node:async_hooks'
|
|
2
|
+
import { blocksHaveRetainedImage, isOffloadedImageBlock, offloadedImagePlaceholder } from './image-offload-compat.js'
|
|
2
3
|
|
|
3
4
|
const visionBackendScope = new AsyncLocalStorage()
|
|
4
5
|
const wrappedContexts = new WeakMap()
|
|
@@ -127,12 +128,8 @@ async function resolveCredential(ctx, plan) {
|
|
|
127
128
|
return undefined
|
|
128
129
|
}
|
|
129
130
|
|
|
130
|
-
function contentHasImage(
|
|
131
|
-
return (messages ?? []).some((message) =>
|
|
132
|
-
if (!Array.isArray(message?.content)) return false
|
|
133
|
-
if (typeof core?.blocksHaveImage === 'function') return core.blocksHaveImage(message.content)
|
|
134
|
-
return message.content.some((block) => block?.type === 'image')
|
|
135
|
-
})
|
|
131
|
+
function contentHasImage(_core, messages) {
|
|
132
|
+
return (messages ?? []).some((message) => blocksHaveRetainedImage(message?.content))
|
|
136
133
|
}
|
|
137
134
|
|
|
138
135
|
async function toOpenAIMessages(ctx, messages) {
|
|
@@ -151,6 +148,10 @@ async function toOpenAIMessages(ctx, messages) {
|
|
|
151
148
|
continue
|
|
152
149
|
}
|
|
153
150
|
if (block?.type === 'image' && block.attachment) {
|
|
151
|
+
if (isOffloadedImageBlock(block)) {
|
|
152
|
+
converted.push({ type: 'text', text: offloadedImagePlaceholder(block) })
|
|
153
|
+
continue
|
|
154
|
+
}
|
|
154
155
|
const stored = await attachments.readImage(block.attachment)
|
|
155
156
|
const mediaType = block.attachment.mediaType || stored?.mediaType || 'image/png'
|
|
156
157
|
const data = Buffer.from(stored.data).toString('base64')
|
|
@@ -1,8 +1,19 @@
|
|
|
1
1
|
import { createHash } from 'node:crypto'
|
|
2
|
+
import { createSessionTurnResolver } from './session-turn-resolver.js'
|
|
2
3
|
|
|
3
|
-
function
|
|
4
|
+
function legacyTurnNumberOf(session) {
|
|
4
5
|
try {
|
|
5
|
-
|
|
6
|
+
// dsh 0.1.2-alpha.4 removed the bare `session.events` array in favor of
|
|
7
|
+
// `session.snapshotEvents()`; mirror legacySessionEvents
|
|
8
|
+
// (session-vision-index.js), which prefers the snapshot when both surfaces
|
|
9
|
+
// exist — transitional Hosts keep the bare array but leave it stale, so a
|
|
10
|
+
// bare-array-first read would resolve a stale turn on them.
|
|
11
|
+
let events
|
|
12
|
+
if (typeof session?.snapshotEvents === 'function') {
|
|
13
|
+
events = session.snapshotEvents()
|
|
14
|
+
} else {
|
|
15
|
+
events = session?.events
|
|
16
|
+
}
|
|
6
17
|
if (!Array.isArray(events)) return 0
|
|
7
18
|
const last = events.findLast((event) => event && event.type === 'turn/start')
|
|
8
19
|
return last && Number.isInteger(last.data?.turn) ? last.data.turn : 0
|
|
@@ -19,8 +30,15 @@ function sessionIdOf(session) {
|
|
|
19
30
|
}
|
|
20
31
|
}
|
|
21
32
|
|
|
22
|
-
export function visionBreakerScopeOf(session) {
|
|
23
|
-
|
|
33
|
+
export function visionBreakerScopeOf(session, turnOf) {
|
|
34
|
+
let turn
|
|
35
|
+
try {
|
|
36
|
+
if (typeof turnOf === 'function') turn = turnOf(session)
|
|
37
|
+
} catch {
|
|
38
|
+
turn = undefined
|
|
39
|
+
}
|
|
40
|
+
if (!Number.isInteger(turn) || turn < 0) turn = legacyTurnNumberOf(session)
|
|
41
|
+
return `${sessionIdOf(session)}:${turn}`
|
|
24
42
|
}
|
|
25
43
|
|
|
26
44
|
function credentialFingerprintOf(value) {
|
|
@@ -76,7 +94,9 @@ function healthFromGate(gate) {
|
|
|
76
94
|
* giving shadow code mutation authority. `capture()` is called only while
|
|
77
95
|
* core.apply() constructs the breaker; `healthForCandidate()` uses peek().
|
|
78
96
|
*/
|
|
79
|
-
export function createVisionBreakerShadowHealth(ctx) {
|
|
97
|
+
export function createVisionBreakerShadowHealth(ctx, options = {}) {
|
|
98
|
+
const sessionTurnResolver = options.sessionTurnResolver ?? createSessionTurnResolver(ctx)
|
|
99
|
+
const turnOf = sessionTurnResolver.turnOf
|
|
80
100
|
let breaker
|
|
81
101
|
return {
|
|
82
102
|
capture(value) {
|
|
@@ -88,7 +108,7 @@ export function createVisionBreakerShadowHealth(ctx) {
|
|
|
88
108
|
const key = typeof candidate?.key === 'string' ? candidate.key : undefined
|
|
89
109
|
if (!key) return undefined
|
|
90
110
|
const fingerprint = await visionBreakerFingerprintForCandidate(ctx, candidate)
|
|
91
|
-
const scope = visionBreakerScopeOf(context.session)
|
|
111
|
+
const scope = visionBreakerScopeOf(context.session, turnOf)
|
|
92
112
|
return healthFromGate(breaker.peek(key, fingerprint, scope))
|
|
93
113
|
},
|
|
94
114
|
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
export const VISION_SUCCESS_GUIDANCE =
|
|
2
|
+
'Vision Router trusted execution metadata: the preceding vision_describe call SUCCEEDED. ' +
|
|
3
|
+
'This metadata is not image content. If that successful evidence resolves the visual fact needed for the current task, stop making additional vision calls for that fact and continue the remaining user task from this evidence. ' +
|
|
4
|
+
'If no non-visual work remains, answer now in exactly the format the user requested; when the user asks for only one value or short phrase, do not add Markdown, explanation, or a restatement. ' +
|
|
5
|
+
'Call another vision tool only for a specific unresolved visual fact. For verbatim or exact-text tasks such as identifiers, amounts, times, version strings, or code, preserve every visible symbol, prefix, suffix, and punctuation from the most faithful evidence already collected; a later semantic summary must not overwrite a more exact transcription. Focused OCR or localization is still appropriate when character identity remains uncertain.'
|
|
6
|
+
|
|
7
|
+
function isVisionFailureEnvelope(value) {
|
|
8
|
+
const text = value.trim()
|
|
9
|
+
if (!text.startsWith('{')) return false
|
|
10
|
+
try {
|
|
11
|
+
const parsed = JSON.parse(text)
|
|
12
|
+
return (
|
|
13
|
+
parsed !== null &&
|
|
14
|
+
typeof parsed === 'object' &&
|
|
15
|
+
parsed.ok === false &&
|
|
16
|
+
typeof parsed.code === 'string' &&
|
|
17
|
+
/^VISION_[A-Z0-9_]+$/u.test(parsed.code)
|
|
18
|
+
)
|
|
19
|
+
} catch {
|
|
20
|
+
return false
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export function visionDescribeSuccessContext(exec, result) {
|
|
25
|
+
if (
|
|
26
|
+
exec?.name !== 'vision_describe' ||
|
|
27
|
+
!exec?.agent ||
|
|
28
|
+
result?.isError !== false ||
|
|
29
|
+
typeof result.value !== 'string' ||
|
|
30
|
+
result.value.trim().length === 0 ||
|
|
31
|
+
isVisionFailureEnvelope(result.value)
|
|
32
|
+
) return undefined
|
|
33
|
+
return {
|
|
34
|
+
role: 'user',
|
|
35
|
+
id: `vision-router-evidence-success-${String(exec.callId)}`,
|
|
36
|
+
content: [{ type: 'text', text: VISION_SUCCESS_GUIDANCE }],
|
|
37
|
+
source: { kind: 'plugin', plugin: 'dsh-vision-router' },
|
|
38
|
+
}
|
|
39
|
+
}
|
package/lib/vision-resilience.js
CHANGED
|
@@ -376,12 +376,14 @@ export function createVisionTurnMemory({
|
|
|
376
376
|
maxScopes = 64,
|
|
377
377
|
maxSessions = maxScopes,
|
|
378
378
|
maxAttemptsPerScope = 64,
|
|
379
|
+
maxEvidencePerScope = 32,
|
|
379
380
|
} = {}) {
|
|
380
381
|
const scopes = new Map() // scope -> { failedKinds:Set, attempted: [], allFailed:false }
|
|
381
382
|
const lastScopeBySession = new Map() // bounded LRU sessionId -> scope
|
|
382
383
|
const scopeLimit = Math.max(1, Math.floor(Number(maxScopes) || 64))
|
|
383
384
|
const sessionLimit = Math.max(1, Math.floor(Number(maxSessions) || scopeLimit))
|
|
384
385
|
const attemptLimit = Math.max(1, Math.floor(Number(maxAttemptsPerScope) || 64))
|
|
386
|
+
const evidenceLimit = Math.max(1, Math.floor(Number(maxEvidencePerScope) || 32))
|
|
385
387
|
|
|
386
388
|
const dropScope = (scope) => {
|
|
387
389
|
scopes.delete(scope)
|
|
@@ -410,7 +412,14 @@ export function createVisionTurnMemory({
|
|
|
410
412
|
const entry = (scope) => {
|
|
411
413
|
let hit = scopes.get(scope)
|
|
412
414
|
if (hit === undefined) {
|
|
413
|
-
hit = {
|
|
415
|
+
hit = {
|
|
416
|
+
failedKinds: new Set(),
|
|
417
|
+
attempted: [],
|
|
418
|
+
allFailed: false,
|
|
419
|
+
localOcr: new Map(),
|
|
420
|
+
derivedArtifacts: new Map(),
|
|
421
|
+
degradedRefinements: new Map(),
|
|
422
|
+
}
|
|
414
423
|
scopes.set(scope, hit)
|
|
415
424
|
pruneScopes()
|
|
416
425
|
} else {
|
|
@@ -453,6 +462,69 @@ export function createVisionTurnMemory({
|
|
|
453
462
|
return hit === undefined ? [] : [...hit.attempted]
|
|
454
463
|
},
|
|
455
464
|
|
|
465
|
+
resolveSource(scope, sourceKey) {
|
|
466
|
+
const key = typeof sourceKey === 'string' ? sourceKey : ''
|
|
467
|
+
const hit = scopes.get(scope)
|
|
468
|
+
if (hit === undefined || key === '') return key
|
|
469
|
+
for (const [artifactToken, rootSource] of hit.derivedArtifacts) {
|
|
470
|
+
if (key === artifactToken || key.includes(artifactToken)) return rootSource
|
|
471
|
+
}
|
|
472
|
+
return key
|
|
473
|
+
},
|
|
474
|
+
|
|
475
|
+
recordLocalOcr(scope, sourceKey, evidence = {}) {
|
|
476
|
+
const hit = entry(scope)
|
|
477
|
+
const key = this.resolveSource(scope, sourceKey)
|
|
478
|
+
if (key === '') return
|
|
479
|
+
hit.localOcr.delete(key)
|
|
480
|
+
hit.localOcr.set(key, Object.freeze({ uncertain: evidence.uncertain === true }))
|
|
481
|
+
while (hit.localOcr.size > evidenceLimit) hit.localOcr.delete(hit.localOcr.keys().next().value)
|
|
482
|
+
},
|
|
483
|
+
|
|
484
|
+
hasLocalOcr(scope, sourceKey) {
|
|
485
|
+
const hit = scopes.get(scope)
|
|
486
|
+
if (hit === undefined) return false
|
|
487
|
+
return hit.localOcr.has(this.resolveSource(scope, sourceKey))
|
|
488
|
+
},
|
|
489
|
+
|
|
490
|
+
degradedEvidenceTokens(scope) {
|
|
491
|
+
const hit = scopes.get(scope)
|
|
492
|
+
if (hit === undefined || !hit.allFailed || hit.localOcr.size === 0) return []
|
|
493
|
+
const tokens = [...hit.localOcr.keys()]
|
|
494
|
+
for (const [artifactToken, rootSource] of hit.derivedArtifacts) {
|
|
495
|
+
if (hit.localOcr.has(rootSource)) tokens.push(artifactToken)
|
|
496
|
+
}
|
|
497
|
+
return tokens
|
|
498
|
+
},
|
|
499
|
+
|
|
500
|
+
recordDerivedArtifact(scope, sourceKey, artifactToken) {
|
|
501
|
+
const token = typeof artifactToken === 'string' ? artifactToken : ''
|
|
502
|
+
if (token === '') return
|
|
503
|
+
const hit = entry(scope)
|
|
504
|
+
const rootSource = this.resolveSource(scope, sourceKey)
|
|
505
|
+
if (rootSource === '') return
|
|
506
|
+
hit.derivedArtifacts.delete(token)
|
|
507
|
+
hit.derivedArtifacts.set(token, rootSource)
|
|
508
|
+
while (hit.derivedArtifacts.size > evidenceLimit) hit.derivedArtifacts.delete(hit.derivedArtifacts.keys().next().value)
|
|
509
|
+
},
|
|
510
|
+
|
|
511
|
+
degradedRefinementCount(scope, sourceKey) {
|
|
512
|
+
const hit = scopes.get(scope)
|
|
513
|
+
if (hit === undefined) return 0
|
|
514
|
+
return hit.degradedRefinements.get(this.resolveSource(scope, sourceKey)) ?? 0
|
|
515
|
+
},
|
|
516
|
+
|
|
517
|
+
recordDegradedRefinement(scope, sourceKey) {
|
|
518
|
+
const hit = entry(scope)
|
|
519
|
+
const key = this.resolveSource(scope, sourceKey)
|
|
520
|
+
if (key === '') return 0
|
|
521
|
+
const count = (hit.degradedRefinements.get(key) ?? 0) + 1
|
|
522
|
+
hit.degradedRefinements.delete(key)
|
|
523
|
+
hit.degradedRefinements.set(key, count)
|
|
524
|
+
while (hit.degradedRefinements.size > evidenceLimit) hit.degradedRefinements.delete(hit.degradedRefinements.keys().next().value)
|
|
525
|
+
return count
|
|
526
|
+
},
|
|
527
|
+
|
|
456
528
|
stats() {
|
|
457
529
|
return { scopes: scopes.size, sessions: lastScopeBySession.size }
|
|
458
530
|
},
|