dsh-vision-router 2.1.0 → 2.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/docs/releases/v2.1.1.md +19 -0
- package/docs/releases/v2.1.2.md +30 -0
- package/lib/agent-request-route-authority.js +94 -0
- package/lib/client-host-compat-prelude.js +29 -17
- package/lib/delegated-call-config.js +27 -0
- package/lib/doctor.js +45 -4
- package/lib/grounding-coordinate-frame.js +93 -0
- package/lib/grounding-coordinate-runtime.js +246 -0
- package/lib/legacy-core-vision-policy-bridge.js +17 -146
- package/lib/runtime-composition.js +43 -10
- package/lib/session-surface-policy.js +59 -32
- package/lib/session-vision-mode-authority.js +254 -0
- package/lib/session-vision-mode-boundary.js +408 -0
- package/lib/twin-image-capability-fallback.js +428 -0
- package/lib/vision-attachment-handle-runtime.js +212 -0
- package/lib/vision-tool-runtime-boundary.js +224 -5
- package/package.json +6 -5
|
@@ -0,0 +1,246 @@
|
|
|
1
|
+
import { randomUUID } from 'node:crypto'
|
|
2
|
+
import { unlink } from 'node:fs/promises'
|
|
3
|
+
import path from 'node:path'
|
|
4
|
+
import { writeArtifactFile } from './artifact-boundary.js'
|
|
5
|
+
import {
|
|
6
|
+
createGroundingFrame,
|
|
7
|
+
groundingFrameBoxToSource,
|
|
8
|
+
} from './grounding-coordinate-frame.js'
|
|
9
|
+
import { wrapVisionAttachmentHandleDefinition } from './vision-attachment-handle-runtime.js'
|
|
10
|
+
|
|
11
|
+
const GROUNDING_TOOL_NAMES = new Set(['vision_ground', 'vision_detect'])
|
|
12
|
+
let sharpPromise
|
|
13
|
+
|
|
14
|
+
function loadSharp() {
|
|
15
|
+
if (!sharpPromise) {
|
|
16
|
+
sharpPromise = import('sharp').then((mod) => mod.default ?? mod).catch((error) => {
|
|
17
|
+
sharpPromise = undefined
|
|
18
|
+
throw error
|
|
19
|
+
})
|
|
20
|
+
}
|
|
21
|
+
return sharpPromise
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
function workspaceOf(exec) {
|
|
25
|
+
const cwd = exec?.agent?.session?.header?.cwd
|
|
26
|
+
return typeof cwd === 'string' && cwd !== '' ? cwd : process.cwd()
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
function artifactsDirOf(config) {
|
|
30
|
+
return typeof config?.artifactsDir === 'string' && config.artifactsDir !== ''
|
|
31
|
+
? config.artifactsDir
|
|
32
|
+
: '.dsh-vision-router/artifacts'
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
function contextService(ctx, name) {
|
|
36
|
+
try {
|
|
37
|
+
if (typeof ctx?.get === 'function') return ctx.get(name)
|
|
38
|
+
} catch {
|
|
39
|
+
return undefined
|
|
40
|
+
}
|
|
41
|
+
return ctx?.[name]
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
async function readSourceBytes(ctx, core, sessionVisionIndex, exec, image) {
|
|
45
|
+
const source = String(image ?? '')
|
|
46
|
+
if (core?.isAttachmentIdInput?.(source)) {
|
|
47
|
+
const session = exec?.agent?.session
|
|
48
|
+
const ref = sessionVisionIndex?.lookupAttachment?.(session, source.trim())
|
|
49
|
+
if (ref === undefined) {
|
|
50
|
+
throw new Error(
|
|
51
|
+
`vision-router: unknown attachment id "${source}" (it must come from an image uploaded in this conversation)`,
|
|
52
|
+
)
|
|
53
|
+
}
|
|
54
|
+
const attachments = contextService(ctx, 'attachments')
|
|
55
|
+
if (!attachments || typeof attachments.readImage !== 'function') {
|
|
56
|
+
throw new Error('vision-router: the attachment service is not available in this deployment')
|
|
57
|
+
}
|
|
58
|
+
const stored = await attachments.readImage(ref, exec?.signal)
|
|
59
|
+
if (!stored?.data) throw new Error(`vision-router: failed to read attachment ${source}`)
|
|
60
|
+
return Buffer.from(stored.data)
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
const fs = contextService(ctx, 'fs')
|
|
64
|
+
if (!fs || typeof fs.resolve !== 'function' || typeof fs.readBytes !== 'function') {
|
|
65
|
+
throw new Error('vision-router: the fs service is not available')
|
|
66
|
+
}
|
|
67
|
+
const target = await fs.resolve(source)
|
|
68
|
+
return Buffer.from(await fs.readBytes(target, undefined, 20 * 1024 * 1024))
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
async function buildGroundingFrame(bytes) {
|
|
72
|
+
const sharp = await loadSharp()
|
|
73
|
+
const meta = await sharp(bytes, { failOn: 'none' }).metadata()
|
|
74
|
+
const width = Number(meta.width ?? 0)
|
|
75
|
+
const height = Number(meta.height ?? 0)
|
|
76
|
+
if (width <= 0 || height <= 0) throw new Error('could not read image dimensions')
|
|
77
|
+
const frame = createGroundingFrame(width, height)
|
|
78
|
+
const framed = await sharp(bytes, { failOn: 'none' })
|
|
79
|
+
.resize(frame.renderedWidth, frame.renderedHeight, { fit: 'fill' })
|
|
80
|
+
.extend({
|
|
81
|
+
top: frame.top,
|
|
82
|
+
bottom: frame.bottom,
|
|
83
|
+
left: frame.left,
|
|
84
|
+
right: frame.right,
|
|
85
|
+
background: { r: 0, g: 0, b: 0, alpha: 1 },
|
|
86
|
+
})
|
|
87
|
+
.png()
|
|
88
|
+
.toBuffer()
|
|
89
|
+
return { frame, framed, width, height }
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
function parseToolResult(raw) {
|
|
93
|
+
if (typeof raw !== 'string' || raw.trim() === '') return undefined
|
|
94
|
+
try {
|
|
95
|
+
const parsed = JSON.parse(raw)
|
|
96
|
+
return parsed && typeof parsed === 'object' && !Array.isArray(parsed) ? parsed : undefined
|
|
97
|
+
} catch {
|
|
98
|
+
return undefined
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
async function removeInternalFrame(workspace, publishedPath) {
|
|
103
|
+
if (typeof publishedPath !== 'string' || publishedPath === '') return
|
|
104
|
+
const target = path.isAbsolute(publishedPath) ? publishedPath : path.resolve(workspace, publishedPath)
|
|
105
|
+
try {
|
|
106
|
+
await unlink(target)
|
|
107
|
+
} catch {
|
|
108
|
+
// The internal frame is best-effort cleanup. Artifact retention may already
|
|
109
|
+
// have removed it after the delegated tool completed.
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
async function executeInGroundingFrame({
|
|
114
|
+
ctx,
|
|
115
|
+
core,
|
|
116
|
+
config,
|
|
117
|
+
sessionVisionIndex,
|
|
118
|
+
def,
|
|
119
|
+
execute,
|
|
120
|
+
args,
|
|
121
|
+
exec,
|
|
122
|
+
}) {
|
|
123
|
+
const source = String(args?.image ?? '')
|
|
124
|
+
const bytes = await readSourceBytes(ctx, core, sessionVisionIndex, exec, source)
|
|
125
|
+
const { frame, framed, width, height } = await buildGroundingFrame(bytes)
|
|
126
|
+
const workspace = workspaceOf(exec)
|
|
127
|
+
const artifactsDir = artifactsDirOf(config)
|
|
128
|
+
const internalName = `.grounding-frame-${randomUUID()}.png`
|
|
129
|
+
const framePath = await writeArtifactFile(workspace, artifactsDir, internalName, framed)
|
|
130
|
+
|
|
131
|
+
try {
|
|
132
|
+
const raw = await execute(
|
|
133
|
+
{
|
|
134
|
+
...(args ?? {}),
|
|
135
|
+
image: framePath,
|
|
136
|
+
// Core's annotation would be in the protocol frame. Re-publish only an
|
|
137
|
+
// original-raster annotation after the deterministic inverse transform.
|
|
138
|
+
annotate: false,
|
|
139
|
+
},
|
|
140
|
+
exec,
|
|
141
|
+
)
|
|
142
|
+
const delegated = parseToolResult(raw)
|
|
143
|
+
if (!delegated || delegated.ok === false) return raw
|
|
144
|
+
|
|
145
|
+
if (def.name === 'vision_ground') {
|
|
146
|
+
const mapped = groundingFrameBoxToSource(delegated, frame)
|
|
147
|
+
if (!mapped) {
|
|
148
|
+
throw new Error('vision_ground: the model box falls entirely outside the letterboxed source raster')
|
|
149
|
+
}
|
|
150
|
+
const result = { ...mapped, width, height }
|
|
151
|
+
if (args?.annotate !== false) {
|
|
152
|
+
if (typeof core?.annotateBoxBuffer !== 'function') {
|
|
153
|
+
throw new Error('vision_ground: core annotation helper is unavailable')
|
|
154
|
+
}
|
|
155
|
+
const annotated = await core.annotateBoxBuffer(bytes, mapped)
|
|
156
|
+
const stem = core?.artifactStemOf?.(source, 'ground') ?? `image-ground-${Date.now()}`
|
|
157
|
+
result.annotatedPath = await writeArtifactFile(workspace, artifactsDir, `${stem}.png`, annotated)
|
|
158
|
+
}
|
|
159
|
+
return JSON.stringify(result)
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
const elements = []
|
|
163
|
+
for (const item of Array.isArray(delegated.elements) ? delegated.elements : []) {
|
|
164
|
+
const mapped = groundingFrameBoxToSource(item?.box, frame)
|
|
165
|
+
if (!mapped) continue
|
|
166
|
+
elements.push({
|
|
167
|
+
...item,
|
|
168
|
+
number: elements.length + 1,
|
|
169
|
+
box: mapped,
|
|
170
|
+
})
|
|
171
|
+
}
|
|
172
|
+
const result = { width, height, elements }
|
|
173
|
+
if (args?.annotate !== false && elements.length > 0) {
|
|
174
|
+
if (typeof core?.annotateBoxesBuffer !== 'function') {
|
|
175
|
+
throw new Error('vision_detect: core annotation helper is unavailable')
|
|
176
|
+
}
|
|
177
|
+
const annotated = await core.annotateBoxesBuffer(bytes, elements.map((item) => item.box))
|
|
178
|
+
const stem = core?.artifactStemOf?.(source, 'detect') ?? `image-detect-${Date.now()}`
|
|
179
|
+
result.annotatedPath = await writeArtifactFile(workspace, artifactsDir, `${stem}.png`, annotated)
|
|
180
|
+
}
|
|
181
|
+
return JSON.stringify(result)
|
|
182
|
+
} finally {
|
|
183
|
+
await removeInternalFrame(workspace, framePath)
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
function wrapGroundingDefinition(ctx, options, def) {
|
|
188
|
+
if (!def || !GROUNDING_TOOL_NAMES.has(def.name) || typeof def.execute !== 'function') return def
|
|
189
|
+
const execute = def.execute
|
|
190
|
+
return {
|
|
191
|
+
...def,
|
|
192
|
+
execute(args, exec) {
|
|
193
|
+
return executeInGroundingFrame({
|
|
194
|
+
ctx,
|
|
195
|
+
...options,
|
|
196
|
+
def,
|
|
197
|
+
execute,
|
|
198
|
+
args,
|
|
199
|
+
exec,
|
|
200
|
+
})
|
|
201
|
+
},
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
/**
|
|
206
|
+
* Give vision_ground / vision_detect one explicit provider-independent
|
|
207
|
+
* coordinate protocol: the model always receives an exact 1000x1000
|
|
208
|
+
* letterboxed raster, while callers always receive Host-canonical source
|
|
209
|
+
* pixels. The same registration seam also canonicalizes DSH text-only image
|
|
210
|
+
* handles before any Vision Router tool sees them, so short model-facing
|
|
211
|
+
* sha256 prefixes can never fall through into filesystem-path resolution.
|
|
212
|
+
* Core still owns provider selection, fallback, retry and failure semantics.
|
|
213
|
+
*/
|
|
214
|
+
export function contextWithGroundingCoordinateFrame(ctx, options = {}) {
|
|
215
|
+
if (!ctx || (typeof ctx !== 'object' && typeof ctx !== 'function')) return ctx
|
|
216
|
+
const sourceTools = ctx.tools ?? contextService(ctx, 'tools')
|
|
217
|
+
if (!sourceTools || typeof sourceTools.register !== 'function') return ctx
|
|
218
|
+
|
|
219
|
+
const tools = new Proxy(sourceTools, {
|
|
220
|
+
get(target, property) {
|
|
221
|
+
if (property !== 'register') {
|
|
222
|
+
const value = Reflect.get(target, property, target)
|
|
223
|
+
return typeof value === 'function' ? value.bind(target) : value
|
|
224
|
+
}
|
|
225
|
+
const register = Reflect.get(target, property, target)
|
|
226
|
+
return (def, ...rest) => {
|
|
227
|
+
const grounded = wrapGroundingDefinition(ctx, options, def)
|
|
228
|
+
const attachmentResolved = wrapVisionAttachmentHandleDefinition(grounded, options)
|
|
229
|
+
return register.call(target, attachmentResolved, ...rest)
|
|
230
|
+
}
|
|
231
|
+
},
|
|
232
|
+
})
|
|
233
|
+
|
|
234
|
+
return new Proxy(ctx, {
|
|
235
|
+
get(target, property) {
|
|
236
|
+
if (property === 'tools') return tools
|
|
237
|
+
if (property === 'get') {
|
|
238
|
+
const get = Reflect.get(target, property, target)
|
|
239
|
+
if (typeof get !== 'function') return get
|
|
240
|
+
return (name, ...rest) => name === 'tools' ? tools : get.call(target, name, ...rest)
|
|
241
|
+
}
|
|
242
|
+
const value = Reflect.get(target, property, target)
|
|
243
|
+
return typeof value === 'function' ? value.bind(target) : value
|
|
244
|
+
},
|
|
245
|
+
})
|
|
246
|
+
}
|
|
@@ -1,150 +1,21 @@
|
|
|
1
|
-
import { currentSessionSurfacePolicy } from './session-surface-policy.js'
|
|
2
|
-
import { knownSessionVisionMemory } from './session-vision-state.js'
|
|
3
|
-
|
|
4
|
-
function isObject(value) {
|
|
5
|
-
return value !== null && typeof value === 'object'
|
|
6
|
-
}
|
|
7
|
-
|
|
8
|
-
function collectAttachmentIds(messages) {
|
|
9
|
-
const ids = []
|
|
10
|
-
const seen = new Set()
|
|
11
|
-
const pending = []
|
|
12
|
-
for (const message of messages ?? []) {
|
|
13
|
-
if (message && Array.isArray(message.content)) pending.push(...message.content)
|
|
14
|
-
}
|
|
15
|
-
while (pending.length > 0) {
|
|
16
|
-
const block = pending.pop()
|
|
17
|
-
if (!block || typeof block !== 'object') continue
|
|
18
|
-
if (block.type === 'image') {
|
|
19
|
-
const ref = block.attachment
|
|
20
|
-
const raw = ref && (ref.attachmentId ?? ref.id)
|
|
21
|
-
if (raw !== undefined && raw !== null) {
|
|
22
|
-
const id = String(raw)
|
|
23
|
-
if (id !== '' && !seen.has(id)) {
|
|
24
|
-
seen.add(id)
|
|
25
|
-
ids.push(id)
|
|
26
|
-
}
|
|
27
|
-
}
|
|
28
|
-
}
|
|
29
|
-
if (Array.isArray(block.content)) pending.push(...block.content)
|
|
30
|
-
}
|
|
31
|
-
return ids.reverse()
|
|
32
|
-
}
|
|
33
|
-
|
|
34
|
-
function appendVisionRouterAttachmentHint(payload, decision, config) {
|
|
35
|
-
const policy = currentSessionSurfacePolicy(config)
|
|
36
|
-
if (decision?.kind === 'reject' || policy.ownership !== 'vision-router-owned') {
|
|
37
|
-
return decision
|
|
38
|
-
}
|
|
39
|
-
|
|
40
|
-
// Vision Router-owned wrappers may deliberately preserve raw pixels when the
|
|
41
|
-
// delegated source model is itself multimodal. The provider wire carries the
|
|
42
|
-
// bytes, but not DSH's durable attachmentId, so a model that chooses a
|
|
43
|
-
// precision Vision Router tool would otherwise have to guess an id. Surface
|
|
44
|
-
// the exact current-turn ids as read-only model context while leaving the raw
|
|
45
|
-
// image blocks and the session-scoped lookup fence unchanged.
|
|
46
|
-
const source = Array.isArray(payload?.messages) ? payload.messages : []
|
|
47
|
-
const ids = collectAttachmentIds(source)
|
|
48
|
-
if (ids.length === 0) return decision
|
|
49
|
-
|
|
50
|
-
const baseMessages = Array.isArray(decision?.messages)
|
|
51
|
-
? decision.messages
|
|
52
|
-
: source
|
|
53
|
-
const turn = Number.isInteger(payload?.turn) ? payload.turn : 'current'
|
|
54
|
-
const step = Number.isInteger(payload?.step) ? payload.step : 'step'
|
|
55
|
-
const hintId = `vision-router-attachment-refs-${turn}-${step}`
|
|
56
|
-
if (baseMessages.some((message) => message?.id === hintId)) return decision
|
|
57
|
-
|
|
58
|
-
const quoted = ids.map((id) => `"${id}"`).join(', ')
|
|
59
|
-
const hint = {
|
|
60
|
-
role: 'user',
|
|
61
|
-
id: hintId,
|
|
62
|
-
content: [{
|
|
63
|
-
type: 'text',
|
|
64
|
-
text: `Vision Router attachment references for the image(s) in this step: ${quoted}. If a Vision Router tool requires attachmentIds or an attachment-id image argument, use only these exact ids. Never guess or invent an attachment id.`,
|
|
65
|
-
}],
|
|
66
|
-
source: { kind: 'plugin', plugin: 'dsh-vision-router' },
|
|
67
|
-
}
|
|
68
|
-
const messages = [...baseMessages, hint]
|
|
69
|
-
return isObject(decision)
|
|
70
|
-
? { ...decision, messages }
|
|
71
|
-
: { kind: 'continue', messages }
|
|
72
|
-
}
|
|
73
|
-
|
|
74
|
-
function rewriteTextOnlyDecision(payload, decision, rewriteHistoryImages, config) {
|
|
75
|
-
const policy = currentSessionSurfacePolicy(config)
|
|
76
|
-
if (
|
|
77
|
-
decision?.kind === 'reject' ||
|
|
78
|
-
policy.rewriteCurrentImages !== true ||
|
|
79
|
-
typeof rewriteHistoryImages !== 'function'
|
|
80
|
-
) {
|
|
81
|
-
return decision
|
|
82
|
-
}
|
|
83
|
-
|
|
84
|
-
const source = Array.isArray(decision?.messages)
|
|
85
|
-
? decision.messages
|
|
86
|
-
: Array.isArray(payload?.messages)
|
|
87
|
-
? payload.messages
|
|
88
|
-
: undefined
|
|
89
|
-
if (!source) return decision
|
|
90
|
-
|
|
91
|
-
// Core registers the exact SessionMemoryView while processing this same
|
|
92
|
-
// pre-step. Reuse it here so a text-only fallback preserves cached visual
|
|
93
|
-
// descriptions instead of degrading them back to a generic attachment marker.
|
|
94
|
-
const memory = knownSessionVisionMemory(payload?.agent?.session)
|
|
95
|
-
const rewritten = rewriteHistoryImages(source, memory)
|
|
96
|
-
const messages = rewritten?.messages
|
|
97
|
-
if (!Array.isArray(messages) || messages === source) return decision
|
|
98
|
-
if (isObject(decision)) return { ...decision, messages }
|
|
99
|
-
return { kind: 'continue', messages }
|
|
100
|
-
}
|
|
101
|
-
|
|
102
1
|
/**
|
|
103
|
-
*
|
|
104
|
-
*
|
|
2
|
+
* Retired pre-step compatibility shell.
|
|
3
|
+
*
|
|
4
|
+
* This module remains as a stable internal import while the 2.x composition is
|
|
5
|
+
* still being collapsed, but it deliberately owns no runtime behavior.
|
|
6
|
+
*
|
|
7
|
+
* Historical versions intercepted `agent/pre-step` for two model-only concerns:
|
|
8
|
+
* rewriting text-only image messages into `[attached image: ...]` markers and
|
|
9
|
+
* appending a synthetic attachment-id user message for Vision Router wrappers.
|
|
10
|
+
* DSH persists every admitted pre-step message as `user/message`, so both
|
|
11
|
+
* transforms crossed the durable transcript boundary and could surface in the
|
|
12
|
+
* Web conversation. User-owned image messages must instead remain byte-for-byte
|
|
13
|
+
* Session facts; request-only image projection belongs to the selected adapter.
|
|
105
14
|
*
|
|
106
|
-
*
|
|
107
|
-
*
|
|
108
|
-
*
|
|
109
|
-
* rewrite and surfaces current durable attachment ids for a Vision Router-owned
|
|
110
|
-
* route. Every other context service, Settings object, injected child and config
|
|
111
|
-
* value passes through unchanged.
|
|
15
|
+
* Keep this function identity-only until runtime-composition removes the import
|
|
16
|
+
* in a dedicated closure cleanup. No Settings/config/service identity is
|
|
17
|
+
* impersonated and no `agent/pre-step` listener is installed here.
|
|
112
18
|
*/
|
|
113
|
-
export function installLegacyCoreVisionPolicyBridge(
|
|
114
|
-
ctx,
|
|
115
|
-
config = {},
|
|
116
|
-
{ rewriteHistoryImages } = {},
|
|
117
|
-
) {
|
|
118
|
-
if (!isObject(ctx)) return { ctx, config }
|
|
119
|
-
|
|
120
|
-
const wrappedCtx = new Proxy(ctx, {
|
|
121
|
-
get(target, property) {
|
|
122
|
-
if (property === 'on') {
|
|
123
|
-
const on = Reflect.get(target, property, target)
|
|
124
|
-
if (typeof on !== 'function') return on
|
|
125
|
-
return (event, handler, ...rest) => {
|
|
126
|
-
if (event !== 'agent/pre-step' || typeof handler !== 'function') {
|
|
127
|
-
return on.call(target, event, handler, ...rest)
|
|
128
|
-
}
|
|
129
|
-
return on.call(target, event, async function legacyCoreVisionPolicyPreStep(payload, next) {
|
|
130
|
-
const decision = await handler.call(this, payload, next)
|
|
131
|
-
const rewritten = rewriteTextOnlyDecision(
|
|
132
|
-
payload,
|
|
133
|
-
decision,
|
|
134
|
-
rewriteHistoryImages,
|
|
135
|
-
config,
|
|
136
|
-
)
|
|
137
|
-
return appendVisionRouterAttachmentHint(payload, rewritten, config)
|
|
138
|
-
}, ...rest)
|
|
139
|
-
}
|
|
140
|
-
}
|
|
141
|
-
const value = Reflect.get(target, property, target)
|
|
142
|
-
return typeof value === 'function' ? value.bind(target) : value
|
|
143
|
-
},
|
|
144
|
-
})
|
|
145
|
-
|
|
146
|
-
return {
|
|
147
|
-
ctx: wrappedCtx,
|
|
148
|
-
config,
|
|
149
|
-
}
|
|
19
|
+
export function installLegacyCoreVisionPolicyBridge(ctx, config = {}) {
|
|
20
|
+
return { ctx, config }
|
|
150
21
|
}
|
|
@@ -15,6 +15,7 @@ import { createCoreVisionSurfaceRuntime } from './core-vision-surface.js'
|
|
|
15
15
|
import { installSessionVisionIndexBoundary } from './session-vision-index.js'
|
|
16
16
|
import { createSessionVisionRuntime } from './session-vision-runtime.js'
|
|
17
17
|
import { installLegacyCoreVisionPolicyBridge } from './legacy-core-vision-policy-bridge.js'
|
|
18
|
+
import { installSessionVisionModeBoundary } from './session-vision-mode-boundary.js'
|
|
18
19
|
import { installPiAiBridgeWireCompat } from './pi-ai-bridge-wire-compat.js'
|
|
19
20
|
import { installLiveModelDiscovery } from './live-model-discovery.js'
|
|
20
21
|
import { installVisionModelRegistry } from './vision-model-registry.js'
|
|
@@ -28,6 +29,11 @@ import { installTesseractExecFileCompat } from './tesseract-exec-compat.js'
|
|
|
28
29
|
import { installLocalMutationRouteBoundary } from './web-capability-boundary.js'
|
|
29
30
|
import { installScreenshotSourceBoundary } from './screenshot-source-boundary.js'
|
|
30
31
|
import { installVisionToolRuntimeBoundary } from './vision-tool-runtime-boundary.js'
|
|
32
|
+
import {
|
|
33
|
+
configureAgentRequestRouteAuthority,
|
|
34
|
+
contextWithAgentRequestRouteAuthority,
|
|
35
|
+
} from './agent-request-route-authority.js'
|
|
36
|
+
import { contextWithGroundingCoordinateFrame } from './grounding-coordinate-runtime.js'
|
|
31
37
|
import { installVisionRoutingRuntime } from './vision-routing-runtime.js'
|
|
32
38
|
import { createCapabilityProfileStore } from './vision-capability-probe.js'
|
|
33
39
|
import { installCapabilityBenchmarkService } from './vision-capability-benchmark-service.js'
|
|
@@ -186,25 +192,32 @@ export function applyVisionRuntimeComposition(ctx, config = {}, core) {
|
|
|
186
192
|
runtimeI18nCore,
|
|
187
193
|
{ logger: logging.logger, index: sessionVisionRuntime.index },
|
|
188
194
|
)
|
|
189
|
-
//
|
|
190
|
-
//
|
|
191
|
-
//
|
|
195
|
+
// The retired bridge stays identity-only. Session mode is a new explicit
|
|
196
|
+
// boundary because it owns a different concern: per-Agent visibility and
|
|
197
|
+
// execution authority derived from DSH's modelSelection projection.
|
|
192
198
|
const legacyCoreCompat = installLegacyCoreVisionPolicyBridge(
|
|
193
199
|
sessionIndexCtx,
|
|
194
200
|
nativeImageCompat.config,
|
|
195
201
|
{ rewriteHistoryImages: core.rewriteHistoryImages },
|
|
196
202
|
)
|
|
203
|
+
const sessionVisionModeCompat = installSessionVisionModeBoundary(
|
|
204
|
+
legacyCoreCompat.ctx,
|
|
205
|
+
legacyCoreCompat.config,
|
|
206
|
+
)
|
|
197
207
|
|
|
198
208
|
// Final structured-flow guard sits closest to core.apply so it sees the
|
|
199
209
|
// actual tool registrations and pre-step listener. The diagnostic observer
|
|
200
210
|
// remains immediately inside it so existing timeout/budget semantics stay
|
|
201
211
|
// unchanged.
|
|
202
212
|
const limitDiagnosticCtx = installVisionLimitDiagnostics(
|
|
203
|
-
|
|
204
|
-
|
|
213
|
+
sessionVisionModeCompat.ctx,
|
|
214
|
+
sessionVisionModeCompat.config,
|
|
205
215
|
logging.logger,
|
|
206
216
|
)
|
|
207
|
-
const structuredCtx = installStructuredFlowHardening(
|
|
217
|
+
const structuredCtx = installStructuredFlowHardening(
|
|
218
|
+
limitDiagnosticCtx,
|
|
219
|
+
sessionVisionModeCompat.config,
|
|
220
|
+
)
|
|
208
221
|
const backgroundProfiling = installBackgroundCapabilityProfiling(
|
|
209
222
|
structuredCtx,
|
|
210
223
|
runtimeConfig,
|
|
@@ -249,11 +262,21 @@ export function applyVisionRuntimeComposition(ctx, config = {}, core) {
|
|
|
249
262
|
logger: logging.logger,
|
|
250
263
|
})
|
|
251
264
|
|
|
265
|
+
// Ground/detect own one explicit raster-coordinate boundary. It wraps only
|
|
266
|
+
// the tool-registration view, before runtime-performance observation. That
|
|
267
|
+
// leaves adapter sampling on the same execution seam while backend preflight
|
|
268
|
+
// remains the final outer Core-visible policy boundary.
|
|
269
|
+
const groundingCoordinateCtx = contextWithGroundingCoordinateFrame(executionCtx, {
|
|
270
|
+
core,
|
|
271
|
+
config: runtimeConfig,
|
|
272
|
+
sessionVisionIndex: sessionVisionRuntime.index,
|
|
273
|
+
})
|
|
274
|
+
|
|
252
275
|
// Only real visual-tool adapter streams are timed, and only while live Auto
|
|
253
276
|
// authority permits future-routing observation. Benchmark/background calls
|
|
254
277
|
// have no visual-tool scope and cannot contaminate this store.
|
|
255
278
|
const performanceCtx = contextWithVisionRuntimePerformance(
|
|
256
|
-
|
|
279
|
+
groundingCoordinateCtx,
|
|
257
280
|
runtimePerformanceStore,
|
|
258
281
|
{
|
|
259
282
|
logger: logging.logger,
|
|
@@ -295,6 +318,16 @@ export function applyVisionRuntimeComposition(ctx, config = {}, core) {
|
|
|
295
318
|
})
|
|
296
319
|
installTesseractExecFileCompat(backendRuntimeCtx)
|
|
297
320
|
|
|
321
|
+
// Core owns exactly one agent/request routing hook. Protect the completed
|
|
322
|
+
// provider/model handoff at that event boundary so future route-switch logic
|
|
323
|
+
// cannot accidentally carry source-model call defaults into the target.
|
|
324
|
+
configureAgentRequestRouteAuthority(backendRuntimeCtx, {
|
|
325
|
+
wrapperRoute: runtimeConfig.wrapperRoute,
|
|
326
|
+
chainRoute: runtimeConfig.chainRoute,
|
|
327
|
+
logger: logging.logger,
|
|
328
|
+
})
|
|
329
|
+
const coreRequestAuthorityCtx = contextWithAgentRequestRouteAuthority(backendRuntimeCtx)
|
|
330
|
+
|
|
298
331
|
try {
|
|
299
332
|
const c = hardenedConfig && typeof hardenedConfig === 'object' ? hardenedConfig : {}
|
|
300
333
|
const local = c.localOllama && typeof c.localOllama === 'object' ? c.localOllama : {}
|
|
@@ -316,8 +349,8 @@ export function applyVisionRuntimeComposition(ctx, config = {}, core) {
|
|
|
316
349
|
withVisionCircuitBreakerObserver(
|
|
317
350
|
breakerShadowHealth.capture,
|
|
318
351
|
() => core.apply(
|
|
319
|
-
|
|
320
|
-
|
|
352
|
+
coreRequestAuthorityCtx,
|
|
353
|
+
sessionVisionModeCompat.config,
|
|
321
354
|
{
|
|
322
355
|
sessionVision: sessionVisionRuntime,
|
|
323
356
|
coreVisionSurface: coreVisionSurfaceRuntime,
|
|
@@ -345,4 +378,4 @@ export function applyVisionRuntimeComposition(ctx, config = {}, core) {
|
|
|
345
378
|
)
|
|
346
379
|
throw error
|
|
347
380
|
}
|
|
348
|
-
}
|
|
381
|
+
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { currentSessionVisionPolicy } from './native-image-coexistence.js'
|
|
2
|
+
import { currentSessionVisionModeAuthority } from './session-vision-mode-authority.js'
|
|
2
3
|
|
|
3
4
|
const OWNERSHIP = Object.freeze({
|
|
4
5
|
PLUGIN_OWNED: 'vision-router-owned',
|
|
@@ -22,62 +23,87 @@ function normalizedOwnership(policy) {
|
|
|
22
23
|
* Resolve one immutable, read-only snapshot of the Vision Router capability
|
|
23
24
|
* surface for the current session.
|
|
24
25
|
*
|
|
25
|
-
*
|
|
26
|
-
*
|
|
27
|
-
*
|
|
28
|
-
*
|
|
26
|
+
* Image ownership and Vision-mode authority are intentionally distinct. The
|
|
27
|
+
* native-image policy answers who can consume raw pixels; the mode authority
|
|
28
|
+
* answers whether this Session explicitly selected a Vision Router-owned
|
|
29
|
+
* wrapper/twin for the current step. Only the latter can grant plugin tools or
|
|
30
|
+
* automatic visual work. This prevents a historical image, a native multimodal
|
|
31
|
+
* source model, or a stale request header from silently turning Vision back on.
|
|
32
|
+
*
|
|
33
|
+
* Durable transcript invariant: once a real session policy exists, Core may
|
|
34
|
+
* observe/index image blocks but may never replace a user-owned image message
|
|
35
|
+
* before the Agent loop appends it to the Session log. Image projection belongs
|
|
36
|
+
* at the adapter/request boundary. The historical `rewriteCurrentImages` bit is
|
|
37
|
+
* therefore intentionally ignored even when an older policy producer still
|
|
38
|
+
* exposes it; the compatibility field below is permanently false.
|
|
29
39
|
*/
|
|
30
40
|
export function resolveSessionSurfacePolicy({
|
|
31
41
|
visionPolicy,
|
|
42
|
+
visionModeAuthority,
|
|
32
43
|
config = {},
|
|
33
44
|
schemaBootstrapping = false,
|
|
34
45
|
} = {}) {
|
|
35
46
|
const source = isObject(visionPolicy) ? visionPolicy : undefined
|
|
47
|
+
const authority = isObject(visionModeAuthority) ? visionModeAuthority : undefined
|
|
36
48
|
const value = isObject(config) ? config : {}
|
|
37
49
|
const ownership = normalizedOwnership(source)
|
|
38
|
-
const native = ownership === OWNERSHIP.NATIVE
|
|
39
50
|
const pluginOwned = ownership === OWNERSHIP.PLUGIN_OWNED
|
|
40
|
-
const textOnly = ownership === OWNERSHIP.TEXT_ONLY
|
|
41
51
|
|
|
42
|
-
//
|
|
43
|
-
//
|
|
44
|
-
// explicit
|
|
45
|
-
|
|
46
|
-
const
|
|
47
|
-
|
|
48
|
-
|
|
52
|
+
// During schema bootstrap there is no real Session policy, so global Settings
|
|
53
|
+
// still decide which definitions Core registers. During a real step, an
|
|
54
|
+
// explicit authority snapshot wins; direct helper callers fall back to the
|
|
55
|
+
// already-classified plugin ownership instead of inventing an enabled mode.
|
|
56
|
+
const visionModeEnabled = authority !== undefined
|
|
57
|
+
? authority.enabled === true
|
|
58
|
+
: source === undefined
|
|
59
|
+
? true
|
|
60
|
+
: pluginOwned
|
|
61
|
+
|
|
62
|
+
// No active session policy means no session-specific preservation authority.
|
|
63
|
+
// During a real pre-step, every ownership result — including explicit
|
|
64
|
+
// text-only and unknown — preserves the durable user message unchanged.
|
|
65
|
+
const preserveRawImages = source !== undefined
|
|
66
|
+
const rewriteCurrentImages = false
|
|
67
|
+
const allowStructuredBootstrap = visionModeEnabled
|
|
68
|
+
const allowGenericAutoMount = visionModeEnabled
|
|
49
69
|
|
|
50
70
|
const surface = Object.freeze({
|
|
51
71
|
preserveRawImages,
|
|
52
72
|
rewriteCurrentImages,
|
|
53
|
-
visionTools: value.tool !== false,
|
|
73
|
+
visionTools: value.tool !== false && visionModeEnabled,
|
|
54
74
|
structuredBootstrap:
|
|
55
|
-
value.structuredVisionBootstrap === true &&
|
|
75
|
+
value.structuredVisionBootstrap === true && visionModeEnabled,
|
|
56
76
|
genericAutoMount:
|
|
57
|
-
value.autoActivateOnImage !== false &&
|
|
77
|
+
value.autoActivateOnImage !== false && visionModeEnabled,
|
|
58
78
|
instantDescribe:
|
|
59
|
-
value.instantDescribe !== false &&
|
|
79
|
+
value.instantDescribe !== false && visionModeEnabled,
|
|
60
80
|
})
|
|
61
81
|
|
|
62
82
|
const overrides = {}
|
|
63
|
-
if (schemaBootstrapping === true && value.tool === false)
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
83
|
+
if (schemaBootstrapping === true && source === undefined && value.tool === false) {
|
|
84
|
+
overrides.tool = true
|
|
85
|
+
}
|
|
86
|
+
// Core's historical rewriteImages switch controls an agent/pre-step message
|
|
87
|
+
// transform. Disable it for every real session policy so neither current nor
|
|
88
|
+
// historical user image blocks can be persisted as internal attachment text.
|
|
89
|
+
// Vision Router-owned adapters still perform their private request projection.
|
|
90
|
+
if (source !== undefined && value.rewriteImages !== false) overrides.rewriteImages = false
|
|
91
|
+
|
|
92
|
+
// The composer/model selection is the Session authority. When it is OFF,
|
|
93
|
+
// suppress every automatic/plugin-owned visual surface while preserving the
|
|
94
|
+
// durable image itself. Settings remain unchanged; these are turn-local Core
|
|
95
|
+
// projections only.
|
|
96
|
+
if (source !== undefined && !visionModeEnabled) {
|
|
97
|
+
if (value.tool !== false) overrides.tool = false
|
|
98
|
+
if (value.instantDescribe !== false) overrides.instantDescribe = false
|
|
99
|
+
if (value.autoActivateOnImage !== false) overrides.autoActivateOnImage = false
|
|
100
|
+
if (value.structuredVisionBootstrap !== false) overrides.structuredVisionBootstrap = false
|
|
69
101
|
}
|
|
70
102
|
|
|
71
103
|
return Object.freeze({
|
|
72
104
|
ownership,
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
pluginOwned ||
|
|
76
|
-
textOnly ||
|
|
77
|
-
surface.visionTools ||
|
|
78
|
-
surface.structuredBootstrap ||
|
|
79
|
-
surface.genericAutoMount
|
|
80
|
-
),
|
|
105
|
+
visionModeEnabled,
|
|
106
|
+
participates: source !== undefined,
|
|
81
107
|
preserveRawImages,
|
|
82
108
|
rewriteCurrentImages,
|
|
83
109
|
allowStructuredBootstrap,
|
|
@@ -87,10 +113,11 @@ export function resolveSessionSurfacePolicy({
|
|
|
87
113
|
})
|
|
88
114
|
}
|
|
89
115
|
|
|
90
|
-
/** Read the turn-local ownership
|
|
116
|
+
/** Read the turn-local ownership + mode snapshots and resolve the Core surface. */
|
|
91
117
|
export function currentSessionSurfacePolicy(config = {}, options = {}) {
|
|
92
118
|
return resolveSessionSurfacePolicy({
|
|
93
119
|
visionPolicy: currentSessionVisionPolicy(),
|
|
120
|
+
visionModeAuthority: currentSessionVisionModeAuthority(),
|
|
94
121
|
config,
|
|
95
122
|
schemaBootstrapping: options?.schemaBootstrapping === true,
|
|
96
123
|
})
|