dsh-vision-router 2.2.7 → 2.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -8
- package/README.zh.md +8 -8
- package/docs/architecture/2.3.0-compat-retirement.md +147 -0
- package/docs/architecture/2.3.0-execution-plan.md +410 -0
- package/docs/architecture/2.3.0-final-audit.md +279 -0
- package/docs/architecture/2.3.0-reopened-plan.md +720 -0
- package/docs/architecture/compat-inventory.md +26 -34
- package/docs/architecture/dsh-020-rc2-adaptation.md +23 -0
- package/docs/architecture/dsh-compatibility-matrix.md +65 -54
- package/docs/architecture/dsh-support-window.md +29 -21
- package/docs/architecture/final-closure-audit.md +2 -0
- package/docs/architecture/host-first-proxy-convergence.md +1 -1
- package/docs/releases/v2.2.8.md +25 -0
- package/docs/releases/v2.3.0.md +53 -0
- package/entry.js +8 -56
- package/index.js +161 -691
- package/lib/catalog-corrections.js +1 -2
- package/lib/client-host-compat-prelude.js +7 -12
- package/lib/client-maintenance-actions.js +145 -0
- package/lib/client-presentation-boundary-main.js +82 -32
- package/lib/client-presentation-boundary.js +60 -40
- package/lib/client-script-carrier.js +90 -0
- package/lib/client.js +248 -146
- package/lib/core-apply-lifecycle.js +47 -0
- package/lib/core-primitives.js +88 -135
- package/lib/desktop-screenshot-tool.js +262 -0
- package/lib/dsh-contract-compat.js +11 -26
- package/lib/dsh-support-window.js +4 -4
- package/lib/guide-vision-toggle-highlight.js +7 -10
- package/lib/http-compat.js +1 -2
- package/lib/live-model-client-prelude.js +7 -14
- package/lib/local-remote-settings-permission.js +41 -16
- package/lib/maintenance-routes.js +136 -0
- package/lib/public-config.js +57 -0
- package/lib/public-entry.js +2 -7
- package/lib/remote-settings-bridge.js +11 -0
- package/lib/remote-settings-risk-client-boundary.js +155 -0
- package/lib/remote-settings-risk-confirmation.js +17 -105
- package/lib/runtime-composition.js +105 -30
- package/lib/runtime-i18n-boundary.js +10 -11
- package/lib/session-message-source-compat.js +7 -3
- package/lib/session-vision-index.js +154 -9
- package/lib/session-vision-mode-boundary.js +43 -8
- package/lib/settings-client-loader-lifecycle.js +129 -0
- package/lib/settings-factory-lifecycle.js +25 -14
- package/lib/settings-ia-client-prelude.js +11 -11
- package/lib/settings-limit-client-prelude.js +7 -9
- package/lib/sharp-runtime.js +0 -7
- package/lib/strict-live-model-client-prelude.js +7 -10
- package/lib/structured-flow-hardening.js +3 -2
- package/lib/v2-settings-ia-integration.js +21 -6
- package/lib/vision-backend-runtime-policy.js +3 -2
- package/lib/vision-background-benchmark.js +3 -5
- package/lib/vision-capability-benchmark-client.js +7 -10
- package/lib/vision-capability-benchmark-service.js +1 -0
- package/lib/vision-diagnostics-routes.js +216 -0
- package/lib/vision-evidence-guidance.js +3 -1
- package/lib/vision-exact-check-client.js +11 -6
- package/lib/vision-model-visibility-boundary-main.js +7 -11
- package/lib/vision-model-visibility-boundary.js +7 -12
- package/lib/vision-provider-transport.js +0 -29
- package/lib/vision-routing-settings-prelude.js +7 -11
- package/lib/vision-toggle-root-hardening.js +50 -8
- package/lib/vision-tool-runtime-boundary.js +5 -1
- package/lib/vision-turn-budget-client-prelude.js +7 -10
- package/lib/web/remote-settings-client.js +9 -14
- package/package.json +23 -13
- package/lib/android-attachment-compat.js +0 -244
- package/lib/settings-client-rc8-lifecycle.js +0 -402
package/lib/core-primitives.js
CHANGED
|
@@ -1,94 +1,51 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
1
|
+
|
|
2
|
+
|
|
3
3
|
import path from 'node:path'
|
|
4
|
-
|
|
5
|
-
import { DeepSeekAdapter, resolveAdapterOptions } from '@deepseek-ai/dsh-llm-deepseek'
|
|
6
|
-
import { getOrCreateAnonymousUserId } from '@deepseek-ai/dsh-anonymous-user-id'
|
|
4
|
+
|
|
7
5
|
import { existsSync } from 'node:fs'
|
|
8
6
|
import { execFile } from 'node:child_process'
|
|
9
7
|
import { Worker } from 'node:worker_threads'
|
|
10
8
|
import { createRequire } from 'node:module'
|
|
11
9
|
import { pathToFileURL } from 'node:url'
|
|
12
10
|
import { promisify } from 'node:util'
|
|
13
|
-
import {
|
|
11
|
+
import { fetchWithOpenAICompatibility } from './http-compat.js'
|
|
14
12
|
import {
|
|
15
13
|
directSessionAffinityHeaders,
|
|
16
|
-
isOfficialOpenCodeGoUrl,
|
|
17
|
-
openCodeSessionAffinityHeaderForUrl,
|
|
18
14
|
rawSessionIdentity,
|
|
19
|
-
sessionIdentityOf,
|
|
20
15
|
} from './session-affinity.js'
|
|
21
|
-
import { runWithVisionSessionAffinity
|
|
16
|
+
import { runWithVisionSessionAffinity } from './session-affinity-runtime.js'
|
|
22
17
|
import {
|
|
23
|
-
routingCorrectionFor,
|
|
24
|
-
toAnthropicMessages,
|
|
25
18
|
callAnthropicCompatible,
|
|
26
19
|
anthropicMediaType,
|
|
27
20
|
} from './catalog-corrections.js'
|
|
28
|
-
|
|
29
|
-
import { probeLocalBackends } from './local-connection-probe.js'
|
|
30
|
-
import { detectDshSelfUpdatePlan, runDshPluginUpdate } from './self-update.js'
|
|
21
|
+
|
|
31
22
|
import {
|
|
32
|
-
classifyVisionFailure,
|
|
33
|
-
createDeadline,
|
|
34
23
|
combineSignals,
|
|
35
|
-
createVisionCircuitBreaker,
|
|
36
|
-
createVisionTurnMemory,
|
|
37
|
-
buildVisionFailure,
|
|
38
|
-
ensureSentencePunctuation,
|
|
39
|
-
resultCodeForKinds,
|
|
40
24
|
qwenKeyEndpointHint,
|
|
41
25
|
kindForHttpStatus,
|
|
42
|
-
VISION_FAILURE_KINDS,
|
|
43
|
-
VISION_RESULT_CODES,
|
|
44
26
|
} from './vision-resilience.js'
|
|
45
|
-
|
|
46
|
-
import {
|
|
47
|
-
|
|
48
|
-
import {
|
|
49
|
-
normalizeStructuredBootstrapResult,
|
|
50
|
-
structuredBootstrapMemory,
|
|
51
|
-
structuredBootstrapQuestion,
|
|
52
|
-
} from './structured-bootstrap.js'
|
|
53
|
-
import { planMixedBranches, renderMixedGuidance } from './mixed-router.js'
|
|
54
|
-
import { renderDepthGuidance } from './depth-guidance.js'
|
|
55
|
-
import { assertNoRepetitionLoop } from './repetition-guard.js'
|
|
56
|
-
import { compareRgbaStreams } from './pixel-diff-stream.js'
|
|
27
|
+
|
|
28
|
+
import { createHash } from 'node:crypto'
|
|
29
|
+
|
|
57
30
|
import {
|
|
58
|
-
boundedOcrTiles,
|
|
59
31
|
defaultImageResourceGovernor,
|
|
60
32
|
estimateImageOperationBytes,
|
|
61
33
|
scaleBox,
|
|
62
34
|
scaledDimensions,
|
|
63
35
|
} from './image-resource-governor.js'
|
|
64
|
-
|
|
65
|
-
import { createSessionVisionStateStore } from './session-vision-state.js'
|
|
36
|
+
|
|
66
37
|
import {
|
|
67
38
|
ERROR_RESPONSE_MAX_BYTES,
|
|
68
|
-
METADATA_RESPONSE_MAX_BYTES,
|
|
69
39
|
MODEL_RESPONSE_MAX_BYTES,
|
|
70
40
|
readResponseJsonBounded,
|
|
71
41
|
readResponseTextBounded,
|
|
72
42
|
} from './http-body-limit.js'
|
|
73
|
-
|
|
43
|
+
|
|
74
44
|
import { stripTrailingSlashes } from './string-normalization.js'
|
|
75
45
|
import { isOffloadedImageBlock, offloadedImagePlaceholder } from './image-offload-compat.js'
|
|
76
46
|
import { proxyHostMatchesAny } from './proxy-routing.js'
|
|
77
|
-
import { parseVersionComparator } from './version-range.js'
|
|
78
|
-
import { captureWindowsDesktop } from './windows-desktop-capture.js'
|
|
79
|
-
import {
|
|
80
|
-
sharpPromise,
|
|
81
|
-
sharpWarningHook,
|
|
82
|
-
registerSharpWarningHook,
|
|
83
|
-
warnSharp,
|
|
84
|
-
parseVersionParts,
|
|
85
|
-
compareVersionParts,
|
|
86
|
-
versionSatisfies,
|
|
87
|
-
sharpPeerRangeCache,
|
|
88
|
-
sharpPeerRange,
|
|
89
|
-
loadSharp,
|
|
90
|
-
} from './sharp-runtime.js'
|
|
91
47
|
|
|
48
|
+
import { loadSharp } from './sharp-runtime.js'
|
|
92
49
|
export const IMAGE_EXTENSIONS = {
|
|
93
50
|
png: 'image/png',
|
|
94
51
|
jpg: 'image/jpeg',
|
|
@@ -109,7 +66,6 @@ export function mediaTypeOf(path) {
|
|
|
109
66
|
*/
|
|
110
67
|
export { depthLimitFor } from './depth-guidance.js'
|
|
111
68
|
|
|
112
|
-
|
|
113
69
|
/**
|
|
114
70
|
* Detect the image format from magic bytes instead of the file extension.
|
|
115
71
|
* Attachments are stored as content-addressed files WITHOUT an extension,
|
|
@@ -1268,29 +1224,52 @@ export function bitmapOfGray(raw, width, height, threshold = 128) {
|
|
|
1268
1224
|
return out
|
|
1269
1225
|
}
|
|
1270
1226
|
|
|
1227
|
+
function visionAbortReason(signal, fallback = 'vision operation aborted') {
|
|
1228
|
+
if (signal?.reason instanceof Error) return signal.reason
|
|
1229
|
+
const error = new Error(fallback)
|
|
1230
|
+
error.name = 'AbortError'
|
|
1231
|
+
error.code = 'ABORT_ERR'
|
|
1232
|
+
return error
|
|
1233
|
+
}
|
|
1234
|
+
|
|
1235
|
+
function throwIfVisionAborted(signal, fallback) {
|
|
1236
|
+
if (signal?.aborted) throw visionAbortReason(signal, fallback)
|
|
1237
|
+
}
|
|
1238
|
+
|
|
1239
|
+
function isVisionAbort(error, signal) {
|
|
1240
|
+
return signal?.aborted === true || error?.name === 'AbortError' || error?.code === 'ABORT_ERR'
|
|
1241
|
+
}
|
|
1242
|
+
|
|
1271
1243
|
/** Vectorize an image buffer into an SVG string via potrace posterization. */
|
|
1272
|
-
export function posterizeSvg(bytes, steps = 4, fillStrategy = 'dominant', timeoutMs = 60000) {
|
|
1244
|
+
export function posterizeSvg(bytes, steps = 4, fillStrategy = 'dominant', timeoutMs = 60000, options = {}) {
|
|
1273
1245
|
// potrace is CPU-bound and runs its computation in long synchronous
|
|
1274
1246
|
// chunks: on the main thread it blocks the whole dsh process (other
|
|
1275
1247
|
// sessions time out) and a setTimeout-based timeout can NEVER fire while
|
|
1276
1248
|
// the loop is blocked. Run it in a worker thread instead — the main loop
|
|
1277
1249
|
// stays responsive, and a timeout hard-terminates the worker.
|
|
1250
|
+
const signal = options?.signal
|
|
1278
1251
|
return new Promise((resolve, reject) => {
|
|
1279
1252
|
let settled = false
|
|
1280
1253
|
let worker
|
|
1254
|
+
let timer
|
|
1255
|
+
let abortListener
|
|
1281
1256
|
const finish = (error, svg) => {
|
|
1282
1257
|
if (settled) return
|
|
1283
1258
|
settled = true
|
|
1284
|
-
clearTimeout(timer)
|
|
1259
|
+
if (timer !== undefined) clearTimeout(timer)
|
|
1260
|
+
if (signal && abortListener) signal.removeEventListener?.('abort', abortListener)
|
|
1285
1261
|
void worker?.terminate()
|
|
1286
1262
|
if (error) reject(error)
|
|
1287
1263
|
else resolve(svg)
|
|
1288
1264
|
}
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1265
|
+
abortListener = () => finish(visionAbortReason(signal, 'potrace aborted'))
|
|
1266
|
+
if (signal?.aborted) {
|
|
1267
|
+
finish(visionAbortReason(signal, 'potrace aborted'))
|
|
1268
|
+
return
|
|
1269
|
+
}
|
|
1270
|
+
signal?.addEventListener?.('abort', abortListener, { once: true })
|
|
1271
|
+
timer = setTimeout(() => {
|
|
1272
|
+
finish(
|
|
1294
1273
|
new Error(
|
|
1295
1274
|
'potrace timed out — the image is too large or too complex; crop it to the target region first',
|
|
1296
1275
|
),
|
|
@@ -1347,23 +1326,30 @@ export function posterizeSvg(bytes, steps = 4, fillStrategy = 'dominant', timeou
|
|
|
1347
1326
|
* @param palette - [{ hex, count, share }] from quantizeColors, ordered by
|
|
1348
1327
|
* share descending.
|
|
1349
1328
|
*/
|
|
1350
|
-
export function posterizeSvgColor(data, info, palette, timeoutMs = 60000) {
|
|
1329
|
+
export function posterizeSvgColor(data, info, palette, timeoutMs = 60000, options = {}) {
|
|
1330
|
+
const signal = options?.signal
|
|
1351
1331
|
return new Promise((resolve, reject) => {
|
|
1352
1332
|
let settled = false
|
|
1353
1333
|
let worker
|
|
1334
|
+
let timer
|
|
1335
|
+
let abortListener
|
|
1354
1336
|
const finish = (error, svg) => {
|
|
1355
1337
|
if (settled) return
|
|
1356
1338
|
settled = true
|
|
1357
|
-
clearTimeout(timer)
|
|
1339
|
+
if (timer !== undefined) clearTimeout(timer)
|
|
1340
|
+
if (signal && abortListener) signal.removeEventListener?.('abort', abortListener)
|
|
1358
1341
|
void worker?.terminate()
|
|
1359
1342
|
if (error) reject(error)
|
|
1360
1343
|
else resolve(svg)
|
|
1361
1344
|
}
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
|
|
1366
|
-
|
|
1345
|
+
abortListener = () => finish(visionAbortReason(signal, 'color trace aborted'))
|
|
1346
|
+
if (signal?.aborted) {
|
|
1347
|
+
finish(visionAbortReason(signal, 'color trace aborted'))
|
|
1348
|
+
return
|
|
1349
|
+
}
|
|
1350
|
+
signal?.addEventListener?.('abort', abortListener, { once: true })
|
|
1351
|
+
timer = setTimeout(() => {
|
|
1352
|
+
finish(
|
|
1367
1353
|
new Error(
|
|
1368
1354
|
'color trace timed out — the image is too large or too complex; crop it to the target region first',
|
|
1369
1355
|
),
|
|
@@ -1473,13 +1459,21 @@ export function resolveVisionOcrEngine(requestedEngine, configuredEngine = 'auto
|
|
|
1473
1459
|
}
|
|
1474
1460
|
|
|
1475
1461
|
/** OCR image bytes with a local tesseract binary (chi_sim+eng) when available. */
|
|
1476
|
-
export async function ocrWithTesseract(bytes, timeoutMs = 60000) {
|
|
1477
|
-
const exec = promisify(execFile)
|
|
1462
|
+
export async function ocrWithTesseract(bytes, timeoutMs = 60000, options = {}) {
|
|
1463
|
+
const exec = typeof options.exec === 'function' ? options.exec : promisify(execFile)
|
|
1464
|
+
const signal = options.signal
|
|
1465
|
+
throwIfVisionAborted(signal, 'tesseract OCR aborted')
|
|
1478
1466
|
const { stdout } = await exec(
|
|
1479
1467
|
'tesseract',
|
|
1480
1468
|
['stdin', 'stdout', '-l', 'chi_sim+eng', '--psm', '6'],
|
|
1481
|
-
{
|
|
1469
|
+
{
|
|
1470
|
+
timeout: Math.min(timeoutMs, 60000),
|
|
1471
|
+
maxBuffer: 32 * 1024 * 1024,
|
|
1472
|
+
input: bytes,
|
|
1473
|
+
...(signal === undefined ? {} : { signal }),
|
|
1474
|
+
},
|
|
1482
1475
|
)
|
|
1476
|
+
throwIfVisionAborted(signal, 'tesseract OCR aborted')
|
|
1483
1477
|
return String(stdout ?? '')
|
|
1484
1478
|
}
|
|
1485
1479
|
|
|
@@ -1543,6 +1537,7 @@ export function selectTesseractOcrCandidate(candidates) {
|
|
|
1543
1537
|
|
|
1544
1538
|
export async function ocrWithTesseractAdaptive(bytes, timeoutMs = 12000, options = {}) {
|
|
1545
1539
|
const exec = typeof options.exec === 'function' ? options.exec : promisify(execFile)
|
|
1540
|
+
const signal = options.signal
|
|
1546
1541
|
const requestedBudget = Number(timeoutMs)
|
|
1547
1542
|
const budgetMs = Number.isFinite(requestedBudget) && requestedBudget > 0
|
|
1548
1543
|
? Math.min(60000, requestedBudget)
|
|
@@ -1551,20 +1546,25 @@ export async function ocrWithTesseractAdaptive(bytes, timeoutMs = 12000, options
|
|
|
1551
1546
|
const attemptedPsms = []
|
|
1552
1547
|
const remaining = () => Math.max(1, budgetMs - (Date.now() - started))
|
|
1553
1548
|
const run = async (psm) => {
|
|
1549
|
+
throwIfVisionAborted(signal, 'adaptive tesseract OCR aborted')
|
|
1554
1550
|
attemptedPsms.push(psm)
|
|
1555
1551
|
const common = ['stdin', 'stdout', '-l', 'chi_sim+eng', '--psm', String(psm)]
|
|
1556
1552
|
const execOptions = () => ({
|
|
1557
1553
|
timeout: remaining(),
|
|
1558
1554
|
maxBuffer: 32 * 1024 * 1024,
|
|
1559
1555
|
input: bytes,
|
|
1556
|
+
...(signal === undefined ? {} : { signal }),
|
|
1560
1557
|
})
|
|
1561
1558
|
const { stdout: plain } = await exec('tesseract', common, execOptions())
|
|
1559
|
+
throwIfVisionAborted(signal, 'adaptive tesseract OCR aborted')
|
|
1562
1560
|
const text = String(plain ?? '').trim()
|
|
1563
1561
|
if (text === '') return assessTesseractOcr('', '', psm)
|
|
1564
1562
|
try {
|
|
1565
1563
|
const { stdout: tsv } = await exec('tesseract', [...common, 'tsv'], execOptions())
|
|
1564
|
+
throwIfVisionAborted(signal, 'adaptive tesseract OCR aborted')
|
|
1566
1565
|
return assessTesseractOcr(text, String(tsv ?? ''), psm)
|
|
1567
|
-
} catch {
|
|
1566
|
+
} catch (error) {
|
|
1567
|
+
if (isVisionAbort(error, signal)) throw error
|
|
1568
1568
|
return { psm, text, words: 0, quality: 0, riskyTokens: [], uncertain: true }
|
|
1569
1569
|
}
|
|
1570
1570
|
}
|
|
@@ -1577,7 +1577,8 @@ export async function ocrWithTesseractAdaptive(bytes, timeoutMs = 12000, options
|
|
|
1577
1577
|
if (Date.now() - started >= budgetMs) break
|
|
1578
1578
|
try {
|
|
1579
1579
|
candidates.push(await run(psm))
|
|
1580
|
-
} catch {
|
|
1580
|
+
} catch (error) {
|
|
1581
|
+
if (isVisionAbort(error, signal)) throw error
|
|
1581
1582
|
// Keep the first successful local OCR result when an optional review pass fails.
|
|
1582
1583
|
}
|
|
1583
1584
|
}
|
|
@@ -2096,6 +2097,7 @@ export async function callLocalBackend(provider, messages, options = {}) {
|
|
|
2096
2097
|
maxTokens,
|
|
2097
2098
|
signal: options.signal,
|
|
2098
2099
|
allowKeyless: true,
|
|
2100
|
+
providerTransport: options.providerTransport,
|
|
2099
2101
|
system: system.join('\n').trim(),
|
|
2100
2102
|
...(rawSessionIdentity(options.sessionId) === undefined
|
|
2101
2103
|
? {}
|
|
@@ -2109,6 +2111,7 @@ export async function callLocalBackend(provider, messages, options = {}) {
|
|
|
2109
2111
|
}
|
|
2110
2112
|
return callOpenAICompatible(provider, messages, {
|
|
2111
2113
|
maxTokens,
|
|
2114
|
+
providerTransport: options.providerTransport,
|
|
2112
2115
|
signal: options.signal,
|
|
2113
2116
|
...(rawSessionIdentity(options.sessionId) === undefined
|
|
2114
2117
|
? {}
|
|
@@ -2291,7 +2294,7 @@ export async function callOpenAICompatible(provider, messages, options = {}) {
|
|
|
2291
2294
|
body: JSON.stringify(body),
|
|
2292
2295
|
...(options.signal === undefined ? {} : { signal: options.signal }),
|
|
2293
2296
|
},
|
|
2294
|
-
{ active: true, providerName: provider.name },
|
|
2297
|
+
{ active: true, providerName: provider.name, providerTransport: options.providerTransport },
|
|
2295
2298
|
)
|
|
2296
2299
|
const response = await request()
|
|
2297
2300
|
if (!response.ok) {
|
|
@@ -2414,61 +2417,6 @@ async function visionAnswer(llm, options) {
|
|
|
2414
2417
|
})
|
|
2415
2418
|
}
|
|
2416
2419
|
|
|
2417
|
-
/** Environment shim for `resolveAdapterOptions`: `{ get: (name) => ({ value }) }`. */
|
|
2418
|
-
export function launchEnvironmentLike(env) {
|
|
2419
|
-
const map = env ?? {}
|
|
2420
|
-
return {
|
|
2421
|
-
get(name) {
|
|
2422
|
-
return Object.prototype.hasOwnProperty.call(map, name) ? { value: map[name] } : undefined
|
|
2423
|
-
},
|
|
2424
|
-
}
|
|
2425
|
-
}
|
|
2426
|
-
|
|
2427
|
-
/**
|
|
2428
|
-
* Legacy-only DeepSeek reconstruction for Host contracts that do not own the
|
|
2429
|
-
* official provider lifecycle. Modern Hosts keep `deepseek-official` Host-owned
|
|
2430
|
-
* because request-local attachment/file/image-access seams cannot be cloned
|
|
2431
|
-
* safely here. Remove this helper when the legacy takeover window is retired.
|
|
2432
|
-
*/
|
|
2433
|
-
export function createNativeDeepSeekAdapter(ctx) {
|
|
2434
|
-
const env = launchEnvironmentLike(
|
|
2435
|
-
typeof process !== 'undefined' && process.env ? process.env : {},
|
|
2436
|
-
)
|
|
2437
|
-
const options = () => {
|
|
2438
|
-
let raw
|
|
2439
|
-
try {
|
|
2440
|
-
const settings = ctx.get('settings')
|
|
2441
|
-
raw = settings && settings.get ? settings.get('llm-deepseek') : undefined
|
|
2442
|
-
} catch {
|
|
2443
|
-
raw = undefined
|
|
2444
|
-
}
|
|
2445
|
-
return resolveAdapterOptions(raw ?? {}, env)
|
|
2446
|
-
}
|
|
2447
|
-
const resolveApiKey = async (connection) => {
|
|
2448
|
-
const ref = connection.apiKeyEnv
|
|
2449
|
-
const credentials = ctx.get('credentials')
|
|
2450
|
-
if (credentials !== undefined) {
|
|
2451
|
-
try {
|
|
2452
|
-
const hit = await credentials.resolve(ref)
|
|
2453
|
-
if (hit && typeof hit.value === 'string' && hit.value.length > 0) return hit.value
|
|
2454
|
-
} catch {
|
|
2455
|
-
/* fall through to the environment */
|
|
2456
|
-
}
|
|
2457
|
-
}
|
|
2458
|
-
const ambient = env.get(ref)
|
|
2459
|
-
if (ambient !== undefined && typeof ambient.value === 'string' && ambient.value.length > 0) {
|
|
2460
|
-
return ambient.value
|
|
2461
|
-
}
|
|
2462
|
-
throw new Error(`vision-router: no API key for the native DeepSeek route (${ref})`)
|
|
2463
|
-
}
|
|
2464
|
-
let userId
|
|
2465
|
-
const resolveUserId = () => {
|
|
2466
|
-
if (userId === undefined) userId = getOrCreateAnonymousUserId()
|
|
2467
|
-
return userId
|
|
2468
|
-
}
|
|
2469
|
-
return new DeepSeekAdapter({ options, resolveApiKey, resolveUserId })
|
|
2470
|
-
}
|
|
2471
|
-
|
|
2472
2420
|
/**
|
|
2473
2421
|
* dsh-vision 并入:本地识别提示模板。
|
|
2474
2422
|
* `plain` = 平铺描述;`structured` = 结构化识别(【初步判断】/【细节】/
|
|
@@ -2595,7 +2543,11 @@ export async function buildInstantLocalMap(ctx, messages, provider, options = {}
|
|
|
2595
2543
|
const text = await callLocalBackend(
|
|
2596
2544
|
currentProvider,
|
|
2597
2545
|
[{ role: 'user', content }],
|
|
2598
|
-
{
|
|
2546
|
+
{
|
|
2547
|
+
maxTokens: currentProvider.maxTokens ?? 2048,
|
|
2548
|
+
signal,
|
|
2549
|
+
providerTransport: options.providerTransport,
|
|
2550
|
+
},
|
|
2599
2551
|
)
|
|
2600
2552
|
return { id, ok: true, text, elapsedMs: Date.now() - startedAt }
|
|
2601
2553
|
} catch (error) {
|
|
@@ -2673,7 +2625,7 @@ export async function buildInstantLocalMap(ctx, messages, provider, options = {}
|
|
|
2673
2625
|
* 标记;provider/style/timeout 也可以是 getter,让设置页保存后下一次 stream
|
|
2674
2626
|
* 立即读取新值,无需重启或重建 adapter。
|
|
2675
2627
|
*/
|
|
2676
|
-
export function createWrapperStreamBody(ctx, { imageMemory, delegateProvider, preserveImageInput, instantLocal, instantLocalStyle, instantLocalTimeoutMs, instantLocalMaxPixels }) {
|
|
2628
|
+
export function createWrapperStreamBody(ctx, { imageMemory, delegateProvider, preserveImageInput, instantLocal, instantLocalStyle, instantLocalTimeoutMs, instantLocalMaxPixels, providerTransport }) {
|
|
2677
2629
|
// issue #103: the reasoning level is a per-session picker choice (the chat
|
|
2678
2630
|
// page's bottom-right selector), but the host can drop reasoningEffort from
|
|
2679
2631
|
// the later steps of a multi-step turn once the twin metadata lacks a
|
|
@@ -2709,6 +2661,7 @@ export function createWrapperStreamBody(ctx, { imageMemory, delegateProvider, pr
|
|
|
2709
2661
|
memory: imageMemory,
|
|
2710
2662
|
timeoutMs: liveValue(instantLocalTimeoutMs),
|
|
2711
2663
|
downscaleMaxPixels: liveValue(instantLocalMaxPixels),
|
|
2664
|
+
providerTransport,
|
|
2712
2665
|
})
|
|
2713
2666
|
: undefined
|
|
2714
2667
|
// Rewrite image blocks ANYWHERE in the model input — including inside
|
|
@@ -2801,7 +2754,7 @@ export function createWrapperStreamBody(ctx, { imageMemory, delegateProvider, pr
|
|
|
2801
2754
|
* route). Any other route name (e.g. the `deepseek-vision` alias) advertises
|
|
2802
2755
|
* no models, so it stays functional but invisible in the picker.
|
|
2803
2756
|
*/
|
|
2804
|
-
export function createStealthAdapter(ctx, { native, imageMemory, pairs, chainRoute, delegateProvider, instantLocal, instantLocalStyle, instantLocalTimeoutMs, instantLocalMaxPixels }) {
|
|
2757
|
+
export function createStealthAdapter(ctx, { native, imageMemory, pairs, chainRoute, delegateProvider, instantLocal, instantLocalStyle, instantLocalTimeoutMs, instantLocalMaxPixels, providerTransport }) {
|
|
2805
2758
|
return {
|
|
2806
2759
|
providerInfo(provider) {
|
|
2807
2760
|
return { id: provider, name: 'DeepSeek' }
|
|
@@ -2822,7 +2775,7 @@ export function createStealthAdapter(ctx, { native, imageMemory, pairs, chainRou
|
|
|
2822
2775
|
const base = await native.resolveModel(provider, model, signal)
|
|
2823
2776
|
return { ...base, provider, inputModalities: ['text', 'image'] }
|
|
2824
2777
|
},
|
|
2825
|
-
...createWrapperStreamBody(ctx, { imageMemory, delegateProvider, instantLocal, instantLocalStyle, instantLocalTimeoutMs, instantLocalMaxPixels }),
|
|
2778
|
+
...createWrapperStreamBody(ctx, { imageMemory, delegateProvider, instantLocal, instantLocalStyle, instantLocalTimeoutMs, instantLocalMaxPixels, providerTransport }),
|
|
2826
2779
|
}
|
|
2827
2780
|
}
|
|
2828
2781
|
|
|
@@ -0,0 +1,262 @@
|
|
|
1
|
+
import { execFile } from 'node:child_process'
|
|
2
|
+
import { existsSync } from 'node:fs'
|
|
3
|
+
import { readFile, unlink } from 'node:fs/promises'
|
|
4
|
+
import { tmpdir } from 'node:os'
|
|
5
|
+
import path from 'node:path'
|
|
6
|
+
import { promisify } from 'node:util'
|
|
7
|
+
|
|
8
|
+
import {
|
|
9
|
+
callLocalBackend,
|
|
10
|
+
localDescribePrompt,
|
|
11
|
+
localProvidersOf,
|
|
12
|
+
toOpenAIContent,
|
|
13
|
+
} from './core-primitives.js'
|
|
14
|
+
import { loadSharp } from './sharp-runtime.js'
|
|
15
|
+
import { captureWindowsDesktop } from './windows-desktop-capture.js'
|
|
16
|
+
import { currentVisionTurnBudgetSignal } from './turn-budget-context.js'
|
|
17
|
+
import { combineSignals } from './vision-resilience.js'
|
|
18
|
+
|
|
19
|
+
function abortReason(signal, fallback = 'vision_screenshot aborted') {
|
|
20
|
+
if (signal?.reason instanceof Error) return signal.reason
|
|
21
|
+
const error = new Error(fallback)
|
|
22
|
+
error.name = 'AbortError'
|
|
23
|
+
error.code = 'ABORT_ERR'
|
|
24
|
+
return error
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
function throwIfAborted(signal) {
|
|
28
|
+
if (signal?.aborted) throw abortReason(signal)
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
function isAbortFailure(error) {
|
|
32
|
+
return error?.name === 'AbortError' || error?.code === 'ABORT_ERR'
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export async function captureDesktopPng(target, timeoutMs, signal, options = {}) {
|
|
36
|
+
const platform = options.platform ?? process.platform
|
|
37
|
+
const run = options.execFileAsync ?? promisify(execFile)
|
|
38
|
+
const captureWindows = options.captureWindowsDesktop ?? captureWindowsDesktop
|
|
39
|
+
throwIfAborted(signal)
|
|
40
|
+
if (platform === 'win32') {
|
|
41
|
+
// Own DPI-aware capture at the implementation boundary rather than
|
|
42
|
+
// depending on a process-wide execFile argument rewrite.
|
|
43
|
+
await captureWindows(target, { timeoutMs, signal })
|
|
44
|
+
throwIfAborted(signal)
|
|
45
|
+
return platform
|
|
46
|
+
}
|
|
47
|
+
if (platform === 'darwin') {
|
|
48
|
+
// Without -m, screencapture writes one file per display while the tool owns
|
|
49
|
+
// exactly one artifact path, so request the main display explicitly.
|
|
50
|
+
await run('screencapture', ['-x', '-m', target], {
|
|
51
|
+
timeout: timeoutMs,
|
|
52
|
+
windowsHide: true,
|
|
53
|
+
...(signal === undefined ? {} : { signal }),
|
|
54
|
+
})
|
|
55
|
+
throwIfAborted(signal)
|
|
56
|
+
return platform
|
|
57
|
+
}
|
|
58
|
+
try {
|
|
59
|
+
await run('import', ['-window', 'root', target], {
|
|
60
|
+
timeout: timeoutMs,
|
|
61
|
+
...(signal === undefined ? {} : { signal }),
|
|
62
|
+
})
|
|
63
|
+
} catch (error) {
|
|
64
|
+
if (signal?.aborted || isAbortFailure(error)) throw error
|
|
65
|
+
await run('scrot', [target], {
|
|
66
|
+
timeout: timeoutMs,
|
|
67
|
+
...(signal === undefined ? {} : { signal }),
|
|
68
|
+
})
|
|
69
|
+
}
|
|
70
|
+
throwIfAborted(signal)
|
|
71
|
+
return platform
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
async function downscaleForIdentification(data) {
|
|
75
|
+
try {
|
|
76
|
+
const sharp = await loadSharp()
|
|
77
|
+
if (!sharp) return data
|
|
78
|
+
const downscaled = await sharp(data, { failOn: 'none' })
|
|
79
|
+
.resize({ width: 1280, height: 1280, fit: 'inside', withoutEnlargement: true })
|
|
80
|
+
.png()
|
|
81
|
+
.toBuffer()
|
|
82
|
+
return downscaled.length > 0 && downscaled.length < data.length ? downscaled : data
|
|
83
|
+
} catch {
|
|
84
|
+
return data
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
async function identifyDesktopCapture(data, {
|
|
89
|
+
current,
|
|
90
|
+
timeoutMs,
|
|
91
|
+
instantLocalStyle,
|
|
92
|
+
providerTransport,
|
|
93
|
+
signal,
|
|
94
|
+
} = {}) {
|
|
95
|
+
throwIfAborted(signal)
|
|
96
|
+
const locals = localProvidersOf(current())
|
|
97
|
+
if (locals.length === 0) {
|
|
98
|
+
return {
|
|
99
|
+
identifyError: 'no local vision backend enabled (localOllama / localLmStudio); enable one to use identify',
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
const startedAt = Date.now()
|
|
104
|
+
const identifyBytes = await downscaleForIdentification(data)
|
|
105
|
+
throwIfAborted(signal)
|
|
106
|
+
const result = {}
|
|
107
|
+
if (identifyBytes !== data) {
|
|
108
|
+
result.identifyDownscaled = {
|
|
109
|
+
originalBytes: data.length,
|
|
110
|
+
sentBytes: identifyBytes.length,
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
const content = toOpenAIContent(
|
|
115
|
+
[{ type: 'image', attachment: { mediaType: 'image/png', data: identifyBytes } }],
|
|
116
|
+
() => identifyBytes,
|
|
117
|
+
)
|
|
118
|
+
content.push({ type: 'text', text: localDescribePrompt(instantLocalStyle()) })
|
|
119
|
+
|
|
120
|
+
const deadlineAt = Date.now() + timeoutMs()
|
|
121
|
+
const errors = []
|
|
122
|
+
for (let index = 0; index < locals.length; index++) {
|
|
123
|
+
throwIfAborted(signal)
|
|
124
|
+
const local = locals[index]
|
|
125
|
+
const remainingMs = deadlineAt - Date.now()
|
|
126
|
+
if (remainingMs <= 0) break
|
|
127
|
+
// Reserve a fair share for later local backends. A connected but hung
|
|
128
|
+
// first backend must not consume the entire fallback budget.
|
|
129
|
+
const roundBudgetMs = Math.max(1, Math.floor(remainingMs / (locals.length - index)))
|
|
130
|
+
const controller = new AbortController()
|
|
131
|
+
const roundSignal = signal
|
|
132
|
+
? AbortSignal.any([signal, controller.signal])
|
|
133
|
+
: controller.signal
|
|
134
|
+
const timer = setTimeout(() => controller.abort(), roundBudgetMs)
|
|
135
|
+
try {
|
|
136
|
+
const identified = await callLocalBackend(
|
|
137
|
+
local,
|
|
138
|
+
[{ role: 'user', content }],
|
|
139
|
+
{
|
|
140
|
+
maxTokens: local.maxTokens ?? 2048,
|
|
141
|
+
signal: roundSignal,
|
|
142
|
+
providerTransport,
|
|
143
|
+
},
|
|
144
|
+
)
|
|
145
|
+
if (typeof identified === 'string' && identified.trim() !== '') {
|
|
146
|
+
result.identified = identified.trim()
|
|
147
|
+
result.identifiedBy = local.name
|
|
148
|
+
result.elapsedSec = Math.max(1, Math.round((Date.now() - startedAt) / 1000))
|
|
149
|
+
return result
|
|
150
|
+
}
|
|
151
|
+
errors.push(`${local.name}: empty response`)
|
|
152
|
+
} catch (error) {
|
|
153
|
+
if (signal?.aborted) throw abortReason(signal)
|
|
154
|
+
errors.push(`${local.name}: ${error && error.message ? error.message : String(error)}`)
|
|
155
|
+
} finally {
|
|
156
|
+
clearTimeout(timer)
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
result.identifyError = errors.length > 0
|
|
161
|
+
? errors.join('; ').slice(0, 1000)
|
|
162
|
+
: 'local vision identification timed out before a backend could respond'
|
|
163
|
+
return result
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
/**
|
|
167
|
+
* Build the privacy-sensitive desktop screenshot tool definition.
|
|
168
|
+
*
|
|
169
|
+
* Tool registration/exposure remains owned by Core + LocalVisionStabilizer;
|
|
170
|
+
* this module owns only the platform capture and optional local-recognition
|
|
171
|
+
* implementation behind that existing permission boundary.
|
|
172
|
+
*/
|
|
173
|
+
export function createDesktopScreenshotTool({
|
|
174
|
+
current,
|
|
175
|
+
timeoutMs,
|
|
176
|
+
saveArtifact,
|
|
177
|
+
stringOutput,
|
|
178
|
+
instantLocalStyle,
|
|
179
|
+
providerTransport,
|
|
180
|
+
captureDesktop = captureDesktopPng,
|
|
181
|
+
} = {}) {
|
|
182
|
+
if (
|
|
183
|
+
typeof current !== 'function' ||
|
|
184
|
+
typeof timeoutMs !== 'function' ||
|
|
185
|
+
typeof saveArtifact !== 'function' ||
|
|
186
|
+
typeof captureDesktop !== 'function'
|
|
187
|
+
) {
|
|
188
|
+
throw new TypeError('desktop screenshot tool requires live config, timeout, artifact and capture callbacks')
|
|
189
|
+
}
|
|
190
|
+
if (typeof instantLocalStyle !== 'function') {
|
|
191
|
+
throw new TypeError('desktop screenshot tool requires a live local describe style')
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
return {
|
|
195
|
+
name: 'vision_screenshot',
|
|
196
|
+
description:
|
|
197
|
+
'Capture the user\'s desktop screen as a PNG artifact (the virtual screen on Windows; the main display on macOS; the root display on Linux). ' +
|
|
198
|
+
'Windows: per-monitor-DPI-aware PowerShell capture; macOS: screencapture; Linux: ImageMagick import (falls back to scrot; either command must be installed). ' +
|
|
199
|
+
'This privacy-sensitive tool is disabled by default and works only after the user explicitly enables Desktop screenshot in Vision Router settings. ' +
|
|
200
|
+
'Use it when you need to see what is on the user\'s screen right now — e.g. their current GUI, an app, or a page outside this browser. ' +
|
|
201
|
+
'Optional identify=true also runs local recognition on the capture using the enabled local backends (Ollama, then LM Studio) and returns the description alongside the path.',
|
|
202
|
+
parameters: {
|
|
203
|
+
type: 'object',
|
|
204
|
+
properties: {
|
|
205
|
+
identify: {
|
|
206
|
+
type: 'boolean',
|
|
207
|
+
description:
|
|
208
|
+
'Also recognize the captured screen with enabled local vision backends (Ollama, then LM Studio) and return the description text with the path. Default false.',
|
|
209
|
+
},
|
|
210
|
+
},
|
|
211
|
+
additionalProperties: false,
|
|
212
|
+
},
|
|
213
|
+
output: stringOutput,
|
|
214
|
+
async execute(args, exec) {
|
|
215
|
+
if (current().desktopScreenshot !== true) {
|
|
216
|
+
throw new Error(
|
|
217
|
+
'vision_screenshot is disabled; enable Desktop screenshot explicitly in Vision Router settings before use',
|
|
218
|
+
)
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
const tmp = path.join(
|
|
222
|
+
tmpdir(),
|
|
223
|
+
`vision-screenshot-${Date.now()}-${Math.floor(Math.random() * 1e9)}.png`,
|
|
224
|
+
)
|
|
225
|
+
try {
|
|
226
|
+
const signal = combineSignals(currentVisionTurnBudgetSignal(), exec?.signal)
|
|
227
|
+
throwIfAborted(signal)
|
|
228
|
+
const platform = await captureDesktop(tmp, timeoutMs(), signal)
|
|
229
|
+
throwIfAborted(signal)
|
|
230
|
+
if (!existsSync(tmp)) {
|
|
231
|
+
throw new Error(
|
|
232
|
+
`vision_screenshot: no output produced on ${platform} (is a screen available?)`,
|
|
233
|
+
)
|
|
234
|
+
}
|
|
235
|
+
const data = await readFile(tmp, signal === undefined ? undefined : { signal })
|
|
236
|
+
throwIfAborted(signal)
|
|
237
|
+
let identification
|
|
238
|
+
if (args.identify === true) {
|
|
239
|
+
identification = await identifyDesktopCapture(data, {
|
|
240
|
+
current,
|
|
241
|
+
timeoutMs,
|
|
242
|
+
instantLocalStyle,
|
|
243
|
+
providerTransport,
|
|
244
|
+
signal,
|
|
245
|
+
})
|
|
246
|
+
}
|
|
247
|
+
// Artifact publication is the final commit point. If optional local
|
|
248
|
+
// identification is cancelled, no successful screenshot result should
|
|
249
|
+
// already have escaped into the managed artifact store.
|
|
250
|
+
throwIfAborted(signal)
|
|
251
|
+
const target = await saveArtifact(exec, `screenshot-${Date.now()}.png`, data)
|
|
252
|
+
return JSON.stringify({
|
|
253
|
+
path: target,
|
|
254
|
+
bytes: data.length,
|
|
255
|
+
...(identification ?? {}),
|
|
256
|
+
})
|
|
257
|
+
} finally {
|
|
258
|
+
try { await unlink(tmp) } catch { /* best effort cleanup */ }
|
|
259
|
+
}
|
|
260
|
+
},
|
|
261
|
+
}
|
|
262
|
+
}
|