dsh-vision-router 2.2.7 → 2.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. package/README.md +8 -8
  2. package/README.zh.md +8 -8
  3. package/docs/architecture/2.3.0-compat-retirement.md +147 -0
  4. package/docs/architecture/2.3.0-execution-plan.md +410 -0
  5. package/docs/architecture/2.3.0-final-audit.md +279 -0
  6. package/docs/architecture/2.3.0-reopened-plan.md +720 -0
  7. package/docs/architecture/compat-inventory.md +26 -34
  8. package/docs/architecture/dsh-020-rc2-adaptation.md +23 -0
  9. package/docs/architecture/dsh-compatibility-matrix.md +65 -54
  10. package/docs/architecture/dsh-support-window.md +29 -21
  11. package/docs/architecture/final-closure-audit.md +2 -0
  12. package/docs/architecture/host-first-proxy-convergence.md +1 -1
  13. package/docs/releases/v2.2.8.md +25 -0
  14. package/docs/releases/v2.3.0.md +53 -0
  15. package/entry.js +8 -56
  16. package/index.js +161 -691
  17. package/lib/catalog-corrections.js +1 -2
  18. package/lib/client-host-compat-prelude.js +7 -12
  19. package/lib/client-maintenance-actions.js +145 -0
  20. package/lib/client-presentation-boundary-main.js +82 -32
  21. package/lib/client-presentation-boundary.js +60 -40
  22. package/lib/client-script-carrier.js +90 -0
  23. package/lib/client.js +248 -146
  24. package/lib/core-apply-lifecycle.js +47 -0
  25. package/lib/core-primitives.js +88 -135
  26. package/lib/desktop-screenshot-tool.js +262 -0
  27. package/lib/dsh-contract-compat.js +11 -26
  28. package/lib/dsh-support-window.js +4 -4
  29. package/lib/guide-vision-toggle-highlight.js +7 -10
  30. package/lib/http-compat.js +1 -2
  31. package/lib/live-model-client-prelude.js +7 -14
  32. package/lib/local-remote-settings-permission.js +41 -16
  33. package/lib/maintenance-routes.js +136 -0
  34. package/lib/public-config.js +57 -0
  35. package/lib/public-entry.js +2 -7
  36. package/lib/remote-settings-bridge.js +11 -0
  37. package/lib/remote-settings-risk-client-boundary.js +155 -0
  38. package/lib/remote-settings-risk-confirmation.js +17 -105
  39. package/lib/runtime-composition.js +105 -30
  40. package/lib/runtime-i18n-boundary.js +10 -11
  41. package/lib/session-message-source-compat.js +7 -3
  42. package/lib/session-vision-index.js +154 -9
  43. package/lib/session-vision-mode-boundary.js +43 -8
  44. package/lib/settings-client-loader-lifecycle.js +129 -0
  45. package/lib/settings-factory-lifecycle.js +25 -14
  46. package/lib/settings-ia-client-prelude.js +11 -11
  47. package/lib/settings-limit-client-prelude.js +7 -9
  48. package/lib/sharp-runtime.js +0 -7
  49. package/lib/strict-live-model-client-prelude.js +7 -10
  50. package/lib/structured-flow-hardening.js +3 -2
  51. package/lib/v2-settings-ia-integration.js +21 -6
  52. package/lib/vision-backend-runtime-policy.js +3 -2
  53. package/lib/vision-background-benchmark.js +3 -5
  54. package/lib/vision-capability-benchmark-client.js +7 -10
  55. package/lib/vision-capability-benchmark-service.js +1 -0
  56. package/lib/vision-diagnostics-routes.js +216 -0
  57. package/lib/vision-evidence-guidance.js +3 -1
  58. package/lib/vision-exact-check-client.js +11 -6
  59. package/lib/vision-model-visibility-boundary-main.js +7 -11
  60. package/lib/vision-model-visibility-boundary.js +7 -12
  61. package/lib/vision-provider-transport.js +0 -29
  62. package/lib/vision-routing-settings-prelude.js +7 -11
  63. package/lib/vision-toggle-root-hardening.js +50 -8
  64. package/lib/vision-tool-runtime-boundary.js +5 -1
  65. package/lib/vision-turn-budget-client-prelude.js +7 -10
  66. package/lib/web/remote-settings-client.js +9 -14
  67. package/package.json +23 -13
  68. package/lib/android-attachment-compat.js +0 -244
  69. package/lib/settings-client-rc8-lifecycle.js +0 -402
@@ -1,94 +1,51 @@
1
- import z from '@deepseek-ai/schemastery'
2
- import { mkdir, writeFile, readFile, unlink } from 'node:fs/promises'
1
+
2
+
3
3
  import path from 'node:path'
4
- import { tmpdir } from 'node:os'
5
- import { DeepSeekAdapter, resolveAdapterOptions } from '@deepseek-ai/dsh-llm-deepseek'
6
- import { getOrCreateAnonymousUserId } from '@deepseek-ai/dsh-anonymous-user-id'
4
+
7
5
  import { existsSync } from 'node:fs'
8
6
  import { execFile } from 'node:child_process'
9
7
  import { Worker } from 'node:worker_threads'
10
8
  import { createRequire } from 'node:module'
11
9
  import { pathToFileURL } from 'node:url'
12
10
  import { promisify } from 'node:util'
13
- import { appendPromptToImageOnlyMessage, fetchWithOpenAICompatibility } from './http-compat.js'
11
+ import { fetchWithOpenAICompatibility } from './http-compat.js'
14
12
  import {
15
13
  directSessionAffinityHeaders,
16
- isOfficialOpenCodeGoUrl,
17
- openCodeSessionAffinityHeaderForUrl,
18
14
  rawSessionIdentity,
19
- sessionIdentityOf,
20
15
  } from './session-affinity.js'
21
- import { runWithVisionSessionAffinity, streamWithVisionSessionAffinity } from './session-affinity-runtime.js'
16
+ import { runWithVisionSessionAffinity } from './session-affinity-runtime.js'
22
17
  import {
23
- routingCorrectionFor,
24
- toAnthropicMessages,
25
18
  callAnthropicCompatible,
26
19
  anthropicMediaType,
27
20
  } from './catalog-corrections.js'
28
- import { createCachedUpdateChecker } from './update-check.js'
29
- import { probeLocalBackends } from './local-connection-probe.js'
30
- import { detectDshSelfUpdatePlan, runDshPluginUpdate } from './self-update.js'
21
+
31
22
  import {
32
- classifyVisionFailure,
33
- createDeadline,
34
23
  combineSignals,
35
- createVisionCircuitBreaker,
36
- createVisionTurnMemory,
37
- buildVisionFailure,
38
- ensureSentencePunctuation,
39
- resultCodeForKinds,
40
24
  qwenKeyEndpointHint,
41
25
  kindForHttpStatus,
42
- VISION_FAILURE_KINDS,
43
- VISION_RESULT_CODES,
44
26
  } from './vision-resilience.js'
45
- import { currentVisionExecutionOrder } from './vision-execution-order.js'
46
- import { applyVisionExecutionOrder } from './vision-execution-order-apply.js'
47
- import { createHash, randomBytes } from 'node:crypto'
48
- import {
49
- normalizeStructuredBootstrapResult,
50
- structuredBootstrapMemory,
51
- structuredBootstrapQuestion,
52
- } from './structured-bootstrap.js'
53
- import { planMixedBranches, renderMixedGuidance } from './mixed-router.js'
54
- import { renderDepthGuidance } from './depth-guidance.js'
55
- import { assertNoRepetitionLoop } from './repetition-guard.js'
56
- import { compareRgbaStreams } from './pixel-diff-stream.js'
27
+
28
+ import { createHash } from 'node:crypto'
29
+
57
30
  import {
58
- boundedOcrTiles,
59
31
  defaultImageResourceGovernor,
60
32
  estimateImageOperationBytes,
61
33
  scaleBox,
62
34
  scaledDimensions,
63
35
  } from './image-resource-governor.js'
64
- import { createSessionVisionIndex } from './session-vision-index.js'
65
- import { createSessionVisionStateStore } from './session-vision-state.js'
36
+
66
37
  import {
67
38
  ERROR_RESPONSE_MAX_BYTES,
68
- METADATA_RESPONSE_MAX_BYTES,
69
39
  MODEL_RESPONSE_MAX_BYTES,
70
40
  readResponseJsonBounded,
71
41
  readResponseTextBounded,
72
42
  } from './http-body-limit.js'
73
- import { writeArtifactFile } from './artifact-boundary.js'
43
+
74
44
  import { stripTrailingSlashes } from './string-normalization.js'
75
45
  import { isOffloadedImageBlock, offloadedImagePlaceholder } from './image-offload-compat.js'
76
46
  import { proxyHostMatchesAny } from './proxy-routing.js'
77
- import { parseVersionComparator } from './version-range.js'
78
- import { captureWindowsDesktop } from './windows-desktop-capture.js'
79
- import {
80
- sharpPromise,
81
- sharpWarningHook,
82
- registerSharpWarningHook,
83
- warnSharp,
84
- parseVersionParts,
85
- compareVersionParts,
86
- versionSatisfies,
87
- sharpPeerRangeCache,
88
- sharpPeerRange,
89
- loadSharp,
90
- } from './sharp-runtime.js'
91
47
 
48
+ import { loadSharp } from './sharp-runtime.js'
92
49
  export const IMAGE_EXTENSIONS = {
93
50
  png: 'image/png',
94
51
  jpg: 'image/jpeg',
@@ -109,7 +66,6 @@ export function mediaTypeOf(path) {
109
66
  */
110
67
  export { depthLimitFor } from './depth-guidance.js'
111
68
 
112
-
113
69
  /**
114
70
  * Detect the image format from magic bytes instead of the file extension.
115
71
  * Attachments are stored as content-addressed files WITHOUT an extension,
@@ -1268,29 +1224,52 @@ export function bitmapOfGray(raw, width, height, threshold = 128) {
1268
1224
  return out
1269
1225
  }
1270
1226
 
1227
+ function visionAbortReason(signal, fallback = 'vision operation aborted') {
1228
+ if (signal?.reason instanceof Error) return signal.reason
1229
+ const error = new Error(fallback)
1230
+ error.name = 'AbortError'
1231
+ error.code = 'ABORT_ERR'
1232
+ return error
1233
+ }
1234
+
1235
+ function throwIfVisionAborted(signal, fallback) {
1236
+ if (signal?.aborted) throw visionAbortReason(signal, fallback)
1237
+ }
1238
+
1239
+ function isVisionAbort(error, signal) {
1240
+ return signal?.aborted === true || error?.name === 'AbortError' || error?.code === 'ABORT_ERR'
1241
+ }
1242
+
1271
1243
  /** Vectorize an image buffer into an SVG string via potrace posterization. */
1272
- export function posterizeSvg(bytes, steps = 4, fillStrategy = 'dominant', timeoutMs = 60000) {
1244
+ export function posterizeSvg(bytes, steps = 4, fillStrategy = 'dominant', timeoutMs = 60000, options = {}) {
1273
1245
  // potrace is CPU-bound and runs its computation in long synchronous
1274
1246
  // chunks: on the main thread it blocks the whole dsh process (other
1275
1247
  // sessions time out) and a setTimeout-based timeout can NEVER fire while
1276
1248
  // the loop is blocked. Run it in a worker thread instead — the main loop
1277
1249
  // stays responsive, and a timeout hard-terminates the worker.
1250
+ const signal = options?.signal
1278
1251
  return new Promise((resolve, reject) => {
1279
1252
  let settled = false
1280
1253
  let worker
1254
+ let timer
1255
+ let abortListener
1281
1256
  const finish = (error, svg) => {
1282
1257
  if (settled) return
1283
1258
  settled = true
1284
- clearTimeout(timer)
1259
+ if (timer !== undefined) clearTimeout(timer)
1260
+ if (signal && abortListener) signal.removeEventListener?.('abort', abortListener)
1285
1261
  void worker?.terminate()
1286
1262
  if (error) reject(error)
1287
1263
  else resolve(svg)
1288
1264
  }
1289
- const timer = setTimeout(() => {
1290
- if (settled) return
1291
- settled = true
1292
- void worker?.terminate()
1293
- reject(
1265
+ abortListener = () => finish(visionAbortReason(signal, 'potrace aborted'))
1266
+ if (signal?.aborted) {
1267
+ finish(visionAbortReason(signal, 'potrace aborted'))
1268
+ return
1269
+ }
1270
+ signal?.addEventListener?.('abort', abortListener, { once: true })
1271
+ timer = setTimeout(() => {
1272
+ finish(
1294
1273
  new Error(
1295
1274
  'potrace timed out — the image is too large or too complex; crop it to the target region first',
1296
1275
  ),
@@ -1347,23 +1326,30 @@ export function posterizeSvg(bytes, steps = 4, fillStrategy = 'dominant', timeou
1347
1326
  * @param palette - [{ hex, count, share }] from quantizeColors, ordered by
1348
1327
  * share descending.
1349
1328
  */
1350
- export function posterizeSvgColor(data, info, palette, timeoutMs = 60000) {
1329
+ export function posterizeSvgColor(data, info, palette, timeoutMs = 60000, options = {}) {
1330
+ const signal = options?.signal
1351
1331
  return new Promise((resolve, reject) => {
1352
1332
  let settled = false
1353
1333
  let worker
1334
+ let timer
1335
+ let abortListener
1354
1336
  const finish = (error, svg) => {
1355
1337
  if (settled) return
1356
1338
  settled = true
1357
- clearTimeout(timer)
1339
+ if (timer !== undefined) clearTimeout(timer)
1340
+ if (signal && abortListener) signal.removeEventListener?.('abort', abortListener)
1358
1341
  void worker?.terminate()
1359
1342
  if (error) reject(error)
1360
1343
  else resolve(svg)
1361
1344
  }
1362
- const timer = setTimeout(() => {
1363
- if (settled) return
1364
- settled = true
1365
- void worker?.terminate()
1366
- reject(
1345
+ abortListener = () => finish(visionAbortReason(signal, 'color trace aborted'))
1346
+ if (signal?.aborted) {
1347
+ finish(visionAbortReason(signal, 'color trace aborted'))
1348
+ return
1349
+ }
1350
+ signal?.addEventListener?.('abort', abortListener, { once: true })
1351
+ timer = setTimeout(() => {
1352
+ finish(
1367
1353
  new Error(
1368
1354
  'color trace timed out — the image is too large or too complex; crop it to the target region first',
1369
1355
  ),
@@ -1473,13 +1459,21 @@ export function resolveVisionOcrEngine(requestedEngine, configuredEngine = 'auto
1473
1459
  }
1474
1460
 
1475
1461
  /** OCR image bytes with a local tesseract binary (chi_sim+eng) when available. */
1476
- export async function ocrWithTesseract(bytes, timeoutMs = 60000) {
1477
- const exec = promisify(execFile)
1462
+ export async function ocrWithTesseract(bytes, timeoutMs = 60000, options = {}) {
1463
+ const exec = typeof options.exec === 'function' ? options.exec : promisify(execFile)
1464
+ const signal = options.signal
1465
+ throwIfVisionAborted(signal, 'tesseract OCR aborted')
1478
1466
  const { stdout } = await exec(
1479
1467
  'tesseract',
1480
1468
  ['stdin', 'stdout', '-l', 'chi_sim+eng', '--psm', '6'],
1481
- { timeout: Math.min(timeoutMs, 60000), maxBuffer: 32 * 1024 * 1024, input: bytes },
1469
+ {
1470
+ timeout: Math.min(timeoutMs, 60000),
1471
+ maxBuffer: 32 * 1024 * 1024,
1472
+ input: bytes,
1473
+ ...(signal === undefined ? {} : { signal }),
1474
+ },
1482
1475
  )
1476
+ throwIfVisionAborted(signal, 'tesseract OCR aborted')
1483
1477
  return String(stdout ?? '')
1484
1478
  }
1485
1479
 
@@ -1543,6 +1537,7 @@ export function selectTesseractOcrCandidate(candidates) {
1543
1537
 
1544
1538
  export async function ocrWithTesseractAdaptive(bytes, timeoutMs = 12000, options = {}) {
1545
1539
  const exec = typeof options.exec === 'function' ? options.exec : promisify(execFile)
1540
+ const signal = options.signal
1546
1541
  const requestedBudget = Number(timeoutMs)
1547
1542
  const budgetMs = Number.isFinite(requestedBudget) && requestedBudget > 0
1548
1543
  ? Math.min(60000, requestedBudget)
@@ -1551,20 +1546,25 @@ export async function ocrWithTesseractAdaptive(bytes, timeoutMs = 12000, options
1551
1546
  const attemptedPsms = []
1552
1547
  const remaining = () => Math.max(1, budgetMs - (Date.now() - started))
1553
1548
  const run = async (psm) => {
1549
+ throwIfVisionAborted(signal, 'adaptive tesseract OCR aborted')
1554
1550
  attemptedPsms.push(psm)
1555
1551
  const common = ['stdin', 'stdout', '-l', 'chi_sim+eng', '--psm', String(psm)]
1556
1552
  const execOptions = () => ({
1557
1553
  timeout: remaining(),
1558
1554
  maxBuffer: 32 * 1024 * 1024,
1559
1555
  input: bytes,
1556
+ ...(signal === undefined ? {} : { signal }),
1560
1557
  })
1561
1558
  const { stdout: plain } = await exec('tesseract', common, execOptions())
1559
+ throwIfVisionAborted(signal, 'adaptive tesseract OCR aborted')
1562
1560
  const text = String(plain ?? '').trim()
1563
1561
  if (text === '') return assessTesseractOcr('', '', psm)
1564
1562
  try {
1565
1563
  const { stdout: tsv } = await exec('tesseract', [...common, 'tsv'], execOptions())
1564
+ throwIfVisionAborted(signal, 'adaptive tesseract OCR aborted')
1566
1565
  return assessTesseractOcr(text, String(tsv ?? ''), psm)
1567
- } catch {
1566
+ } catch (error) {
1567
+ if (isVisionAbort(error, signal)) throw error
1568
1568
  return { psm, text, words: 0, quality: 0, riskyTokens: [], uncertain: true }
1569
1569
  }
1570
1570
  }
@@ -1577,7 +1577,8 @@ export async function ocrWithTesseractAdaptive(bytes, timeoutMs = 12000, options
1577
1577
  if (Date.now() - started >= budgetMs) break
1578
1578
  try {
1579
1579
  candidates.push(await run(psm))
1580
- } catch {
1580
+ } catch (error) {
1581
+ if (isVisionAbort(error, signal)) throw error
1581
1582
  // Keep the first successful local OCR result when an optional review pass fails.
1582
1583
  }
1583
1584
  }
@@ -2096,6 +2097,7 @@ export async function callLocalBackend(provider, messages, options = {}) {
2096
2097
  maxTokens,
2097
2098
  signal: options.signal,
2098
2099
  allowKeyless: true,
2100
+ providerTransport: options.providerTransport,
2099
2101
  system: system.join('\n').trim(),
2100
2102
  ...(rawSessionIdentity(options.sessionId) === undefined
2101
2103
  ? {}
@@ -2109,6 +2111,7 @@ export async function callLocalBackend(provider, messages, options = {}) {
2109
2111
  }
2110
2112
  return callOpenAICompatible(provider, messages, {
2111
2113
  maxTokens,
2114
+ providerTransport: options.providerTransport,
2112
2115
  signal: options.signal,
2113
2116
  ...(rawSessionIdentity(options.sessionId) === undefined
2114
2117
  ? {}
@@ -2291,7 +2294,7 @@ export async function callOpenAICompatible(provider, messages, options = {}) {
2291
2294
  body: JSON.stringify(body),
2292
2295
  ...(options.signal === undefined ? {} : { signal: options.signal }),
2293
2296
  },
2294
- { active: true, providerName: provider.name },
2297
+ { active: true, providerName: provider.name, providerTransport: options.providerTransport },
2295
2298
  )
2296
2299
  const response = await request()
2297
2300
  if (!response.ok) {
@@ -2414,61 +2417,6 @@ async function visionAnswer(llm, options) {
2414
2417
  })
2415
2418
  }
2416
2419
 
2417
- /** Environment shim for `resolveAdapterOptions`: `{ get: (name) => ({ value }) }`. */
2418
- export function launchEnvironmentLike(env) {
2419
- const map = env ?? {}
2420
- return {
2421
- get(name) {
2422
- return Object.prototype.hasOwnProperty.call(map, name) ? { value: map[name] } : undefined
2423
- },
2424
- }
2425
- }
2426
-
2427
- /**
2428
- * Legacy-only DeepSeek reconstruction for Host contracts that do not own the
2429
- * official provider lifecycle. Modern Hosts keep `deepseek-official` Host-owned
2430
- * because request-local attachment/file/image-access seams cannot be cloned
2431
- * safely here. Remove this helper when the legacy takeover window is retired.
2432
- */
2433
- export function createNativeDeepSeekAdapter(ctx) {
2434
- const env = launchEnvironmentLike(
2435
- typeof process !== 'undefined' && process.env ? process.env : {},
2436
- )
2437
- const options = () => {
2438
- let raw
2439
- try {
2440
- const settings = ctx.get('settings')
2441
- raw = settings && settings.get ? settings.get('llm-deepseek') : undefined
2442
- } catch {
2443
- raw = undefined
2444
- }
2445
- return resolveAdapterOptions(raw ?? {}, env)
2446
- }
2447
- const resolveApiKey = async (connection) => {
2448
- const ref = connection.apiKeyEnv
2449
- const credentials = ctx.get('credentials')
2450
- if (credentials !== undefined) {
2451
- try {
2452
- const hit = await credentials.resolve(ref)
2453
- if (hit && typeof hit.value === 'string' && hit.value.length > 0) return hit.value
2454
- } catch {
2455
- /* fall through to the environment */
2456
- }
2457
- }
2458
- const ambient = env.get(ref)
2459
- if (ambient !== undefined && typeof ambient.value === 'string' && ambient.value.length > 0) {
2460
- return ambient.value
2461
- }
2462
- throw new Error(`vision-router: no API key for the native DeepSeek route (${ref})`)
2463
- }
2464
- let userId
2465
- const resolveUserId = () => {
2466
- if (userId === undefined) userId = getOrCreateAnonymousUserId()
2467
- return userId
2468
- }
2469
- return new DeepSeekAdapter({ options, resolveApiKey, resolveUserId })
2470
- }
2471
-
2472
2420
  /**
2473
2421
  * dsh-vision 并入:本地识别提示模板。
2474
2422
  * `plain` = 平铺描述;`structured` = 结构化识别(【初步判断】/【细节】/
@@ -2595,7 +2543,11 @@ export async function buildInstantLocalMap(ctx, messages, provider, options = {}
2595
2543
  const text = await callLocalBackend(
2596
2544
  currentProvider,
2597
2545
  [{ role: 'user', content }],
2598
- { maxTokens: currentProvider.maxTokens ?? 2048, signal },
2546
+ {
2547
+ maxTokens: currentProvider.maxTokens ?? 2048,
2548
+ signal,
2549
+ providerTransport: options.providerTransport,
2550
+ },
2599
2551
  )
2600
2552
  return { id, ok: true, text, elapsedMs: Date.now() - startedAt }
2601
2553
  } catch (error) {
@@ -2673,7 +2625,7 @@ export async function buildInstantLocalMap(ctx, messages, provider, options = {}
2673
2625
  * 标记;provider/style/timeout 也可以是 getter,让设置页保存后下一次 stream
2674
2626
  * 立即读取新值,无需重启或重建 adapter。
2675
2627
  */
2676
- export function createWrapperStreamBody(ctx, { imageMemory, delegateProvider, preserveImageInput, instantLocal, instantLocalStyle, instantLocalTimeoutMs, instantLocalMaxPixels }) {
2628
+ export function createWrapperStreamBody(ctx, { imageMemory, delegateProvider, preserveImageInput, instantLocal, instantLocalStyle, instantLocalTimeoutMs, instantLocalMaxPixels, providerTransport }) {
2677
2629
  // issue #103: the reasoning level is a per-session picker choice (the chat
2678
2630
  // page's bottom-right selector), but the host can drop reasoningEffort from
2679
2631
  // the later steps of a multi-step turn once the twin metadata lacks a
@@ -2709,6 +2661,7 @@ export function createWrapperStreamBody(ctx, { imageMemory, delegateProvider, pr
2709
2661
  memory: imageMemory,
2710
2662
  timeoutMs: liveValue(instantLocalTimeoutMs),
2711
2663
  downscaleMaxPixels: liveValue(instantLocalMaxPixels),
2664
+ providerTransport,
2712
2665
  })
2713
2666
  : undefined
2714
2667
  // Rewrite image blocks ANYWHERE in the model input — including inside
@@ -2801,7 +2754,7 @@ export function createWrapperStreamBody(ctx, { imageMemory, delegateProvider, pr
2801
2754
  * route). Any other route name (e.g. the `deepseek-vision` alias) advertises
2802
2755
  * no models, so it stays functional but invisible in the picker.
2803
2756
  */
2804
- export function createStealthAdapter(ctx, { native, imageMemory, pairs, chainRoute, delegateProvider, instantLocal, instantLocalStyle, instantLocalTimeoutMs, instantLocalMaxPixels }) {
2757
+ export function createStealthAdapter(ctx, { native, imageMemory, pairs, chainRoute, delegateProvider, instantLocal, instantLocalStyle, instantLocalTimeoutMs, instantLocalMaxPixels, providerTransport }) {
2805
2758
  return {
2806
2759
  providerInfo(provider) {
2807
2760
  return { id: provider, name: 'DeepSeek' }
@@ -2822,7 +2775,7 @@ export function createStealthAdapter(ctx, { native, imageMemory, pairs, chainRou
2822
2775
  const base = await native.resolveModel(provider, model, signal)
2823
2776
  return { ...base, provider, inputModalities: ['text', 'image'] }
2824
2777
  },
2825
- ...createWrapperStreamBody(ctx, { imageMemory, delegateProvider, instantLocal, instantLocalStyle, instantLocalTimeoutMs, instantLocalMaxPixels }),
2778
+ ...createWrapperStreamBody(ctx, { imageMemory, delegateProvider, instantLocal, instantLocalStyle, instantLocalTimeoutMs, instantLocalMaxPixels, providerTransport }),
2826
2779
  }
2827
2780
  }
2828
2781
 
@@ -0,0 +1,262 @@
1
+ import { execFile } from 'node:child_process'
2
+ import { existsSync } from 'node:fs'
3
+ import { readFile, unlink } from 'node:fs/promises'
4
+ import { tmpdir } from 'node:os'
5
+ import path from 'node:path'
6
+ import { promisify } from 'node:util'
7
+
8
+ import {
9
+ callLocalBackend,
10
+ localDescribePrompt,
11
+ localProvidersOf,
12
+ toOpenAIContent,
13
+ } from './core-primitives.js'
14
+ import { loadSharp } from './sharp-runtime.js'
15
+ import { captureWindowsDesktop } from './windows-desktop-capture.js'
16
+ import { currentVisionTurnBudgetSignal } from './turn-budget-context.js'
17
+ import { combineSignals } from './vision-resilience.js'
18
+
19
+ function abortReason(signal, fallback = 'vision_screenshot aborted') {
20
+ if (signal?.reason instanceof Error) return signal.reason
21
+ const error = new Error(fallback)
22
+ error.name = 'AbortError'
23
+ error.code = 'ABORT_ERR'
24
+ return error
25
+ }
26
+
27
+ function throwIfAborted(signal) {
28
+ if (signal?.aborted) throw abortReason(signal)
29
+ }
30
+
31
+ function isAbortFailure(error) {
32
+ return error?.name === 'AbortError' || error?.code === 'ABORT_ERR'
33
+ }
34
+
35
+ export async function captureDesktopPng(target, timeoutMs, signal, options = {}) {
36
+ const platform = options.platform ?? process.platform
37
+ const run = options.execFileAsync ?? promisify(execFile)
38
+ const captureWindows = options.captureWindowsDesktop ?? captureWindowsDesktop
39
+ throwIfAborted(signal)
40
+ if (platform === 'win32') {
41
+ // Own DPI-aware capture at the implementation boundary rather than
42
+ // depending on a process-wide execFile argument rewrite.
43
+ await captureWindows(target, { timeoutMs, signal })
44
+ throwIfAborted(signal)
45
+ return platform
46
+ }
47
+ if (platform === 'darwin') {
48
+ // Without -m, screencapture writes one file per display while the tool owns
49
+ // exactly one artifact path, so request the main display explicitly.
50
+ await run('screencapture', ['-x', '-m', target], {
51
+ timeout: timeoutMs,
52
+ windowsHide: true,
53
+ ...(signal === undefined ? {} : { signal }),
54
+ })
55
+ throwIfAborted(signal)
56
+ return platform
57
+ }
58
+ try {
59
+ await run('import', ['-window', 'root', target], {
60
+ timeout: timeoutMs,
61
+ ...(signal === undefined ? {} : { signal }),
62
+ })
63
+ } catch (error) {
64
+ if (signal?.aborted || isAbortFailure(error)) throw error
65
+ await run('scrot', [target], {
66
+ timeout: timeoutMs,
67
+ ...(signal === undefined ? {} : { signal }),
68
+ })
69
+ }
70
+ throwIfAborted(signal)
71
+ return platform
72
+ }
73
+
74
+ async function downscaleForIdentification(data) {
75
+ try {
76
+ const sharp = await loadSharp()
77
+ if (!sharp) return data
78
+ const downscaled = await sharp(data, { failOn: 'none' })
79
+ .resize({ width: 1280, height: 1280, fit: 'inside', withoutEnlargement: true })
80
+ .png()
81
+ .toBuffer()
82
+ return downscaled.length > 0 && downscaled.length < data.length ? downscaled : data
83
+ } catch {
84
+ return data
85
+ }
86
+ }
87
+
88
+ async function identifyDesktopCapture(data, {
89
+ current,
90
+ timeoutMs,
91
+ instantLocalStyle,
92
+ providerTransport,
93
+ signal,
94
+ } = {}) {
95
+ throwIfAborted(signal)
96
+ const locals = localProvidersOf(current())
97
+ if (locals.length === 0) {
98
+ return {
99
+ identifyError: 'no local vision backend enabled (localOllama / localLmStudio); enable one to use identify',
100
+ }
101
+ }
102
+
103
+ const startedAt = Date.now()
104
+ const identifyBytes = await downscaleForIdentification(data)
105
+ throwIfAborted(signal)
106
+ const result = {}
107
+ if (identifyBytes !== data) {
108
+ result.identifyDownscaled = {
109
+ originalBytes: data.length,
110
+ sentBytes: identifyBytes.length,
111
+ }
112
+ }
113
+
114
+ const content = toOpenAIContent(
115
+ [{ type: 'image', attachment: { mediaType: 'image/png', data: identifyBytes } }],
116
+ () => identifyBytes,
117
+ )
118
+ content.push({ type: 'text', text: localDescribePrompt(instantLocalStyle()) })
119
+
120
+ const deadlineAt = Date.now() + timeoutMs()
121
+ const errors = []
122
+ for (let index = 0; index < locals.length; index++) {
123
+ throwIfAborted(signal)
124
+ const local = locals[index]
125
+ const remainingMs = deadlineAt - Date.now()
126
+ if (remainingMs <= 0) break
127
+ // Reserve a fair share for later local backends. A connected but hung
128
+ // first backend must not consume the entire fallback budget.
129
+ const roundBudgetMs = Math.max(1, Math.floor(remainingMs / (locals.length - index)))
130
+ const controller = new AbortController()
131
+ const roundSignal = signal
132
+ ? AbortSignal.any([signal, controller.signal])
133
+ : controller.signal
134
+ const timer = setTimeout(() => controller.abort(), roundBudgetMs)
135
+ try {
136
+ const identified = await callLocalBackend(
137
+ local,
138
+ [{ role: 'user', content }],
139
+ {
140
+ maxTokens: local.maxTokens ?? 2048,
141
+ signal: roundSignal,
142
+ providerTransport,
143
+ },
144
+ )
145
+ if (typeof identified === 'string' && identified.trim() !== '') {
146
+ result.identified = identified.trim()
147
+ result.identifiedBy = local.name
148
+ result.elapsedSec = Math.max(1, Math.round((Date.now() - startedAt) / 1000))
149
+ return result
150
+ }
151
+ errors.push(`${local.name}: empty response`)
152
+ } catch (error) {
153
+ if (signal?.aborted) throw abortReason(signal)
154
+ errors.push(`${local.name}: ${error && error.message ? error.message : String(error)}`)
155
+ } finally {
156
+ clearTimeout(timer)
157
+ }
158
+ }
159
+
160
+ result.identifyError = errors.length > 0
161
+ ? errors.join('; ').slice(0, 1000)
162
+ : 'local vision identification timed out before a backend could respond'
163
+ return result
164
+ }
165
+
166
+ /**
167
+ * Build the privacy-sensitive desktop screenshot tool definition.
168
+ *
169
+ * Tool registration/exposure remains owned by Core + LocalVisionStabilizer;
170
+ * this module owns only the platform capture and optional local-recognition
171
+ * implementation behind that existing permission boundary.
172
+ */
173
+ export function createDesktopScreenshotTool({
174
+ current,
175
+ timeoutMs,
176
+ saveArtifact,
177
+ stringOutput,
178
+ instantLocalStyle,
179
+ providerTransport,
180
+ captureDesktop = captureDesktopPng,
181
+ } = {}) {
182
+ if (
183
+ typeof current !== 'function' ||
184
+ typeof timeoutMs !== 'function' ||
185
+ typeof saveArtifact !== 'function' ||
186
+ typeof captureDesktop !== 'function'
187
+ ) {
188
+ throw new TypeError('desktop screenshot tool requires live config, timeout, artifact and capture callbacks')
189
+ }
190
+ if (typeof instantLocalStyle !== 'function') {
191
+ throw new TypeError('desktop screenshot tool requires a live local describe style')
192
+ }
193
+
194
+ return {
195
+ name: 'vision_screenshot',
196
+ description:
197
+ 'Capture the user\'s desktop screen as a PNG artifact (the virtual screen on Windows; the main display on macOS; the root display on Linux). ' +
198
+ 'Windows: per-monitor-DPI-aware PowerShell capture; macOS: screencapture; Linux: ImageMagick import (falls back to scrot; either command must be installed). ' +
199
+ 'This privacy-sensitive tool is disabled by default and works only after the user explicitly enables Desktop screenshot in Vision Router settings. ' +
200
+ 'Use it when you need to see what is on the user\'s screen right now — e.g. their current GUI, an app, or a page outside this browser. ' +
201
+ 'Optional identify=true also runs local recognition on the capture using the enabled local backends (Ollama, then LM Studio) and returns the description alongside the path.',
202
+ parameters: {
203
+ type: 'object',
204
+ properties: {
205
+ identify: {
206
+ type: 'boolean',
207
+ description:
208
+ 'Also recognize the captured screen with enabled local vision backends (Ollama, then LM Studio) and return the description text with the path. Default false.',
209
+ },
210
+ },
211
+ additionalProperties: false,
212
+ },
213
+ output: stringOutput,
214
+ async execute(args, exec) {
215
+ if (current().desktopScreenshot !== true) {
216
+ throw new Error(
217
+ 'vision_screenshot is disabled; enable Desktop screenshot explicitly in Vision Router settings before use',
218
+ )
219
+ }
220
+
221
+ const tmp = path.join(
222
+ tmpdir(),
223
+ `vision-screenshot-${Date.now()}-${Math.floor(Math.random() * 1e9)}.png`,
224
+ )
225
+ try {
226
+ const signal = combineSignals(currentVisionTurnBudgetSignal(), exec?.signal)
227
+ throwIfAborted(signal)
228
+ const platform = await captureDesktop(tmp, timeoutMs(), signal)
229
+ throwIfAborted(signal)
230
+ if (!existsSync(tmp)) {
231
+ throw new Error(
232
+ `vision_screenshot: no output produced on ${platform} (is a screen available?)`,
233
+ )
234
+ }
235
+ const data = await readFile(tmp, signal === undefined ? undefined : { signal })
236
+ throwIfAborted(signal)
237
+ let identification
238
+ if (args.identify === true) {
239
+ identification = await identifyDesktopCapture(data, {
240
+ current,
241
+ timeoutMs,
242
+ instantLocalStyle,
243
+ providerTransport,
244
+ signal,
245
+ })
246
+ }
247
+ // Artifact publication is the final commit point. If optional local
248
+ // identification is cancelled, no successful screenshot result should
249
+ // already have escaped into the managed artifact store.
250
+ throwIfAborted(signal)
251
+ const target = await saveArtifact(exec, `screenshot-${Date.now()}.png`, data)
252
+ return JSON.stringify({
253
+ path: target,
254
+ bytes: data.length,
255
+ ...(identification ?? {}),
256
+ })
257
+ } finally {
258
+ try { await unlink(tmp) } catch { /* best effort cleanup */ }
259
+ }
260
+ },
261
+ }
262
+ }