dsh-vision-router 3.2.0 → 3.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. package/README.md +6 -6
  2. package/README.zh.md +21 -6
  3. package/THIRD_PARTY_NOTICES.md +167 -0
  4. package/assets/licenses/apache-2.0.txt +202 -0
  5. package/assets/licenses/llvm-mingw-LICENSE.txt +279 -0
  6. package/assets/licenses/mingw-w64-runtime-COPYING.txt +265 -0
  7. package/assets/licenses/moxcms-BSD-3-Clause.txt +26 -0
  8. package/assets/licenses/ncnn-LICENSE.txt +86 -0
  9. package/assets/licenses/roots-BSD-2-Clause.txt +24 -0
  10. package/assets/licenses/unicode-ident-Unicode-3.0.txt +39 -0
  11. package/assets/licenses/vtracer-MIT.txt +25 -0
  12. package/assets/licenses/zlib-rs-Zlib.txt +19 -0
  13. package/docs/architecture/dsh-support-window.md +25 -6
  14. package/docs/compatibility/dsh-0.2.2-rc2-audit.md +40 -0
  15. package/docs/compatibility/dsh-0.2.2-rc3-audit.md +131 -0
  16. package/docs/compliance/vtracer-alpha4-provenance.md +47 -0
  17. package/docs/compliance/vtracer-alpha4-reconstructed-Cargo.lock +567 -0
  18. package/docs/compliance/vtracer-alpha4-rust-inventory.json +421 -0
  19. package/docs/compliance/vtracer-alpha4-rust-license-texts.md +13018 -0
  20. package/docs/releases/v3.2.1.md +21 -0
  21. package/docs/releases/v3.2.2.md +25 -0
  22. package/docs/releases/v3.3.0.md +34 -0
  23. package/docs/remote-settings.md +83 -8
  24. package/index.js +280 -169
  25. package/lib/client.js +17 -17
  26. package/lib/core-primitives.js +155 -214
  27. package/lib/dsh-support-window.js +6 -3
  28. package/lib/native-ocr-annotation-risk.js +33 -0
  29. package/lib/native-ocr-canary.js +156 -0
  30. package/lib/native-ocr-chemistry-review.js +37 -0
  31. package/lib/native-ocr-document.js +243 -0
  32. package/lib/native-ocr-installation.js +17 -0
  33. package/lib/native-ocr-integrity.js +93 -0
  34. package/lib/native-ocr-question-labels.js +167 -0
  35. package/lib/native-ocr-supervisor.js +268 -0
  36. package/lib/remote-settings-bridge.js +93 -57
  37. package/lib/remote-settings-risk-client-boundary.js +2 -2
  38. package/lib/runtime-reliability.js +7 -0
  39. package/lib/session-query-readers.js +8 -2
  40. package/lib/session-vision-attachment-index.js +61 -14
  41. package/lib/settings-client-017-compat.js +4 -2
  42. package/lib/settings-ia-client-prelude.js +26 -10
  43. package/lib/tesseract-image-preprocess.js +35 -0
  44. package/lib/trace-vtracer-process-runtime.js +133 -0
  45. package/lib/trace-vtracer-process.js +43 -0
  46. package/lib/trace-vtracer-worker.js +163 -0
  47. package/lib/vision-attachment-handle-runtime.js +11 -2
  48. package/lib/vision-backend-runtime-policy.js +5 -3
  49. package/lib/vision-image-input-reader.js +67 -0
  50. package/lib/vision-image-input-verdict.js +24 -3
  51. package/lib/vision-tool-runtime-boundary.js +7 -10
  52. package/lib/vision-tool-stage.js +58 -0
  53. package/lib/web/dsh-settings-017-root-transport.js +52 -6
  54. package/lib/web-capability-boundary.js +6 -0
  55. package/package.json +10 -6
  56. package/scripts/install-native-ocr.mjs +144 -0
package/index.js CHANGED
@@ -118,6 +118,9 @@ import z from '@deepseek-ai/schemastery'
118
118
 
119
119
  import path from 'node:path'
120
120
  import { existsSync } from 'node:fs'
121
+ import { runExperimentalNativeOcrCanary } from './lib/native-ocr-canary.js'
122
+ import { localNativeOcrInstallPaths } from './lib/native-ocr-installation.js'
123
+ import { prepareTesseractSubtitleRetry } from './lib/tesseract-image-preprocess.js'
121
124
 
122
125
  import { pathToFileURL } from 'node:url'
123
126
  import { appendPromptToImageOnlyMessage } from './lib/http-compat.js'
@@ -151,6 +154,8 @@ import {
151
154
  import { currentVisionExecutionOrder } from './lib/vision-execution-order.js'
152
155
  import { applyVisionExecutionOrder } from './lib/vision-execution-order-apply.js'
153
156
  import { currentVisionTurnBudgetSignal } from './lib/turn-budget-context.js'
157
+ import { createVisionToolStageTrace } from './lib/vision-tool-stage.js'
158
+ import { createVisionImageInputReader } from './lib/vision-image-input-reader.js'
154
159
  import { parseGroundingBoxResponse, assessGroundingBox } from './lib/grounding-box-recovery.js'
155
160
  import { createHash } from 'node:crypto'
156
161
  import {
@@ -311,14 +316,14 @@ export const Config = z.object({
311
316
  // (its own cap) and the vision-model fallback only the rest — never two
312
317
  // full timeouts added together.
313
318
  ocrTimeoutMs: z.number().step(1).min(1000).max(120000).default(30000),
314
- // Default engine for vision_ocr. Per-call engine=tesseract|vision remains authoritative.
319
+ // Auto prefers verified owner-installed PP-OCRv6 when supported; explicit engine wins.
315
320
  ocrEngine: z.union(['auto', 'tesseract', 'vision']).default('auto'),
316
321
  proxy: z.string().default(''),
317
322
  proxyHosts: z.array(z.string()).default([...DEFAULT_PROXY_HOSTS]),
318
- // Remote browsers are intentionally unable to use DSH's broad settings.*
319
- // plane. This narrow Vision Router bridge is opt-in and still uses DSH's
320
- // trusted-host transport fence. Only a loopback/local settings page may
321
- // change this permission; the remote bridge rejects writes to the field.
323
+ // DSH versions expose different browser Settings transports. DVR's remote
324
+ // RPC is opt-in, but once acknowledged it edits the same Host-owned config
325
+ // namespace and schema as the local browser. The permission itself is
326
+ // managed by risk confirmation or local revocation, not ordinary remote writes.
322
327
  allowRemoteSettings: z.boolean().default(false),
323
328
  // Privacy policy switch: when enabled, Vision Router executes image-model
324
329
  // requests only against loopback visual endpoints (Ollama, LM Studio, or
@@ -470,6 +475,8 @@ import {
470
475
  resolveVisionOcrEngine,
471
476
  ocrWithTesseract,
472
477
  ocrWithTesseractAdaptive,
478
+ shouldVerifyUncertainAutoOcr,
479
+ verifyUncertainAutoOcrResult,
473
480
  estimateTokens,
474
481
  estimateMessages,
475
482
  trimMessagesToBudget,
@@ -565,6 +572,8 @@ export {
565
572
  resolveVisionOcrEngine,
566
573
  ocrWithTesseract,
567
574
  ocrWithTesseractAdaptive,
575
+ shouldVerifyUncertainAutoOcr,
576
+ verifyUncertainAutoOcrResult,
568
577
  estimateTokens,
569
578
  estimateMessages,
570
579
  trimMessagesToBudget,
@@ -1023,20 +1032,35 @@ export function apply(ctx, config = {}, runtime = {}) {
1023
1032
  }
1024
1033
  if (attachments === undefined) continue
1025
1034
  try {
1026
- const stored = await attachments.readImage(block.attachment)
1035
+ throwIfVisionTaskAborted(options.signal)
1036
+ const stored = await attachments.readImage(block.attachment, options.signal)
1037
+ throwIfVisionTaskAborted(options.signal)
1027
1038
  // Last-mile guard: never send oversized images to the vision
1028
1039
  // endpoint — encoder cost scales with pixels and dominates
1029
1040
  // tool-call latency on retina screenshots.
1030
1041
  let bytes = stored.data
1031
1042
  if (downscaleEnabled() && bytes && bytes.length > 0) {
1032
1043
  bytes = await downscaleImage(bytes, downscaleMaxPixels())
1044
+ throwIfVisionTaskAborted(options.signal)
1033
1045
  }
1034
1046
  content.push(...toOpenAIContent([block], () => bytes))
1035
1047
  } catch (error) {
1036
- ctx.logger?.warn(
1037
- 'vision-http: failed to read image attachment: %s',
1038
- error && error.message ? error.message : String(error),
1039
- )
1048
+ throwIfVisionTaskAborted(options.signal, error)
1049
+ // A text-only completion here would falsely claim the model
1050
+ // saw pixels. Fail this backend immediately and let the
1051
+ // normal vision fallback chain choose another backend.
1052
+ ctx.logger?.warn('vision-http: retained image delivery failed; refusing text-only downgrade')
1053
+ yield {
1054
+ type: 'finish',
1055
+ reason: {
1056
+ kind: 'error',
1057
+ failure: {
1058
+ code: 'VISION_IMAGE_DELIVERY_UNAVAILABLE',
1059
+ message: 'vision-http: retained image could not be delivered to the model',
1060
+ },
1061
+ },
1062
+ }
1063
+ return
1040
1064
  }
1041
1065
  } else if (block && block.type === 'tool-result') {
1042
1066
  // The OpenAI wire has no tool-call frames to hang a `role: tool`
@@ -1701,7 +1725,9 @@ export function apply(ctx, config = {}, runtime = {}) {
1701
1725
  }
1702
1726
  const content = []
1703
1727
  for (const block of blocks) {
1704
- const stored = await attachments.readImage(block.attachment)
1728
+ throwIfVisionTaskAborted(options.signal)
1729
+ const stored = await attachments.readImage(block.attachment, options.signal)
1730
+ throwIfVisionTaskAborted(options.signal)
1705
1731
  content.push(...toOpenAIContent([block], () => stored.data))
1706
1732
  }
1707
1733
  return callOpenAIWithProviderTransport(
@@ -1797,14 +1823,18 @@ export function apply(ctx, config = {}, runtime = {}) {
1797
1823
  throw new Error('corrected route: the attachment service is not registered')
1798
1824
  }
1799
1825
  const bytesOf = async (attachment) => {
1800
- const stored = await attachments.readImage(attachment)
1826
+ throwIfVisionTaskAborted(options.signal)
1827
+ const stored = await attachments.readImage(attachment, options.signal)
1828
+ throwIfVisionTaskAborted(options.signal)
1801
1829
  let bytes = stored.data
1802
1830
  if (downscaleEnabled() && bytes && bytes.length > 0) {
1803
1831
  bytes = await downscaleImage(bytes, downscaleMaxPixels())
1832
+ throwIfVisionTaskAborted(options.signal)
1804
1833
  }
1805
1834
  return bytes
1806
1835
  }
1807
1836
  const anthropic = await toAnthropicMessages(messages, bytesOf)
1837
+ throwIfVisionTaskAborted(options.signal)
1808
1838
  if (anthropic.messages.length === 0) {
1809
1839
  throw new Error(`corrected route "${pair.provider}/${pair.model}": no representable content to send`)
1810
1840
  }
@@ -2402,8 +2432,8 @@ export function apply(ctx, config = {}, runtime = {}) {
2402
2432
  // Host-owned proxy overrides are scoped by lib/legacy-global-proxy-boundary.js.
2403
2433
  // Core no longer owns or installs a process-wide proxy fetch implementation.
2404
2434
 
2405
- const resolveAttachment = (session, id) => sessionVisionIndex.resolveAttachment(session, id)
2406
- const resolveAttachments = (session, ids) => sessionVisionIndex.resolveAttachments(session, ids)
2435
+ const resolveAttachment = (session, id, signal) => sessionVisionIndex.resolveAttachment(session, id, signal)
2436
+ const resolveAttachments = (session, ids, signal) => sessionVisionIndex.resolveAttachments(session, ids, signal)
2407
2437
 
2408
2438
  // Session-local routing handoff between pre-step and agent/request. Modern
2409
2439
  // Hosts store one exact async raw-log tail seq; legacy Hosts retain only the
@@ -2880,121 +2910,103 @@ export function apply(ctx, config = {}, runtime = {}) {
2880
2910
  }
2881
2911
  const blocks = []
2882
2912
  const contentIds = []
2883
-
2913
+ // One shared deadline for the WHOLE task: image preflight, every
2914
+ // provider, fallback and JSON correction draw from one clock.
2915
+ // Previously the clock started after SessionQuery/image loading,
2916
+ // leaving attachmentIds vulnerable to the Host's ~120s watchdog.
2917
+ const hostSignal = combineSignals(currentVisionTurnBudgetSignal(), exec?.signal)
2918
+ const deadline = createDeadline(visionTaskTimeoutMs())
2919
+ const taskSignal = combineSignals(hostSignal, deadline.signal())
2920
+ const stage = createVisionToolStageTrace(ctx.logger, 'vision_describe', taskSignal)
2921
+ try {
2884
2922
  const paths = Array.isArray(args.paths) ? args.paths : []
2885
2923
  const attachmentIds = Array.isArray(args.attachmentIds) ? args.attachmentIds : []
2886
2924
  if (paths.length + attachmentIds.length === 0 || paths.length + attachmentIds.length > 4) {
2887
2925
  throw new Error('vision_describe: provide 1-4 images via paths and/or attachmentIds')
2888
2926
  }
2889
- // Preserve only durable upload ids for a deterministic offline fallback.
2890
- // Never expose or guess the attachment store's private filesystem path.
2891
- const materializableAttachmentIds = [...new Set([
2892
- ...attachmentIds.map((id) => String(id)).filter((id) => isAttachmentIdInput(id)),
2893
- ...paths.map((item) => String(item)).filter((item) => isAttachmentIdInput(item)),
2927
+ // Both argument forms now use the exact same read -> normalize ->
2928
+ // downscale -> save path. The old attachmentIds-only path could retain
2929
+ // an oversized original ref after resize publication failed.
2930
+ const sources = [...paths, ...attachmentIds]
2931
+ const materializableAttachmentIds = [...new Set(
2932
+ sources.map((value) => String(value)).filter((value) => isAttachmentIdInput(value)),
2933
+ )]
2934
+ // attachmentIds is always an attachment namespace, even for a legacy
2935
+ // Host whose authorized IDs do not yet have a sha256: prefix.
2936
+ const requestedRefs = [...new Set([
2937
+ ...attachmentIds.map((value) => String(value)),
2938
+ ...paths.map((value) => String(value)).filter((value) => isAttachmentIdInput(value)),
2894
2939
  ])]
2895
- const attachmentSession = exec && exec.agent && exec.agent.session
2896
- const resolvedAttachmentRefs = materializableAttachmentIds.length > 0
2897
- ? await resolveAttachments(attachmentSession, materializableAttachmentIds)
2940
+ const attachmentSession = exec?.agent?.session
2941
+ const resolvedAttachmentRefs = requestedRefs.length > 0
2942
+ ? await stage('resolve-upload-refs', () =>
2943
+ resolveAttachments(attachmentSession, requestedRefs, taskSignal))
2898
2944
  : new Map()
2899
2945
 
2900
- for (const path of paths) {
2946
+ for (const [index, item] of sources.entries()) {
2947
+ const source = String(item)
2948
+ const attachmentInput = index >= paths.length
2901
2949
  let bytes
2902
2950
  let mediaType
2903
2951
  try {
2904
- // readImageBytes accepts both filesystem paths and attachment ids
2905
- // ("sha256:..."), so a model that passes an uploaded image's id as
2906
- // a path gets the right pixels instead of a not-found error.
2907
- ;({ bytes, mediaType } = await readImageBytes(exec, path, resolvedAttachmentRefs))
2952
+ ;({ bytes, mediaType } = await stage('read-image-' + index, () =>
2953
+ readImageBytes(exec, source, resolvedAttachmentRefs, attachmentInput)))
2908
2954
  } catch (error) {
2909
2955
  throw new Error(
2910
- `vision_describe: failed to read ${path} (${error && error.message ? error.message : String(error)})`,
2956
+ `vision_describe: failed to read ${source} (${error && error.message ? error.message : String(error)})`,
2957
+ { cause: error },
2911
2958
  )
2912
2959
  }
2913
2960
  if (downscaleEnabled()) {
2914
- const resized = await downscaleImage(bytes, downscaleMaxPixels())
2961
+ const resized = await stage('downscale-image-' + index, () =>
2962
+ downscaleImage(bytes, downscaleMaxPixels(), { signal: taskSignal }))
2915
2963
  if (resized !== bytes) {
2916
- ctx.logger?.info('vision-router: downscaled %s for the vision call', path)
2964
+ ctx.logger?.info('vision-router: downscaled vision input index=%d', index)
2917
2965
  }
2918
2966
  bytes = resized
2919
2967
  }
2968
+ // Always publish canonical bytes, including an uploaded image.
2969
+ // Host adapters then read the verified/downsized version, never a
2970
+ // stale or mismatched original attachment reference.
2920
2971
  let ref
2921
2972
  try {
2922
- ref = await attachments.saveImage({
2973
+ ref = await stage('save-normalized-image-' + index, () => attachments.saveImage({
2923
2974
  data: bytes,
2924
2975
  mediaType,
2925
- ...(isAttachmentIdInput(path) || basenameOf(path) === undefined
2976
+ ...(attachmentInput || isAttachmentIdInput(source) || basenameOf(source) === undefined
2926
2977
  ? {}
2927
- : { name: basenameOf(path) }),
2928
- })
2978
+ : { name: basenameOf(source) }),
2979
+ }))
2929
2980
  } catch (error) {
2930
2981
  throw new Error(
2931
- `vision_describe: image ${path} was rejected (${error && error.message ? error.message : String(error)})`,
2982
+ `vision_describe: image ${source} was rejected (${error && error.message ? error.message : String(error)})`,
2983
+ { cause: error },
2932
2984
  )
2933
2985
  }
2986
+ throwIfVisionTaskAborted(taskSignal)
2934
2987
  contentIds.push(String(ref.attachmentId))
2935
2988
  blocks.push({ type: 'image', attachment: ref })
2936
2989
  }
2937
2990
 
2938
- for (const id of attachmentIds) {
2939
- const attachmentId = String(id)
2940
- const ref = isAttachmentIdInput(attachmentId)
2941
- ? resolvedAttachmentRefs.get(attachmentId)
2942
- : await resolveAttachment(attachmentSession, attachmentId)
2943
- if (ref === undefined) {
2944
- throw new Error(
2945
- `vision_describe: unknown attachment id "${id}" (it must come from an image uploaded in this conversation)`,
2946
- )
2947
- }
2948
- let stored
2949
- try {
2950
- stored = await attachments.readImage(ref)
2951
- } catch (error) {
2952
- throw new Error(
2953
- `vision_describe: failed to read attachment ${id} (${error && error.message ? error.message : String(error)})`,
2954
- )
2955
- }
2956
- // Downscale oversized uploads before the vision call: retina
2957
- // screenshots easily reach 10MP+ and the vision encoder's cost
2958
- // scales with pixels — a full-size upload is the dominant part of
2959
- // the tool-call latency. Re-save a resized attachment so the
2960
- // adapter reads the small one.
2961
- if (downscaleEnabled() && stored.data && stored.data.length > 0) {
2962
- const resized = await downscaleImage(stored.data, downscaleMaxPixels())
2963
- if (resized !== stored.data) {
2964
- try {
2965
- const resizedRef = await attachments.saveImage({
2966
- data: resized,
2967
- mediaType: stored.ref && stored.ref.mediaType ? stored.ref.mediaType : 'image/png',
2968
- ...(stored.ref && stored.ref.name ? { name: stored.ref.name } : {}),
2969
- })
2970
- stored = { ref: resizedRef, data: resized }
2971
- ctx.logger?.info('vision-router: downscaled attachment %s for the vision call', id)
2972
- } catch {
2973
- stored = { ...stored, data: resized }
2974
- }
2975
- }
2976
- }
2977
- contentIds.push(String(ref.attachmentId))
2978
- blocks.push({ type: 'image', attachment: stored.ref })
2979
- }
2980
-
2981
2991
  const question = String(args.question ?? '')
2982
2992
  const wantJson = args.json === true
2983
2993
  // Keep the adapter path and direct OpenAI-compatible HTTP path on the
2984
2994
  // exact same prompt, including the structured JSON evidence contract.
2985
2995
  const promptText = visionDescribePrompt(question, wantJson)
2986
- const usablePairs = await resolveToolVisionPairs()
2996
+ const usablePairs = await stage('resolve-vision-backends', () => resolveToolVisionPairs())
2987
2997
  // Freeze this task's direct fallback list: settings changes take effect
2988
2998
  // on the next call, never halfway through one fallback walk.
2989
2999
  const httpFallbacks = httpProviders()
2990
3000
  const rejectedPairs = []
3001
+ let backendPreflightIndex = 0
2991
3002
  for (const pair of pairs()) {
2992
3003
  if (!pair || pair.provider === HTTP_ROUTE) continue
2993
3004
  if (!adapterAvailable(ctx.llm, pair.provider)) {
2994
3005
  rejectedPairs.push(`${pair.provider}/${pair.model}: provider adapter is not registered`)
2995
3006
  continue
2996
3007
  }
2997
- const capability = await resolveVisionBackendCapability(pair.provider, pair.model)
3008
+ const capability = await stage('backend-preflight-' + backendPreflightIndex++, () =>
3009
+ resolveVisionBackendCapability(pair.provider, pair.model))
2998
3010
  if (capability.attemptable === false) {
2999
3011
  rejectedPairs.push(
3000
3012
  `${pair.provider}/${pair.model}: structurally unavailable (${capability.reason ?? 'unknown reason'})`,
@@ -3030,9 +3042,7 @@ export function apply(ctx, config = {}, runtime = {}) {
3030
3042
  )
3031
3043
  }
3032
3044
 
3033
- // One shared deadline for the WHOLE task: every provider, fallback and
3034
- // the JSON-correction retry draws from this single budget.
3035
- const deadline = createDeadline(visionTaskTimeoutMs())
3045
+ // The same clock already started before attachment resolution.
3036
3046
  const baseMessages = [
3037
3047
  {
3038
3048
  role: 'user',
@@ -3051,7 +3061,7 @@ export function apply(ctx, config = {}, runtime = {}) {
3051
3061
  0,
3052
3062
  )
3053
3063
 
3054
- for (const pair of usablePairs) {
3064
+ for (const [pairIndex, pair] of usablePairs.entries()) {
3055
3065
  if (localOnlyVisionEnabled(current()) && !isLocalBackendPair(pair)) {
3056
3066
  errors.push(`${pair.provider}/${pair.model}: skipped (local-only vision policy)`)
3057
3067
  continue
@@ -3064,7 +3074,8 @@ export function apply(ctx, config = {}, runtime = {}) {
3064
3074
  break
3065
3075
  }
3066
3076
  const backendKey = `${pair.provider}/${pair.model}`
3067
- const fingerprint = await credentialFingerprintFor({ provider: pair.provider })
3077
+ const fingerprint = await stage('adapter-auth-' + pairIndex, () =>
3078
+ credentialFingerprintFor({ provider: pair.provider }))
3068
3079
  const gate = visionBreaker.inspect(backendKey, fingerprint, scope)
3069
3080
  if (gate.blocked) {
3070
3081
  errors.push(`${backendKey}: skipped (circuit open: ${gate.reason})`)
@@ -3089,19 +3100,20 @@ export function apply(ctx, config = {}, runtime = {}) {
3089
3100
  )
3090
3101
  : timeoutMs()
3091
3102
  const signal = combineSignals(
3092
- currentVisionTurnBudgetSignal(),
3103
+ hostSignal,
3093
3104
  deadline.signal(),
3094
3105
  AbortSignal.timeout(attemptBudgetMs),
3095
3106
  )
3096
- const capability = await resolveVisionBackendCapability(pair.provider, pair.model)
3097
- let text = await callVisionPairWithOptionalBridge(pair, messages, {
3107
+ const capability = await stage('adapter-capability-' + pairIndex, () =>
3108
+ resolveVisionBackendCapability(pair.provider, pair.model))
3109
+ let text = await stage('adapter-request-' + pairIndex, () => callVisionPairWithOptionalBridge(pair, messages, {
3098
3110
  maxTokens: 4096,
3099
3111
  signal,
3100
3112
  sessionId,
3101
3113
  capability,
3102
3114
  bridgeBlocks: blocks,
3103
3115
  bridgeInstruction: promptText,
3104
- })
3116
+ }))
3105
3117
  assertNoRepetitionLoop(text, backendKey)
3106
3118
  ctx.logger?.info(
3107
3119
  'vision-router: vision call %s ok (%d chars, %d ms)',
@@ -3133,7 +3145,7 @@ ctx.logger?.info(
3133
3145
  },
3134
3146
  ]
3135
3147
  attemptStarted = Date.now()
3136
- text = await callVisionPairWithOptionalBridge(pair, messages, {
3148
+ text = await stage('adapter-json-correction-' + pairIndex, () => callVisionPairWithOptionalBridge(pair, messages, {
3137
3149
  maxTokens: 4096,
3138
3150
  signal,
3139
3151
  sessionId,
@@ -3141,7 +3153,7 @@ ctx.logger?.info(
3141
3153
  bridgeBlocks: blocks,
3142
3154
  bridgeInstruction:
3143
3155
  promptText + '\n\nThat output was not valid JSON. Respond with ONLY a valid JSON object now.',
3144
- })
3156
+ }))
3145
3157
  assertNoRepetitionLoop(text, backendKey)
3146
3158
  ctx.logger?.info(
3147
3159
  'vision-router: vision call %s ok after JSON correction (%d chars, %d ms)',
@@ -3171,7 +3183,7 @@ ctx.logger?.info(
3171
3183
  // A user/Host cancel is not a backend outage. Otherwise the
3172
3184
  // canceled attempt poisons this turn's breaker/failure memory and
3173
3185
  // a healthy retry returns BACKEND_UNAVAILABLE_THIS_TURN.
3174
- throwIfVisionTaskAborted(currentVisionTurnBudgetSignal(), error)
3186
+ throwIfVisionTaskAborted(hostSignal, error)
3175
3187
  const classification = classifyVisionFailure(error)
3176
3188
  consecutiveServerFailures = classification.kind === VISION_FAILURE_KINDS.SERVER
3177
3189
  ? consecutiveServerFailures + 1 : 0
@@ -3197,7 +3209,7 @@ ctx.logger?.info(
3197
3209
  // Direct HTTP providers (built-in keyless OVHcloud by default) are the
3198
3210
  // final fallbacks: they bypass the harness llm service entirely, so the
3199
3211
  // anonymous free endpoint works without any credential.
3200
- for (const provider of httpFallbacks) {
3212
+ for (const [httpIndex, provider] of httpFallbacks.entries()) {
3201
3213
  if (localOnlyVisionEnabled(current()) && !isLoopbackVisionBaseURL(provider?.baseURL)) {
3202
3214
  errors.push(`http:${provider?.name}/${provider?.model}: skipped (local-only vision policy)`)
3203
3215
  continue
@@ -3210,7 +3222,8 @@ ctx.logger?.info(
3210
3222
  break
3211
3223
  }
3212
3224
  const backendKey = `http:${provider.name}/${provider.model}`
3213
- const fingerprint = await credentialFingerprintFor({ kind: 'http', apiKeyEnv: provider.apiKeyEnv })
3225
+ const fingerprint = await stage('http-auth-' + httpIndex, () =>
3226
+ credentialFingerprintFor({ kind: 'http', apiKeyEnv: provider.apiKeyEnv }))
3214
3227
  const gate = visionBreaker.inspect(backendKey, fingerprint, scope)
3215
3228
  if (gate.blocked) {
3216
3229
  errors.push(`${backendKey}: skipped (circuit open: ${gate.reason})`)
@@ -3222,15 +3235,19 @@ ctx.logger?.info(
3222
3235
  let attemptStarted = Date.now()
3223
3236
  try {
3224
3237
  // Precompute bytes once per block (attachments.readImage is async).
3225
- const openAIBlocks = []
3226
- for (const block of blocks) {
3227
- if (block.type === 'image' && block.attachment) {
3228
- const stored = await attachments.readImage(block.attachment)
3229
- openAIBlocks.push(toOpenAIContent([block], () => stored.data)[0])
3230
- } else {
3231
- openAIBlocks.push({ type: 'text', text: block.text })
3238
+ const openAIBlocks = await stage('http-image-delivery-' + httpIndex, async () => {
3239
+ const converted = []
3240
+ for (const block of blocks) {
3241
+ if (block.type === 'image' && block.attachment) {
3242
+ const stored = await attachments.readImage(block.attachment, taskSignal)
3243
+ throwIfVisionTaskAborted(taskSignal)
3244
+ converted.push(toOpenAIContent([block], () => stored.data)[0])
3245
+ } else {
3246
+ converted.push({ type: 'text', text: block.text })
3247
+ }
3232
3248
  }
3233
- }
3249
+ return converted
3250
+ })
3234
3251
  // Direct HTTP providers must receive the same image + question as
3235
3252
  // adapter-backed providers. Some endpoints (e.g. Zhipu GLM) reject
3236
3253
  // a pure-image user message even when permissive endpoints accept it.
@@ -3248,12 +3265,15 @@ ctx.logger?.info(
3248
3265
  consecutiveServerFailures,
3249
3266
  )
3250
3267
  const attemptSignal = combineSignals(
3251
- currentVisionTurnBudgetSignal(),
3268
+ hostSignal,
3252
3269
  deadline.signal(),
3253
3270
  AbortSignal.timeout(attemptBudgetMs),
3254
3271
  )
3255
3272
  const askHttp = async (correction) => {
3256
- const answer = await awaitVisionBackendAttempt(attemptSignal, () => callOpenAIWithProviderTransport(
3273
+ const phase = correction === undefined
3274
+ ? 'http-request-' + httpIndex : 'http-json-correction-' + httpIndex
3275
+ const answer = await stage(phase, () =>
3276
+ awaitVisionBackendAttempt(attemptSignal, () => callOpenAIWithProviderTransport(
3257
3277
  provider,
3258
3278
  correction === undefined
3259
3279
  ? openAIBaseMessages
@@ -3267,7 +3287,7 @@ ctx.logger?.info(
3267
3287
  sessionId,
3268
3288
  resolveCredential,
3269
3289
  },
3270
- ))
3290
+ )))
3271
3291
  return answer
3272
3292
  }
3273
3293
  let text = await askHttp(undefined)
@@ -3319,7 +3339,7 @@ ctx.logger?.info(
3319
3339
  // A user/Host cancel is not a backend outage. Otherwise the
3320
3340
  // canceled attempt poisons this turn's breaker/failure memory and
3321
3341
  // a healthy retry returns BACKEND_UNAVAILABLE_THIS_TURN.
3322
- throwIfVisionTaskAborted(currentVisionTurnBudgetSignal(), error)
3342
+ throwIfVisionTaskAborted(hostSignal, error)
3323
3343
  const classification = classifyVisionFailure(error)
3324
3344
  consecutiveServerFailures = classification.kind === VISION_FAILURE_KINDS.SERVER
3325
3345
  ? consecutiveServerFailures + 1 : 0
@@ -3367,6 +3387,18 @@ ctx.logger?.info(
3367
3387
  }
3368
3388
  }
3369
3389
  return JSON.stringify(baseFailure)
3390
+ } catch (error) {
3391
+ // The caller's explicit cancellation is not a model outage.
3392
+ if (hostSignal?.aborted) throw hostSignal.reason ?? error
3393
+ if (deadline.expired() || taskSignal?.aborted) {
3394
+ return JSON.stringify(buildVisionFailure({
3395
+ code: VISION_RESULT_CODES.TIMEOUT,
3396
+ retryable: false,
3397
+ reason: 'Vision task deadline exhausted while preparing or processing the image.',
3398
+ }))
3399
+ }
3400
+ throw error
3401
+ }
3370
3402
  },
3371
3403
  }
3372
3404
  deepToolDefs.push(visionDescribeTool)
@@ -3498,55 +3530,9 @@ ctx.logger?.info(
3498
3530
  return String(attachmentIds[0]).trim()
3499
3531
  }
3500
3532
 
3501
- const readImageBytes = async (exec, imagePath, resolvedAttachmentRefs) => {
3502
- const input = String(imagePath ?? '')
3503
- let bytes
3504
- let storedMediaType
3505
- if (isAttachmentIdInput(input)) {
3506
- // Uploaded images reach the tool arguments as durable attachment ids
3507
- // ("sha256:..."); resolve them through the session's recorded upload
3508
- // index instead of treating the id as a filesystem path.
3509
- const attachments = ctx.get('attachments')
3510
- if (attachments === undefined) {
3511
- throw new Error('vision-router: the attachment service is not available in this deployment')
3512
- }
3513
- const session = exec && exec.agent && exec.agent.session
3514
- const canonicalInput = input.trim()
3515
- const ref = resolvedAttachmentRefs instanceof Map
3516
- ? resolvedAttachmentRefs.get(canonicalInput)
3517
- : await resolveAttachment(session, canonicalInput)
3518
- if (ref === undefined) {
3519
- throw new Error(
3520
- `vision-router: unknown attachment id "${input}" (it must come from an image uploaded in this conversation)`,
3521
- )
3522
- }
3523
- let stored
3524
- try {
3525
- stored = await attachments.readImage(ref)
3526
- } catch (error) {
3527
- throw new Error(
3528
- `vision-router: failed to read attachment ${input} (${error && error.message ? error.message : String(error)})`,
3529
- )
3530
- }
3531
- bytes = stored.data
3532
- if (stored.ref && typeof stored.ref.mediaType === 'string') {
3533
- storedMediaType = stored.ref.mediaType
3534
- }
3535
- } else {
3536
- const fs = ctx.get('fs')
3537
- if (fs === undefined) throw new Error('vision-router: the fs service is not available')
3538
- const target = await fs.resolve(input)
3539
- bytes = await fs.readBytes(target, undefined, 20 * 1024 * 1024)
3540
- }
3541
- // Attachments are stored as content-addressed files without an
3542
- // extension: sniff the format from the bytes, and fall back to the
3543
- // stored ref / extension only when sniffing cannot decide.
3544
- const mediaType = sniffMediaType(bytes) ?? storedMediaType ?? mediaTypeOf(input)
3545
- if (mediaType === undefined) {
3546
- throw new Error(`unsupported image format ${input} (png/jpeg/webp/gif only)`)
3547
- }
3548
- return { bytes, mediaType }
3549
- }
3533
+ const readImageBytes = createVisionImageInputReader(
3534
+ ctx, resolveAttachment, throwIfVisionTaskAborted,
3535
+ )
3550
3536
 
3551
3537
  const imageDims = async (bytes) => {
3552
3538
  const sharp = await loadSharp()
@@ -4362,9 +4348,10 @@ ctx.logger?.info(
4362
4348
  name: 'vision_ocr',
4363
4349
  description:
4364
4350
  'Transcribe TEXT from an image. ENGINE POLICY: explicit engine=tesseract or engine=vision always wins. ' +
4365
- 'Otherwise the configured OCR engine policy applies. The default auto policy tries local Tesseract ' +
4366
- '(chi_sim+eng) first — fast, free, offline — then falls back to a vision model if local OCR fails or ' +
4367
- 'returns no text. Structured 1+x follow-up does not change the selected policy. Returns the text and ' +
4351
+ 'Otherwise the configured OCR engine policy applies. Auto first prefers a locally installed, SHA-verified ' +
4352
+ 'PP-OCRv6/ncnn on macOS ARM64 or Windows x64, then Tesseract (chi_sim+eng), and finally a vision model ' +
4353
+ 'when local OCR fails, returns no text, or remains uncertain (subject to permission and budget). Structured 1+x follow-up does not change the selected policy. ' +
4354
+ 'Returns the text and ' +
4368
4355
  'which engine produced it. ' +
4369
4356
  'SCOPE: vision_ocr reads letters, it does NOT recognize people, objects or scenes. Never use it ' +
4370
4357
  'as a fallback when vision_describe fails to identify who/what is in a picture ("这是谁" / ' +
@@ -4394,7 +4381,7 @@ ctx.logger?.info(
4394
4381
  },
4395
4382
  engine: {
4396
4383
  type: 'string',
4397
- description: '"auto": use the configured OCR engine policy (default policy is local Tesseract first, then vision fallback); explicit "tesseract"/"vision" always overrides the configured default for this call.',
4384
+ description: '"auto": use the configured OCR engine policy: prefer installed SHA-verified PP-OCRv6 on macOS ARM64 or Windows x64, otherwise local Tesseract; permitted vision fallback on local failure or uncertain text; explicit "tesseract"/"vision" always overrides the configured policy.',
4398
4385
  },
4399
4386
  },
4400
4387
  additionalProperties: false,
@@ -4423,6 +4410,83 @@ ctx.logger?.info(
4423
4410
  // gets a capped slice (never more than 12s), the vision model only the
4424
4411
  // remainder. The two timeouts can never stack into a multi-minute wait.
4425
4412
  const deadline = createDeadline(ocrBudgetMs())
4413
+ // Prefer an owner-installed and SHA-pinned PP-OCRv6 worker in auto mode.
4414
+ // No models or native binaries are bundled with the npm package.
4415
+ // The isolated legacy preview override remains for existing lab tests.
4416
+ const nativeAssets = process.env.DVR_NCNN_OCR_CANARY === '1'
4417
+ ? {
4418
+ binaryPath: process.env.DVR_NCNN_OCR_EXECUTABLE,
4419
+ modelDir: process.env.DVR_NCNN_OCR_MODELS,
4420
+ }
4421
+ : (() => {
4422
+ const installed = localNativeOcrInstallPaths()
4423
+ return installed && existsSync(installed.binaryPath) ? installed : null
4424
+ })()
4425
+ if (engine === 'auto' && nativeAssets !== null) {
4426
+ try {
4427
+ const native = await runExperimentalNativeOcrCanary(bytes, {
4428
+ binaryPath: nativeAssets.binaryPath,
4429
+ modelDir: nativeAssets.modelDir,
4430
+ loadSharp,
4431
+ signal: taskSignal,
4432
+ timeoutMs: Math.min(21000, Math.max(1, Math.floor(deadline.remaining() * .75))),
4433
+ })
4434
+ throwIfVisionTaskAborted(taskSignal)
4435
+ if (native.text.trim()) {
4436
+ if (session) {
4437
+ visionTurnMemory.recordLocalOcr(
4438
+ visionScopeOf(session),
4439
+ visionEvidenceSourceKey(imageInput),
4440
+ { uncertain: native.uncertain === true },
4441
+ )
4442
+ }
4443
+ const nativeResult = {
4444
+ engine: 'ncnn',
4445
+ text: native.text.trim(),
4446
+ uncertain: native.uncertain === true,
4447
+ ...(native.uncertain
4448
+ ? { review: {
4449
+ documentRefined: native.documentRefined === true,
4450
+ regions: Array.isArray(native.regions) ? native.regions.length : undefined,
4451
+ numericOrFormulaRisks: Array.isArray(native.reviewQueue)
4452
+ ? native.reviewQueue.slice(0, 12) : undefined,
4453
+ reviewTruncated: Array.isArray(native.reviewQueue) && native.reviewQueue.length > 12,
4454
+ } }
4455
+ : {}),
4456
+ }
4457
+ // Installing ncnn must not turn uncertain, nonempty local OCR
4458
+ // into an unreviewable final answer. Never bypass privacy,
4459
+ // breaker or the shared one-task deadline to verify it.
4460
+ const reviewed = await verifyUncertainAutoOcrResult(nativeResult, {
4461
+ engine,
4462
+ localOnlyVision: localOnlyVisionEnabled(current()),
4463
+ degraded: degraded.active,
4464
+ remainingMs: deadline.remaining(),
4465
+ verifyVision: () => answerVisionForTool(
4466
+ exec, bytes, mediaType,
4467
+ '请核查图片中的原文,尤其是易混淆的数字、化学式和底部字幕;不确定时不要猜。只输出文字。',
4468
+ { deadline },
4469
+ ),
4470
+ checkAbort: error => throwIfVisionTaskAborted(taskSignal, error),
4471
+ })
4472
+ return JSON.stringify(reviewed)
4473
+ }
4474
+ } catch (error) {
4475
+ throwIfVisionTaskAborted(taskSignal, error)
4476
+ // Native resource saturation must not fork extra local workers.
4477
+ if (error?.code === 'OCR_BUSY' || error?.code === 'OCR_QUEUE_TIMEOUT') {
4478
+ return JSON.stringify({
4479
+ engine: 'none', ok: false,
4480
+ code: error.code, text: '', retryable: false,
4481
+ reason: 'local OCR worker capacity is exhausted',
4482
+ })
4483
+ }
4484
+ ctx.logger?.warn(
4485
+ 'vision-router: local PP-OCRv6 unavailable; using Tesseract/vision policy ' +
4486
+ '(' + String(error?.code ?? 'OCR_NATIVE_FAILED').slice(0, 60) + ')',
4487
+ )
4488
+ }
4489
+ }
4426
4490
  const tesseractSlice = Math.min(12000, deadline.remaining())
4427
4491
  if (engine !== 'vision') {
4428
4492
  let localAttempted = false
@@ -4431,7 +4495,11 @@ ctx.logger?.info(
4431
4495
  const local = await ocrWithTesseractAdaptive(
4432
4496
  bytes,
4433
4497
  tesseractSlice,
4434
- { signal: taskSignal },
4498
+ {
4499
+ signal: taskSignal,
4500
+ preprocess: (source, context) =>
4501
+ prepareTesseractSubtitleRetry(source, { ...context, loadSharp }),
4502
+ },
4435
4503
  )
4436
4504
  if (local.text.trim() !== '') {
4437
4505
  if (session) {
@@ -4441,7 +4509,7 @@ ctx.logger?.info(
4441
4509
  { uncertain: local.uncertain === true },
4442
4510
  )
4443
4511
  }
4444
- return JSON.stringify({
4512
+ const localResult = {
4445
4513
  engine: 'tesseract',
4446
4514
  text: local.text.trim(),
4447
4515
  uncertain: local.uncertain === true,
@@ -4452,10 +4520,48 @@ ctx.logger?.info(
4452
4520
  attemptedPsms: local.attemptedPsms,
4453
4521
  quality: Number(local.quality.toFixed(2)),
4454
4522
  riskyTokens: local.riskyTokens,
4523
+ ...(local.variant && local.variant !== 'original'
4524
+ ? { preprocessing: local.variant } : {}),
4455
4525
  },
4456
4526
  }
4457
4527
  : {}),
4458
- })
4528
+ }
4529
+ // A nonempty but low-confidence transcript is not proof of
4530
+ // accuracy. In auto ONLY, try the existing consent-controlled
4531
+ // vision route while retaining local evidence if unavailable.
4532
+ if (shouldVerifyUncertainAutoOcr({
4533
+ engine,
4534
+ uncertain: local.uncertain,
4535
+ localOnlyVision: localOnlyVisionEnabled(current()),
4536
+ degraded: degraded.active,
4537
+ remainingMs: deadline.remaining(),
4538
+ })) {
4539
+ try {
4540
+ const vision = await answerVisionForTool(
4541
+ exec, bytes, mediaType,
4542
+ '请严格辨认图片上的原文,尤其是底部字幕或浅色文字;不确定时不要猜。只输出文字。',
4543
+ { deadline },
4544
+ )
4545
+ throwIfVisionTaskAborted(taskSignal)
4546
+ if (vision.ok !== false && typeof vision.text === 'string' &&
4547
+ vision.text.trim()) {
4548
+ return JSON.stringify({
4549
+ engine: 'vision',
4550
+ text: vision.text.trim(),
4551
+ review: {
4552
+ fallbackReason: 'low_confidence_local_ocr',
4553
+ localText: localResult.text.slice(0, 1000),
4554
+ localQuality: localResult.review?.quality,
4555
+ },
4556
+ })
4557
+ }
4558
+ } catch (error) {
4559
+ throwIfVisionTaskAborted(taskSignal, error)
4560
+ // The existing local transcript remains available and
4561
+ // retains its uncertainty if visual fallback fails.
4562
+ }
4563
+ }
4564
+ return JSON.stringify(localResult)
4459
4565
  }
4460
4566
  if (engine === 'tesseract') return JSON.stringify({ engine: 'tesseract', text: '', uncertain: true })
4461
4567
  } catch (error) {
@@ -4720,8 +4826,8 @@ ctx.logger?.info(
4720
4826
  deepToolDefs.push({
4721
4827
  name: 'vision_trace',
4722
4828
  description:
4723
- 'Vectorize an image (icon/logo) into an SVG via a local potrace pipeline (no Python). ' +
4724
- 'Default: COLOR-preserving vectorization — one path per dominant color with fill="#rrggbb". ' +
4829
+ 'Vectorize an image (icon/logo) into an SVG via a local VTracer/WASM worker (no Python). ' +
4830
+ 'Default: color-preserving vectorization with a limited color palette. ' +
4725
4831
  'Set color=false for the layered grayscale posterization, where `steps` (1-16, default 4) ' +
4726
4832
  'controls levels. Writes the SVG as an artifact.',
4727
4833
  parameters: {
@@ -4761,16 +4867,21 @@ ctx.logger?.info(
4761
4867
  const colors = Number.isInteger(args.colors) && args.colors > 0 ? Math.min(args.colors, 16) : 8
4762
4868
  const raw = await sharp(traceBytes, { failOn: 'none' }).ensureAlpha().raw().toBuffer({ resolveWithObject: true })
4763
4869
  throwIfVisionTaskAborted(taskSignal)
4764
- const palette = quantizeColors(raw.data, colors)
4765
- colorCount = palette.length
4766
- svg = await posterizeSvgColor(raw.data, raw.info, palette, timeoutMs(), { signal: taskSignal })
4870
+ // User-facing colors is a strict maximum, not an exact
4871
+ // palette. VTracer chooses the colors for better gradient/alpha
4872
+ // fidelity; explicit palette callers use the separate array form.
4873
+ svg = await posterizeSvgColor(raw.data, raw.info, colors, timeoutMs(), { signal: taskSignal })
4874
+ colorCount = Math.min(colors, new Set(
4875
+ [...svg.matchAll(/<path\b[^>]*\bfill="(#[0-9a-fA-F]{6})"/g)]
4876
+ .map((match) => match[1].toLowerCase()),
4877
+ ).size)
4767
4878
  } else {
4768
4879
  svg = await posterizeSvg(traceBytes, steps, 'dominant', timeoutMs(), { signal: taskSignal })
4769
4880
  }
4770
4881
  } catch (error) {
4771
4882
  throwIfVisionTaskAborted(taskSignal, error)
4772
4883
  throw new Error(
4773
- `vision_trace: potrace failed (${error && error.message ? error.message : String(error)})`,
4884
+ `vision_trace: vectorization failed (${error && error.message ? error.message : String(error)})`,
4774
4885
  )
4775
4886
  }
4776
4887
  throwIfVisionTaskAborted(taskSignal)