dsh-vision-router 3.2.1 → 3.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/README.md +5 -4
  2. package/README.zh.md +20 -3
  3. package/THIRD_PARTY_NOTICES.md +60 -0
  4. package/assets/licenses/llvm-mingw-LICENSE.txt +279 -0
  5. package/assets/licenses/mingw-w64-runtime-COPYING.txt +265 -0
  6. package/assets/licenses/ncnn-LICENSE.txt +86 -0
  7. package/docs/architecture/dsh-support-window.md +25 -6
  8. package/docs/compatibility/dsh-0.2.2-rc3-audit.md +131 -0
  9. package/docs/releases/v3.2.2.md +25 -0
  10. package/docs/releases/v3.3.0.md +34 -0
  11. package/docs/releases/v3.4.0.md +41 -0
  12. package/index.js +313 -169
  13. package/lib/client.js +4 -4
  14. package/lib/core-primitives.js +115 -56
  15. package/lib/dsh-support-window.js +6 -3
  16. package/lib/maintenance-routes.js +2 -0
  17. package/lib/managed-tesseract-child.js +33 -0
  18. package/lib/native-ocr-annotation-risk.js +33 -0
  19. package/lib/native-ocr-canary.js +156 -0
  20. package/lib/native-ocr-chemistry-review.js +37 -0
  21. package/lib/native-ocr-document.js +243 -0
  22. package/lib/native-ocr-installation.js +17 -0
  23. package/lib/native-ocr-integrity.js +93 -0
  24. package/lib/native-ocr-question-labels.js +167 -0
  25. package/lib/native-ocr-supervisor.js +268 -0
  26. package/lib/optional-ocr-engines.js +330 -0
  27. package/lib/optional-ocr-routes.js +87 -0
  28. package/lib/runtime-reliability.js +7 -0
  29. package/lib/session-query-readers.js +8 -2
  30. package/lib/session-vision-attachment-index.js +61 -14
  31. package/lib/settings-ia-client-prelude.js +56 -2
  32. package/lib/tesseract-image-preprocess.js +35 -0
  33. package/lib/trace-vtracer-process-runtime.js +133 -0
  34. package/lib/trace-vtracer-process.js +43 -0
  35. package/lib/vision-attachment-handle-runtime.js +11 -2
  36. package/lib/vision-backend-runtime-policy.js +5 -3
  37. package/lib/vision-image-input-reader.js +67 -0
  38. package/lib/vision-image-input-verdict.js +24 -3
  39. package/lib/vision-tool-runtime-boundary.js +7 -10
  40. package/lib/vision-tool-stage.js +58 -0
  41. package/lib/web-capability-boundary.js +4 -0
  42. package/package.json +5 -4
  43. package/scripts/install-native-ocr.mjs +144 -0
package/index.js CHANGED
@@ -118,6 +118,10 @@ import z from '@deepseek-ai/schemastery'
118
118
 
119
119
  import path from 'node:path'
120
120
  import { existsSync } from 'node:fs'
121
+ import { runExperimentalNativeOcrCanary } from './lib/native-ocr-canary.js'
122
+ import { localNativeOcrInstallPaths } from './lib/native-ocr-installation.js'
123
+ import { managedTesseractPaths, runManagedTesseractOcr, systemTesseract } from './lib/optional-ocr-engines.js'
124
+ import { prepareTesseractSubtitleRetry } from './lib/tesseract-image-preprocess.js'
121
125
 
122
126
  import { pathToFileURL } from 'node:url'
123
127
  import { appendPromptToImageOnlyMessage } from './lib/http-compat.js'
@@ -151,6 +155,8 @@ import {
151
155
  import { currentVisionExecutionOrder } from './lib/vision-execution-order.js'
152
156
  import { applyVisionExecutionOrder } from './lib/vision-execution-order-apply.js'
153
157
  import { currentVisionTurnBudgetSignal } from './lib/turn-budget-context.js'
158
+ import { createVisionToolStageTrace } from './lib/vision-tool-stage.js'
159
+ import { createVisionImageInputReader } from './lib/vision-image-input-reader.js'
154
160
  import { parseGroundingBoxResponse, assessGroundingBox } from './lib/grounding-box-recovery.js'
155
161
  import { createHash } from 'node:crypto'
156
162
  import {
@@ -311,7 +317,7 @@ export const Config = z.object({
311
317
  // (its own cap) and the vision-model fallback only the rest — never two
312
318
  // full timeouts added together.
313
319
  ocrTimeoutMs: z.number().step(1).min(1000).max(120000).default(30000),
314
- // Default engine for vision_ocr. Per-call engine=tesseract|vision remains authoritative.
320
+ // Auto prefers verified owner-installed PP-OCRv6 when supported; explicit engine wins.
315
321
  ocrEngine: z.union(['auto', 'tesseract', 'vision']).default('auto'),
316
322
  proxy: z.string().default(''),
317
323
  proxyHosts: z.array(z.string()).default([...DEFAULT_PROXY_HOSTS]),
@@ -470,6 +476,8 @@ import {
470
476
  resolveVisionOcrEngine,
471
477
  ocrWithTesseract,
472
478
  ocrWithTesseractAdaptive,
479
+ shouldVerifyUncertainAutoOcr,
480
+ verifyUncertainAutoOcrResult,
473
481
  estimateTokens,
474
482
  estimateMessages,
475
483
  trimMessagesToBudget,
@@ -565,6 +573,8 @@ export {
565
573
  resolveVisionOcrEngine,
566
574
  ocrWithTesseract,
567
575
  ocrWithTesseractAdaptive,
576
+ shouldVerifyUncertainAutoOcr,
577
+ verifyUncertainAutoOcrResult,
568
578
  estimateTokens,
569
579
  estimateMessages,
570
580
  trimMessagesToBudget,
@@ -1023,20 +1033,35 @@ export function apply(ctx, config = {}, runtime = {}) {
1023
1033
  }
1024
1034
  if (attachments === undefined) continue
1025
1035
  try {
1026
- const stored = await attachments.readImage(block.attachment)
1036
+ throwIfVisionTaskAborted(options.signal)
1037
+ const stored = await attachments.readImage(block.attachment, options.signal)
1038
+ throwIfVisionTaskAborted(options.signal)
1027
1039
  // Last-mile guard: never send oversized images to the vision
1028
1040
  // endpoint — encoder cost scales with pixels and dominates
1029
1041
  // tool-call latency on retina screenshots.
1030
1042
  let bytes = stored.data
1031
1043
  if (downscaleEnabled() && bytes && bytes.length > 0) {
1032
1044
  bytes = await downscaleImage(bytes, downscaleMaxPixels())
1045
+ throwIfVisionTaskAborted(options.signal)
1033
1046
  }
1034
1047
  content.push(...toOpenAIContent([block], () => bytes))
1035
1048
  } catch (error) {
1036
- ctx.logger?.warn(
1037
- 'vision-http: failed to read image attachment: %s',
1038
- error && error.message ? error.message : String(error),
1039
- )
1049
+ throwIfVisionTaskAborted(options.signal, error)
1050
+ // A text-only completion here would falsely claim the model
1051
+ // saw pixels. Fail this backend immediately and let the
1052
+ // normal vision fallback chain choose another backend.
1053
+ ctx.logger?.warn('vision-http: retained image delivery failed; refusing text-only downgrade')
1054
+ yield {
1055
+ type: 'finish',
1056
+ reason: {
1057
+ kind: 'error',
1058
+ failure: {
1059
+ code: 'VISION_IMAGE_DELIVERY_UNAVAILABLE',
1060
+ message: 'vision-http: retained image could not be delivered to the model',
1061
+ },
1062
+ },
1063
+ }
1064
+ return
1040
1065
  }
1041
1066
  } else if (block && block.type === 'tool-result') {
1042
1067
  // The OpenAI wire has no tool-call frames to hang a `role: tool`
@@ -1701,7 +1726,9 @@ export function apply(ctx, config = {}, runtime = {}) {
1701
1726
  }
1702
1727
  const content = []
1703
1728
  for (const block of blocks) {
1704
- const stored = await attachments.readImage(block.attachment)
1729
+ throwIfVisionTaskAborted(options.signal)
1730
+ const stored = await attachments.readImage(block.attachment, options.signal)
1731
+ throwIfVisionTaskAborted(options.signal)
1705
1732
  content.push(...toOpenAIContent([block], () => stored.data))
1706
1733
  }
1707
1734
  return callOpenAIWithProviderTransport(
@@ -1797,14 +1824,18 @@ export function apply(ctx, config = {}, runtime = {}) {
1797
1824
  throw new Error('corrected route: the attachment service is not registered')
1798
1825
  }
1799
1826
  const bytesOf = async (attachment) => {
1800
- const stored = await attachments.readImage(attachment)
1827
+ throwIfVisionTaskAborted(options.signal)
1828
+ const stored = await attachments.readImage(attachment, options.signal)
1829
+ throwIfVisionTaskAborted(options.signal)
1801
1830
  let bytes = stored.data
1802
1831
  if (downscaleEnabled() && bytes && bytes.length > 0) {
1803
1832
  bytes = await downscaleImage(bytes, downscaleMaxPixels())
1833
+ throwIfVisionTaskAborted(options.signal)
1804
1834
  }
1805
1835
  return bytes
1806
1836
  }
1807
1837
  const anthropic = await toAnthropicMessages(messages, bytesOf)
1838
+ throwIfVisionTaskAborted(options.signal)
1808
1839
  if (anthropic.messages.length === 0) {
1809
1840
  throw new Error(`corrected route "${pair.provider}/${pair.model}": no representable content to send`)
1810
1841
  }
@@ -2402,8 +2433,8 @@ export function apply(ctx, config = {}, runtime = {}) {
2402
2433
  // Host-owned proxy overrides are scoped by lib/legacy-global-proxy-boundary.js.
2403
2434
  // Core no longer owns or installs a process-wide proxy fetch implementation.
2404
2435
 
2405
- const resolveAttachment = (session, id) => sessionVisionIndex.resolveAttachment(session, id)
2406
- const resolveAttachments = (session, ids) => sessionVisionIndex.resolveAttachments(session, ids)
2436
+ const resolveAttachment = (session, id, signal) => sessionVisionIndex.resolveAttachment(session, id, signal)
2437
+ const resolveAttachments = (session, ids, signal) => sessionVisionIndex.resolveAttachments(session, ids, signal)
2407
2438
 
2408
2439
  // Session-local routing handoff between pre-step and agent/request. Modern
2409
2440
  // Hosts store one exact async raw-log tail seq; legacy Hosts retain only the
@@ -2880,121 +2911,103 @@ export function apply(ctx, config = {}, runtime = {}) {
2880
2911
  }
2881
2912
  const blocks = []
2882
2913
  const contentIds = []
2883
-
2914
+ // One shared deadline for the WHOLE task: image preflight, every
2915
+ // provider, fallback and JSON correction draw from one clock.
2916
+ // Previously the clock started after SessionQuery/image loading,
2917
+ // leaving attachmentIds vulnerable to the Host's ~120s watchdog.
2918
+ const hostSignal = combineSignals(currentVisionTurnBudgetSignal(), exec?.signal)
2919
+ const deadline = createDeadline(visionTaskTimeoutMs())
2920
+ const taskSignal = combineSignals(hostSignal, deadline.signal())
2921
+ const stage = createVisionToolStageTrace(ctx.logger, 'vision_describe', taskSignal)
2922
+ try {
2884
2923
  const paths = Array.isArray(args.paths) ? args.paths : []
2885
2924
  const attachmentIds = Array.isArray(args.attachmentIds) ? args.attachmentIds : []
2886
2925
  if (paths.length + attachmentIds.length === 0 || paths.length + attachmentIds.length > 4) {
2887
2926
  throw new Error('vision_describe: provide 1-4 images via paths and/or attachmentIds')
2888
2927
  }
2889
- // Preserve only durable upload ids for a deterministic offline fallback.
2890
- // Never expose or guess the attachment store's private filesystem path.
2891
- const materializableAttachmentIds = [...new Set([
2892
- ...attachmentIds.map((id) => String(id)).filter((id) => isAttachmentIdInput(id)),
2893
- ...paths.map((item) => String(item)).filter((item) => isAttachmentIdInput(item)),
2928
+ // Both argument forms now use the exact same read -> normalize ->
2929
+ // downscale -> save path. The old attachmentIds-only path could retain
2930
+ // an oversized original ref after resize publication failed.
2931
+ const sources = [...paths, ...attachmentIds]
2932
+ const materializableAttachmentIds = [...new Set(
2933
+ sources.map((value) => String(value)).filter((value) => isAttachmentIdInput(value)),
2934
+ )]
2935
+ // attachmentIds is always an attachment namespace, even for a legacy
2936
+ // Host whose authorized IDs do not yet have a sha256: prefix.
2937
+ const requestedRefs = [...new Set([
2938
+ ...attachmentIds.map((value) => String(value)),
2939
+ ...paths.map((value) => String(value)).filter((value) => isAttachmentIdInput(value)),
2894
2940
  ])]
2895
- const attachmentSession = exec && exec.agent && exec.agent.session
2896
- const resolvedAttachmentRefs = materializableAttachmentIds.length > 0
2897
- ? await resolveAttachments(attachmentSession, materializableAttachmentIds)
2941
+ const attachmentSession = exec?.agent?.session
2942
+ const resolvedAttachmentRefs = requestedRefs.length > 0
2943
+ ? await stage('resolve-upload-refs', () =>
2944
+ resolveAttachments(attachmentSession, requestedRefs, taskSignal))
2898
2945
  : new Map()
2899
2946
 
2900
- for (const path of paths) {
2947
+ for (const [index, item] of sources.entries()) {
2948
+ const source = String(item)
2949
+ const attachmentInput = index >= paths.length
2901
2950
  let bytes
2902
2951
  let mediaType
2903
2952
  try {
2904
- // readImageBytes accepts both filesystem paths and attachment ids
2905
- // ("sha256:..."), so a model that passes an uploaded image's id as
2906
- // a path gets the right pixels instead of a not-found error.
2907
- ;({ bytes, mediaType } = await readImageBytes(exec, path, resolvedAttachmentRefs))
2953
+ ;({ bytes, mediaType } = await stage('read-image-' + index, () =>
2954
+ readImageBytes(exec, source, resolvedAttachmentRefs, attachmentInput)))
2908
2955
  } catch (error) {
2909
2956
  throw new Error(
2910
- `vision_describe: failed to read ${path} (${error && error.message ? error.message : String(error)})`,
2957
+ `vision_describe: failed to read ${source} (${error && error.message ? error.message : String(error)})`,
2958
+ { cause: error },
2911
2959
  )
2912
2960
  }
2913
2961
  if (downscaleEnabled()) {
2914
- const resized = await downscaleImage(bytes, downscaleMaxPixels())
2962
+ const resized = await stage('downscale-image-' + index, () =>
2963
+ downscaleImage(bytes, downscaleMaxPixels(), { signal: taskSignal }))
2915
2964
  if (resized !== bytes) {
2916
- ctx.logger?.info('vision-router: downscaled %s for the vision call', path)
2965
+ ctx.logger?.info('vision-router: downscaled vision input index=%d', index)
2917
2966
  }
2918
2967
  bytes = resized
2919
2968
  }
2969
+ // Always publish canonical bytes, including an uploaded image.
2970
+ // Host adapters then read the verified/downsized version, never a
2971
+ // stale or mismatched original attachment reference.
2920
2972
  let ref
2921
2973
  try {
2922
- ref = await attachments.saveImage({
2974
+ ref = await stage('save-normalized-image-' + index, () => attachments.saveImage({
2923
2975
  data: bytes,
2924
2976
  mediaType,
2925
- ...(isAttachmentIdInput(path) || basenameOf(path) === undefined
2977
+ ...(attachmentInput || isAttachmentIdInput(source) || basenameOf(source) === undefined
2926
2978
  ? {}
2927
- : { name: basenameOf(path) }),
2928
- })
2979
+ : { name: basenameOf(source) }),
2980
+ }))
2929
2981
  } catch (error) {
2930
2982
  throw new Error(
2931
- `vision_describe: image ${path} was rejected (${error && error.message ? error.message : String(error)})`,
2983
+ `vision_describe: image ${source} was rejected (${error && error.message ? error.message : String(error)})`,
2984
+ { cause: error },
2932
2985
  )
2933
2986
  }
2987
+ throwIfVisionTaskAborted(taskSignal)
2934
2988
  contentIds.push(String(ref.attachmentId))
2935
2989
  blocks.push({ type: 'image', attachment: ref })
2936
2990
  }
2937
2991
 
2938
- for (const id of attachmentIds) {
2939
- const attachmentId = String(id)
2940
- const ref = isAttachmentIdInput(attachmentId)
2941
- ? resolvedAttachmentRefs.get(attachmentId)
2942
- : await resolveAttachment(attachmentSession, attachmentId)
2943
- if (ref === undefined) {
2944
- throw new Error(
2945
- `vision_describe: unknown attachment id "${id}" (it must come from an image uploaded in this conversation)`,
2946
- )
2947
- }
2948
- let stored
2949
- try {
2950
- stored = await attachments.readImage(ref)
2951
- } catch (error) {
2952
- throw new Error(
2953
- `vision_describe: failed to read attachment ${id} (${error && error.message ? error.message : String(error)})`,
2954
- )
2955
- }
2956
- // Downscale oversized uploads before the vision call: retina
2957
- // screenshots easily reach 10MP+ and the vision encoder's cost
2958
- // scales with pixels — a full-size upload is the dominant part of
2959
- // the tool-call latency. Re-save a resized attachment so the
2960
- // adapter reads the small one.
2961
- if (downscaleEnabled() && stored.data && stored.data.length > 0) {
2962
- const resized = await downscaleImage(stored.data, downscaleMaxPixels())
2963
- if (resized !== stored.data) {
2964
- try {
2965
- const resizedRef = await attachments.saveImage({
2966
- data: resized,
2967
- mediaType: stored.ref && stored.ref.mediaType ? stored.ref.mediaType : 'image/png',
2968
- ...(stored.ref && stored.ref.name ? { name: stored.ref.name } : {}),
2969
- })
2970
- stored = { ref: resizedRef, data: resized }
2971
- ctx.logger?.info('vision-router: downscaled attachment %s for the vision call', id)
2972
- } catch {
2973
- stored = { ...stored, data: resized }
2974
- }
2975
- }
2976
- }
2977
- contentIds.push(String(ref.attachmentId))
2978
- blocks.push({ type: 'image', attachment: stored.ref })
2979
- }
2980
-
2981
2992
  const question = String(args.question ?? '')
2982
2993
  const wantJson = args.json === true
2983
2994
  // Keep the adapter path and direct OpenAI-compatible HTTP path on the
2984
2995
  // exact same prompt, including the structured JSON evidence contract.
2985
2996
  const promptText = visionDescribePrompt(question, wantJson)
2986
- const usablePairs = await resolveToolVisionPairs()
2997
+ const usablePairs = await stage('resolve-vision-backends', () => resolveToolVisionPairs())
2987
2998
  // Freeze this task's direct fallback list: settings changes take effect
2988
2999
  // on the next call, never halfway through one fallback walk.
2989
3000
  const httpFallbacks = httpProviders()
2990
3001
  const rejectedPairs = []
3002
+ let backendPreflightIndex = 0
2991
3003
  for (const pair of pairs()) {
2992
3004
  if (!pair || pair.provider === HTTP_ROUTE) continue
2993
3005
  if (!adapterAvailable(ctx.llm, pair.provider)) {
2994
3006
  rejectedPairs.push(`${pair.provider}/${pair.model}: provider adapter is not registered`)
2995
3007
  continue
2996
3008
  }
2997
- const capability = await resolveVisionBackendCapability(pair.provider, pair.model)
3009
+ const capability = await stage('backend-preflight-' + backendPreflightIndex++, () =>
3010
+ resolveVisionBackendCapability(pair.provider, pair.model))
2998
3011
  if (capability.attemptable === false) {
2999
3012
  rejectedPairs.push(
3000
3013
  `${pair.provider}/${pair.model}: structurally unavailable (${capability.reason ?? 'unknown reason'})`,
@@ -3030,9 +3043,7 @@ export function apply(ctx, config = {}, runtime = {}) {
3030
3043
  )
3031
3044
  }
3032
3045
 
3033
- // One shared deadline for the WHOLE task: every provider, fallback and
3034
- // the JSON-correction retry draws from this single budget.
3035
- const deadline = createDeadline(visionTaskTimeoutMs())
3046
+ // The same clock already started before attachment resolution.
3036
3047
  const baseMessages = [
3037
3048
  {
3038
3049
  role: 'user',
@@ -3051,7 +3062,7 @@ export function apply(ctx, config = {}, runtime = {}) {
3051
3062
  0,
3052
3063
  )
3053
3064
 
3054
- for (const pair of usablePairs) {
3065
+ for (const [pairIndex, pair] of usablePairs.entries()) {
3055
3066
  if (localOnlyVisionEnabled(current()) && !isLocalBackendPair(pair)) {
3056
3067
  errors.push(`${pair.provider}/${pair.model}: skipped (local-only vision policy)`)
3057
3068
  continue
@@ -3064,7 +3075,8 @@ export function apply(ctx, config = {}, runtime = {}) {
3064
3075
  break
3065
3076
  }
3066
3077
  const backendKey = `${pair.provider}/${pair.model}`
3067
- const fingerprint = await credentialFingerprintFor({ provider: pair.provider })
3078
+ const fingerprint = await stage('adapter-auth-' + pairIndex, () =>
3079
+ credentialFingerprintFor({ provider: pair.provider }))
3068
3080
  const gate = visionBreaker.inspect(backendKey, fingerprint, scope)
3069
3081
  if (gate.blocked) {
3070
3082
  errors.push(`${backendKey}: skipped (circuit open: ${gate.reason})`)
@@ -3089,19 +3101,20 @@ export function apply(ctx, config = {}, runtime = {}) {
3089
3101
  )
3090
3102
  : timeoutMs()
3091
3103
  const signal = combineSignals(
3092
- currentVisionTurnBudgetSignal(),
3104
+ hostSignal,
3093
3105
  deadline.signal(),
3094
3106
  AbortSignal.timeout(attemptBudgetMs),
3095
3107
  )
3096
- const capability = await resolveVisionBackendCapability(pair.provider, pair.model)
3097
- let text = await callVisionPairWithOptionalBridge(pair, messages, {
3108
+ const capability = await stage('adapter-capability-' + pairIndex, () =>
3109
+ resolveVisionBackendCapability(pair.provider, pair.model))
3110
+ let text = await stage('adapter-request-' + pairIndex, () => callVisionPairWithOptionalBridge(pair, messages, {
3098
3111
  maxTokens: 4096,
3099
3112
  signal,
3100
3113
  sessionId,
3101
3114
  capability,
3102
3115
  bridgeBlocks: blocks,
3103
3116
  bridgeInstruction: promptText,
3104
- })
3117
+ }))
3105
3118
  assertNoRepetitionLoop(text, backendKey)
3106
3119
  ctx.logger?.info(
3107
3120
  'vision-router: vision call %s ok (%d chars, %d ms)',
@@ -3133,7 +3146,7 @@ ctx.logger?.info(
3133
3146
  },
3134
3147
  ]
3135
3148
  attemptStarted = Date.now()
3136
- text = await callVisionPairWithOptionalBridge(pair, messages, {
3149
+ text = await stage('adapter-json-correction-' + pairIndex, () => callVisionPairWithOptionalBridge(pair, messages, {
3137
3150
  maxTokens: 4096,
3138
3151
  signal,
3139
3152
  sessionId,
@@ -3141,7 +3154,7 @@ ctx.logger?.info(
3141
3154
  bridgeBlocks: blocks,
3142
3155
  bridgeInstruction:
3143
3156
  promptText + '\n\nThat output was not valid JSON. Respond with ONLY a valid JSON object now.',
3144
- })
3157
+ }))
3145
3158
  assertNoRepetitionLoop(text, backendKey)
3146
3159
  ctx.logger?.info(
3147
3160
  'vision-router: vision call %s ok after JSON correction (%d chars, %d ms)',
@@ -3171,7 +3184,7 @@ ctx.logger?.info(
3171
3184
  // A user/Host cancel is not a backend outage. Otherwise the
3172
3185
  // canceled attempt poisons this turn's breaker/failure memory and
3173
3186
  // a healthy retry returns BACKEND_UNAVAILABLE_THIS_TURN.
3174
- throwIfVisionTaskAborted(currentVisionTurnBudgetSignal(), error)
3187
+ throwIfVisionTaskAborted(hostSignal, error)
3175
3188
  const classification = classifyVisionFailure(error)
3176
3189
  consecutiveServerFailures = classification.kind === VISION_FAILURE_KINDS.SERVER
3177
3190
  ? consecutiveServerFailures + 1 : 0
@@ -3197,7 +3210,7 @@ ctx.logger?.info(
3197
3210
  // Direct HTTP providers (built-in keyless OVHcloud by default) are the
3198
3211
  // final fallbacks: they bypass the harness llm service entirely, so the
3199
3212
  // anonymous free endpoint works without any credential.
3200
- for (const provider of httpFallbacks) {
3213
+ for (const [httpIndex, provider] of httpFallbacks.entries()) {
3201
3214
  if (localOnlyVisionEnabled(current()) && !isLoopbackVisionBaseURL(provider?.baseURL)) {
3202
3215
  errors.push(`http:${provider?.name}/${provider?.model}: skipped (local-only vision policy)`)
3203
3216
  continue
@@ -3210,7 +3223,8 @@ ctx.logger?.info(
3210
3223
  break
3211
3224
  }
3212
3225
  const backendKey = `http:${provider.name}/${provider.model}`
3213
- const fingerprint = await credentialFingerprintFor({ kind: 'http', apiKeyEnv: provider.apiKeyEnv })
3226
+ const fingerprint = await stage('http-auth-' + httpIndex, () =>
3227
+ credentialFingerprintFor({ kind: 'http', apiKeyEnv: provider.apiKeyEnv }))
3214
3228
  const gate = visionBreaker.inspect(backendKey, fingerprint, scope)
3215
3229
  if (gate.blocked) {
3216
3230
  errors.push(`${backendKey}: skipped (circuit open: ${gate.reason})`)
@@ -3222,15 +3236,19 @@ ctx.logger?.info(
3222
3236
  let attemptStarted = Date.now()
3223
3237
  try {
3224
3238
  // Precompute bytes once per block (attachments.readImage is async).
3225
- const openAIBlocks = []
3226
- for (const block of blocks) {
3227
- if (block.type === 'image' && block.attachment) {
3228
- const stored = await attachments.readImage(block.attachment)
3229
- openAIBlocks.push(toOpenAIContent([block], () => stored.data)[0])
3230
- } else {
3231
- openAIBlocks.push({ type: 'text', text: block.text })
3239
+ const openAIBlocks = await stage('http-image-delivery-' + httpIndex, async () => {
3240
+ const converted = []
3241
+ for (const block of blocks) {
3242
+ if (block.type === 'image' && block.attachment) {
3243
+ const stored = await attachments.readImage(block.attachment, taskSignal)
3244
+ throwIfVisionTaskAborted(taskSignal)
3245
+ converted.push(toOpenAIContent([block], () => stored.data)[0])
3246
+ } else {
3247
+ converted.push({ type: 'text', text: block.text })
3248
+ }
3232
3249
  }
3233
- }
3250
+ return converted
3251
+ })
3234
3252
  // Direct HTTP providers must receive the same image + question as
3235
3253
  // adapter-backed providers. Some endpoints (e.g. Zhipu GLM) reject
3236
3254
  // a pure-image user message even when permissive endpoints accept it.
@@ -3248,12 +3266,15 @@ ctx.logger?.info(
3248
3266
  consecutiveServerFailures,
3249
3267
  )
3250
3268
  const attemptSignal = combineSignals(
3251
- currentVisionTurnBudgetSignal(),
3269
+ hostSignal,
3252
3270
  deadline.signal(),
3253
3271
  AbortSignal.timeout(attemptBudgetMs),
3254
3272
  )
3255
3273
  const askHttp = async (correction) => {
3256
- const answer = await awaitVisionBackendAttempt(attemptSignal, () => callOpenAIWithProviderTransport(
3274
+ const phase = correction === undefined
3275
+ ? 'http-request-' + httpIndex : 'http-json-correction-' + httpIndex
3276
+ const answer = await stage(phase, () =>
3277
+ awaitVisionBackendAttempt(attemptSignal, () => callOpenAIWithProviderTransport(
3257
3278
  provider,
3258
3279
  correction === undefined
3259
3280
  ? openAIBaseMessages
@@ -3267,7 +3288,7 @@ ctx.logger?.info(
3267
3288
  sessionId,
3268
3289
  resolveCredential,
3269
3290
  },
3270
- ))
3291
+ )))
3271
3292
  return answer
3272
3293
  }
3273
3294
  let text = await askHttp(undefined)
@@ -3319,7 +3340,7 @@ ctx.logger?.info(
3319
3340
  // A user/Host cancel is not a backend outage. Otherwise the
3320
3341
  // canceled attempt poisons this turn's breaker/failure memory and
3321
3342
  // a healthy retry returns BACKEND_UNAVAILABLE_THIS_TURN.
3322
- throwIfVisionTaskAborted(currentVisionTurnBudgetSignal(), error)
3343
+ throwIfVisionTaskAborted(hostSignal, error)
3323
3344
  const classification = classifyVisionFailure(error)
3324
3345
  consecutiveServerFailures = classification.kind === VISION_FAILURE_KINDS.SERVER
3325
3346
  ? consecutiveServerFailures + 1 : 0
@@ -3367,6 +3388,18 @@ ctx.logger?.info(
3367
3388
  }
3368
3389
  }
3369
3390
  return JSON.stringify(baseFailure)
3391
+ } catch (error) {
3392
+ // The caller's explicit cancellation is not a model outage.
3393
+ if (hostSignal?.aborted) throw hostSignal.reason ?? error
3394
+ if (deadline.expired() || taskSignal?.aborted) {
3395
+ return JSON.stringify(buildVisionFailure({
3396
+ code: VISION_RESULT_CODES.TIMEOUT,
3397
+ retryable: false,
3398
+ reason: 'Vision task deadline exhausted while preparing or processing the image.',
3399
+ }))
3400
+ }
3401
+ throw error
3402
+ }
3370
3403
  },
3371
3404
  }
3372
3405
  deepToolDefs.push(visionDescribeTool)
@@ -3498,55 +3531,9 @@ ctx.logger?.info(
3498
3531
  return String(attachmentIds[0]).trim()
3499
3532
  }
3500
3533
 
3501
- const readImageBytes = async (exec, imagePath, resolvedAttachmentRefs) => {
3502
- const input = String(imagePath ?? '')
3503
- let bytes
3504
- let storedMediaType
3505
- if (isAttachmentIdInput(input)) {
3506
- // Uploaded images reach the tool arguments as durable attachment ids
3507
- // ("sha256:..."); resolve them through the session's recorded upload
3508
- // index instead of treating the id as a filesystem path.
3509
- const attachments = ctx.get('attachments')
3510
- if (attachments === undefined) {
3511
- throw new Error('vision-router: the attachment service is not available in this deployment')
3512
- }
3513
- const session = exec && exec.agent && exec.agent.session
3514
- const canonicalInput = input.trim()
3515
- const ref = resolvedAttachmentRefs instanceof Map
3516
- ? resolvedAttachmentRefs.get(canonicalInput)
3517
- : await resolveAttachment(session, canonicalInput)
3518
- if (ref === undefined) {
3519
- throw new Error(
3520
- `vision-router: unknown attachment id "${input}" (it must come from an image uploaded in this conversation)`,
3521
- )
3522
- }
3523
- let stored
3524
- try {
3525
- stored = await attachments.readImage(ref)
3526
- } catch (error) {
3527
- throw new Error(
3528
- `vision-router: failed to read attachment ${input} (${error && error.message ? error.message : String(error)})`,
3529
- )
3530
- }
3531
- bytes = stored.data
3532
- if (stored.ref && typeof stored.ref.mediaType === 'string') {
3533
- storedMediaType = stored.ref.mediaType
3534
- }
3535
- } else {
3536
- const fs = ctx.get('fs')
3537
- if (fs === undefined) throw new Error('vision-router: the fs service is not available')
3538
- const target = await fs.resolve(input)
3539
- bytes = await fs.readBytes(target, undefined, 20 * 1024 * 1024)
3540
- }
3541
- // Attachments are stored as content-addressed files without an
3542
- // extension: sniff the format from the bytes, and fall back to the
3543
- // stored ref / extension only when sniffing cannot decide.
3544
- const mediaType = sniffMediaType(bytes) ?? storedMediaType ?? mediaTypeOf(input)
3545
- if (mediaType === undefined) {
3546
- throw new Error(`unsupported image format ${input} (png/jpeg/webp/gif only)`)
3547
- }
3548
- return { bytes, mediaType }
3549
- }
3534
+ const readImageBytes = createVisionImageInputReader(
3535
+ ctx, resolveAttachment, throwIfVisionTaskAborted,
3536
+ )
3550
3537
 
3551
3538
  const imageDims = async (bytes) => {
3552
3539
  const sharp = await loadSharp()
@@ -4362,9 +4349,10 @@ ctx.logger?.info(
4362
4349
  name: 'vision_ocr',
4363
4350
  description:
4364
4351
  'Transcribe TEXT from an image. ENGINE POLICY: explicit engine=tesseract or engine=vision always wins. ' +
4365
- 'Otherwise the configured OCR engine policy applies. The default auto policy tries local Tesseract ' +
4366
- '(chi_sim+eng) first — fast, free, offline — then falls back to a vision model if local OCR fails or ' +
4367
- 'returns no text. Structured 1+x follow-up does not change the selected policy. Returns the text and ' +
4352
+ 'Otherwise the configured OCR engine policy applies. Auto first prefers a locally installed, SHA-verified ' +
4353
+ 'PP-OCRv6/ncnn on macOS ARM64 or Windows x64, then Tesseract (chi_sim+eng), and finally a vision model ' +
4354
+ 'when local OCR fails, returns no text, or remains uncertain (subject to permission and budget). Structured 1+x follow-up does not change the selected policy. ' +
4355
+ 'Returns the text and ' +
4368
4356
  'which engine produced it. ' +
4369
4357
  'SCOPE: vision_ocr reads letters, it does NOT recognize people, objects or scenes. Never use it ' +
4370
4358
  'as a fallback when vision_describe fails to identify who/what is in a picture ("这是谁" / ' +
@@ -4394,7 +4382,7 @@ ctx.logger?.info(
4394
4382
  },
4395
4383
  engine: {
4396
4384
  type: 'string',
4397
- description: '"auto": use the configured OCR engine policy (default policy is local Tesseract first, then vision fallback); explicit "tesseract"/"vision" always overrides the configured default for this call.',
4385
+ description: '"auto": use the configured OCR engine policy: prefer installed SHA-verified PP-OCRv6 on macOS ARM64 or Windows x64, otherwise local Tesseract; permitted vision fallback on local failure or uncertain text; explicit "tesseract"/"vision" always overrides the configured policy.',
4398
4386
  },
4399
4387
  },
4400
4388
  additionalProperties: false,
@@ -4423,16 +4411,120 @@ ctx.logger?.info(
4423
4411
  // gets a capped slice (never more than 12s), the vision model only the
4424
4412
  // remainder. The two timeouts can never stack into a multi-minute wait.
4425
4413
  const deadline = createDeadline(ocrBudgetMs())
4414
+ // Prefer an owner-installed and SHA-pinned PP-OCRv6 worker in auto mode.
4415
+ // No models or native binaries are bundled with the npm package.
4416
+ // The isolated legacy preview override remains for existing lab tests.
4417
+ const nativeAssets = process.env.DVR_NCNN_OCR_CANARY === '1'
4418
+ ? {
4419
+ binaryPath: process.env.DVR_NCNN_OCR_EXECUTABLE,
4420
+ modelDir: process.env.DVR_NCNN_OCR_MODELS,
4421
+ }
4422
+ : (() => {
4423
+ const installed = localNativeOcrInstallPaths()
4424
+ return installed && existsSync(installed.binaryPath) ? installed : null
4425
+ })()
4426
+ if (engine === 'auto' && nativeAssets !== null) {
4427
+ try {
4428
+ const native = await runExperimentalNativeOcrCanary(bytes, {
4429
+ binaryPath: nativeAssets.binaryPath,
4430
+ modelDir: nativeAssets.modelDir,
4431
+ loadSharp,
4432
+ signal: taskSignal,
4433
+ timeoutMs: Math.min(21000, Math.max(1, Math.floor(deadline.remaining() * .75))),
4434
+ })
4435
+ throwIfVisionTaskAborted(taskSignal)
4436
+ if (native.text.trim()) {
4437
+ if (session) {
4438
+ visionTurnMemory.recordLocalOcr(
4439
+ visionScopeOf(session),
4440
+ visionEvidenceSourceKey(imageInput),
4441
+ { uncertain: native.uncertain === true },
4442
+ )
4443
+ }
4444
+ const nativeResult = {
4445
+ engine: 'ncnn',
4446
+ text: native.text.trim(),
4447
+ uncertain: native.uncertain === true,
4448
+ ...(native.uncertain
4449
+ ? { review: {
4450
+ documentRefined: native.documentRefined === true,
4451
+ regions: Array.isArray(native.regions) ? native.regions.length : undefined,
4452
+ numericOrFormulaRisks: Array.isArray(native.reviewQueue)
4453
+ ? native.reviewQueue.slice(0, 12) : undefined,
4454
+ reviewTruncated: Array.isArray(native.reviewQueue) && native.reviewQueue.length > 12,
4455
+ } }
4456
+ : {}),
4457
+ }
4458
+ // Installing ncnn must not turn uncertain, nonempty local OCR
4459
+ // into an unreviewable final answer. Never bypass privacy,
4460
+ // breaker or the shared one-task deadline to verify it.
4461
+ const reviewed = await verifyUncertainAutoOcrResult(nativeResult, {
4462
+ engine,
4463
+ localOnlyVision: localOnlyVisionEnabled(current()),
4464
+ degraded: degraded.active,
4465
+ remainingMs: deadline.remaining(),
4466
+ verifyVision: () => answerVisionForTool(
4467
+ exec, bytes, mediaType,
4468
+ '请核查图片中的原文,尤其是易混淆的数字、化学式和底部字幕;不确定时不要猜。只输出文字。',
4469
+ { deadline },
4470
+ ),
4471
+ checkAbort: error => throwIfVisionTaskAborted(taskSignal, error),
4472
+ })
4473
+ return JSON.stringify(reviewed)
4474
+ }
4475
+ } catch (error) {
4476
+ throwIfVisionTaskAborted(taskSignal, error)
4477
+ // Native resource saturation must not fork extra local workers.
4478
+ if (error?.code === 'OCR_BUSY' || error?.code === 'OCR_QUEUE_TIMEOUT') {
4479
+ return JSON.stringify({
4480
+ engine: 'none', ok: false,
4481
+ code: error.code, text: '', retryable: false,
4482
+ reason: 'local OCR worker capacity is exhausted',
4483
+ })
4484
+ }
4485
+ ctx.logger?.warn(
4486
+ 'vision-router: local PP-OCRv6 unavailable; using Tesseract/vision policy ' +
4487
+ '(' + String(error?.code ?? 'OCR_NATIVE_FAILED').slice(0, 60) + ')',
4488
+ )
4489
+ }
4490
+ }
4426
4491
  const tesseractSlice = Math.min(12000, deadline.remaining())
4427
4492
  if (engine !== 'vision') {
4428
4493
  let localAttempted = false
4429
4494
  try {
4430
4495
  localAttempted = true
4431
- const local = await ocrWithTesseractAdaptive(
4432
- bytes,
4433
- tesseractSlice,
4434
- { signal: taskSignal },
4435
- )
4496
+ const managed = existsSync(managedTesseractPaths().manifest)
4497
+ let local
4498
+ if (managed) {
4499
+ try {
4500
+ const result = await runManagedTesseractOcr(bytes, {
4501
+ timeoutMs: Math.min(tesseractSlice, deadline.remaining()), signal: taskSignal,
4502
+ })
4503
+ const quality = Math.min(100, Math.max(0, result.confidence))
4504
+ local = {
4505
+ text: result.text, quality,
4506
+ uncertain: quality < 85 || /[0-9OIlL]{3,}/.test(result.text),
4507
+ psm: 6, attemptedPsms: [6],
4508
+ riskyTokens: [], variant: 'tesseract-js-wasm',
4509
+ }
4510
+ } catch (error) {
4511
+ throwIfVisionTaskAborted(taskSignal, error)
4512
+ ctx.logger?.warn('vision-router: managed Tesseract failed; trying system Tesseract')
4513
+ }
4514
+ }
4515
+ if ((!local || !local.text.trim()) && (!managed || await systemTesseract())) {
4516
+ const native = await ocrWithTesseractAdaptive(
4517
+ bytes,
4518
+ Math.min(tesseractSlice, deadline.remaining()),
4519
+ {
4520
+ signal: taskSignal,
4521
+ preprocess: (source, context) =>
4522
+ prepareTesseractSubtitleRetry(source, { ...context, loadSharp }),
4523
+ },
4524
+ )
4525
+ if (!local || native.text.trim()) local = native
4526
+ }
4527
+ if (!local) local = { text: '', uncertain: true, quality: 0, psm: 6, attemptedPsms: [6], riskyTokens: [] }
4436
4528
  if (local.text.trim() !== '') {
4437
4529
  if (session) {
4438
4530
  visionTurnMemory.recordLocalOcr(
@@ -4441,7 +4533,7 @@ ctx.logger?.info(
4441
4533
  { uncertain: local.uncertain === true },
4442
4534
  )
4443
4535
  }
4444
- return JSON.stringify({
4536
+ const localResult = {
4445
4537
  engine: 'tesseract',
4446
4538
  text: local.text.trim(),
4447
4539
  uncertain: local.uncertain === true,
@@ -4452,10 +4544,48 @@ ctx.logger?.info(
4452
4544
  attemptedPsms: local.attemptedPsms,
4453
4545
  quality: Number(local.quality.toFixed(2)),
4454
4546
  riskyTokens: local.riskyTokens,
4547
+ ...(local.variant && local.variant !== 'original'
4548
+ ? { preprocessing: local.variant } : {}),
4455
4549
  },
4456
4550
  }
4457
4551
  : {}),
4458
- })
4552
+ }
4553
+ // A nonempty but low-confidence transcript is not proof of
4554
+ // accuracy. In auto ONLY, try the existing consent-controlled
4555
+ // vision route while retaining local evidence if unavailable.
4556
+ if (shouldVerifyUncertainAutoOcr({
4557
+ engine,
4558
+ uncertain: local.uncertain,
4559
+ localOnlyVision: localOnlyVisionEnabled(current()),
4560
+ degraded: degraded.active,
4561
+ remainingMs: deadline.remaining(),
4562
+ })) {
4563
+ try {
4564
+ const vision = await answerVisionForTool(
4565
+ exec, bytes, mediaType,
4566
+ '请严格辨认图片上的原文,尤其是底部字幕或浅色文字;不确定时不要猜。只输出文字。',
4567
+ { deadline },
4568
+ )
4569
+ throwIfVisionTaskAborted(taskSignal)
4570
+ if (vision.ok !== false && typeof vision.text === 'string' &&
4571
+ vision.text.trim()) {
4572
+ return JSON.stringify({
4573
+ engine: 'vision',
4574
+ text: vision.text.trim(),
4575
+ review: {
4576
+ fallbackReason: 'low_confidence_local_ocr',
4577
+ localText: localResult.text.slice(0, 1000),
4578
+ localQuality: localResult.review?.quality,
4579
+ },
4580
+ })
4581
+ }
4582
+ } catch (error) {
4583
+ throwIfVisionTaskAborted(taskSignal, error)
4584
+ // The existing local transcript remains available and
4585
+ // retains its uncertainty if visual fallback fails.
4586
+ }
4587
+ }
4588
+ return JSON.stringify(localResult)
4459
4589
  }
4460
4590
  if (engine === 'tesseract') return JSON.stringify({ engine: 'tesseract', text: '', uncertain: true })
4461
4591
  } catch (error) {
@@ -4598,12 +4728,26 @@ ctx.logger?.info(
4598
4728
  let used = 'none'
4599
4729
  if (engine !== 'vision') {
4600
4730
  try {
4601
- const out = await ocrWithTesseract(
4602
- chunk,
4603
- Math.min(12000, deadline.remaining()),
4604
- { signal: taskSignal },
4605
- )
4606
- text = out.trim()
4731
+ const hasManagedTesseract = existsSync(managedTesseractPaths().manifest)
4732
+ if (hasManagedTesseract) {
4733
+ try {
4734
+ const managed = await runManagedTesseractOcr(chunk, {
4735
+ timeoutMs: Math.min(12000, deadline.remaining()), signal: taskSignal,
4736
+ })
4737
+ text = managed.text.trim()
4738
+ } catch (error) {
4739
+ throwIfVisionTaskAborted(taskSignal, error)
4740
+ ctx.logger?.warn('vision-router: managed Tesseract chunk failed; checking for system binary')
4741
+ }
4742
+ }
4743
+ if (!text && (!hasManagedTesseract || await systemTesseract())) {
4744
+ const out = await ocrWithTesseract(
4745
+ chunk,
4746
+ Math.min(12000, deadline.remaining()),
4747
+ { signal: taskSignal },
4748
+ )
4749
+ text = out.trim()
4750
+ }
4607
4751
  used = 'tesseract'
4608
4752
  } catch (error) {
4609
4753
  throwIfVisionTaskAborted(taskSignal, error)