dsh-vision-router 3.2.1 → 3.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -4
- package/README.zh.md +20 -3
- package/THIRD_PARTY_NOTICES.md +60 -0
- package/assets/licenses/llvm-mingw-LICENSE.txt +279 -0
- package/assets/licenses/mingw-w64-runtime-COPYING.txt +265 -0
- package/assets/licenses/ncnn-LICENSE.txt +86 -0
- package/docs/architecture/dsh-support-window.md +25 -6
- package/docs/compatibility/dsh-0.2.2-rc3-audit.md +131 -0
- package/docs/releases/v3.2.2.md +25 -0
- package/docs/releases/v3.3.0.md +34 -0
- package/docs/releases/v3.4.0.md +41 -0
- package/index.js +313 -169
- package/lib/client.js +4 -4
- package/lib/core-primitives.js +115 -56
- package/lib/dsh-support-window.js +6 -3
- package/lib/maintenance-routes.js +2 -0
- package/lib/managed-tesseract-child.js +33 -0
- package/lib/native-ocr-annotation-risk.js +33 -0
- package/lib/native-ocr-canary.js +156 -0
- package/lib/native-ocr-chemistry-review.js +37 -0
- package/lib/native-ocr-document.js +243 -0
- package/lib/native-ocr-installation.js +17 -0
- package/lib/native-ocr-integrity.js +93 -0
- package/lib/native-ocr-question-labels.js +167 -0
- package/lib/native-ocr-supervisor.js +268 -0
- package/lib/optional-ocr-engines.js +330 -0
- package/lib/optional-ocr-routes.js +87 -0
- package/lib/runtime-reliability.js +7 -0
- package/lib/session-query-readers.js +8 -2
- package/lib/session-vision-attachment-index.js +61 -14
- package/lib/settings-ia-client-prelude.js +56 -2
- package/lib/tesseract-image-preprocess.js +35 -0
- package/lib/trace-vtracer-process-runtime.js +133 -0
- package/lib/trace-vtracer-process.js +43 -0
- package/lib/vision-attachment-handle-runtime.js +11 -2
- package/lib/vision-backend-runtime-policy.js +5 -3
- package/lib/vision-image-input-reader.js +67 -0
- package/lib/vision-image-input-verdict.js +24 -3
- package/lib/vision-tool-runtime-boundary.js +7 -10
- package/lib/vision-tool-stage.js +58 -0
- package/lib/web-capability-boundary.js +4 -0
- package/package.json +5 -4
- package/scripts/install-native-ocr.mjs +144 -0
package/index.js
CHANGED
|
@@ -118,6 +118,10 @@ import z from '@deepseek-ai/schemastery'
|
|
|
118
118
|
|
|
119
119
|
import path from 'node:path'
|
|
120
120
|
import { existsSync } from 'node:fs'
|
|
121
|
+
import { runExperimentalNativeOcrCanary } from './lib/native-ocr-canary.js'
|
|
122
|
+
import { localNativeOcrInstallPaths } from './lib/native-ocr-installation.js'
|
|
123
|
+
import { managedTesseractPaths, runManagedTesseractOcr, systemTesseract } from './lib/optional-ocr-engines.js'
|
|
124
|
+
import { prepareTesseractSubtitleRetry } from './lib/tesseract-image-preprocess.js'
|
|
121
125
|
|
|
122
126
|
import { pathToFileURL } from 'node:url'
|
|
123
127
|
import { appendPromptToImageOnlyMessage } from './lib/http-compat.js'
|
|
@@ -151,6 +155,8 @@ import {
|
|
|
151
155
|
import { currentVisionExecutionOrder } from './lib/vision-execution-order.js'
|
|
152
156
|
import { applyVisionExecutionOrder } from './lib/vision-execution-order-apply.js'
|
|
153
157
|
import { currentVisionTurnBudgetSignal } from './lib/turn-budget-context.js'
|
|
158
|
+
import { createVisionToolStageTrace } from './lib/vision-tool-stage.js'
|
|
159
|
+
import { createVisionImageInputReader } from './lib/vision-image-input-reader.js'
|
|
154
160
|
import { parseGroundingBoxResponse, assessGroundingBox } from './lib/grounding-box-recovery.js'
|
|
155
161
|
import { createHash } from 'node:crypto'
|
|
156
162
|
import {
|
|
@@ -311,7 +317,7 @@ export const Config = z.object({
|
|
|
311
317
|
// (its own cap) and the vision-model fallback only the rest — never two
|
|
312
318
|
// full timeouts added together.
|
|
313
319
|
ocrTimeoutMs: z.number().step(1).min(1000).max(120000).default(30000),
|
|
314
|
-
//
|
|
320
|
+
// Auto prefers verified owner-installed PP-OCRv6 when supported; explicit engine wins.
|
|
315
321
|
ocrEngine: z.union(['auto', 'tesseract', 'vision']).default('auto'),
|
|
316
322
|
proxy: z.string().default(''),
|
|
317
323
|
proxyHosts: z.array(z.string()).default([...DEFAULT_PROXY_HOSTS]),
|
|
@@ -470,6 +476,8 @@ import {
|
|
|
470
476
|
resolveVisionOcrEngine,
|
|
471
477
|
ocrWithTesseract,
|
|
472
478
|
ocrWithTesseractAdaptive,
|
|
479
|
+
shouldVerifyUncertainAutoOcr,
|
|
480
|
+
verifyUncertainAutoOcrResult,
|
|
473
481
|
estimateTokens,
|
|
474
482
|
estimateMessages,
|
|
475
483
|
trimMessagesToBudget,
|
|
@@ -565,6 +573,8 @@ export {
|
|
|
565
573
|
resolveVisionOcrEngine,
|
|
566
574
|
ocrWithTesseract,
|
|
567
575
|
ocrWithTesseractAdaptive,
|
|
576
|
+
shouldVerifyUncertainAutoOcr,
|
|
577
|
+
verifyUncertainAutoOcrResult,
|
|
568
578
|
estimateTokens,
|
|
569
579
|
estimateMessages,
|
|
570
580
|
trimMessagesToBudget,
|
|
@@ -1023,20 +1033,35 @@ export function apply(ctx, config = {}, runtime = {}) {
|
|
|
1023
1033
|
}
|
|
1024
1034
|
if (attachments === undefined) continue
|
|
1025
1035
|
try {
|
|
1026
|
-
|
|
1036
|
+
throwIfVisionTaskAborted(options.signal)
|
|
1037
|
+
const stored = await attachments.readImage(block.attachment, options.signal)
|
|
1038
|
+
throwIfVisionTaskAborted(options.signal)
|
|
1027
1039
|
// Last-mile guard: never send oversized images to the vision
|
|
1028
1040
|
// endpoint — encoder cost scales with pixels and dominates
|
|
1029
1041
|
// tool-call latency on retina screenshots.
|
|
1030
1042
|
let bytes = stored.data
|
|
1031
1043
|
if (downscaleEnabled() && bytes && bytes.length > 0) {
|
|
1032
1044
|
bytes = await downscaleImage(bytes, downscaleMaxPixels())
|
|
1045
|
+
throwIfVisionTaskAborted(options.signal)
|
|
1033
1046
|
}
|
|
1034
1047
|
content.push(...toOpenAIContent([block], () => bytes))
|
|
1035
1048
|
} catch (error) {
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1049
|
+
throwIfVisionTaskAborted(options.signal, error)
|
|
1050
|
+
// A text-only completion here would falsely claim the model
|
|
1051
|
+
// saw pixels. Fail this backend immediately and let the
|
|
1052
|
+
// normal vision fallback chain choose another backend.
|
|
1053
|
+
ctx.logger?.warn('vision-http: retained image delivery failed; refusing text-only downgrade')
|
|
1054
|
+
yield {
|
|
1055
|
+
type: 'finish',
|
|
1056
|
+
reason: {
|
|
1057
|
+
kind: 'error',
|
|
1058
|
+
failure: {
|
|
1059
|
+
code: 'VISION_IMAGE_DELIVERY_UNAVAILABLE',
|
|
1060
|
+
message: 'vision-http: retained image could not be delivered to the model',
|
|
1061
|
+
},
|
|
1062
|
+
},
|
|
1063
|
+
}
|
|
1064
|
+
return
|
|
1040
1065
|
}
|
|
1041
1066
|
} else if (block && block.type === 'tool-result') {
|
|
1042
1067
|
// The OpenAI wire has no tool-call frames to hang a `role: tool`
|
|
@@ -1701,7 +1726,9 @@ export function apply(ctx, config = {}, runtime = {}) {
|
|
|
1701
1726
|
}
|
|
1702
1727
|
const content = []
|
|
1703
1728
|
for (const block of blocks) {
|
|
1704
|
-
|
|
1729
|
+
throwIfVisionTaskAborted(options.signal)
|
|
1730
|
+
const stored = await attachments.readImage(block.attachment, options.signal)
|
|
1731
|
+
throwIfVisionTaskAborted(options.signal)
|
|
1705
1732
|
content.push(...toOpenAIContent([block], () => stored.data))
|
|
1706
1733
|
}
|
|
1707
1734
|
return callOpenAIWithProviderTransport(
|
|
@@ -1797,14 +1824,18 @@ export function apply(ctx, config = {}, runtime = {}) {
|
|
|
1797
1824
|
throw new Error('corrected route: the attachment service is not registered')
|
|
1798
1825
|
}
|
|
1799
1826
|
const bytesOf = async (attachment) => {
|
|
1800
|
-
|
|
1827
|
+
throwIfVisionTaskAborted(options.signal)
|
|
1828
|
+
const stored = await attachments.readImage(attachment, options.signal)
|
|
1829
|
+
throwIfVisionTaskAborted(options.signal)
|
|
1801
1830
|
let bytes = stored.data
|
|
1802
1831
|
if (downscaleEnabled() && bytes && bytes.length > 0) {
|
|
1803
1832
|
bytes = await downscaleImage(bytes, downscaleMaxPixels())
|
|
1833
|
+
throwIfVisionTaskAborted(options.signal)
|
|
1804
1834
|
}
|
|
1805
1835
|
return bytes
|
|
1806
1836
|
}
|
|
1807
1837
|
const anthropic = await toAnthropicMessages(messages, bytesOf)
|
|
1838
|
+
throwIfVisionTaskAborted(options.signal)
|
|
1808
1839
|
if (anthropic.messages.length === 0) {
|
|
1809
1840
|
throw new Error(`corrected route "${pair.provider}/${pair.model}": no representable content to send`)
|
|
1810
1841
|
}
|
|
@@ -2402,8 +2433,8 @@ export function apply(ctx, config = {}, runtime = {}) {
|
|
|
2402
2433
|
// Host-owned proxy overrides are scoped by lib/legacy-global-proxy-boundary.js.
|
|
2403
2434
|
// Core no longer owns or installs a process-wide proxy fetch implementation.
|
|
2404
2435
|
|
|
2405
|
-
const resolveAttachment = (session, id) => sessionVisionIndex.resolveAttachment(session, id)
|
|
2406
|
-
const resolveAttachments = (session, ids) => sessionVisionIndex.resolveAttachments(session, ids)
|
|
2436
|
+
const resolveAttachment = (session, id, signal) => sessionVisionIndex.resolveAttachment(session, id, signal)
|
|
2437
|
+
const resolveAttachments = (session, ids, signal) => sessionVisionIndex.resolveAttachments(session, ids, signal)
|
|
2407
2438
|
|
|
2408
2439
|
// Session-local routing handoff between pre-step and agent/request. Modern
|
|
2409
2440
|
// Hosts store one exact async raw-log tail seq; legacy Hosts retain only the
|
|
@@ -2880,121 +2911,103 @@ export function apply(ctx, config = {}, runtime = {}) {
|
|
|
2880
2911
|
}
|
|
2881
2912
|
const blocks = []
|
|
2882
2913
|
const contentIds = []
|
|
2883
|
-
|
|
2914
|
+
// One shared deadline for the WHOLE task: image preflight, every
|
|
2915
|
+
// provider, fallback and JSON correction draw from one clock.
|
|
2916
|
+
// Previously the clock started after SessionQuery/image loading,
|
|
2917
|
+
// leaving attachmentIds vulnerable to the Host's ~120s watchdog.
|
|
2918
|
+
const hostSignal = combineSignals(currentVisionTurnBudgetSignal(), exec?.signal)
|
|
2919
|
+
const deadline = createDeadline(visionTaskTimeoutMs())
|
|
2920
|
+
const taskSignal = combineSignals(hostSignal, deadline.signal())
|
|
2921
|
+
const stage = createVisionToolStageTrace(ctx.logger, 'vision_describe', taskSignal)
|
|
2922
|
+
try {
|
|
2884
2923
|
const paths = Array.isArray(args.paths) ? args.paths : []
|
|
2885
2924
|
const attachmentIds = Array.isArray(args.attachmentIds) ? args.attachmentIds : []
|
|
2886
2925
|
if (paths.length + attachmentIds.length === 0 || paths.length + attachmentIds.length > 4) {
|
|
2887
2926
|
throw new Error('vision_describe: provide 1-4 images via paths and/or attachmentIds')
|
|
2888
2927
|
}
|
|
2889
|
-
//
|
|
2890
|
-
//
|
|
2891
|
-
|
|
2892
|
-
|
|
2893
|
-
|
|
2928
|
+
// Both argument forms now use the exact same read -> normalize ->
|
|
2929
|
+
// downscale -> save path. The old attachmentIds-only path could retain
|
|
2930
|
+
// an oversized original ref after resize publication failed.
|
|
2931
|
+
const sources = [...paths, ...attachmentIds]
|
|
2932
|
+
const materializableAttachmentIds = [...new Set(
|
|
2933
|
+
sources.map((value) => String(value)).filter((value) => isAttachmentIdInput(value)),
|
|
2934
|
+
)]
|
|
2935
|
+
// attachmentIds is always an attachment namespace, even for a legacy
|
|
2936
|
+
// Host whose authorized IDs do not yet have a sha256: prefix.
|
|
2937
|
+
const requestedRefs = [...new Set([
|
|
2938
|
+
...attachmentIds.map((value) => String(value)),
|
|
2939
|
+
...paths.map((value) => String(value)).filter((value) => isAttachmentIdInput(value)),
|
|
2894
2940
|
])]
|
|
2895
|
-
const attachmentSession = exec
|
|
2896
|
-
const resolvedAttachmentRefs =
|
|
2897
|
-
? await
|
|
2941
|
+
const attachmentSession = exec?.agent?.session
|
|
2942
|
+
const resolvedAttachmentRefs = requestedRefs.length > 0
|
|
2943
|
+
? await stage('resolve-upload-refs', () =>
|
|
2944
|
+
resolveAttachments(attachmentSession, requestedRefs, taskSignal))
|
|
2898
2945
|
: new Map()
|
|
2899
2946
|
|
|
2900
|
-
for (const
|
|
2947
|
+
for (const [index, item] of sources.entries()) {
|
|
2948
|
+
const source = String(item)
|
|
2949
|
+
const attachmentInput = index >= paths.length
|
|
2901
2950
|
let bytes
|
|
2902
2951
|
let mediaType
|
|
2903
2952
|
try {
|
|
2904
|
-
|
|
2905
|
-
|
|
2906
|
-
// a path gets the right pixels instead of a not-found error.
|
|
2907
|
-
;({ bytes, mediaType } = await readImageBytes(exec, path, resolvedAttachmentRefs))
|
|
2953
|
+
;({ bytes, mediaType } = await stage('read-image-' + index, () =>
|
|
2954
|
+
readImageBytes(exec, source, resolvedAttachmentRefs, attachmentInput)))
|
|
2908
2955
|
} catch (error) {
|
|
2909
2956
|
throw new Error(
|
|
2910
|
-
`vision_describe: failed to read ${
|
|
2957
|
+
`vision_describe: failed to read ${source} (${error && error.message ? error.message : String(error)})`,
|
|
2958
|
+
{ cause: error },
|
|
2911
2959
|
)
|
|
2912
2960
|
}
|
|
2913
2961
|
if (downscaleEnabled()) {
|
|
2914
|
-
const resized = await
|
|
2962
|
+
const resized = await stage('downscale-image-' + index, () =>
|
|
2963
|
+
downscaleImage(bytes, downscaleMaxPixels(), { signal: taskSignal }))
|
|
2915
2964
|
if (resized !== bytes) {
|
|
2916
|
-
ctx.logger?.info('vision-router: downscaled
|
|
2965
|
+
ctx.logger?.info('vision-router: downscaled vision input index=%d', index)
|
|
2917
2966
|
}
|
|
2918
2967
|
bytes = resized
|
|
2919
2968
|
}
|
|
2969
|
+
// Always publish canonical bytes, including an uploaded image.
|
|
2970
|
+
// Host adapters then read the verified/downsized version, never a
|
|
2971
|
+
// stale or mismatched original attachment reference.
|
|
2920
2972
|
let ref
|
|
2921
2973
|
try {
|
|
2922
|
-
ref = await attachments.saveImage({
|
|
2974
|
+
ref = await stage('save-normalized-image-' + index, () => attachments.saveImage({
|
|
2923
2975
|
data: bytes,
|
|
2924
2976
|
mediaType,
|
|
2925
|
-
...(isAttachmentIdInput(
|
|
2977
|
+
...(attachmentInput || isAttachmentIdInput(source) || basenameOf(source) === undefined
|
|
2926
2978
|
? {}
|
|
2927
|
-
: { name: basenameOf(
|
|
2928
|
-
})
|
|
2979
|
+
: { name: basenameOf(source) }),
|
|
2980
|
+
}))
|
|
2929
2981
|
} catch (error) {
|
|
2930
2982
|
throw new Error(
|
|
2931
|
-
`vision_describe: image ${
|
|
2983
|
+
`vision_describe: image ${source} was rejected (${error && error.message ? error.message : String(error)})`,
|
|
2984
|
+
{ cause: error },
|
|
2932
2985
|
)
|
|
2933
2986
|
}
|
|
2987
|
+
throwIfVisionTaskAborted(taskSignal)
|
|
2934
2988
|
contentIds.push(String(ref.attachmentId))
|
|
2935
2989
|
blocks.push({ type: 'image', attachment: ref })
|
|
2936
2990
|
}
|
|
2937
2991
|
|
|
2938
|
-
for (const id of attachmentIds) {
|
|
2939
|
-
const attachmentId = String(id)
|
|
2940
|
-
const ref = isAttachmentIdInput(attachmentId)
|
|
2941
|
-
? resolvedAttachmentRefs.get(attachmentId)
|
|
2942
|
-
: await resolveAttachment(attachmentSession, attachmentId)
|
|
2943
|
-
if (ref === undefined) {
|
|
2944
|
-
throw new Error(
|
|
2945
|
-
`vision_describe: unknown attachment id "${id}" (it must come from an image uploaded in this conversation)`,
|
|
2946
|
-
)
|
|
2947
|
-
}
|
|
2948
|
-
let stored
|
|
2949
|
-
try {
|
|
2950
|
-
stored = await attachments.readImage(ref)
|
|
2951
|
-
} catch (error) {
|
|
2952
|
-
throw new Error(
|
|
2953
|
-
`vision_describe: failed to read attachment ${id} (${error && error.message ? error.message : String(error)})`,
|
|
2954
|
-
)
|
|
2955
|
-
}
|
|
2956
|
-
// Downscale oversized uploads before the vision call: retina
|
|
2957
|
-
// screenshots easily reach 10MP+ and the vision encoder's cost
|
|
2958
|
-
// scales with pixels — a full-size upload is the dominant part of
|
|
2959
|
-
// the tool-call latency. Re-save a resized attachment so the
|
|
2960
|
-
// adapter reads the small one.
|
|
2961
|
-
if (downscaleEnabled() && stored.data && stored.data.length > 0) {
|
|
2962
|
-
const resized = await downscaleImage(stored.data, downscaleMaxPixels())
|
|
2963
|
-
if (resized !== stored.data) {
|
|
2964
|
-
try {
|
|
2965
|
-
const resizedRef = await attachments.saveImage({
|
|
2966
|
-
data: resized,
|
|
2967
|
-
mediaType: stored.ref && stored.ref.mediaType ? stored.ref.mediaType : 'image/png',
|
|
2968
|
-
...(stored.ref && stored.ref.name ? { name: stored.ref.name } : {}),
|
|
2969
|
-
})
|
|
2970
|
-
stored = { ref: resizedRef, data: resized }
|
|
2971
|
-
ctx.logger?.info('vision-router: downscaled attachment %s for the vision call', id)
|
|
2972
|
-
} catch {
|
|
2973
|
-
stored = { ...stored, data: resized }
|
|
2974
|
-
}
|
|
2975
|
-
}
|
|
2976
|
-
}
|
|
2977
|
-
contentIds.push(String(ref.attachmentId))
|
|
2978
|
-
blocks.push({ type: 'image', attachment: stored.ref })
|
|
2979
|
-
}
|
|
2980
|
-
|
|
2981
2992
|
const question = String(args.question ?? '')
|
|
2982
2993
|
const wantJson = args.json === true
|
|
2983
2994
|
// Keep the adapter path and direct OpenAI-compatible HTTP path on the
|
|
2984
2995
|
// exact same prompt, including the structured JSON evidence contract.
|
|
2985
2996
|
const promptText = visionDescribePrompt(question, wantJson)
|
|
2986
|
-
const usablePairs = await resolveToolVisionPairs()
|
|
2997
|
+
const usablePairs = await stage('resolve-vision-backends', () => resolveToolVisionPairs())
|
|
2987
2998
|
// Freeze this task's direct fallback list: settings changes take effect
|
|
2988
2999
|
// on the next call, never halfway through one fallback walk.
|
|
2989
3000
|
const httpFallbacks = httpProviders()
|
|
2990
3001
|
const rejectedPairs = []
|
|
3002
|
+
let backendPreflightIndex = 0
|
|
2991
3003
|
for (const pair of pairs()) {
|
|
2992
3004
|
if (!pair || pair.provider === HTTP_ROUTE) continue
|
|
2993
3005
|
if (!adapterAvailable(ctx.llm, pair.provider)) {
|
|
2994
3006
|
rejectedPairs.push(`${pair.provider}/${pair.model}: provider adapter is not registered`)
|
|
2995
3007
|
continue
|
|
2996
3008
|
}
|
|
2997
|
-
const capability = await
|
|
3009
|
+
const capability = await stage('backend-preflight-' + backendPreflightIndex++, () =>
|
|
3010
|
+
resolveVisionBackendCapability(pair.provider, pair.model))
|
|
2998
3011
|
if (capability.attemptable === false) {
|
|
2999
3012
|
rejectedPairs.push(
|
|
3000
3013
|
`${pair.provider}/${pair.model}: structurally unavailable (${capability.reason ?? 'unknown reason'})`,
|
|
@@ -3030,9 +3043,7 @@ export function apply(ctx, config = {}, runtime = {}) {
|
|
|
3030
3043
|
)
|
|
3031
3044
|
}
|
|
3032
3045
|
|
|
3033
|
-
//
|
|
3034
|
-
// the JSON-correction retry draws from this single budget.
|
|
3035
|
-
const deadline = createDeadline(visionTaskTimeoutMs())
|
|
3046
|
+
// The same clock already started before attachment resolution.
|
|
3036
3047
|
const baseMessages = [
|
|
3037
3048
|
{
|
|
3038
3049
|
role: 'user',
|
|
@@ -3051,7 +3062,7 @@ export function apply(ctx, config = {}, runtime = {}) {
|
|
|
3051
3062
|
0,
|
|
3052
3063
|
)
|
|
3053
3064
|
|
|
3054
|
-
for (const pair of usablePairs) {
|
|
3065
|
+
for (const [pairIndex, pair] of usablePairs.entries()) {
|
|
3055
3066
|
if (localOnlyVisionEnabled(current()) && !isLocalBackendPair(pair)) {
|
|
3056
3067
|
errors.push(`${pair.provider}/${pair.model}: skipped (local-only vision policy)`)
|
|
3057
3068
|
continue
|
|
@@ -3064,7 +3075,8 @@ export function apply(ctx, config = {}, runtime = {}) {
|
|
|
3064
3075
|
break
|
|
3065
3076
|
}
|
|
3066
3077
|
const backendKey = `${pair.provider}/${pair.model}`
|
|
3067
|
-
const fingerprint = await
|
|
3078
|
+
const fingerprint = await stage('adapter-auth-' + pairIndex, () =>
|
|
3079
|
+
credentialFingerprintFor({ provider: pair.provider }))
|
|
3068
3080
|
const gate = visionBreaker.inspect(backendKey, fingerprint, scope)
|
|
3069
3081
|
if (gate.blocked) {
|
|
3070
3082
|
errors.push(`${backendKey}: skipped (circuit open: ${gate.reason})`)
|
|
@@ -3089,19 +3101,20 @@ export function apply(ctx, config = {}, runtime = {}) {
|
|
|
3089
3101
|
)
|
|
3090
3102
|
: timeoutMs()
|
|
3091
3103
|
const signal = combineSignals(
|
|
3092
|
-
|
|
3104
|
+
hostSignal,
|
|
3093
3105
|
deadline.signal(),
|
|
3094
3106
|
AbortSignal.timeout(attemptBudgetMs),
|
|
3095
3107
|
)
|
|
3096
|
-
const capability = await
|
|
3097
|
-
|
|
3108
|
+
const capability = await stage('adapter-capability-' + pairIndex, () =>
|
|
3109
|
+
resolveVisionBackendCapability(pair.provider, pair.model))
|
|
3110
|
+
let text = await stage('adapter-request-' + pairIndex, () => callVisionPairWithOptionalBridge(pair, messages, {
|
|
3098
3111
|
maxTokens: 4096,
|
|
3099
3112
|
signal,
|
|
3100
3113
|
sessionId,
|
|
3101
3114
|
capability,
|
|
3102
3115
|
bridgeBlocks: blocks,
|
|
3103
3116
|
bridgeInstruction: promptText,
|
|
3104
|
-
})
|
|
3117
|
+
}))
|
|
3105
3118
|
assertNoRepetitionLoop(text, backendKey)
|
|
3106
3119
|
ctx.logger?.info(
|
|
3107
3120
|
'vision-router: vision call %s ok (%d chars, %d ms)',
|
|
@@ -3133,7 +3146,7 @@ ctx.logger?.info(
|
|
|
3133
3146
|
},
|
|
3134
3147
|
]
|
|
3135
3148
|
attemptStarted = Date.now()
|
|
3136
|
-
text = await callVisionPairWithOptionalBridge(pair, messages, {
|
|
3149
|
+
text = await stage('adapter-json-correction-' + pairIndex, () => callVisionPairWithOptionalBridge(pair, messages, {
|
|
3137
3150
|
maxTokens: 4096,
|
|
3138
3151
|
signal,
|
|
3139
3152
|
sessionId,
|
|
@@ -3141,7 +3154,7 @@ ctx.logger?.info(
|
|
|
3141
3154
|
bridgeBlocks: blocks,
|
|
3142
3155
|
bridgeInstruction:
|
|
3143
3156
|
promptText + '\n\nThat output was not valid JSON. Respond with ONLY a valid JSON object now.',
|
|
3144
|
-
})
|
|
3157
|
+
}))
|
|
3145
3158
|
assertNoRepetitionLoop(text, backendKey)
|
|
3146
3159
|
ctx.logger?.info(
|
|
3147
3160
|
'vision-router: vision call %s ok after JSON correction (%d chars, %d ms)',
|
|
@@ -3171,7 +3184,7 @@ ctx.logger?.info(
|
|
|
3171
3184
|
// A user/Host cancel is not a backend outage. Otherwise the
|
|
3172
3185
|
// canceled attempt poisons this turn's breaker/failure memory and
|
|
3173
3186
|
// a healthy retry returns BACKEND_UNAVAILABLE_THIS_TURN.
|
|
3174
|
-
throwIfVisionTaskAborted(
|
|
3187
|
+
throwIfVisionTaskAborted(hostSignal, error)
|
|
3175
3188
|
const classification = classifyVisionFailure(error)
|
|
3176
3189
|
consecutiveServerFailures = classification.kind === VISION_FAILURE_KINDS.SERVER
|
|
3177
3190
|
? consecutiveServerFailures + 1 : 0
|
|
@@ -3197,7 +3210,7 @@ ctx.logger?.info(
|
|
|
3197
3210
|
// Direct HTTP providers (built-in keyless OVHcloud by default) are the
|
|
3198
3211
|
// final fallbacks: they bypass the harness llm service entirely, so the
|
|
3199
3212
|
// anonymous free endpoint works without any credential.
|
|
3200
|
-
for (const provider of httpFallbacks) {
|
|
3213
|
+
for (const [httpIndex, provider] of httpFallbacks.entries()) {
|
|
3201
3214
|
if (localOnlyVisionEnabled(current()) && !isLoopbackVisionBaseURL(provider?.baseURL)) {
|
|
3202
3215
|
errors.push(`http:${provider?.name}/${provider?.model}: skipped (local-only vision policy)`)
|
|
3203
3216
|
continue
|
|
@@ -3210,7 +3223,8 @@ ctx.logger?.info(
|
|
|
3210
3223
|
break
|
|
3211
3224
|
}
|
|
3212
3225
|
const backendKey = `http:${provider.name}/${provider.model}`
|
|
3213
|
-
const fingerprint = await
|
|
3226
|
+
const fingerprint = await stage('http-auth-' + httpIndex, () =>
|
|
3227
|
+
credentialFingerprintFor({ kind: 'http', apiKeyEnv: provider.apiKeyEnv }))
|
|
3214
3228
|
const gate = visionBreaker.inspect(backendKey, fingerprint, scope)
|
|
3215
3229
|
if (gate.blocked) {
|
|
3216
3230
|
errors.push(`${backendKey}: skipped (circuit open: ${gate.reason})`)
|
|
@@ -3222,15 +3236,19 @@ ctx.logger?.info(
|
|
|
3222
3236
|
let attemptStarted = Date.now()
|
|
3223
3237
|
try {
|
|
3224
3238
|
// Precompute bytes once per block (attachments.readImage is async).
|
|
3225
|
-
const openAIBlocks =
|
|
3226
|
-
|
|
3227
|
-
|
|
3228
|
-
|
|
3229
|
-
|
|
3230
|
-
|
|
3231
|
-
|
|
3239
|
+
const openAIBlocks = await stage('http-image-delivery-' + httpIndex, async () => {
|
|
3240
|
+
const converted = []
|
|
3241
|
+
for (const block of blocks) {
|
|
3242
|
+
if (block.type === 'image' && block.attachment) {
|
|
3243
|
+
const stored = await attachments.readImage(block.attachment, taskSignal)
|
|
3244
|
+
throwIfVisionTaskAborted(taskSignal)
|
|
3245
|
+
converted.push(toOpenAIContent([block], () => stored.data)[0])
|
|
3246
|
+
} else {
|
|
3247
|
+
converted.push({ type: 'text', text: block.text })
|
|
3248
|
+
}
|
|
3232
3249
|
}
|
|
3233
|
-
|
|
3250
|
+
return converted
|
|
3251
|
+
})
|
|
3234
3252
|
// Direct HTTP providers must receive the same image + question as
|
|
3235
3253
|
// adapter-backed providers. Some endpoints (e.g. Zhipu GLM) reject
|
|
3236
3254
|
// a pure-image user message even when permissive endpoints accept it.
|
|
@@ -3248,12 +3266,15 @@ ctx.logger?.info(
|
|
|
3248
3266
|
consecutiveServerFailures,
|
|
3249
3267
|
)
|
|
3250
3268
|
const attemptSignal = combineSignals(
|
|
3251
|
-
|
|
3269
|
+
hostSignal,
|
|
3252
3270
|
deadline.signal(),
|
|
3253
3271
|
AbortSignal.timeout(attemptBudgetMs),
|
|
3254
3272
|
)
|
|
3255
3273
|
const askHttp = async (correction) => {
|
|
3256
|
-
const
|
|
3274
|
+
const phase = correction === undefined
|
|
3275
|
+
? 'http-request-' + httpIndex : 'http-json-correction-' + httpIndex
|
|
3276
|
+
const answer = await stage(phase, () =>
|
|
3277
|
+
awaitVisionBackendAttempt(attemptSignal, () => callOpenAIWithProviderTransport(
|
|
3257
3278
|
provider,
|
|
3258
3279
|
correction === undefined
|
|
3259
3280
|
? openAIBaseMessages
|
|
@@ -3267,7 +3288,7 @@ ctx.logger?.info(
|
|
|
3267
3288
|
sessionId,
|
|
3268
3289
|
resolveCredential,
|
|
3269
3290
|
},
|
|
3270
|
-
))
|
|
3291
|
+
)))
|
|
3271
3292
|
return answer
|
|
3272
3293
|
}
|
|
3273
3294
|
let text = await askHttp(undefined)
|
|
@@ -3319,7 +3340,7 @@ ctx.logger?.info(
|
|
|
3319
3340
|
// A user/Host cancel is not a backend outage. Otherwise the
|
|
3320
3341
|
// canceled attempt poisons this turn's breaker/failure memory and
|
|
3321
3342
|
// a healthy retry returns BACKEND_UNAVAILABLE_THIS_TURN.
|
|
3322
|
-
throwIfVisionTaskAborted(
|
|
3343
|
+
throwIfVisionTaskAborted(hostSignal, error)
|
|
3323
3344
|
const classification = classifyVisionFailure(error)
|
|
3324
3345
|
consecutiveServerFailures = classification.kind === VISION_FAILURE_KINDS.SERVER
|
|
3325
3346
|
? consecutiveServerFailures + 1 : 0
|
|
@@ -3367,6 +3388,18 @@ ctx.logger?.info(
|
|
|
3367
3388
|
}
|
|
3368
3389
|
}
|
|
3369
3390
|
return JSON.stringify(baseFailure)
|
|
3391
|
+
} catch (error) {
|
|
3392
|
+
// The caller's explicit cancellation is not a model outage.
|
|
3393
|
+
if (hostSignal?.aborted) throw hostSignal.reason ?? error
|
|
3394
|
+
if (deadline.expired() || taskSignal?.aborted) {
|
|
3395
|
+
return JSON.stringify(buildVisionFailure({
|
|
3396
|
+
code: VISION_RESULT_CODES.TIMEOUT,
|
|
3397
|
+
retryable: false,
|
|
3398
|
+
reason: 'Vision task deadline exhausted while preparing or processing the image.',
|
|
3399
|
+
}))
|
|
3400
|
+
}
|
|
3401
|
+
throw error
|
|
3402
|
+
}
|
|
3370
3403
|
},
|
|
3371
3404
|
}
|
|
3372
3405
|
deepToolDefs.push(visionDescribeTool)
|
|
@@ -3498,55 +3531,9 @@ ctx.logger?.info(
|
|
|
3498
3531
|
return String(attachmentIds[0]).trim()
|
|
3499
3532
|
}
|
|
3500
3533
|
|
|
3501
|
-
const readImageBytes =
|
|
3502
|
-
|
|
3503
|
-
|
|
3504
|
-
let storedMediaType
|
|
3505
|
-
if (isAttachmentIdInput(input)) {
|
|
3506
|
-
// Uploaded images reach the tool arguments as durable attachment ids
|
|
3507
|
-
// ("sha256:..."); resolve them through the session's recorded upload
|
|
3508
|
-
// index instead of treating the id as a filesystem path.
|
|
3509
|
-
const attachments = ctx.get('attachments')
|
|
3510
|
-
if (attachments === undefined) {
|
|
3511
|
-
throw new Error('vision-router: the attachment service is not available in this deployment')
|
|
3512
|
-
}
|
|
3513
|
-
const session = exec && exec.agent && exec.agent.session
|
|
3514
|
-
const canonicalInput = input.trim()
|
|
3515
|
-
const ref = resolvedAttachmentRefs instanceof Map
|
|
3516
|
-
? resolvedAttachmentRefs.get(canonicalInput)
|
|
3517
|
-
: await resolveAttachment(session, canonicalInput)
|
|
3518
|
-
if (ref === undefined) {
|
|
3519
|
-
throw new Error(
|
|
3520
|
-
`vision-router: unknown attachment id "${input}" (it must come from an image uploaded in this conversation)`,
|
|
3521
|
-
)
|
|
3522
|
-
}
|
|
3523
|
-
let stored
|
|
3524
|
-
try {
|
|
3525
|
-
stored = await attachments.readImage(ref)
|
|
3526
|
-
} catch (error) {
|
|
3527
|
-
throw new Error(
|
|
3528
|
-
`vision-router: failed to read attachment ${input} (${error && error.message ? error.message : String(error)})`,
|
|
3529
|
-
)
|
|
3530
|
-
}
|
|
3531
|
-
bytes = stored.data
|
|
3532
|
-
if (stored.ref && typeof stored.ref.mediaType === 'string') {
|
|
3533
|
-
storedMediaType = stored.ref.mediaType
|
|
3534
|
-
}
|
|
3535
|
-
} else {
|
|
3536
|
-
const fs = ctx.get('fs')
|
|
3537
|
-
if (fs === undefined) throw new Error('vision-router: the fs service is not available')
|
|
3538
|
-
const target = await fs.resolve(input)
|
|
3539
|
-
bytes = await fs.readBytes(target, undefined, 20 * 1024 * 1024)
|
|
3540
|
-
}
|
|
3541
|
-
// Attachments are stored as content-addressed files without an
|
|
3542
|
-
// extension: sniff the format from the bytes, and fall back to the
|
|
3543
|
-
// stored ref / extension only when sniffing cannot decide.
|
|
3544
|
-
const mediaType = sniffMediaType(bytes) ?? storedMediaType ?? mediaTypeOf(input)
|
|
3545
|
-
if (mediaType === undefined) {
|
|
3546
|
-
throw new Error(`unsupported image format ${input} (png/jpeg/webp/gif only)`)
|
|
3547
|
-
}
|
|
3548
|
-
return { bytes, mediaType }
|
|
3549
|
-
}
|
|
3534
|
+
const readImageBytes = createVisionImageInputReader(
|
|
3535
|
+
ctx, resolveAttachment, throwIfVisionTaskAborted,
|
|
3536
|
+
)
|
|
3550
3537
|
|
|
3551
3538
|
const imageDims = async (bytes) => {
|
|
3552
3539
|
const sharp = await loadSharp()
|
|
@@ -4362,9 +4349,10 @@ ctx.logger?.info(
|
|
|
4362
4349
|
name: 'vision_ocr',
|
|
4363
4350
|
description:
|
|
4364
4351
|
'Transcribe TEXT from an image. ENGINE POLICY: explicit engine=tesseract or engine=vision always wins. ' +
|
|
4365
|
-
'Otherwise the configured OCR engine policy applies.
|
|
4366
|
-
'
|
|
4367
|
-
'returns no text. Structured 1+x follow-up does not change the selected policy.
|
|
4352
|
+
'Otherwise the configured OCR engine policy applies. Auto first prefers a locally installed, SHA-verified ' +
|
|
4353
|
+
'PP-OCRv6/ncnn on macOS ARM64 or Windows x64, then Tesseract (chi_sim+eng), and finally a vision model ' +
|
|
4354
|
+
'when local OCR fails, returns no text, or remains uncertain (subject to permission and budget). Structured 1+x follow-up does not change the selected policy. ' +
|
|
4355
|
+
'Returns the text and ' +
|
|
4368
4356
|
'which engine produced it. ' +
|
|
4369
4357
|
'SCOPE: vision_ocr reads letters, it does NOT recognize people, objects or scenes. Never use it ' +
|
|
4370
4358
|
'as a fallback when vision_describe fails to identify who/what is in a picture ("这是谁" / ' +
|
|
@@ -4394,7 +4382,7 @@ ctx.logger?.info(
|
|
|
4394
4382
|
},
|
|
4395
4383
|
engine: {
|
|
4396
4384
|
type: 'string',
|
|
4397
|
-
description: '"auto": use the configured OCR engine policy
|
|
4385
|
+
description: '"auto": use the configured OCR engine policy: prefer installed SHA-verified PP-OCRv6 on macOS ARM64 or Windows x64, otherwise local Tesseract; permitted vision fallback on local failure or uncertain text; explicit "tesseract"/"vision" always overrides the configured policy.',
|
|
4398
4386
|
},
|
|
4399
4387
|
},
|
|
4400
4388
|
additionalProperties: false,
|
|
@@ -4423,16 +4411,120 @@ ctx.logger?.info(
|
|
|
4423
4411
|
// gets a capped slice (never more than 12s), the vision model only the
|
|
4424
4412
|
// remainder. The two timeouts can never stack into a multi-minute wait.
|
|
4425
4413
|
const deadline = createDeadline(ocrBudgetMs())
|
|
4414
|
+
// Prefer an owner-installed and SHA-pinned PP-OCRv6 worker in auto mode.
|
|
4415
|
+
// No models or native binaries are bundled with the npm package.
|
|
4416
|
+
// The isolated legacy preview override remains for existing lab tests.
|
|
4417
|
+
const nativeAssets = process.env.DVR_NCNN_OCR_CANARY === '1'
|
|
4418
|
+
? {
|
|
4419
|
+
binaryPath: process.env.DVR_NCNN_OCR_EXECUTABLE,
|
|
4420
|
+
modelDir: process.env.DVR_NCNN_OCR_MODELS,
|
|
4421
|
+
}
|
|
4422
|
+
: (() => {
|
|
4423
|
+
const installed = localNativeOcrInstallPaths()
|
|
4424
|
+
return installed && existsSync(installed.binaryPath) ? installed : null
|
|
4425
|
+
})()
|
|
4426
|
+
if (engine === 'auto' && nativeAssets !== null) {
|
|
4427
|
+
try {
|
|
4428
|
+
const native = await runExperimentalNativeOcrCanary(bytes, {
|
|
4429
|
+
binaryPath: nativeAssets.binaryPath,
|
|
4430
|
+
modelDir: nativeAssets.modelDir,
|
|
4431
|
+
loadSharp,
|
|
4432
|
+
signal: taskSignal,
|
|
4433
|
+
timeoutMs: Math.min(21000, Math.max(1, Math.floor(deadline.remaining() * .75))),
|
|
4434
|
+
})
|
|
4435
|
+
throwIfVisionTaskAborted(taskSignal)
|
|
4436
|
+
if (native.text.trim()) {
|
|
4437
|
+
if (session) {
|
|
4438
|
+
visionTurnMemory.recordLocalOcr(
|
|
4439
|
+
visionScopeOf(session),
|
|
4440
|
+
visionEvidenceSourceKey(imageInput),
|
|
4441
|
+
{ uncertain: native.uncertain === true },
|
|
4442
|
+
)
|
|
4443
|
+
}
|
|
4444
|
+
const nativeResult = {
|
|
4445
|
+
engine: 'ncnn',
|
|
4446
|
+
text: native.text.trim(),
|
|
4447
|
+
uncertain: native.uncertain === true,
|
|
4448
|
+
...(native.uncertain
|
|
4449
|
+
? { review: {
|
|
4450
|
+
documentRefined: native.documentRefined === true,
|
|
4451
|
+
regions: Array.isArray(native.regions) ? native.regions.length : undefined,
|
|
4452
|
+
numericOrFormulaRisks: Array.isArray(native.reviewQueue)
|
|
4453
|
+
? native.reviewQueue.slice(0, 12) : undefined,
|
|
4454
|
+
reviewTruncated: Array.isArray(native.reviewQueue) && native.reviewQueue.length > 12,
|
|
4455
|
+
} }
|
|
4456
|
+
: {}),
|
|
4457
|
+
}
|
|
4458
|
+
// Installing ncnn must not turn uncertain, nonempty local OCR
|
|
4459
|
+
// into an unreviewable final answer. Never bypass privacy,
|
|
4460
|
+
// breaker or the shared one-task deadline to verify it.
|
|
4461
|
+
const reviewed = await verifyUncertainAutoOcrResult(nativeResult, {
|
|
4462
|
+
engine,
|
|
4463
|
+
localOnlyVision: localOnlyVisionEnabled(current()),
|
|
4464
|
+
degraded: degraded.active,
|
|
4465
|
+
remainingMs: deadline.remaining(),
|
|
4466
|
+
verifyVision: () => answerVisionForTool(
|
|
4467
|
+
exec, bytes, mediaType,
|
|
4468
|
+
'请核查图片中的原文,尤其是易混淆的数字、化学式和底部字幕;不确定时不要猜。只输出文字。',
|
|
4469
|
+
{ deadline },
|
|
4470
|
+
),
|
|
4471
|
+
checkAbort: error => throwIfVisionTaskAborted(taskSignal, error),
|
|
4472
|
+
})
|
|
4473
|
+
return JSON.stringify(reviewed)
|
|
4474
|
+
}
|
|
4475
|
+
} catch (error) {
|
|
4476
|
+
throwIfVisionTaskAborted(taskSignal, error)
|
|
4477
|
+
// Native resource saturation must not fork extra local workers.
|
|
4478
|
+
if (error?.code === 'OCR_BUSY' || error?.code === 'OCR_QUEUE_TIMEOUT') {
|
|
4479
|
+
return JSON.stringify({
|
|
4480
|
+
engine: 'none', ok: false,
|
|
4481
|
+
code: error.code, text: '', retryable: false,
|
|
4482
|
+
reason: 'local OCR worker capacity is exhausted',
|
|
4483
|
+
})
|
|
4484
|
+
}
|
|
4485
|
+
ctx.logger?.warn(
|
|
4486
|
+
'vision-router: local PP-OCRv6 unavailable; using Tesseract/vision policy ' +
|
|
4487
|
+
'(' + String(error?.code ?? 'OCR_NATIVE_FAILED').slice(0, 60) + ')',
|
|
4488
|
+
)
|
|
4489
|
+
}
|
|
4490
|
+
}
|
|
4426
4491
|
const tesseractSlice = Math.min(12000, deadline.remaining())
|
|
4427
4492
|
if (engine !== 'vision') {
|
|
4428
4493
|
let localAttempted = false
|
|
4429
4494
|
try {
|
|
4430
4495
|
localAttempted = true
|
|
4431
|
-
const
|
|
4432
|
-
|
|
4433
|
-
|
|
4434
|
-
{
|
|
4435
|
-
|
|
4496
|
+
const managed = existsSync(managedTesseractPaths().manifest)
|
|
4497
|
+
let local
|
|
4498
|
+
if (managed) {
|
|
4499
|
+
try {
|
|
4500
|
+
const result = await runManagedTesseractOcr(bytes, {
|
|
4501
|
+
timeoutMs: Math.min(tesseractSlice, deadline.remaining()), signal: taskSignal,
|
|
4502
|
+
})
|
|
4503
|
+
const quality = Math.min(100, Math.max(0, result.confidence))
|
|
4504
|
+
local = {
|
|
4505
|
+
text: result.text, quality,
|
|
4506
|
+
uncertain: quality < 85 || /[0-9OIlL]{3,}/.test(result.text),
|
|
4507
|
+
psm: 6, attemptedPsms: [6],
|
|
4508
|
+
riskyTokens: [], variant: 'tesseract-js-wasm',
|
|
4509
|
+
}
|
|
4510
|
+
} catch (error) {
|
|
4511
|
+
throwIfVisionTaskAborted(taskSignal, error)
|
|
4512
|
+
ctx.logger?.warn('vision-router: managed Tesseract failed; trying system Tesseract')
|
|
4513
|
+
}
|
|
4514
|
+
}
|
|
4515
|
+
if ((!local || !local.text.trim()) && (!managed || await systemTesseract())) {
|
|
4516
|
+
const native = await ocrWithTesseractAdaptive(
|
|
4517
|
+
bytes,
|
|
4518
|
+
Math.min(tesseractSlice, deadline.remaining()),
|
|
4519
|
+
{
|
|
4520
|
+
signal: taskSignal,
|
|
4521
|
+
preprocess: (source, context) =>
|
|
4522
|
+
prepareTesseractSubtitleRetry(source, { ...context, loadSharp }),
|
|
4523
|
+
},
|
|
4524
|
+
)
|
|
4525
|
+
if (!local || native.text.trim()) local = native
|
|
4526
|
+
}
|
|
4527
|
+
if (!local) local = { text: '', uncertain: true, quality: 0, psm: 6, attemptedPsms: [6], riskyTokens: [] }
|
|
4436
4528
|
if (local.text.trim() !== '') {
|
|
4437
4529
|
if (session) {
|
|
4438
4530
|
visionTurnMemory.recordLocalOcr(
|
|
@@ -4441,7 +4533,7 @@ ctx.logger?.info(
|
|
|
4441
4533
|
{ uncertain: local.uncertain === true },
|
|
4442
4534
|
)
|
|
4443
4535
|
}
|
|
4444
|
-
|
|
4536
|
+
const localResult = {
|
|
4445
4537
|
engine: 'tesseract',
|
|
4446
4538
|
text: local.text.trim(),
|
|
4447
4539
|
uncertain: local.uncertain === true,
|
|
@@ -4452,10 +4544,48 @@ ctx.logger?.info(
|
|
|
4452
4544
|
attemptedPsms: local.attemptedPsms,
|
|
4453
4545
|
quality: Number(local.quality.toFixed(2)),
|
|
4454
4546
|
riskyTokens: local.riskyTokens,
|
|
4547
|
+
...(local.variant && local.variant !== 'original'
|
|
4548
|
+
? { preprocessing: local.variant } : {}),
|
|
4455
4549
|
},
|
|
4456
4550
|
}
|
|
4457
4551
|
: {}),
|
|
4458
|
-
}
|
|
4552
|
+
}
|
|
4553
|
+
// A nonempty but low-confidence transcript is not proof of
|
|
4554
|
+
// accuracy. In auto ONLY, try the existing consent-controlled
|
|
4555
|
+
// vision route while retaining local evidence if unavailable.
|
|
4556
|
+
if (shouldVerifyUncertainAutoOcr({
|
|
4557
|
+
engine,
|
|
4558
|
+
uncertain: local.uncertain,
|
|
4559
|
+
localOnlyVision: localOnlyVisionEnabled(current()),
|
|
4560
|
+
degraded: degraded.active,
|
|
4561
|
+
remainingMs: deadline.remaining(),
|
|
4562
|
+
})) {
|
|
4563
|
+
try {
|
|
4564
|
+
const vision = await answerVisionForTool(
|
|
4565
|
+
exec, bytes, mediaType,
|
|
4566
|
+
'请严格辨认图片上的原文,尤其是底部字幕或浅色文字;不确定时不要猜。只输出文字。',
|
|
4567
|
+
{ deadline },
|
|
4568
|
+
)
|
|
4569
|
+
throwIfVisionTaskAborted(taskSignal)
|
|
4570
|
+
if (vision.ok !== false && typeof vision.text === 'string' &&
|
|
4571
|
+
vision.text.trim()) {
|
|
4572
|
+
return JSON.stringify({
|
|
4573
|
+
engine: 'vision',
|
|
4574
|
+
text: vision.text.trim(),
|
|
4575
|
+
review: {
|
|
4576
|
+
fallbackReason: 'low_confidence_local_ocr',
|
|
4577
|
+
localText: localResult.text.slice(0, 1000),
|
|
4578
|
+
localQuality: localResult.review?.quality,
|
|
4579
|
+
},
|
|
4580
|
+
})
|
|
4581
|
+
}
|
|
4582
|
+
} catch (error) {
|
|
4583
|
+
throwIfVisionTaskAborted(taskSignal, error)
|
|
4584
|
+
// The existing local transcript remains available and
|
|
4585
|
+
// retains its uncertainty if visual fallback fails.
|
|
4586
|
+
}
|
|
4587
|
+
}
|
|
4588
|
+
return JSON.stringify(localResult)
|
|
4459
4589
|
}
|
|
4460
4590
|
if (engine === 'tesseract') return JSON.stringify({ engine: 'tesseract', text: '', uncertain: true })
|
|
4461
4591
|
} catch (error) {
|
|
@@ -4598,12 +4728,26 @@ ctx.logger?.info(
|
|
|
4598
4728
|
let used = 'none'
|
|
4599
4729
|
if (engine !== 'vision') {
|
|
4600
4730
|
try {
|
|
4601
|
-
const
|
|
4602
|
-
|
|
4603
|
-
|
|
4604
|
-
|
|
4605
|
-
|
|
4606
|
-
|
|
4731
|
+
const hasManagedTesseract = existsSync(managedTesseractPaths().manifest)
|
|
4732
|
+
if (hasManagedTesseract) {
|
|
4733
|
+
try {
|
|
4734
|
+
const managed = await runManagedTesseractOcr(chunk, {
|
|
4735
|
+
timeoutMs: Math.min(12000, deadline.remaining()), signal: taskSignal,
|
|
4736
|
+
})
|
|
4737
|
+
text = managed.text.trim()
|
|
4738
|
+
} catch (error) {
|
|
4739
|
+
throwIfVisionTaskAborted(taskSignal, error)
|
|
4740
|
+
ctx.logger?.warn('vision-router: managed Tesseract chunk failed; checking for system binary')
|
|
4741
|
+
}
|
|
4742
|
+
}
|
|
4743
|
+
if (!text && (!hasManagedTesseract || await systemTesseract())) {
|
|
4744
|
+
const out = await ocrWithTesseract(
|
|
4745
|
+
chunk,
|
|
4746
|
+
Math.min(12000, deadline.remaining()),
|
|
4747
|
+
{ signal: taskSignal },
|
|
4748
|
+
)
|
|
4749
|
+
text = out.trim()
|
|
4750
|
+
}
|
|
4607
4751
|
used = 'tesseract'
|
|
4608
4752
|
} catch (error) {
|
|
4609
4753
|
throwIfVisionTaskAborted(taskSignal, error)
|