dsh-vision-router 3.2.0 → 3.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -6
- package/README.zh.md +21 -6
- package/THIRD_PARTY_NOTICES.md +167 -0
- package/assets/licenses/apache-2.0.txt +202 -0
- package/assets/licenses/llvm-mingw-LICENSE.txt +279 -0
- package/assets/licenses/mingw-w64-runtime-COPYING.txt +265 -0
- package/assets/licenses/moxcms-BSD-3-Clause.txt +26 -0
- package/assets/licenses/ncnn-LICENSE.txt +86 -0
- package/assets/licenses/roots-BSD-2-Clause.txt +24 -0
- package/assets/licenses/unicode-ident-Unicode-3.0.txt +39 -0
- package/assets/licenses/vtracer-MIT.txt +25 -0
- package/assets/licenses/zlib-rs-Zlib.txt +19 -0
- package/docs/architecture/dsh-support-window.md +25 -6
- package/docs/compatibility/dsh-0.2.2-rc2-audit.md +40 -0
- package/docs/compatibility/dsh-0.2.2-rc3-audit.md +131 -0
- package/docs/compliance/vtracer-alpha4-provenance.md +47 -0
- package/docs/compliance/vtracer-alpha4-reconstructed-Cargo.lock +567 -0
- package/docs/compliance/vtracer-alpha4-rust-inventory.json +421 -0
- package/docs/compliance/vtracer-alpha4-rust-license-texts.md +13018 -0
- package/docs/releases/v3.2.1.md +21 -0
- package/docs/releases/v3.2.2.md +25 -0
- package/docs/releases/v3.3.0.md +34 -0
- package/docs/remote-settings.md +83 -8
- package/index.js +280 -169
- package/lib/client.js +17 -17
- package/lib/core-primitives.js +155 -214
- package/lib/dsh-support-window.js +6 -3
- package/lib/native-ocr-annotation-risk.js +33 -0
- package/lib/native-ocr-canary.js +156 -0
- package/lib/native-ocr-chemistry-review.js +37 -0
- package/lib/native-ocr-document.js +243 -0
- package/lib/native-ocr-installation.js +17 -0
- package/lib/native-ocr-integrity.js +93 -0
- package/lib/native-ocr-question-labels.js +167 -0
- package/lib/native-ocr-supervisor.js +268 -0
- package/lib/remote-settings-bridge.js +93 -57
- package/lib/remote-settings-risk-client-boundary.js +2 -2
- package/lib/runtime-reliability.js +7 -0
- package/lib/session-query-readers.js +8 -2
- package/lib/session-vision-attachment-index.js +61 -14
- package/lib/settings-client-017-compat.js +4 -2
- package/lib/settings-ia-client-prelude.js +26 -10
- package/lib/tesseract-image-preprocess.js +35 -0
- package/lib/trace-vtracer-process-runtime.js +133 -0
- package/lib/trace-vtracer-process.js +43 -0
- package/lib/trace-vtracer-worker.js +163 -0
- package/lib/vision-attachment-handle-runtime.js +11 -2
- package/lib/vision-backend-runtime-policy.js +5 -3
- package/lib/vision-image-input-reader.js +67 -0
- package/lib/vision-image-input-verdict.js +24 -3
- package/lib/vision-tool-runtime-boundary.js +7 -10
- package/lib/vision-tool-stage.js +58 -0
- package/lib/web/dsh-settings-017-root-transport.js +52 -6
- package/lib/web-capability-boundary.js +6 -0
- package/package.json +10 -6
- package/scripts/install-native-ocr.mjs +144 -0
package/index.js
CHANGED
|
@@ -118,6 +118,9 @@ import z from '@deepseek-ai/schemastery'
|
|
|
118
118
|
|
|
119
119
|
import path from 'node:path'
|
|
120
120
|
import { existsSync } from 'node:fs'
|
|
121
|
+
import { runExperimentalNativeOcrCanary } from './lib/native-ocr-canary.js'
|
|
122
|
+
import { localNativeOcrInstallPaths } from './lib/native-ocr-installation.js'
|
|
123
|
+
import { prepareTesseractSubtitleRetry } from './lib/tesseract-image-preprocess.js'
|
|
121
124
|
|
|
122
125
|
import { pathToFileURL } from 'node:url'
|
|
123
126
|
import { appendPromptToImageOnlyMessage } from './lib/http-compat.js'
|
|
@@ -151,6 +154,8 @@ import {
|
|
|
151
154
|
import { currentVisionExecutionOrder } from './lib/vision-execution-order.js'
|
|
152
155
|
import { applyVisionExecutionOrder } from './lib/vision-execution-order-apply.js'
|
|
153
156
|
import { currentVisionTurnBudgetSignal } from './lib/turn-budget-context.js'
|
|
157
|
+
import { createVisionToolStageTrace } from './lib/vision-tool-stage.js'
|
|
158
|
+
import { createVisionImageInputReader } from './lib/vision-image-input-reader.js'
|
|
154
159
|
import { parseGroundingBoxResponse, assessGroundingBox } from './lib/grounding-box-recovery.js'
|
|
155
160
|
import { createHash } from 'node:crypto'
|
|
156
161
|
import {
|
|
@@ -311,14 +316,14 @@ export const Config = z.object({
|
|
|
311
316
|
// (its own cap) and the vision-model fallback only the rest — never two
|
|
312
317
|
// full timeouts added together.
|
|
313
318
|
ocrTimeoutMs: z.number().step(1).min(1000).max(120000).default(30000),
|
|
314
|
-
//
|
|
319
|
+
// Auto prefers verified owner-installed PP-OCRv6 when supported; explicit engine wins.
|
|
315
320
|
ocrEngine: z.union(['auto', 'tesseract', 'vision']).default('auto'),
|
|
316
321
|
proxy: z.string().default(''),
|
|
317
322
|
proxyHosts: z.array(z.string()).default([...DEFAULT_PROXY_HOSTS]),
|
|
318
|
-
//
|
|
319
|
-
//
|
|
320
|
-
//
|
|
321
|
-
//
|
|
323
|
+
// DSH versions expose different browser Settings transports. DVR's remote
|
|
324
|
+
// RPC is opt-in, but once acknowledged it edits the same Host-owned config
|
|
325
|
+
// namespace and schema as the local browser. The permission itself is
|
|
326
|
+
// managed by risk confirmation or local revocation, not ordinary remote writes.
|
|
322
327
|
allowRemoteSettings: z.boolean().default(false),
|
|
323
328
|
// Privacy policy switch: when enabled, Vision Router executes image-model
|
|
324
329
|
// requests only against loopback visual endpoints (Ollama, LM Studio, or
|
|
@@ -470,6 +475,8 @@ import {
|
|
|
470
475
|
resolveVisionOcrEngine,
|
|
471
476
|
ocrWithTesseract,
|
|
472
477
|
ocrWithTesseractAdaptive,
|
|
478
|
+
shouldVerifyUncertainAutoOcr,
|
|
479
|
+
verifyUncertainAutoOcrResult,
|
|
473
480
|
estimateTokens,
|
|
474
481
|
estimateMessages,
|
|
475
482
|
trimMessagesToBudget,
|
|
@@ -565,6 +572,8 @@ export {
|
|
|
565
572
|
resolveVisionOcrEngine,
|
|
566
573
|
ocrWithTesseract,
|
|
567
574
|
ocrWithTesseractAdaptive,
|
|
575
|
+
shouldVerifyUncertainAutoOcr,
|
|
576
|
+
verifyUncertainAutoOcrResult,
|
|
568
577
|
estimateTokens,
|
|
569
578
|
estimateMessages,
|
|
570
579
|
trimMessagesToBudget,
|
|
@@ -1023,20 +1032,35 @@ export function apply(ctx, config = {}, runtime = {}) {
|
|
|
1023
1032
|
}
|
|
1024
1033
|
if (attachments === undefined) continue
|
|
1025
1034
|
try {
|
|
1026
|
-
|
|
1035
|
+
throwIfVisionTaskAborted(options.signal)
|
|
1036
|
+
const stored = await attachments.readImage(block.attachment, options.signal)
|
|
1037
|
+
throwIfVisionTaskAborted(options.signal)
|
|
1027
1038
|
// Last-mile guard: never send oversized images to the vision
|
|
1028
1039
|
// endpoint — encoder cost scales with pixels and dominates
|
|
1029
1040
|
// tool-call latency on retina screenshots.
|
|
1030
1041
|
let bytes = stored.data
|
|
1031
1042
|
if (downscaleEnabled() && bytes && bytes.length > 0) {
|
|
1032
1043
|
bytes = await downscaleImage(bytes, downscaleMaxPixels())
|
|
1044
|
+
throwIfVisionTaskAborted(options.signal)
|
|
1033
1045
|
}
|
|
1034
1046
|
content.push(...toOpenAIContent([block], () => bytes))
|
|
1035
1047
|
} catch (error) {
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1048
|
+
throwIfVisionTaskAborted(options.signal, error)
|
|
1049
|
+
// A text-only completion here would falsely claim the model
|
|
1050
|
+
// saw pixels. Fail this backend immediately and let the
|
|
1051
|
+
// normal vision fallback chain choose another backend.
|
|
1052
|
+
ctx.logger?.warn('vision-http: retained image delivery failed; refusing text-only downgrade')
|
|
1053
|
+
yield {
|
|
1054
|
+
type: 'finish',
|
|
1055
|
+
reason: {
|
|
1056
|
+
kind: 'error',
|
|
1057
|
+
failure: {
|
|
1058
|
+
code: 'VISION_IMAGE_DELIVERY_UNAVAILABLE',
|
|
1059
|
+
message: 'vision-http: retained image could not be delivered to the model',
|
|
1060
|
+
},
|
|
1061
|
+
},
|
|
1062
|
+
}
|
|
1063
|
+
return
|
|
1040
1064
|
}
|
|
1041
1065
|
} else if (block && block.type === 'tool-result') {
|
|
1042
1066
|
// The OpenAI wire has no tool-call frames to hang a `role: tool`
|
|
@@ -1701,7 +1725,9 @@ export function apply(ctx, config = {}, runtime = {}) {
|
|
|
1701
1725
|
}
|
|
1702
1726
|
const content = []
|
|
1703
1727
|
for (const block of blocks) {
|
|
1704
|
-
|
|
1728
|
+
throwIfVisionTaskAborted(options.signal)
|
|
1729
|
+
const stored = await attachments.readImage(block.attachment, options.signal)
|
|
1730
|
+
throwIfVisionTaskAborted(options.signal)
|
|
1705
1731
|
content.push(...toOpenAIContent([block], () => stored.data))
|
|
1706
1732
|
}
|
|
1707
1733
|
return callOpenAIWithProviderTransport(
|
|
@@ -1797,14 +1823,18 @@ export function apply(ctx, config = {}, runtime = {}) {
|
|
|
1797
1823
|
throw new Error('corrected route: the attachment service is not registered')
|
|
1798
1824
|
}
|
|
1799
1825
|
const bytesOf = async (attachment) => {
|
|
1800
|
-
|
|
1826
|
+
throwIfVisionTaskAborted(options.signal)
|
|
1827
|
+
const stored = await attachments.readImage(attachment, options.signal)
|
|
1828
|
+
throwIfVisionTaskAborted(options.signal)
|
|
1801
1829
|
let bytes = stored.data
|
|
1802
1830
|
if (downscaleEnabled() && bytes && bytes.length > 0) {
|
|
1803
1831
|
bytes = await downscaleImage(bytes, downscaleMaxPixels())
|
|
1832
|
+
throwIfVisionTaskAborted(options.signal)
|
|
1804
1833
|
}
|
|
1805
1834
|
return bytes
|
|
1806
1835
|
}
|
|
1807
1836
|
const anthropic = await toAnthropicMessages(messages, bytesOf)
|
|
1837
|
+
throwIfVisionTaskAborted(options.signal)
|
|
1808
1838
|
if (anthropic.messages.length === 0) {
|
|
1809
1839
|
throw new Error(`corrected route "${pair.provider}/${pair.model}": no representable content to send`)
|
|
1810
1840
|
}
|
|
@@ -2402,8 +2432,8 @@ export function apply(ctx, config = {}, runtime = {}) {
|
|
|
2402
2432
|
// Host-owned proxy overrides are scoped by lib/legacy-global-proxy-boundary.js.
|
|
2403
2433
|
// Core no longer owns or installs a process-wide proxy fetch implementation.
|
|
2404
2434
|
|
|
2405
|
-
const resolveAttachment = (session, id) => sessionVisionIndex.resolveAttachment(session, id)
|
|
2406
|
-
const resolveAttachments = (session, ids) => sessionVisionIndex.resolveAttachments(session, ids)
|
|
2435
|
+
const resolveAttachment = (session, id, signal) => sessionVisionIndex.resolveAttachment(session, id, signal)
|
|
2436
|
+
const resolveAttachments = (session, ids, signal) => sessionVisionIndex.resolveAttachments(session, ids, signal)
|
|
2407
2437
|
|
|
2408
2438
|
// Session-local routing handoff between pre-step and agent/request. Modern
|
|
2409
2439
|
// Hosts store one exact async raw-log tail seq; legacy Hosts retain only the
|
|
@@ -2880,121 +2910,103 @@ export function apply(ctx, config = {}, runtime = {}) {
|
|
|
2880
2910
|
}
|
|
2881
2911
|
const blocks = []
|
|
2882
2912
|
const contentIds = []
|
|
2883
|
-
|
|
2913
|
+
// One shared deadline for the WHOLE task: image preflight, every
|
|
2914
|
+
// provider, fallback and JSON correction draw from one clock.
|
|
2915
|
+
// Previously the clock started after SessionQuery/image loading,
|
|
2916
|
+
// leaving attachmentIds vulnerable to the Host's ~120s watchdog.
|
|
2917
|
+
const hostSignal = combineSignals(currentVisionTurnBudgetSignal(), exec?.signal)
|
|
2918
|
+
const deadline = createDeadline(visionTaskTimeoutMs())
|
|
2919
|
+
const taskSignal = combineSignals(hostSignal, deadline.signal())
|
|
2920
|
+
const stage = createVisionToolStageTrace(ctx.logger, 'vision_describe', taskSignal)
|
|
2921
|
+
try {
|
|
2884
2922
|
const paths = Array.isArray(args.paths) ? args.paths : []
|
|
2885
2923
|
const attachmentIds = Array.isArray(args.attachmentIds) ? args.attachmentIds : []
|
|
2886
2924
|
if (paths.length + attachmentIds.length === 0 || paths.length + attachmentIds.length > 4) {
|
|
2887
2925
|
throw new Error('vision_describe: provide 1-4 images via paths and/or attachmentIds')
|
|
2888
2926
|
}
|
|
2889
|
-
//
|
|
2890
|
-
//
|
|
2891
|
-
|
|
2892
|
-
|
|
2893
|
-
|
|
2927
|
+
// Both argument forms now use the exact same read -> normalize ->
|
|
2928
|
+
// downscale -> save path. The old attachmentIds-only path could retain
|
|
2929
|
+
// an oversized original ref after resize publication failed.
|
|
2930
|
+
const sources = [...paths, ...attachmentIds]
|
|
2931
|
+
const materializableAttachmentIds = [...new Set(
|
|
2932
|
+
sources.map((value) => String(value)).filter((value) => isAttachmentIdInput(value)),
|
|
2933
|
+
)]
|
|
2934
|
+
// attachmentIds is always an attachment namespace, even for a legacy
|
|
2935
|
+
// Host whose authorized IDs do not yet have a sha256: prefix.
|
|
2936
|
+
const requestedRefs = [...new Set([
|
|
2937
|
+
...attachmentIds.map((value) => String(value)),
|
|
2938
|
+
...paths.map((value) => String(value)).filter((value) => isAttachmentIdInput(value)),
|
|
2894
2939
|
])]
|
|
2895
|
-
const attachmentSession = exec
|
|
2896
|
-
const resolvedAttachmentRefs =
|
|
2897
|
-
? await
|
|
2940
|
+
const attachmentSession = exec?.agent?.session
|
|
2941
|
+
const resolvedAttachmentRefs = requestedRefs.length > 0
|
|
2942
|
+
? await stage('resolve-upload-refs', () =>
|
|
2943
|
+
resolveAttachments(attachmentSession, requestedRefs, taskSignal))
|
|
2898
2944
|
: new Map()
|
|
2899
2945
|
|
|
2900
|
-
for (const
|
|
2946
|
+
for (const [index, item] of sources.entries()) {
|
|
2947
|
+
const source = String(item)
|
|
2948
|
+
const attachmentInput = index >= paths.length
|
|
2901
2949
|
let bytes
|
|
2902
2950
|
let mediaType
|
|
2903
2951
|
try {
|
|
2904
|
-
|
|
2905
|
-
|
|
2906
|
-
// a path gets the right pixels instead of a not-found error.
|
|
2907
|
-
;({ bytes, mediaType } = await readImageBytes(exec, path, resolvedAttachmentRefs))
|
|
2952
|
+
;({ bytes, mediaType } = await stage('read-image-' + index, () =>
|
|
2953
|
+
readImageBytes(exec, source, resolvedAttachmentRefs, attachmentInput)))
|
|
2908
2954
|
} catch (error) {
|
|
2909
2955
|
throw new Error(
|
|
2910
|
-
`vision_describe: failed to read ${
|
|
2956
|
+
`vision_describe: failed to read ${source} (${error && error.message ? error.message : String(error)})`,
|
|
2957
|
+
{ cause: error },
|
|
2911
2958
|
)
|
|
2912
2959
|
}
|
|
2913
2960
|
if (downscaleEnabled()) {
|
|
2914
|
-
const resized = await
|
|
2961
|
+
const resized = await stage('downscale-image-' + index, () =>
|
|
2962
|
+
downscaleImage(bytes, downscaleMaxPixels(), { signal: taskSignal }))
|
|
2915
2963
|
if (resized !== bytes) {
|
|
2916
|
-
ctx.logger?.info('vision-router: downscaled
|
|
2964
|
+
ctx.logger?.info('vision-router: downscaled vision input index=%d', index)
|
|
2917
2965
|
}
|
|
2918
2966
|
bytes = resized
|
|
2919
2967
|
}
|
|
2968
|
+
// Always publish canonical bytes, including an uploaded image.
|
|
2969
|
+
// Host adapters then read the verified/downsized version, never a
|
|
2970
|
+
// stale or mismatched original attachment reference.
|
|
2920
2971
|
let ref
|
|
2921
2972
|
try {
|
|
2922
|
-
ref = await attachments.saveImage({
|
|
2973
|
+
ref = await stage('save-normalized-image-' + index, () => attachments.saveImage({
|
|
2923
2974
|
data: bytes,
|
|
2924
2975
|
mediaType,
|
|
2925
|
-
...(isAttachmentIdInput(
|
|
2976
|
+
...(attachmentInput || isAttachmentIdInput(source) || basenameOf(source) === undefined
|
|
2926
2977
|
? {}
|
|
2927
|
-
: { name: basenameOf(
|
|
2928
|
-
})
|
|
2978
|
+
: { name: basenameOf(source) }),
|
|
2979
|
+
}))
|
|
2929
2980
|
} catch (error) {
|
|
2930
2981
|
throw new Error(
|
|
2931
|
-
`vision_describe: image ${
|
|
2982
|
+
`vision_describe: image ${source} was rejected (${error && error.message ? error.message : String(error)})`,
|
|
2983
|
+
{ cause: error },
|
|
2932
2984
|
)
|
|
2933
2985
|
}
|
|
2986
|
+
throwIfVisionTaskAborted(taskSignal)
|
|
2934
2987
|
contentIds.push(String(ref.attachmentId))
|
|
2935
2988
|
blocks.push({ type: 'image', attachment: ref })
|
|
2936
2989
|
}
|
|
2937
2990
|
|
|
2938
|
-
for (const id of attachmentIds) {
|
|
2939
|
-
const attachmentId = String(id)
|
|
2940
|
-
const ref = isAttachmentIdInput(attachmentId)
|
|
2941
|
-
? resolvedAttachmentRefs.get(attachmentId)
|
|
2942
|
-
: await resolveAttachment(attachmentSession, attachmentId)
|
|
2943
|
-
if (ref === undefined) {
|
|
2944
|
-
throw new Error(
|
|
2945
|
-
`vision_describe: unknown attachment id "${id}" (it must come from an image uploaded in this conversation)`,
|
|
2946
|
-
)
|
|
2947
|
-
}
|
|
2948
|
-
let stored
|
|
2949
|
-
try {
|
|
2950
|
-
stored = await attachments.readImage(ref)
|
|
2951
|
-
} catch (error) {
|
|
2952
|
-
throw new Error(
|
|
2953
|
-
`vision_describe: failed to read attachment ${id} (${error && error.message ? error.message : String(error)})`,
|
|
2954
|
-
)
|
|
2955
|
-
}
|
|
2956
|
-
// Downscale oversized uploads before the vision call: retina
|
|
2957
|
-
// screenshots easily reach 10MP+ and the vision encoder's cost
|
|
2958
|
-
// scales with pixels — a full-size upload is the dominant part of
|
|
2959
|
-
// the tool-call latency. Re-save a resized attachment so the
|
|
2960
|
-
// adapter reads the small one.
|
|
2961
|
-
if (downscaleEnabled() && stored.data && stored.data.length > 0) {
|
|
2962
|
-
const resized = await downscaleImage(stored.data, downscaleMaxPixels())
|
|
2963
|
-
if (resized !== stored.data) {
|
|
2964
|
-
try {
|
|
2965
|
-
const resizedRef = await attachments.saveImage({
|
|
2966
|
-
data: resized,
|
|
2967
|
-
mediaType: stored.ref && stored.ref.mediaType ? stored.ref.mediaType : 'image/png',
|
|
2968
|
-
...(stored.ref && stored.ref.name ? { name: stored.ref.name } : {}),
|
|
2969
|
-
})
|
|
2970
|
-
stored = { ref: resizedRef, data: resized }
|
|
2971
|
-
ctx.logger?.info('vision-router: downscaled attachment %s for the vision call', id)
|
|
2972
|
-
} catch {
|
|
2973
|
-
stored = { ...stored, data: resized }
|
|
2974
|
-
}
|
|
2975
|
-
}
|
|
2976
|
-
}
|
|
2977
|
-
contentIds.push(String(ref.attachmentId))
|
|
2978
|
-
blocks.push({ type: 'image', attachment: stored.ref })
|
|
2979
|
-
}
|
|
2980
|
-
|
|
2981
2991
|
const question = String(args.question ?? '')
|
|
2982
2992
|
const wantJson = args.json === true
|
|
2983
2993
|
// Keep the adapter path and direct OpenAI-compatible HTTP path on the
|
|
2984
2994
|
// exact same prompt, including the structured JSON evidence contract.
|
|
2985
2995
|
const promptText = visionDescribePrompt(question, wantJson)
|
|
2986
|
-
const usablePairs = await resolveToolVisionPairs()
|
|
2996
|
+
const usablePairs = await stage('resolve-vision-backends', () => resolveToolVisionPairs())
|
|
2987
2997
|
// Freeze this task's direct fallback list: settings changes take effect
|
|
2988
2998
|
// on the next call, never halfway through one fallback walk.
|
|
2989
2999
|
const httpFallbacks = httpProviders()
|
|
2990
3000
|
const rejectedPairs = []
|
|
3001
|
+
let backendPreflightIndex = 0
|
|
2991
3002
|
for (const pair of pairs()) {
|
|
2992
3003
|
if (!pair || pair.provider === HTTP_ROUTE) continue
|
|
2993
3004
|
if (!adapterAvailable(ctx.llm, pair.provider)) {
|
|
2994
3005
|
rejectedPairs.push(`${pair.provider}/${pair.model}: provider adapter is not registered`)
|
|
2995
3006
|
continue
|
|
2996
3007
|
}
|
|
2997
|
-
const capability = await
|
|
3008
|
+
const capability = await stage('backend-preflight-' + backendPreflightIndex++, () =>
|
|
3009
|
+
resolveVisionBackendCapability(pair.provider, pair.model))
|
|
2998
3010
|
if (capability.attemptable === false) {
|
|
2999
3011
|
rejectedPairs.push(
|
|
3000
3012
|
`${pair.provider}/${pair.model}: structurally unavailable (${capability.reason ?? 'unknown reason'})`,
|
|
@@ -3030,9 +3042,7 @@ export function apply(ctx, config = {}, runtime = {}) {
|
|
|
3030
3042
|
)
|
|
3031
3043
|
}
|
|
3032
3044
|
|
|
3033
|
-
//
|
|
3034
|
-
// the JSON-correction retry draws from this single budget.
|
|
3035
|
-
const deadline = createDeadline(visionTaskTimeoutMs())
|
|
3045
|
+
// The same clock already started before attachment resolution.
|
|
3036
3046
|
const baseMessages = [
|
|
3037
3047
|
{
|
|
3038
3048
|
role: 'user',
|
|
@@ -3051,7 +3061,7 @@ export function apply(ctx, config = {}, runtime = {}) {
|
|
|
3051
3061
|
0,
|
|
3052
3062
|
)
|
|
3053
3063
|
|
|
3054
|
-
for (const pair of usablePairs) {
|
|
3064
|
+
for (const [pairIndex, pair] of usablePairs.entries()) {
|
|
3055
3065
|
if (localOnlyVisionEnabled(current()) && !isLocalBackendPair(pair)) {
|
|
3056
3066
|
errors.push(`${pair.provider}/${pair.model}: skipped (local-only vision policy)`)
|
|
3057
3067
|
continue
|
|
@@ -3064,7 +3074,8 @@ export function apply(ctx, config = {}, runtime = {}) {
|
|
|
3064
3074
|
break
|
|
3065
3075
|
}
|
|
3066
3076
|
const backendKey = `${pair.provider}/${pair.model}`
|
|
3067
|
-
const fingerprint = await
|
|
3077
|
+
const fingerprint = await stage('adapter-auth-' + pairIndex, () =>
|
|
3078
|
+
credentialFingerprintFor({ provider: pair.provider }))
|
|
3068
3079
|
const gate = visionBreaker.inspect(backendKey, fingerprint, scope)
|
|
3069
3080
|
if (gate.blocked) {
|
|
3070
3081
|
errors.push(`${backendKey}: skipped (circuit open: ${gate.reason})`)
|
|
@@ -3089,19 +3100,20 @@ export function apply(ctx, config = {}, runtime = {}) {
|
|
|
3089
3100
|
)
|
|
3090
3101
|
: timeoutMs()
|
|
3091
3102
|
const signal = combineSignals(
|
|
3092
|
-
|
|
3103
|
+
hostSignal,
|
|
3093
3104
|
deadline.signal(),
|
|
3094
3105
|
AbortSignal.timeout(attemptBudgetMs),
|
|
3095
3106
|
)
|
|
3096
|
-
const capability = await
|
|
3097
|
-
|
|
3107
|
+
const capability = await stage('adapter-capability-' + pairIndex, () =>
|
|
3108
|
+
resolveVisionBackendCapability(pair.provider, pair.model))
|
|
3109
|
+
let text = await stage('adapter-request-' + pairIndex, () => callVisionPairWithOptionalBridge(pair, messages, {
|
|
3098
3110
|
maxTokens: 4096,
|
|
3099
3111
|
signal,
|
|
3100
3112
|
sessionId,
|
|
3101
3113
|
capability,
|
|
3102
3114
|
bridgeBlocks: blocks,
|
|
3103
3115
|
bridgeInstruction: promptText,
|
|
3104
|
-
})
|
|
3116
|
+
}))
|
|
3105
3117
|
assertNoRepetitionLoop(text, backendKey)
|
|
3106
3118
|
ctx.logger?.info(
|
|
3107
3119
|
'vision-router: vision call %s ok (%d chars, %d ms)',
|
|
@@ -3133,7 +3145,7 @@ ctx.logger?.info(
|
|
|
3133
3145
|
},
|
|
3134
3146
|
]
|
|
3135
3147
|
attemptStarted = Date.now()
|
|
3136
|
-
text = await callVisionPairWithOptionalBridge(pair, messages, {
|
|
3148
|
+
text = await stage('adapter-json-correction-' + pairIndex, () => callVisionPairWithOptionalBridge(pair, messages, {
|
|
3137
3149
|
maxTokens: 4096,
|
|
3138
3150
|
signal,
|
|
3139
3151
|
sessionId,
|
|
@@ -3141,7 +3153,7 @@ ctx.logger?.info(
|
|
|
3141
3153
|
bridgeBlocks: blocks,
|
|
3142
3154
|
bridgeInstruction:
|
|
3143
3155
|
promptText + '\n\nThat output was not valid JSON. Respond with ONLY a valid JSON object now.',
|
|
3144
|
-
})
|
|
3156
|
+
}))
|
|
3145
3157
|
assertNoRepetitionLoop(text, backendKey)
|
|
3146
3158
|
ctx.logger?.info(
|
|
3147
3159
|
'vision-router: vision call %s ok after JSON correction (%d chars, %d ms)',
|
|
@@ -3171,7 +3183,7 @@ ctx.logger?.info(
|
|
|
3171
3183
|
// A user/Host cancel is not a backend outage. Otherwise the
|
|
3172
3184
|
// canceled attempt poisons this turn's breaker/failure memory and
|
|
3173
3185
|
// a healthy retry returns BACKEND_UNAVAILABLE_THIS_TURN.
|
|
3174
|
-
throwIfVisionTaskAborted(
|
|
3186
|
+
throwIfVisionTaskAborted(hostSignal, error)
|
|
3175
3187
|
const classification = classifyVisionFailure(error)
|
|
3176
3188
|
consecutiveServerFailures = classification.kind === VISION_FAILURE_KINDS.SERVER
|
|
3177
3189
|
? consecutiveServerFailures + 1 : 0
|
|
@@ -3197,7 +3209,7 @@ ctx.logger?.info(
|
|
|
3197
3209
|
// Direct HTTP providers (built-in keyless OVHcloud by default) are the
|
|
3198
3210
|
// final fallbacks: they bypass the harness llm service entirely, so the
|
|
3199
3211
|
// anonymous free endpoint works without any credential.
|
|
3200
|
-
for (const provider of httpFallbacks) {
|
|
3212
|
+
for (const [httpIndex, provider] of httpFallbacks.entries()) {
|
|
3201
3213
|
if (localOnlyVisionEnabled(current()) && !isLoopbackVisionBaseURL(provider?.baseURL)) {
|
|
3202
3214
|
errors.push(`http:${provider?.name}/${provider?.model}: skipped (local-only vision policy)`)
|
|
3203
3215
|
continue
|
|
@@ -3210,7 +3222,8 @@ ctx.logger?.info(
|
|
|
3210
3222
|
break
|
|
3211
3223
|
}
|
|
3212
3224
|
const backendKey = `http:${provider.name}/${provider.model}`
|
|
3213
|
-
const fingerprint = await
|
|
3225
|
+
const fingerprint = await stage('http-auth-' + httpIndex, () =>
|
|
3226
|
+
credentialFingerprintFor({ kind: 'http', apiKeyEnv: provider.apiKeyEnv }))
|
|
3214
3227
|
const gate = visionBreaker.inspect(backendKey, fingerprint, scope)
|
|
3215
3228
|
if (gate.blocked) {
|
|
3216
3229
|
errors.push(`${backendKey}: skipped (circuit open: ${gate.reason})`)
|
|
@@ -3222,15 +3235,19 @@ ctx.logger?.info(
|
|
|
3222
3235
|
let attemptStarted = Date.now()
|
|
3223
3236
|
try {
|
|
3224
3237
|
// Precompute bytes once per block (attachments.readImage is async).
|
|
3225
|
-
const openAIBlocks =
|
|
3226
|
-
|
|
3227
|
-
|
|
3228
|
-
|
|
3229
|
-
|
|
3230
|
-
|
|
3231
|
-
|
|
3238
|
+
const openAIBlocks = await stage('http-image-delivery-' + httpIndex, async () => {
|
|
3239
|
+
const converted = []
|
|
3240
|
+
for (const block of blocks) {
|
|
3241
|
+
if (block.type === 'image' && block.attachment) {
|
|
3242
|
+
const stored = await attachments.readImage(block.attachment, taskSignal)
|
|
3243
|
+
throwIfVisionTaskAborted(taskSignal)
|
|
3244
|
+
converted.push(toOpenAIContent([block], () => stored.data)[0])
|
|
3245
|
+
} else {
|
|
3246
|
+
converted.push({ type: 'text', text: block.text })
|
|
3247
|
+
}
|
|
3232
3248
|
}
|
|
3233
|
-
|
|
3249
|
+
return converted
|
|
3250
|
+
})
|
|
3234
3251
|
// Direct HTTP providers must receive the same image + question as
|
|
3235
3252
|
// adapter-backed providers. Some endpoints (e.g. Zhipu GLM) reject
|
|
3236
3253
|
// a pure-image user message even when permissive endpoints accept it.
|
|
@@ -3248,12 +3265,15 @@ ctx.logger?.info(
|
|
|
3248
3265
|
consecutiveServerFailures,
|
|
3249
3266
|
)
|
|
3250
3267
|
const attemptSignal = combineSignals(
|
|
3251
|
-
|
|
3268
|
+
hostSignal,
|
|
3252
3269
|
deadline.signal(),
|
|
3253
3270
|
AbortSignal.timeout(attemptBudgetMs),
|
|
3254
3271
|
)
|
|
3255
3272
|
const askHttp = async (correction) => {
|
|
3256
|
-
const
|
|
3273
|
+
const phase = correction === undefined
|
|
3274
|
+
? 'http-request-' + httpIndex : 'http-json-correction-' + httpIndex
|
|
3275
|
+
const answer = await stage(phase, () =>
|
|
3276
|
+
awaitVisionBackendAttempt(attemptSignal, () => callOpenAIWithProviderTransport(
|
|
3257
3277
|
provider,
|
|
3258
3278
|
correction === undefined
|
|
3259
3279
|
? openAIBaseMessages
|
|
@@ -3267,7 +3287,7 @@ ctx.logger?.info(
|
|
|
3267
3287
|
sessionId,
|
|
3268
3288
|
resolveCredential,
|
|
3269
3289
|
},
|
|
3270
|
-
))
|
|
3290
|
+
)))
|
|
3271
3291
|
return answer
|
|
3272
3292
|
}
|
|
3273
3293
|
let text = await askHttp(undefined)
|
|
@@ -3319,7 +3339,7 @@ ctx.logger?.info(
|
|
|
3319
3339
|
// A user/Host cancel is not a backend outage. Otherwise the
|
|
3320
3340
|
// canceled attempt poisons this turn's breaker/failure memory and
|
|
3321
3341
|
// a healthy retry returns BACKEND_UNAVAILABLE_THIS_TURN.
|
|
3322
|
-
throwIfVisionTaskAborted(
|
|
3342
|
+
throwIfVisionTaskAborted(hostSignal, error)
|
|
3323
3343
|
const classification = classifyVisionFailure(error)
|
|
3324
3344
|
consecutiveServerFailures = classification.kind === VISION_FAILURE_KINDS.SERVER
|
|
3325
3345
|
? consecutiveServerFailures + 1 : 0
|
|
@@ -3367,6 +3387,18 @@ ctx.logger?.info(
|
|
|
3367
3387
|
}
|
|
3368
3388
|
}
|
|
3369
3389
|
return JSON.stringify(baseFailure)
|
|
3390
|
+
} catch (error) {
|
|
3391
|
+
// The caller's explicit cancellation is not a model outage.
|
|
3392
|
+
if (hostSignal?.aborted) throw hostSignal.reason ?? error
|
|
3393
|
+
if (deadline.expired() || taskSignal?.aborted) {
|
|
3394
|
+
return JSON.stringify(buildVisionFailure({
|
|
3395
|
+
code: VISION_RESULT_CODES.TIMEOUT,
|
|
3396
|
+
retryable: false,
|
|
3397
|
+
reason: 'Vision task deadline exhausted while preparing or processing the image.',
|
|
3398
|
+
}))
|
|
3399
|
+
}
|
|
3400
|
+
throw error
|
|
3401
|
+
}
|
|
3370
3402
|
},
|
|
3371
3403
|
}
|
|
3372
3404
|
deepToolDefs.push(visionDescribeTool)
|
|
@@ -3498,55 +3530,9 @@ ctx.logger?.info(
|
|
|
3498
3530
|
return String(attachmentIds[0]).trim()
|
|
3499
3531
|
}
|
|
3500
3532
|
|
|
3501
|
-
const readImageBytes =
|
|
3502
|
-
|
|
3503
|
-
|
|
3504
|
-
let storedMediaType
|
|
3505
|
-
if (isAttachmentIdInput(input)) {
|
|
3506
|
-
// Uploaded images reach the tool arguments as durable attachment ids
|
|
3507
|
-
// ("sha256:..."); resolve them through the session's recorded upload
|
|
3508
|
-
// index instead of treating the id as a filesystem path.
|
|
3509
|
-
const attachments = ctx.get('attachments')
|
|
3510
|
-
if (attachments === undefined) {
|
|
3511
|
-
throw new Error('vision-router: the attachment service is not available in this deployment')
|
|
3512
|
-
}
|
|
3513
|
-
const session = exec && exec.agent && exec.agent.session
|
|
3514
|
-
const canonicalInput = input.trim()
|
|
3515
|
-
const ref = resolvedAttachmentRefs instanceof Map
|
|
3516
|
-
? resolvedAttachmentRefs.get(canonicalInput)
|
|
3517
|
-
: await resolveAttachment(session, canonicalInput)
|
|
3518
|
-
if (ref === undefined) {
|
|
3519
|
-
throw new Error(
|
|
3520
|
-
`vision-router: unknown attachment id "${input}" (it must come from an image uploaded in this conversation)`,
|
|
3521
|
-
)
|
|
3522
|
-
}
|
|
3523
|
-
let stored
|
|
3524
|
-
try {
|
|
3525
|
-
stored = await attachments.readImage(ref)
|
|
3526
|
-
} catch (error) {
|
|
3527
|
-
throw new Error(
|
|
3528
|
-
`vision-router: failed to read attachment ${input} (${error && error.message ? error.message : String(error)})`,
|
|
3529
|
-
)
|
|
3530
|
-
}
|
|
3531
|
-
bytes = stored.data
|
|
3532
|
-
if (stored.ref && typeof stored.ref.mediaType === 'string') {
|
|
3533
|
-
storedMediaType = stored.ref.mediaType
|
|
3534
|
-
}
|
|
3535
|
-
} else {
|
|
3536
|
-
const fs = ctx.get('fs')
|
|
3537
|
-
if (fs === undefined) throw new Error('vision-router: the fs service is not available')
|
|
3538
|
-
const target = await fs.resolve(input)
|
|
3539
|
-
bytes = await fs.readBytes(target, undefined, 20 * 1024 * 1024)
|
|
3540
|
-
}
|
|
3541
|
-
// Attachments are stored as content-addressed files without an
|
|
3542
|
-
// extension: sniff the format from the bytes, and fall back to the
|
|
3543
|
-
// stored ref / extension only when sniffing cannot decide.
|
|
3544
|
-
const mediaType = sniffMediaType(bytes) ?? storedMediaType ?? mediaTypeOf(input)
|
|
3545
|
-
if (mediaType === undefined) {
|
|
3546
|
-
throw new Error(`unsupported image format ${input} (png/jpeg/webp/gif only)`)
|
|
3547
|
-
}
|
|
3548
|
-
return { bytes, mediaType }
|
|
3549
|
-
}
|
|
3533
|
+
const readImageBytes = createVisionImageInputReader(
|
|
3534
|
+
ctx, resolveAttachment, throwIfVisionTaskAborted,
|
|
3535
|
+
)
|
|
3550
3536
|
|
|
3551
3537
|
const imageDims = async (bytes) => {
|
|
3552
3538
|
const sharp = await loadSharp()
|
|
@@ -4362,9 +4348,10 @@ ctx.logger?.info(
|
|
|
4362
4348
|
name: 'vision_ocr',
|
|
4363
4349
|
description:
|
|
4364
4350
|
'Transcribe TEXT from an image. ENGINE POLICY: explicit engine=tesseract or engine=vision always wins. ' +
|
|
4365
|
-
'Otherwise the configured OCR engine policy applies.
|
|
4366
|
-
'
|
|
4367
|
-
'returns no text. Structured 1+x follow-up does not change the selected policy.
|
|
4351
|
+
'Otherwise the configured OCR engine policy applies. Auto first prefers a locally installed, SHA-verified ' +
|
|
4352
|
+
'PP-OCRv6/ncnn on macOS ARM64 or Windows x64, then Tesseract (chi_sim+eng), and finally a vision model ' +
|
|
4353
|
+
'when local OCR fails, returns no text, or remains uncertain (subject to permission and budget). Structured 1+x follow-up does not change the selected policy. ' +
|
|
4354
|
+
'Returns the text and ' +
|
|
4368
4355
|
'which engine produced it. ' +
|
|
4369
4356
|
'SCOPE: vision_ocr reads letters, it does NOT recognize people, objects or scenes. Never use it ' +
|
|
4370
4357
|
'as a fallback when vision_describe fails to identify who/what is in a picture ("这是谁" / ' +
|
|
@@ -4394,7 +4381,7 @@ ctx.logger?.info(
|
|
|
4394
4381
|
},
|
|
4395
4382
|
engine: {
|
|
4396
4383
|
type: 'string',
|
|
4397
|
-
description: '"auto": use the configured OCR engine policy
|
|
4384
|
+
description: '"auto": use the configured OCR engine policy: prefer installed SHA-verified PP-OCRv6 on macOS ARM64 or Windows x64, otherwise local Tesseract; permitted vision fallback on local failure or uncertain text; explicit "tesseract"/"vision" always overrides the configured policy.',
|
|
4398
4385
|
},
|
|
4399
4386
|
},
|
|
4400
4387
|
additionalProperties: false,
|
|
@@ -4423,6 +4410,83 @@ ctx.logger?.info(
|
|
|
4423
4410
|
// gets a capped slice (never more than 12s), the vision model only the
|
|
4424
4411
|
// remainder. The two timeouts can never stack into a multi-minute wait.
|
|
4425
4412
|
const deadline = createDeadline(ocrBudgetMs())
|
|
4413
|
+
// Prefer an owner-installed and SHA-pinned PP-OCRv6 worker in auto mode.
|
|
4414
|
+
// No models or native binaries are bundled with the npm package.
|
|
4415
|
+
// The isolated legacy preview override remains for existing lab tests.
|
|
4416
|
+
const nativeAssets = process.env.DVR_NCNN_OCR_CANARY === '1'
|
|
4417
|
+
? {
|
|
4418
|
+
binaryPath: process.env.DVR_NCNN_OCR_EXECUTABLE,
|
|
4419
|
+
modelDir: process.env.DVR_NCNN_OCR_MODELS,
|
|
4420
|
+
}
|
|
4421
|
+
: (() => {
|
|
4422
|
+
const installed = localNativeOcrInstallPaths()
|
|
4423
|
+
return installed && existsSync(installed.binaryPath) ? installed : null
|
|
4424
|
+
})()
|
|
4425
|
+
if (engine === 'auto' && nativeAssets !== null) {
|
|
4426
|
+
try {
|
|
4427
|
+
const native = await runExperimentalNativeOcrCanary(bytes, {
|
|
4428
|
+
binaryPath: nativeAssets.binaryPath,
|
|
4429
|
+
modelDir: nativeAssets.modelDir,
|
|
4430
|
+
loadSharp,
|
|
4431
|
+
signal: taskSignal,
|
|
4432
|
+
timeoutMs: Math.min(21000, Math.max(1, Math.floor(deadline.remaining() * .75))),
|
|
4433
|
+
})
|
|
4434
|
+
throwIfVisionTaskAborted(taskSignal)
|
|
4435
|
+
if (native.text.trim()) {
|
|
4436
|
+
if (session) {
|
|
4437
|
+
visionTurnMemory.recordLocalOcr(
|
|
4438
|
+
visionScopeOf(session),
|
|
4439
|
+
visionEvidenceSourceKey(imageInput),
|
|
4440
|
+
{ uncertain: native.uncertain === true },
|
|
4441
|
+
)
|
|
4442
|
+
}
|
|
4443
|
+
const nativeResult = {
|
|
4444
|
+
engine: 'ncnn',
|
|
4445
|
+
text: native.text.trim(),
|
|
4446
|
+
uncertain: native.uncertain === true,
|
|
4447
|
+
...(native.uncertain
|
|
4448
|
+
? { review: {
|
|
4449
|
+
documentRefined: native.documentRefined === true,
|
|
4450
|
+
regions: Array.isArray(native.regions) ? native.regions.length : undefined,
|
|
4451
|
+
numericOrFormulaRisks: Array.isArray(native.reviewQueue)
|
|
4452
|
+
? native.reviewQueue.slice(0, 12) : undefined,
|
|
4453
|
+
reviewTruncated: Array.isArray(native.reviewQueue) && native.reviewQueue.length > 12,
|
|
4454
|
+
} }
|
|
4455
|
+
: {}),
|
|
4456
|
+
}
|
|
4457
|
+
// Installing ncnn must not turn uncertain, nonempty local OCR
|
|
4458
|
+
// into an unreviewable final answer. Never bypass privacy,
|
|
4459
|
+
// breaker or the shared one-task deadline to verify it.
|
|
4460
|
+
const reviewed = await verifyUncertainAutoOcrResult(nativeResult, {
|
|
4461
|
+
engine,
|
|
4462
|
+
localOnlyVision: localOnlyVisionEnabled(current()),
|
|
4463
|
+
degraded: degraded.active,
|
|
4464
|
+
remainingMs: deadline.remaining(),
|
|
4465
|
+
verifyVision: () => answerVisionForTool(
|
|
4466
|
+
exec, bytes, mediaType,
|
|
4467
|
+
'请核查图片中的原文,尤其是易混淆的数字、化学式和底部字幕;不确定时不要猜。只输出文字。',
|
|
4468
|
+
{ deadline },
|
|
4469
|
+
),
|
|
4470
|
+
checkAbort: error => throwIfVisionTaskAborted(taskSignal, error),
|
|
4471
|
+
})
|
|
4472
|
+
return JSON.stringify(reviewed)
|
|
4473
|
+
}
|
|
4474
|
+
} catch (error) {
|
|
4475
|
+
throwIfVisionTaskAborted(taskSignal, error)
|
|
4476
|
+
// Native resource saturation must not fork extra local workers.
|
|
4477
|
+
if (error?.code === 'OCR_BUSY' || error?.code === 'OCR_QUEUE_TIMEOUT') {
|
|
4478
|
+
return JSON.stringify({
|
|
4479
|
+
engine: 'none', ok: false,
|
|
4480
|
+
code: error.code, text: '', retryable: false,
|
|
4481
|
+
reason: 'local OCR worker capacity is exhausted',
|
|
4482
|
+
})
|
|
4483
|
+
}
|
|
4484
|
+
ctx.logger?.warn(
|
|
4485
|
+
'vision-router: local PP-OCRv6 unavailable; using Tesseract/vision policy ' +
|
|
4486
|
+
'(' + String(error?.code ?? 'OCR_NATIVE_FAILED').slice(0, 60) + ')',
|
|
4487
|
+
)
|
|
4488
|
+
}
|
|
4489
|
+
}
|
|
4426
4490
|
const tesseractSlice = Math.min(12000, deadline.remaining())
|
|
4427
4491
|
if (engine !== 'vision') {
|
|
4428
4492
|
let localAttempted = false
|
|
@@ -4431,7 +4495,11 @@ ctx.logger?.info(
|
|
|
4431
4495
|
const local = await ocrWithTesseractAdaptive(
|
|
4432
4496
|
bytes,
|
|
4433
4497
|
tesseractSlice,
|
|
4434
|
-
{
|
|
4498
|
+
{
|
|
4499
|
+
signal: taskSignal,
|
|
4500
|
+
preprocess: (source, context) =>
|
|
4501
|
+
prepareTesseractSubtitleRetry(source, { ...context, loadSharp }),
|
|
4502
|
+
},
|
|
4435
4503
|
)
|
|
4436
4504
|
if (local.text.trim() !== '') {
|
|
4437
4505
|
if (session) {
|
|
@@ -4441,7 +4509,7 @@ ctx.logger?.info(
|
|
|
4441
4509
|
{ uncertain: local.uncertain === true },
|
|
4442
4510
|
)
|
|
4443
4511
|
}
|
|
4444
|
-
|
|
4512
|
+
const localResult = {
|
|
4445
4513
|
engine: 'tesseract',
|
|
4446
4514
|
text: local.text.trim(),
|
|
4447
4515
|
uncertain: local.uncertain === true,
|
|
@@ -4452,10 +4520,48 @@ ctx.logger?.info(
|
|
|
4452
4520
|
attemptedPsms: local.attemptedPsms,
|
|
4453
4521
|
quality: Number(local.quality.toFixed(2)),
|
|
4454
4522
|
riskyTokens: local.riskyTokens,
|
|
4523
|
+
...(local.variant && local.variant !== 'original'
|
|
4524
|
+
? { preprocessing: local.variant } : {}),
|
|
4455
4525
|
},
|
|
4456
4526
|
}
|
|
4457
4527
|
: {}),
|
|
4458
|
-
}
|
|
4528
|
+
}
|
|
4529
|
+
// A nonempty but low-confidence transcript is not proof of
|
|
4530
|
+
// accuracy. In auto ONLY, try the existing consent-controlled
|
|
4531
|
+
// vision route while retaining local evidence if unavailable.
|
|
4532
|
+
if (shouldVerifyUncertainAutoOcr({
|
|
4533
|
+
engine,
|
|
4534
|
+
uncertain: local.uncertain,
|
|
4535
|
+
localOnlyVision: localOnlyVisionEnabled(current()),
|
|
4536
|
+
degraded: degraded.active,
|
|
4537
|
+
remainingMs: deadline.remaining(),
|
|
4538
|
+
})) {
|
|
4539
|
+
try {
|
|
4540
|
+
const vision = await answerVisionForTool(
|
|
4541
|
+
exec, bytes, mediaType,
|
|
4542
|
+
'请严格辨认图片上的原文,尤其是底部字幕或浅色文字;不确定时不要猜。只输出文字。',
|
|
4543
|
+
{ deadline },
|
|
4544
|
+
)
|
|
4545
|
+
throwIfVisionTaskAborted(taskSignal)
|
|
4546
|
+
if (vision.ok !== false && typeof vision.text === 'string' &&
|
|
4547
|
+
vision.text.trim()) {
|
|
4548
|
+
return JSON.stringify({
|
|
4549
|
+
engine: 'vision',
|
|
4550
|
+
text: vision.text.trim(),
|
|
4551
|
+
review: {
|
|
4552
|
+
fallbackReason: 'low_confidence_local_ocr',
|
|
4553
|
+
localText: localResult.text.slice(0, 1000),
|
|
4554
|
+
localQuality: localResult.review?.quality,
|
|
4555
|
+
},
|
|
4556
|
+
})
|
|
4557
|
+
}
|
|
4558
|
+
} catch (error) {
|
|
4559
|
+
throwIfVisionTaskAborted(taskSignal, error)
|
|
4560
|
+
// The existing local transcript remains available and
|
|
4561
|
+
// retains its uncertainty if visual fallback fails.
|
|
4562
|
+
}
|
|
4563
|
+
}
|
|
4564
|
+
return JSON.stringify(localResult)
|
|
4459
4565
|
}
|
|
4460
4566
|
if (engine === 'tesseract') return JSON.stringify({ engine: 'tesseract', text: '', uncertain: true })
|
|
4461
4567
|
} catch (error) {
|
|
@@ -4720,8 +4826,8 @@ ctx.logger?.info(
|
|
|
4720
4826
|
deepToolDefs.push({
|
|
4721
4827
|
name: 'vision_trace',
|
|
4722
4828
|
description:
|
|
4723
|
-
'Vectorize an image (icon/logo) into an SVG via a local
|
|
4724
|
-
'Default:
|
|
4829
|
+
'Vectorize an image (icon/logo) into an SVG via a local VTracer/WASM worker (no Python). ' +
|
|
4830
|
+
'Default: color-preserving vectorization with a limited color palette. ' +
|
|
4725
4831
|
'Set color=false for the layered grayscale posterization, where `steps` (1-16, default 4) ' +
|
|
4726
4832
|
'controls levels. Writes the SVG as an artifact.',
|
|
4727
4833
|
parameters: {
|
|
@@ -4761,16 +4867,21 @@ ctx.logger?.info(
|
|
|
4761
4867
|
const colors = Number.isInteger(args.colors) && args.colors > 0 ? Math.min(args.colors, 16) : 8
|
|
4762
4868
|
const raw = await sharp(traceBytes, { failOn: 'none' }).ensureAlpha().raw().toBuffer({ resolveWithObject: true })
|
|
4763
4869
|
throwIfVisionTaskAborted(taskSignal)
|
|
4764
|
-
|
|
4765
|
-
|
|
4766
|
-
|
|
4870
|
+
// User-facing colors is a strict maximum, not an exact
|
|
4871
|
+
// palette. VTracer chooses the colors for better gradient/alpha
|
|
4872
|
+
// fidelity; explicit palette callers use the separate array form.
|
|
4873
|
+
svg = await posterizeSvgColor(raw.data, raw.info, colors, timeoutMs(), { signal: taskSignal })
|
|
4874
|
+
colorCount = Math.min(colors, new Set(
|
|
4875
|
+
[...svg.matchAll(/<path\b[^>]*\bfill="(#[0-9a-fA-F]{6})"/g)]
|
|
4876
|
+
.map((match) => match[1].toLowerCase()),
|
|
4877
|
+
).size)
|
|
4767
4878
|
} else {
|
|
4768
4879
|
svg = await posterizeSvg(traceBytes, steps, 'dominant', timeoutMs(), { signal: taskSignal })
|
|
4769
4880
|
}
|
|
4770
4881
|
} catch (error) {
|
|
4771
4882
|
throwIfVisionTaskAborted(taskSignal, error)
|
|
4772
4883
|
throw new Error(
|
|
4773
|
-
`vision_trace:
|
|
4884
|
+
`vision_trace: vectorization failed (${error && error.message ? error.message : String(error)})`,
|
|
4774
4885
|
)
|
|
4775
4886
|
}
|
|
4776
4887
|
throwIfVisionTaskAborted(taskSignal)
|