dsh-vision-router 3.2.0 → 3.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -6
- package/README.zh.md +21 -6
- package/THIRD_PARTY_NOTICES.md +167 -0
- package/assets/licenses/apache-2.0.txt +202 -0
- package/assets/licenses/llvm-mingw-LICENSE.txt +279 -0
- package/assets/licenses/mingw-w64-runtime-COPYING.txt +265 -0
- package/assets/licenses/moxcms-BSD-3-Clause.txt +26 -0
- package/assets/licenses/ncnn-LICENSE.txt +86 -0
- package/assets/licenses/roots-BSD-2-Clause.txt +24 -0
- package/assets/licenses/unicode-ident-Unicode-3.0.txt +39 -0
- package/assets/licenses/vtracer-MIT.txt +25 -0
- package/assets/licenses/zlib-rs-Zlib.txt +19 -0
- package/docs/architecture/dsh-support-window.md +25 -6
- package/docs/compatibility/dsh-0.2.2-rc2-audit.md +40 -0
- package/docs/compatibility/dsh-0.2.2-rc3-audit.md +131 -0
- package/docs/compliance/vtracer-alpha4-provenance.md +47 -0
- package/docs/compliance/vtracer-alpha4-reconstructed-Cargo.lock +567 -0
- package/docs/compliance/vtracer-alpha4-rust-inventory.json +421 -0
- package/docs/compliance/vtracer-alpha4-rust-license-texts.md +13018 -0
- package/docs/releases/v3.2.1.md +21 -0
- package/docs/releases/v3.2.2.md +25 -0
- package/docs/releases/v3.3.0.md +34 -0
- package/docs/remote-settings.md +83 -8
- package/index.js +280 -169
- package/lib/client.js +17 -17
- package/lib/core-primitives.js +155 -214
- package/lib/dsh-support-window.js +6 -3
- package/lib/native-ocr-annotation-risk.js +33 -0
- package/lib/native-ocr-canary.js +156 -0
- package/lib/native-ocr-chemistry-review.js +37 -0
- package/lib/native-ocr-document.js +243 -0
- package/lib/native-ocr-installation.js +17 -0
- package/lib/native-ocr-integrity.js +93 -0
- package/lib/native-ocr-question-labels.js +167 -0
- package/lib/native-ocr-supervisor.js +268 -0
- package/lib/remote-settings-bridge.js +93 -57
- package/lib/remote-settings-risk-client-boundary.js +2 -2
- package/lib/runtime-reliability.js +7 -0
- package/lib/session-query-readers.js +8 -2
- package/lib/session-vision-attachment-index.js +61 -14
- package/lib/settings-client-017-compat.js +4 -2
- package/lib/settings-ia-client-prelude.js +26 -10
- package/lib/tesseract-image-preprocess.js +35 -0
- package/lib/trace-vtracer-process-runtime.js +133 -0
- package/lib/trace-vtracer-process.js +43 -0
- package/lib/trace-vtracer-worker.js +163 -0
- package/lib/vision-attachment-handle-runtime.js +11 -2
- package/lib/vision-backend-runtime-policy.js +5 -3
- package/lib/vision-image-input-reader.js +67 -0
- package/lib/vision-image-input-verdict.js +24 -3
- package/lib/vision-tool-runtime-boundary.js +7 -10
- package/lib/vision-tool-stage.js +58 -0
- package/lib/web/dsh-settings-017-root-transport.js +52 -6
- package/lib/web-capability-boundary.js +6 -0
- package/package.json +10 -6
- package/scripts/install-native-ocr.mjs +144 -0
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto'
|
|
2
|
+
import { createReadStream } from 'node:fs'
|
|
3
|
+
import { lstat, realpath } from 'node:fs/promises'
|
|
4
|
+
import { isAbsolute } from 'node:path'
|
|
5
|
+
import { performance } from 'node:perf_hooks'
|
|
6
|
+
import { runIsolatedLocalOcr, LocalOcrProcessError } from './native-ocr-supervisor.js'
|
|
7
|
+
import { runExperimentalNativeDocumentOcr } from './native-ocr-document.js'
|
|
8
|
+
import { verifyNativeOcrAssets } from './native-ocr-integrity.js'
|
|
9
|
+
|
|
10
|
+
const MAX_TIME_MS = 22_000
|
|
11
|
+
const MAX_DECODE_INPUT_BYTES = 24 * 1024 * 1024
|
|
12
|
+
const MAX_INPUT_PIXELS = 16_000_000
|
|
13
|
+
// SHA-pinned experimental identities for owner-installed native OCR workers.
|
|
14
|
+
// These hashes do NOT establish public redistribution clearance. A public
|
|
15
|
+
// model/binary archive needs signed clean-CI builds, SBOM and all notices;
|
|
16
|
+
// the JS/npm release must not silently bundle these native assets.
|
|
17
|
+
export const EXPERIMENTAL_OCR_BINARY_SHA256 = Object.freeze({
|
|
18
|
+
'darwin-arm64': '71d535b86b0543e7408b41ed66f45d3bea46e09e0e3f38aa491a32a60085d204',
|
|
19
|
+
'win32-x64': '2e4b96a204c34eebe11f117049b1a3b966f43a8402a66065c28773528cd18a33',
|
|
20
|
+
})
|
|
21
|
+
|
|
22
|
+
function fail(code, description) {
|
|
23
|
+
return new LocalOcrProcessError(code, description)
|
|
24
|
+
}
|
|
25
|
+
async function hashFile(path) {
|
|
26
|
+
const hash = createHash('sha256')
|
|
27
|
+
for await (const buf of createReadStream(path)) hash.update(buf)
|
|
28
|
+
return hash.digest('hex')
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
export async function verifyExperimentalNativeOcrBinary(binaryPath, {
|
|
32
|
+
platform = process.platform,
|
|
33
|
+
arch = process.arch,
|
|
34
|
+
binaryDigests = EXPERIMENTAL_OCR_BINARY_SHA256,
|
|
35
|
+
} = {}) {
|
|
36
|
+
const expected = binaryDigests[platform + '-' + arch]
|
|
37
|
+
if (!expected) throw fail('OCR_UNSUPPORTED_PLATFORM', 'native OCR preview unsupported on this platform')
|
|
38
|
+
if (typeof binaryPath !== 'string' || !isAbsolute(binaryPath)) {
|
|
39
|
+
throw fail('OCR_UNTRUSTED_RUNNER', 'native OCR binary path must be absolute')
|
|
40
|
+
}
|
|
41
|
+
let info
|
|
42
|
+
try {
|
|
43
|
+
info = await lstat(binaryPath)
|
|
44
|
+
if (!info.isFile() || info.isSymbolicLink() || info.size > 32 * 1024 * 1024) {
|
|
45
|
+
throw new Error('not a regular executable')
|
|
46
|
+
}
|
|
47
|
+
// On macOS /tmp is an OS-managed symlink to /private/tmp; requiring
|
|
48
|
+
// literal canonical-path equality would reject legitimate runner files.
|
|
49
|
+
// Reject a symlink at the file itself and pin file content by SHA-256.
|
|
50
|
+
const canonical = await realpath(binaryPath)
|
|
51
|
+
if (!isAbsolute(canonical)) throw new Error('invalid canonical path')
|
|
52
|
+
} catch {
|
|
53
|
+
throw fail('OCR_UNTRUSTED_RUNNER', 'native OCR runner missing or invalid')
|
|
54
|
+
}
|
|
55
|
+
const digest = await hashFile(binaryPath)
|
|
56
|
+
if (digest !== expected) throw fail('OCR_BINARY_INTEGRITY', 'native OCR executable integrity mismatch')
|
|
57
|
+
return { ok: true, digest }
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
function isAbort(signal, error) {
|
|
61
|
+
return signal?.aborted || error?.code === 'OCR_CANCELLED' || error?.name === 'AbortError'
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* Build the same strict RGB24 input used by the native ncnn worker. sharp is
|
|
66
|
+
* Host-owned and supplied lazily; native code does not parse untrusted JPEG
|
|
67
|
+
* and is never given an image file path. Existing image admission happens
|
|
68
|
+
* before calling this function.
|
|
69
|
+
*/
|
|
70
|
+
export async function decodeExperimentalOcrRgb(bytes, {
|
|
71
|
+
loadSharp,
|
|
72
|
+
signal,
|
|
73
|
+
} = {}) {
|
|
74
|
+
// DSH's attachment store returns Uint8Array; filesystem reads may be Buffer.
|
|
75
|
+
// Normalize only byte views, preserving byteOffset/byteLength so no adjacent
|
|
76
|
+
// ArrayBuffer content is decoded or transported to the native process.
|
|
77
|
+
if (!(bytes instanceof Uint8Array) || bytes.byteLength < 8 ||
|
|
78
|
+
bytes.byteLength > MAX_DECODE_INPUT_BYTES) {
|
|
79
|
+
throw fail('OCR_INVALID_INPUT', 'invalid image buffer')
|
|
80
|
+
}
|
|
81
|
+
const imageBuffer = Buffer.isBuffer(bytes)
|
|
82
|
+
? bytes : Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength)
|
|
83
|
+
if (signal?.aborted) throw fail('OCR_CANCELLED', 'OCR cancelled')
|
|
84
|
+
if (typeof loadSharp !== 'function') throw fail('OCR_DECODER_MISSING', 'sharp loader unavailable')
|
|
85
|
+
const sharp = await loadSharp()
|
|
86
|
+
if (signal?.aborted) throw fail('OCR_CANCELLED', 'OCR cancelled')
|
|
87
|
+
const { data, info } = await sharp(imageBuffer, {
|
|
88
|
+
failOn: 'error',
|
|
89
|
+
limitInputPixels: MAX_INPUT_PIXELS,
|
|
90
|
+
sequentialRead: true,
|
|
91
|
+
})
|
|
92
|
+
.rotate()
|
|
93
|
+
.resize({ width: 2048, height: 2048, fit: 'inside', withoutEnlargement: true })
|
|
94
|
+
.removeAlpha()
|
|
95
|
+
.toColourspace('srgb')
|
|
96
|
+
.raw()
|
|
97
|
+
.toBuffer({ resolveWithObject: true })
|
|
98
|
+
if (signal?.aborted) throw fail('OCR_CANCELLED', 'OCR cancelled')
|
|
99
|
+
if (!info || !Number.isSafeInteger(info.width) || !Number.isSafeInteger(info.height) ||
|
|
100
|
+
info.channels !== 3 || info.width < 1 || info.height < 1 ||
|
|
101
|
+
info.width > 4096 || info.height > 4096 ||
|
|
102
|
+
info.width * info.height > 4_000_000 ||
|
|
103
|
+
data.byteLength !== info.width * info.height * 3) {
|
|
104
|
+
throw fail('OCR_INVALID_RGB', 'decoded image outside native OCR limits')
|
|
105
|
+
}
|
|
106
|
+
return { rgb: data, width: info.width, height: info.height }
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/**
|
|
110
|
+
* Called only for explicit isolated canary tests or when auto mode finds an
|
|
111
|
+
* approved owner-installed native worker; absent native assets cannot enable
|
|
112
|
+
* OCR by themselves. The executable's hash AND all 5 model/charset file hashes
|
|
113
|
+
* must match before the child process can be started.
|
|
114
|
+
*
|
|
115
|
+
* Returns full document OCR evidence for the caller to render conservatively.
|
|
116
|
+
* Does not contact any vision service, change user settings or mutate images.
|
|
117
|
+
*/
|
|
118
|
+
export async function runExperimentalNativeOcrCanary(bytes, {
|
|
119
|
+
binaryPath,
|
|
120
|
+
modelDir,
|
|
121
|
+
loadSharp,
|
|
122
|
+
signal,
|
|
123
|
+
timeoutMs = 18_000,
|
|
124
|
+
platform = process.platform,
|
|
125
|
+
arch = process.arch,
|
|
126
|
+
binaryDigests,
|
|
127
|
+
} = {}) {
|
|
128
|
+
if (signal?.aborted) throw fail('OCR_CANCELLED', 'OCR cancelled')
|
|
129
|
+
if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) {
|
|
130
|
+
throw fail('OCR_INVALID_TIMEOUT', 'invalid native OCR timeout')
|
|
131
|
+
}
|
|
132
|
+
const started = performance.now()
|
|
133
|
+
const budget = Math.min(MAX_TIME_MS, Math.floor(timeoutMs))
|
|
134
|
+
const left = () => budget - (performance.now() - started)
|
|
135
|
+
// Cheap gate first: don't decode an untrusted image if model assets are broken.
|
|
136
|
+
await verifyExperimentalNativeOcrBinary(binaryPath, { platform, arch, binaryDigests })
|
|
137
|
+
if (signal?.aborted) throw fail('OCR_CANCELLED', 'OCR cancelled')
|
|
138
|
+
await verifyNativeOcrAssets(modelDir)
|
|
139
|
+
if (signal?.aborted) throw fail('OCR_CANCELLED', 'OCR cancelled')
|
|
140
|
+
const { rgb, width, height } = await decodeExperimentalOcrRgb(bytes, { loadSharp, signal })
|
|
141
|
+
if (left() <= 0) throw fail('OCR_TIMEOUT', 'native OCR time exhausted before inference')
|
|
142
|
+
const nativeRun = (frame, options) => runIsolatedLocalOcr(frame, {
|
|
143
|
+
executable: binaryPath,
|
|
144
|
+
args: [modelDir],
|
|
145
|
+
timeoutMs: Math.min(Math.max(1, left()), options.timeoutMs),
|
|
146
|
+
signal,
|
|
147
|
+
})
|
|
148
|
+
const result = await runExperimentalNativeDocumentOcr(rgb, width, height, {
|
|
149
|
+
nativeRun, signal, timeoutMs: Math.max(1, left()),
|
|
150
|
+
})
|
|
151
|
+
if (signal?.aborted) throw fail('OCR_CANCELLED', 'OCR cancelled')
|
|
152
|
+
if (!result || typeof result.text !== 'string') {
|
|
153
|
+
throw fail('OCR_INVALID_RESULT', 'native OCR returned malformed result')
|
|
154
|
+
}
|
|
155
|
+
return result
|
|
156
|
+
}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Minimal evidence-backed chemistry glyph reconciliation for photographed
|
|
3
|
+
* textbook pages. Never fabricates a number or changes element stoichiometry.
|
|
4
|
+
*/
|
|
5
|
+
const normalizeSpecies = text => text.normalize('NFKC').replace(/\s+/g,'')
|
|
6
|
+
const speciesPattern = /([A-Z][A-Za-z0-9]{1,18})\(1\)/g
|
|
7
|
+
|
|
8
|
+
export function reviewChemicalStateGlyphs(regions) {
|
|
9
|
+
const updates=[]
|
|
10
|
+
const normalized=regions.map(r=>normalizeSpecies(String(r.text??'')))
|
|
11
|
+
for(let i=0;i<regions.length;i++) {
|
|
12
|
+
const row=regions[i]
|
|
13
|
+
if(!row.text || !/[+=→]/.test(row.text))continue
|
|
14
|
+
const input=normalizeSpecies(row.text)
|
|
15
|
+
const matches=[...input.matchAll(speciesPattern)]
|
|
16
|
+
for(const match of matches) {
|
|
17
|
+
const species=match[1]
|
|
18
|
+
// '1' is not a valid physical-state label. Prefer independently
|
|
19
|
+
// recognized '(l)' only when the *same species* occurs elsewhere.
|
|
20
|
+
const expected=species+'(l)'
|
|
21
|
+
const verifiedElsewhere=normalized.some((line,j)=>j!==i && line.includes(expected))
|
|
22
|
+
if(!verifiedElsewhere)continue
|
|
23
|
+
const erroneous=species+'(1)'
|
|
24
|
+
if(!row.text.includes(erroneous))continue
|
|
25
|
+
const corrected=row.text.replace(erroneous,expected)
|
|
26
|
+
if(corrected===row.text)continue
|
|
27
|
+
row.text=corrected
|
|
28
|
+
row.uncertain=true
|
|
29
|
+
row.phaseGlyphReconciled=true
|
|
30
|
+
row.phaseEvidence={before:erroneous,after:expected,
|
|
31
|
+
reason:'same_species_independently_recognized_with_liquid_state'}
|
|
32
|
+
updates.push({index:i,...row.phaseEvidence})
|
|
33
|
+
break
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
return {regions,updates}
|
|
37
|
+
}
|
|
@@ -0,0 +1,243 @@
|
|
|
1
|
+
import { performance } from 'node:perf_hooks'
|
|
2
|
+
import { encodeNativeOcrRgbFrame } from './native-ocr-supervisor.js'
|
|
3
|
+
import { recoverPhotoQuestionHeadings } from './native-ocr-question-labels.js'
|
|
4
|
+
import { reviewChemicalStateGlyphs } from './native-ocr-chemistry-review.js'
|
|
5
|
+
import { detectRedAnnotationNearText } from './native-ocr-annotation-risk.js'
|
|
6
|
+
|
|
7
|
+
const MAX_DOCUMENT_TILES = 8
|
|
8
|
+
const MAX_DOCUMENT_TIME_MS = 24_000
|
|
9
|
+
|
|
10
|
+
function clamp(value,min,max) { return Math.max(min,Math.min(max,value)) }
|
|
11
|
+
|
|
12
|
+
function detectPortraitGutter(rgb,width,height) {
|
|
13
|
+
if (height < 900 || height / width < 1.28) return null
|
|
14
|
+
const from=Math.max(13,Math.round(width*.05))
|
|
15
|
+
const until=Math.min(width-12,Math.round(width*.28))
|
|
16
|
+
const y0=Math.round(height*.20),y1=Math.round(height*.83)
|
|
17
|
+
const scores=[]
|
|
18
|
+
let meanBrightness=0,n=0
|
|
19
|
+
const gray=(pixel)=>.299*rgb[pixel]+.587*rgb[pixel+1]+.114*rgb[pixel+2]
|
|
20
|
+
for (let x=from;x<until;x+=2) {
|
|
21
|
+
let diff=0,count=0
|
|
22
|
+
for (let y=y0;y<y1;y+=6) {
|
|
23
|
+
let left=0,right=0
|
|
24
|
+
const offset=(y*width+x)*3
|
|
25
|
+
for(let d=1;d<=10;d++)left+=gray(offset-3*d)
|
|
26
|
+
for(let d=0;d<10;d++)right+=gray(offset+3*d)
|
|
27
|
+
diff+=Math.abs(right-left)/10
|
|
28
|
+
count++
|
|
29
|
+
if(x===from){meanBrightness+=gray(offset);n++}
|
|
30
|
+
}
|
|
31
|
+
scores.push({x,score:diff/Math.max(1,count)})
|
|
32
|
+
}
|
|
33
|
+
if (!scores.length || meanBrightness/Math.max(1,n)<120) return null
|
|
34
|
+
const sorted=[...scores].sort((a,b)=>a.score-b.score)
|
|
35
|
+
const med=sorted[Math.floor(sorted.length/2)].score
|
|
36
|
+
const best=sorted.at(-1)
|
|
37
|
+
if (best.score < med*1.27 || best.x < from+4 || best.x >= until-4) return null
|
|
38
|
+
// Leave a small safety margin to preserve question numbers and decimals
|
|
39
|
+
// beside the spine; a ten-pixel inward shift proved able to drop a line.
|
|
40
|
+
return clamp(best.x-5,0,Math.floor(width*.25))
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
function cropRGB(rgb,width,rect) {
|
|
44
|
+
const {x,y,w,h}=rect
|
|
45
|
+
const dst=Buffer.allocUnsafe(w*h*3)
|
|
46
|
+
for(let row=0;row<h;row++) {
|
|
47
|
+
const from=((row+y)*width+x)*3
|
|
48
|
+
const to=row*w*3
|
|
49
|
+
dst.set(rgb.subarray(from,from+w*3),to)
|
|
50
|
+
}
|
|
51
|
+
return dst
|
|
52
|
+
}
|
|
53
|
+
function intersection(boxA,boxB) {
|
|
54
|
+
return Math.max(0,Math.min(boxA[2],boxB[2])-Math.max(boxA[0],boxB[0])) *
|
|
55
|
+
Math.max(0,Math.min(boxA[3],boxB[3])-Math.max(boxA[1],boxB[1]))
|
|
56
|
+
}
|
|
57
|
+
function area(b) { return Math.max(0,b[2]-b[0])*Math.max(0,b[3]-b[1]) }
|
|
58
|
+
function centerY(b) {return (b[1]+b[3])/2}
|
|
59
|
+
function isQuestionNumber(value) {
|
|
60
|
+
return /^(?:[1-9]|1[012])[.]?$/.test(value.trim())
|
|
61
|
+
}
|
|
62
|
+
function markRisk(value) {
|
|
63
|
+
return /[A-Za-z][0-9]|[0-9][A-Za-z]|[₀-₉⁰-⁹]|[=→×Δ]/u.test(value)
|
|
64
|
+
}
|
|
65
|
+
function mergeTileRegions(items) {
|
|
66
|
+
const merged=[]
|
|
67
|
+
for(const r of items.sort((a,b)=>centerY(a.box)-centerY(b.box))) {
|
|
68
|
+
if(!Array.isArray(r.box) || r.box.length!==4 ||
|
|
69
|
+
!r.box.every(Number.isFinite) || !r.text?.trim() ||
|
|
70
|
+
!Number.isFinite(r.confidence))continue
|
|
71
|
+
let match=-1
|
|
72
|
+
for(let i=0;i<merged.length;i++) {
|
|
73
|
+
const prev=merged[i]
|
|
74
|
+
const overlap=intersection(r.box,prev.box)
|
|
75
|
+
const iou=overlap/Math.max(1,area(r.box)+area(prev.box)-overlap)
|
|
76
|
+
const overlapWide=Math.max(0,Math.min(r.box[2],prev.box[2])-Math.max(r.box[0],prev.box[0])) /
|
|
77
|
+
Math.max(1,Math.max(r.box[2]-r.box[0],prev.box[2]-prev.box[0]))
|
|
78
|
+
const sameBaseline=Math.abs(centerY(r.box)-centerY(prev.box)) <
|
|
79
|
+
Math.max(12,.55*Math.max(r.box[3]-r.box[1],prev.box[3]-prev.box[1]))
|
|
80
|
+
if (sameBaseline && (iou>.48 || (overlapWide>.75 && overlap/Math.max(1,Math.min(area(r.box),area(prev.box)))>.7))) {
|
|
81
|
+
match=i
|
|
82
|
+
break
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
if(match===-1){merged.push(r);continue}
|
|
86
|
+
const old=merged[match]
|
|
87
|
+
const favorable=(r.edgeClearance>12?1:0)-(old.edgeClearance>12?1:0)
|
|
88
|
+
const selected=favorable>0 || (favorable===0 && (r.confidence>old.confidence ||
|
|
89
|
+
(r.confidence===old.confidence && r.text.length>old.text.length)))
|
|
90
|
+
if(selected)merged[match]={...r,uncertain:r.uncertain ||
|
|
91
|
+
(markRisk(old.text+r.text) && r.text!==old.text)}
|
|
92
|
+
else if(r.text!==old.text && markRisk(old.text+r.text))old.uncertain=true
|
|
93
|
+
}
|
|
94
|
+
// Numbers and the text of their question often form two coincident boxes.
|
|
95
|
+
const kept=merged.filter((r)=>!isQuestionNumber(r.text) ||
|
|
96
|
+
!merged.some((other)=>other!==r &&
|
|
97
|
+
Math.abs(centerY(other.box)-centerY(r.box))<15 &&
|
|
98
|
+
other.text.trim().startsWith(r.text.trim()) &&
|
|
99
|
+
other.text.trim().length>r.text.trim().length+8))
|
|
100
|
+
// The old 'abs(y1-y2)<12 ? x : y' comparator was non-transitive.
|
|
101
|
+
// Quantized reading rows establish a stable total order even with
|
|
102
|
+
// overlapping annotations or adjacent text boxes.
|
|
103
|
+
kept.sort((a,b)=> {
|
|
104
|
+
const cy=centerY(a.box),dy=centerY(b.box)
|
|
105
|
+
const rowA=Math.round(cy/16),rowB=Math.round(dy/16)
|
|
106
|
+
return rowA-rowB || a.box[0]-b.box[0] || cy-dy
|
|
107
|
+
})
|
|
108
|
+
return kept
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
export function planExperimentalDocumentTiles(rgb,width,height,coarse) {
|
|
112
|
+
const regions=Array.isArray(coarse?.regions) ? coarse.regions : []
|
|
113
|
+
if (height/width<1.28 || height<900 || regions.length<9) return []
|
|
114
|
+
const x=detectPortraitGutter(rgb,width,height) ?? 0
|
|
115
|
+
const w=width-x
|
|
116
|
+
const valid=regions.filter(r=>Array.isArray(r.box) &&
|
|
117
|
+
r.box.length===4 && r.box.every(Number.isFinite) &&
|
|
118
|
+
r.box[3]>r.box[1] && r.box[3]-r.box[1]<height*.12)
|
|
119
|
+
if (valid.length<8) return []
|
|
120
|
+
const upper=clamp(Math.floor(Math.min(...valid.map(r=>r.box[1]))-18),0,height-80)
|
|
121
|
+
const lower=clamp(Math.ceil(Math.max(...valid.map(r=>r.box[3]))+30),upper+80,height)
|
|
122
|
+
const tileHeight=clamp(Math.round(w*.34),260,840)
|
|
123
|
+
const overlap=clamp(Math.round(tileHeight*.16),30,85)
|
|
124
|
+
const step=tileHeight-overlap
|
|
125
|
+
if (Math.ceil((lower-upper)/step)>MAX_DOCUMENT_TILES) return []
|
|
126
|
+
const tiles=[]
|
|
127
|
+
for(let y=upper;y<lower;y+=step) {
|
|
128
|
+
const h=Math.min(tileHeight,lower-y)
|
|
129
|
+
if(h<50) break
|
|
130
|
+
tiles.push({x,y,w,h})
|
|
131
|
+
if(y+h>=lower)break
|
|
132
|
+
}
|
|
133
|
+
return tiles.length>=2 ? tiles : []
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/** Experimental document mode. Never replaces the user-selected OCR engine. */
|
|
137
|
+
export async function runExperimentalNativeDocumentOcr(rgb,width,height,{
|
|
138
|
+
nativeRun,timeoutMs=16_000,signal
|
|
139
|
+
}={}) {
|
|
140
|
+
if(!(rgb instanceof Uint8Array))throw new TypeError('OCR RGB pixels required')
|
|
141
|
+
if(typeof nativeRun!=='function')throw new TypeError('nativeRun is required')
|
|
142
|
+
if(!Number.isFinite(timeoutMs) || timeoutMs<=0)
|
|
143
|
+
throw Object.assign(new Error('Invalid OCR deadline'),{code:'OCR_INVALID_TIMEOUT'})
|
|
144
|
+
if(signal?.aborted)
|
|
145
|
+
throw Object.assign(new Error('OCR cancelled'),{code:'OCR_CANCELLED'})
|
|
146
|
+
const full=encodeNativeOcrRgbFrame(rgb,width,height)
|
|
147
|
+
const started=performance.now()
|
|
148
|
+
const budget=clamp(timeoutMs,1,MAX_DOCUMENT_TIME_MS)
|
|
149
|
+
const remaining=()=>Math.max(0,budget-(performance.now()-started))
|
|
150
|
+
const initial=await nativeRun(full,{timeoutMs:Math.max(1,remaining()),signal})
|
|
151
|
+
if(signal?.aborted)
|
|
152
|
+
throw Object.assign(new Error('OCR cancelled'),{code:'OCR_CANCELLED'})
|
|
153
|
+
if(!initial || typeof initial.text!=='string')
|
|
154
|
+
throw Object.assign(new Error('Invalid OCR worker result'),{code:'OCR_INVALID_RESULT'})
|
|
155
|
+
const tiles=planExperimentalDocumentTiles(rgb,width,height,initial)
|
|
156
|
+
if(!tiles.length)return {...initial,tilesUsed:0}
|
|
157
|
+
const items=[]
|
|
158
|
+
let partial=false
|
|
159
|
+
for(const tile of tiles) {
|
|
160
|
+
if(signal?.aborted)throw Object.assign(new Error('OCR cancelled'),{code:'OCR_CANCELLED'})
|
|
161
|
+
if(remaining()<750){partial=true;break}
|
|
162
|
+
const frame=encodeNativeOcrRgbFrame(cropRGB(rgb,width,tile),tile.w,tile.h)
|
|
163
|
+
let out
|
|
164
|
+
try {out=await nativeRun(frame,{timeoutMs:Math.max(1,remaining()),signal})}
|
|
165
|
+
catch(error) {
|
|
166
|
+
if(error?.code==='OCR_CANCELLED' || signal?.aborted)throw error
|
|
167
|
+
partial=true
|
|
168
|
+
break
|
|
169
|
+
}
|
|
170
|
+
if(!Array.isArray(out.regions)){partial=true;break}
|
|
171
|
+
for(const r of out.regions) {
|
|
172
|
+
const [x1,y1,x2,y2]=r.box??[]
|
|
173
|
+
if (![x1,y1,x2,y2].every(Number.isFinite))continue
|
|
174
|
+
items.push({...r,
|
|
175
|
+
box:[tile.x+x1,tile.y+y1,tile.x+x2,tile.y+y2],
|
|
176
|
+
edgeClearance:Math.min(y1,tile.h-y2),
|
|
177
|
+
})
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
if(partial || !items.length)return {...initial,uncertain:true,review:'document_refinement_incomplete',tilesUsed:0}
|
|
181
|
+
const regions=mergeTileRegions(items)
|
|
182
|
+
// In a photographed textbook the printed question numbers can sit left of
|
|
183
|
+
// the detected main text column. Recover them from one bounded left strip.
|
|
184
|
+
// Never replace the full sentence with a truncated strip reading.
|
|
185
|
+
// A narrow gutter probe supplements the detected whole-line text. It cannot
|
|
186
|
+
// assign a question number by itself: the pure matching module requires
|
|
187
|
+
// Chinese content alignment plus monotonic question-number evidence.
|
|
188
|
+
const marginLabels=[]
|
|
189
|
+
if (regions.length>=9 && remaining()>1000) {
|
|
190
|
+
const first=tiles[0]
|
|
191
|
+
const last=tiles.at(-1)
|
|
192
|
+
const sx=clamp(first.x-40,0,width-1)
|
|
193
|
+
const sy=clamp(Math.round(first.y+(last.y+last.h-first.y)*.40),0,height-1)
|
|
194
|
+
const strip={x:sx,y:sy,w:Math.min(210,width-sx),h:last.y+last.h-sy}
|
|
195
|
+
if(strip.w>=75 && strip.h>=110 && strip.w*strip.h<=4_000_000) {
|
|
196
|
+
try {
|
|
197
|
+
const frame=encodeNativeOcrRgbFrame(cropRGB(rgb,width,strip),strip.w,strip.h)
|
|
198
|
+
const left=await nativeRun(frame,{timeoutMs:Math.max(1,remaining()),signal})
|
|
199
|
+
if(Array.isArray(left?.regions)) {
|
|
200
|
+
for(const candidate of left.regions) {
|
|
201
|
+
if(!Array.isArray(candidate.box) || candidate.box.length!==4)continue
|
|
202
|
+
const [x1,y1,x2,y2]=candidate.box
|
|
203
|
+
if(![x1,y1,x2,y2].every(Number.isFinite))continue
|
|
204
|
+
marginLabels.push({
|
|
205
|
+
text: candidate.text,
|
|
206
|
+
confidence: candidate.confidence,
|
|
207
|
+
box:[sx+x1,sy+y1,sx+x2,sy+y2]
|
|
208
|
+
})
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
} catch(error) {
|
|
212
|
+
if(signal?.aborted || error?.code==='OCR_CANCELLED')throw error
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
const recovery=recoverPhotoQuestionHeadings(regions,marginLabels)
|
|
217
|
+
const phaseReview=reviewChemicalStateGlyphs(regions)
|
|
218
|
+
const labelsRecovered=recovery.recovered.length
|
|
219
|
+
// Keep original evidence and localize high-impact uncertainty. An OCR
|
|
220
|
+
// confidence score cannot establish that a decimal point is present.
|
|
221
|
+
const reviewQueue=regions.filter(r=>r.uncertain &&
|
|
222
|
+
/\d/.test(r.text) && /(?:g|kg|kJ|mol|℃|°C|[A-Z]\d)/u.test(r.text))
|
|
223
|
+
.map(r=>{
|
|
224
|
+
const red=detectRedAnnotationNearText(rgb,width,height,r.box)
|
|
225
|
+
const reason=r.formulaDigitReconciled ? 'formula_digit_conflict_between_views' :
|
|
226
|
+
r.phaseGlyphReconciled ? 'chemical_state_glyph_conflict' :
|
|
227
|
+
red.detected ? 'numeric_text_overlapped_by_red_annotation' : 'verify_numeric_or_formula'
|
|
228
|
+
return {
|
|
229
|
+
box:r.box,text:r.text,reason,
|
|
230
|
+
redInkEvidence:red.detected ? red : undefined,
|
|
231
|
+
evidence:r.formulaRecoveryEvidence??r.phaseEvidence??null,
|
|
232
|
+
}
|
|
233
|
+
})
|
|
234
|
+
const text=regions.map(r=>r.text).join('\n')
|
|
235
|
+
const confidence=regions.length ?
|
|
236
|
+
regions.reduce((v,r)=>v+r.confidence,0)/regions.length : 0
|
|
237
|
+
return {text,confidence,regions,uncertain:regions.some(x=>x.uncertain),
|
|
238
|
+
engine:'ncnn',tilesUsed:tiles.length,labelsRecovered,
|
|
239
|
+
formulaDigitsReconciled:recovery.formulaReconciled.length,
|
|
240
|
+
phaseGlyphReconciled:phaseReview.updates.length,
|
|
241
|
+
reviewQueue,
|
|
242
|
+
rejectedQuestionLabelCandidates:recovery.rejected.length,documentRefined:true}
|
|
243
|
+
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { homedir } from 'node:os'
|
|
2
|
+
import { join } from 'node:path'
|
|
3
|
+
|
|
4
|
+
// Owner-installed, integrity-pinned native OCR. Never probe unapproved Linux
|
|
5
|
+
// variants or switch a default unless the named worker actually exists.
|
|
6
|
+
export function localNativeOcrInstallPaths({
|
|
7
|
+
platform = process.platform, arch = process.arch, home = homedir(),
|
|
8
|
+
} = {}) {
|
|
9
|
+
const target = platform + '-' + arch
|
|
10
|
+
if (target !== 'darwin-arm64' && target !== 'win32-x64') return null
|
|
11
|
+
const root = join(home, '.dsh-vision-router', 'ocr', 'ppocrv6-small')
|
|
12
|
+
return Object.freeze({
|
|
13
|
+
root,
|
|
14
|
+
binaryPath: join(root, target === 'win32-x64' ? 'worker.exe' : 'worker'),
|
|
15
|
+
modelDir: join(root, 'models'),
|
|
16
|
+
})
|
|
17
|
+
}
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto'
|
|
2
|
+
import { createReadStream } from 'node:fs'
|
|
3
|
+
import { lstat, realpath } from 'node:fs/promises'
|
|
4
|
+
import { isAbsolute, join, resolve, sep } from 'node:path'
|
|
5
|
+
import { LocalOcrProcessError } from './native-ocr-supervisor.js'
|
|
6
|
+
|
|
7
|
+
// Reproducible PP-OCRv6 small -> ncnn conversion artifacts, pinned in the lab.
|
|
8
|
+
// Source ONNX revisions and license evidence belong in the release NOTICE/SBOM.
|
|
9
|
+
export const EXPERIMENTAL_OCR_MODEL_FILES = Object.freeze({
|
|
10
|
+
'det.ncnn.bin': Object.freeze({
|
|
11
|
+
bytes: 9813748,
|
|
12
|
+
sha256: '720620c709ae18566e037f5ac086075553118c91fa2d289af405060e7d047357',
|
|
13
|
+
}),
|
|
14
|
+
'det.ncnn.param': Object.freeze({
|
|
15
|
+
bytes: 18868,
|
|
16
|
+
sha256: '986dbd2de4746574263bc616295435d9d775cf2092379ab4c0df42143205bfc4',
|
|
17
|
+
}),
|
|
18
|
+
'rec.ncnn.bin': Object.freeze({
|
|
19
|
+
bytes: 21066500,
|
|
20
|
+
sha256: '40f8afb7b9d835e8f5b7d03bd08d74d0f762b27999d05ffdf4644c1c4b24f798',
|
|
21
|
+
}),
|
|
22
|
+
'rec.ncnn.param': Object.freeze({
|
|
23
|
+
bytes: 13845,
|
|
24
|
+
sha256: 'ad92c23ca2cac8cfa414bc2b6b8c4ad73121f53543d7fb1d6953853b05fc5bda',
|
|
25
|
+
}),
|
|
26
|
+
'ppocrv6_dict.txt': Object.freeze({
|
|
27
|
+
bytes: 74947,
|
|
28
|
+
sha256: 'b5f2bfe2bdd9448429e3e82b51c789775d9b42f2403d082b00662eb77e401c5d',
|
|
29
|
+
}),
|
|
30
|
+
})
|
|
31
|
+
|
|
32
|
+
function fail(code, message) {
|
|
33
|
+
return new LocalOcrProcessError(code, message)
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
async function sha256(path) {
|
|
37
|
+
const hash = createHash('sha256')
|
|
38
|
+
for await (const chunk of createReadStream(path)) hash.update(chunk)
|
|
39
|
+
return hash.digest('hex')
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Checks every distributed native model and its character dictionary before
|
|
44
|
+
* starting the native worker. No downloading, rebuilding or replacing files.
|
|
45
|
+
*
|
|
46
|
+
* The real deployment must pin a platform-specific native executable digest
|
|
47
|
+
* as well; this gate addresses the model asset closure only.
|
|
48
|
+
*/
|
|
49
|
+
export async function verifyNativeOcrAssets(modelDir, { files = EXPERIMENTAL_OCR_MODEL_FILES } = {}) {
|
|
50
|
+
if (typeof modelDir !== 'string' || !isAbsolute(modelDir)) {
|
|
51
|
+
throw fail('OCR_MODEL_PATH_INVALID', 'invalid native OCR model directory')
|
|
52
|
+
}
|
|
53
|
+
const dir = resolve(modelDir)
|
|
54
|
+
let canonicalDir
|
|
55
|
+
try {
|
|
56
|
+
const directory = await lstat(dir)
|
|
57
|
+
if (!directory.isDirectory() || directory.isSymbolicLink()) throw new Error('not a real directory')
|
|
58
|
+
canonicalDir = await realpath(dir)
|
|
59
|
+
} catch {
|
|
60
|
+
throw fail('OCR_MODEL_MISSING', 'native OCR model directory unavailable')
|
|
61
|
+
}
|
|
62
|
+
for (const [name, expected] of Object.entries(files)) {
|
|
63
|
+
if (!/^[A-Za-z0-9._-]+$/.test(name) || name === '.' || name === '..') {
|
|
64
|
+
throw fail('OCR_MODEL_MANIFEST_INVALID', 'invalid native OCR manifest filename')
|
|
65
|
+
}
|
|
66
|
+
if (!expected || !Number.isSafeInteger(expected.bytes) ||
|
|
67
|
+
expected.bytes <= 0 || expected.bytes > 32 * 1024 * 1024 ||
|
|
68
|
+
!/^[0-9a-f]{64}$/.test(expected.sha256)) {
|
|
69
|
+
throw fail('OCR_MODEL_MANIFEST_INVALID', 'invalid native OCR manifest entry')
|
|
70
|
+
}
|
|
71
|
+
const path = join(dir, name)
|
|
72
|
+
let stat
|
|
73
|
+
try {
|
|
74
|
+
stat = await lstat(path)
|
|
75
|
+
if (!stat.isFile() || stat.isSymbolicLink()) throw new Error('not a regular file')
|
|
76
|
+
const canonicalFile = await realpath(path)
|
|
77
|
+
if (!canonicalFile.startsWith(canonicalDir + sep)) throw new Error('path escaped model directory')
|
|
78
|
+
} catch {
|
|
79
|
+
throw fail('OCR_MODEL_MISSING', 'required native OCR model asset missing')
|
|
80
|
+
}
|
|
81
|
+
if (stat.size !== expected.bytes) {
|
|
82
|
+
throw fail('OCR_MODEL_INTEGRITY', 'native OCR model asset size mismatch')
|
|
83
|
+
}
|
|
84
|
+
let actual
|
|
85
|
+
try { actual = await sha256(path) } catch {
|
|
86
|
+
throw fail('OCR_MODEL_INTEGRITY', 'native OCR model asset unreadable')
|
|
87
|
+
}
|
|
88
|
+
if (actual !== expected.sha256) {
|
|
89
|
+
throw fail('OCR_MODEL_INTEGRITY', 'native OCR model asset digest mismatch')
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
return { ok: true, checked: Object.keys(files).length }
|
|
93
|
+
}
|