dsh-ab-ocr 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +378 -0
- package/cordis.patch.yml +7 -0
- package/lib/artifacts.d.ts +100 -0
- package/lib/artifacts.d.ts.map +1 -0
- package/lib/artifacts.js +97 -0
- package/lib/artifacts.js.map +1 -0
- package/lib/config.d.ts +77 -0
- package/lib/config.d.ts.map +1 -0
- package/lib/config.js +51 -0
- package/lib/config.js.map +1 -0
- package/lib/documents.d.ts +62 -0
- package/lib/documents.d.ts.map +1 -0
- package/lib/documents.js +173 -0
- package/lib/documents.js.map +1 -0
- package/lib/events.d.ts +161 -0
- package/lib/events.d.ts.map +1 -0
- package/lib/events.js +158 -0
- package/lib/events.js.map +1 -0
- package/lib/filename.d.ts +47 -0
- package/lib/filename.d.ts.map +1 -0
- package/lib/filename.js +77 -0
- package/lib/filename.js.map +1 -0
- package/lib/index.d.ts +85 -0
- package/lib/index.d.ts.map +1 -0
- package/lib/index.js +1761 -0
- package/lib/index.js.map +1 -0
- package/lib/levels.d.ts +24 -0
- package/lib/levels.d.ts.map +1 -0
- package/lib/levels.js +52 -0
- package/lib/levels.js.map +1 -0
- package/lib/plan.d.ts +103 -0
- package/lib/plan.d.ts.map +1 -0
- package/lib/plan.js +210 -0
- package/lib/plan.js.map +1 -0
- package/lib/recognize.d.ts +36 -0
- package/lib/recognize.d.ts.map +1 -0
- package/lib/recognize.js +390 -0
- package/lib/recognize.js.map +1 -0
- package/lib/records.d.ts +91 -0
- package/lib/records.d.ts.map +1 -0
- package/lib/records.js +130 -0
- package/lib/records.js.map +1 -0
- package/lib/render.d.ts +19 -0
- package/lib/render.d.ts.map +1 -0
- package/lib/render.js +45 -0
- package/lib/render.js.map +1 -0
- package/lib/sandbox.d.ts +54 -0
- package/lib/sandbox.d.ts.map +1 -0
- package/lib/sandbox.js +101 -0
- package/lib/sandbox.js.map +1 -0
- package/lib/types.d.ts +147 -0
- package/lib/types.d.ts.map +1 -0
- package/lib/types.js +7 -0
- package/lib/types.js.map +1 -0
- package/lib/worker.d.ts +107 -0
- package/lib/worker.d.ts.map +1 -0
- package/lib/worker.js +143 -0
- package/lib/worker.js.map +1 -0
- package/package.json +98 -0
- package/python/README.md +125 -0
- package/python/assemble.py +358 -0
- package/python/clean.py +197 -0
- package/python/layout.py +403 -0
- package/python/ocr_worker.py +516 -0
- package/python/requirements.txt +16 -0
- package/python/source.py +182 -0
- package/scripts/setup.mjs +251 -0
- package/tsconfig.json +30 -0
- package/tsdown.config.ts +18 -0
package/lib/index.js
ADDED
|
@@ -0,0 +1,1761 @@
|
|
|
1
|
+
import { defineTool } from "@deepseek-ai/dsh-tools";
|
|
2
|
+
import { existsSync } from "node:fs";
|
|
3
|
+
import { basename, dirname, extname, isAbsolute, join, resolve } from "node:path";
|
|
4
|
+
import { fileURLToPath } from "node:url";
|
|
5
|
+
import { createHash } from "node:crypto";
|
|
6
|
+
import { sandboxDenialMarker } from "@deepseek-ai/dsh-sandbox";
|
|
7
|
+
import { spawn } from "node:child_process";
|
|
8
|
+
import z from "@deepseek-ai/schemastery";
|
|
9
|
+
//#region lib/render.js
|
|
10
|
+
/**
|
|
11
|
+
* Model-facing text of one ocr call's result. Pure: it reads only the canonical
|
|
12
|
+
* value the call recorded, so a replayed session renders the same lines.
|
|
13
|
+
* @module @deepseek-ai/dsh-ab-ocr/render
|
|
14
|
+
*/
|
|
15
|
+
/**
|
|
16
|
+
* Describe one recognized document.
|
|
17
|
+
* @param document - the document's canonical value.
|
|
18
|
+
* @returns the document's result lines.
|
|
19
|
+
*/
|
|
20
|
+
function renderDocument(document) {
|
|
21
|
+
const scope = document.pages === document.totalPages ? String(document.pages) : document.pages + " of " + document.totalPages;
|
|
22
|
+
const lines = [
|
|
23
|
+
document.corrections > 0 ? "Corrected " + document.corrections + " heading level(s) in " + document.input : "Recognized " + document.input,
|
|
24
|
+
"Saved to " + document.output,
|
|
25
|
+
"Pages: " + scope + " | lines: " + document.lines + " | headings: " + document.headings + " | characters: " + document.chars,
|
|
26
|
+
"Removed " + document.droppedPageNumbers + " page numbers and " + document.droppedRunningHeads + " running heads; joined " + document.joinedAcrossPages + " paragraph(s) across a page break"
|
|
27
|
+
];
|
|
28
|
+
if (document.pageDir !== "") lines.push("Per-page text: " + document.pageDir);
|
|
29
|
+
lines.push("Outline: " + document.outlinePath + (document.outlineTruncated ? " (incomplete)" : ""));
|
|
30
|
+
return lines.join("\n");
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Compose the model-facing text of one call's result.
|
|
34
|
+
* @param value - the canonical value the call recorded.
|
|
35
|
+
* @returns one block per document, then one per failure.
|
|
36
|
+
*/
|
|
37
|
+
function renderOcr(value) {
|
|
38
|
+
const blocks = value.documents.map(renderDocument);
|
|
39
|
+
for (const failure of value.failures) blocks.push("Failed " + failure.input + "\n" + failure.error);
|
|
40
|
+
if (blocks.length === 0) return "Nothing was recognized.";
|
|
41
|
+
return blocks.join("\n\n---\n\n");
|
|
42
|
+
}
|
|
43
|
+
/** Stem used when a source name compacts to nothing. */
|
|
44
|
+
const FALLBACK_STEM = "document";
|
|
45
|
+
/** Characters no file name may carry: Windows' reserved set, the POSIX separator, and controls. */
|
|
46
|
+
const ILLEGAL_CHARACTERS = /[\u0000-\u001f\u007f-\u009f<>:"/\\|?*]/g;
|
|
47
|
+
/** Device names Windows reserves in every directory, with or without an extension. */
|
|
48
|
+
const RESERVED_DEVICE_NAME = /^(?:con|prn|aux|nul|com[1-9]|lpt[1-9])$/i;
|
|
49
|
+
/** Leading and trailing characters a file name may not carry on Windows. */
|
|
50
|
+
const EDGE_NOISE = /^[\s.\-]+|[\s.\-]+$/g;
|
|
51
|
+
/** Every spacing character Unicode offers, which the source name is stripped of. */
|
|
52
|
+
const SPACES = /[\s\u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000]+/g;
|
|
53
|
+
/**
|
|
54
|
+
* Build the stem one source document's artifacts are named after: the source's
|
|
55
|
+
* own base name with its extension dropped and every space removed.
|
|
56
|
+
* @param sourcePath - the path the model asked to recognize.
|
|
57
|
+
* @returns a stem with no spaces and no characters a file system rejects.
|
|
58
|
+
*/
|
|
59
|
+
function markdownStem(sourcePath) {
|
|
60
|
+
const text = basename(sourcePath);
|
|
61
|
+
const safe = text.slice(0, text.length - extname(text).length).replace(SPACES, "").normalize("NFC").replace(ILLEGAL_CHARACTERS, "-").replace(/-{2,}/g, "-").replace(EDGE_NOISE, "");
|
|
62
|
+
return safe === "" ? FALLBACK_STEM : safe;
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* Build the Markdown file name one source document is saved under.
|
|
66
|
+
* @param sourcePath - the path the model asked to recognize.
|
|
67
|
+
* @returns the stem, with a leading underscore when the name is a reserved
|
|
68
|
+
* Windows device name, carrying the Markdown extension.
|
|
69
|
+
*/
|
|
70
|
+
function markdownFileName(sourcePath) {
|
|
71
|
+
const stem = markdownStem(sourcePath);
|
|
72
|
+
return (RESERVED_DEVICE_NAME.test(stem) ? "_" + stem : stem) + ".md";
|
|
73
|
+
}
|
|
74
|
+
/**
|
|
75
|
+
* Build the Markdown file name one recognition's own revision is saved under,
|
|
76
|
+
* used when the plain name already holds a different document.
|
|
77
|
+
* @param sourcePath - the path the model asked to recognize.
|
|
78
|
+
* @param digest - the recognition's content digest.
|
|
79
|
+
* @returns the source's Markdown name with the digest before its extension.
|
|
80
|
+
*/
|
|
81
|
+
function markdownDigestFileName(sourcePath, digest) {
|
|
82
|
+
const name = markdownFileName(sourcePath);
|
|
83
|
+
return name.slice(0, name.length - 3) + "." + digest + ".md";
|
|
84
|
+
}
|
|
85
|
+
/**
|
|
86
|
+
* Resolve where one source document's Markdown file is written.
|
|
87
|
+
* @param sourcePath - the absolute path the model asked to recognize.
|
|
88
|
+
* @param outputDir - the deployment's directory, or an empty string for the source's own.
|
|
89
|
+
* @returns the absolute path of the Markdown file.
|
|
90
|
+
*/
|
|
91
|
+
function markdownPath(sourcePath, outputDir) {
|
|
92
|
+
return join(outputDir === "" ? dirname(sourcePath) : outputDir, markdownFileName(sourcePath));
|
|
93
|
+
}
|
|
94
|
+
/** How many digits a per-page file index carries, which keeps a 99999-page scan in lexicographic order. */
|
|
95
|
+
const PAGE_INDEX_DIGITS = 5;
|
|
96
|
+
/**
|
|
97
|
+
* Build the file name one page's recognized text is written under.
|
|
98
|
+
* @param index - the page's one-based number.
|
|
99
|
+
* @returns the page file's base name.
|
|
100
|
+
*/
|
|
101
|
+
function pageFileName(index) {
|
|
102
|
+
return "page-" + String(index).padStart(PAGE_INDEX_DIGITS, "0") + ".txt";
|
|
103
|
+
}
|
|
104
|
+
//#endregion
|
|
105
|
+
//#region lib/artifacts.js
|
|
106
|
+
/**
|
|
107
|
+
* Where one source document's artifacts live, and the content digest that names
|
|
108
|
+
* the recognition they belong to.
|
|
109
|
+
*
|
|
110
|
+
* A source named `<stem>` under a base directory `<base>` keeps its Markdown at
|
|
111
|
+
* `<base>/<stem>.md`, its records under `<base>/<pageDirName>/<stem>/`, and one
|
|
112
|
+
* revision of those records under that page root's `<digest>` directory. The
|
|
113
|
+
* digest covers everything a recognition is: the source path and content, the page
|
|
114
|
+
* selection, and the merge options. The same digest therefore names the same record
|
|
115
|
+
* on a repeat call, so the record directory is found without scanning the page root
|
|
116
|
+
* — and a source whose content changed is a different recognition even when its
|
|
117
|
+
* length did not change with it.
|
|
118
|
+
* @module @deepseek-ai/dsh-ab-ocr/artifacts
|
|
119
|
+
*/
|
|
120
|
+
/** File name of the pointer to a document's newest recognition. */
|
|
121
|
+
const LATEST_NAME = "latest.json";
|
|
122
|
+
/** File name of a recognition's record. */
|
|
123
|
+
const RECORD_NAME = "record.json";
|
|
124
|
+
/** File name of a recognition's heading outline. */
|
|
125
|
+
const OUTLINE_NAME = "outline.json";
|
|
126
|
+
/** Characters of the digest that name a record directory. */
|
|
127
|
+
const DIGEST_LENGTH = 12;
|
|
128
|
+
/**
|
|
129
|
+
* Resolve the base directory one source document's artifacts sit under.
|
|
130
|
+
* @param sourcePath - the absolute path the model asked to recognize.
|
|
131
|
+
* @param outputDir - the call's output directory, or an empty string for the source's own.
|
|
132
|
+
* @returns the base directory.
|
|
133
|
+
*/
|
|
134
|
+
function artifactBase(sourcePath, outputDir) {
|
|
135
|
+
return outputDir === "" ? dirname(sourcePath) : outputDir;
|
|
136
|
+
}
|
|
137
|
+
/**
|
|
138
|
+
* Resolve the paths of one source document that do not depend on a recognition.
|
|
139
|
+
* @param sourcePath - the absolute path the model asked to recognize.
|
|
140
|
+
* @param base - the directory the document's artifacts sit under.
|
|
141
|
+
* @param pageDirName - the page directory's name under the base directory.
|
|
142
|
+
* @returns the document's stable artifact paths.
|
|
143
|
+
*/
|
|
144
|
+
function artifactRoot(sourcePath, base, pageDirName) {
|
|
145
|
+
const stem = markdownStem(sourcePath);
|
|
146
|
+
const pageRoot = join(base, pageDirName, stem);
|
|
147
|
+
return {
|
|
148
|
+
base,
|
|
149
|
+
stem,
|
|
150
|
+
document: join(base, markdownFileName(sourcePath)),
|
|
151
|
+
pageRoot,
|
|
152
|
+
latest: join(pageRoot, LATEST_NAME)
|
|
153
|
+
};
|
|
154
|
+
}
|
|
155
|
+
/**
|
|
156
|
+
* Resolve every path one recognition's artifacts are written to.
|
|
157
|
+
* @param sourcePath - the absolute path the model asked to recognize.
|
|
158
|
+
* @param base - the directory the document's artifacts sit under.
|
|
159
|
+
* @param pageDirName - the page directory's name under the base directory.
|
|
160
|
+
* @param digest - the recognition's content digest.
|
|
161
|
+
* @returns the document's artifact paths, including this recognition's revision.
|
|
162
|
+
*/
|
|
163
|
+
function artifactPaths(sourcePath, base, pageDirName, digest) {
|
|
164
|
+
const root = artifactRoot(sourcePath, base, pageDirName);
|
|
165
|
+
const recordDir = join(root.pageRoot, digest);
|
|
166
|
+
return {
|
|
167
|
+
...root,
|
|
168
|
+
digest,
|
|
169
|
+
digestedDocument: join(base, markdownDigestFileName(sourcePath, digest)),
|
|
170
|
+
recordDir,
|
|
171
|
+
record: join(recordDir, RECORD_NAME),
|
|
172
|
+
outline: join(recordDir, OUTLINE_NAME)
|
|
173
|
+
};
|
|
174
|
+
}
|
|
175
|
+
/**
|
|
176
|
+
* Identify one recognition from the values that decided its text.
|
|
177
|
+
*
|
|
178
|
+
* The digest follows the source's content, not merely its length: a source edited
|
|
179
|
+
* without changing size is a different recognition, and an unchanged one is the
|
|
180
|
+
* same recognition however often it is read.
|
|
181
|
+
* @param sourcePath - the absolute path the model asked to recognize.
|
|
182
|
+
* @param stamp - the source metadata the digest covers alongside its content.
|
|
183
|
+
* @param sourceDigest - digest of the source bytes, or an empty string when the
|
|
184
|
+
* worker did not report one.
|
|
185
|
+
* @param pageSelection - the call's page selection, or null for the whole document.
|
|
186
|
+
* @param merge - the merge options the call sends.
|
|
187
|
+
* @returns the first 12 hex characters of the digest.
|
|
188
|
+
*/
|
|
189
|
+
function digest12(sourcePath, stamp, sourceDigest, pageSelection, merge) {
|
|
190
|
+
const material = JSON.stringify([
|
|
191
|
+
sourcePath,
|
|
192
|
+
sourceDigest === "" ? stamp.size : sourceDigest,
|
|
193
|
+
stamp.mtimeMs,
|
|
194
|
+
pageSelection,
|
|
195
|
+
merge
|
|
196
|
+
]);
|
|
197
|
+
return createHash("sha256").update(material, "utf8").digest("hex").slice(0, DIGEST_LENGTH);
|
|
198
|
+
}
|
|
199
|
+
//#endregion
|
|
200
|
+
//#region lib/sandbox.js
|
|
201
|
+
/**
|
|
202
|
+
* The per-call file fence this plugin writes under, and the one place a write
|
|
203
|
+
* is refused.
|
|
204
|
+
*
|
|
205
|
+
* `@deepseek-ai/dsh-fs-sandbox` fences each mutation by a per-call policy, and
|
|
206
|
+
* that policy is the only thing naming the workspace the CALLING SESSION runs
|
|
207
|
+
* in: omitted, the backend falls back to the deployment's root — the directory
|
|
208
|
+
* the server was launched from — and a session working anywhere else has every
|
|
209
|
+
* artifact refused. The tool layer therefore resolves one policy per call and
|
|
210
|
+
* stamps it onto every write.
|
|
211
|
+
*
|
|
212
|
+
* The policy service is read with `ctx.get('sandboxPolicy')` rather than
|
|
213
|
+
* declared in `inject`, for the same reason `ctx.fs` is: a declared injection
|
|
214
|
+
* makes the service a provider requirement of this row. A composition that
|
|
215
|
+
* mounts the confining filesystem WITHOUT the policy service therefore composes
|
|
216
|
+
* cleanly and is refused at the first call, which is the earliest point where
|
|
217
|
+
* its real shape is knowable — see {@link callFence}.
|
|
218
|
+
* @module @deepseek-ai/dsh-ab-ocr/sandbox
|
|
219
|
+
*/
|
|
220
|
+
/**
|
|
221
|
+
* The structured code a confining filesystem reports a refused mutation with.
|
|
222
|
+
* Matched structurally rather than by importing the error class, so a backend
|
|
223
|
+
* that throws its own equivalent is still recognized as a denial.
|
|
224
|
+
*/
|
|
225
|
+
const SANDBOX_DENIED = "FS_SANDBOX_DENIED";
|
|
226
|
+
/**
|
|
227
|
+
* Resolve the policy one call writes under, refusing a composition that could
|
|
228
|
+
* only ever write to the wrong place.
|
|
229
|
+
* @param ctx - the registrant context, carrying the filesystem and the policy service.
|
|
230
|
+
* @param request - the calling session whose cwd is the workspace root, or `undefined` for an agentless call.
|
|
231
|
+
* @returns the fence to stamp onto this call's writes, or undefined when the mounted backend does not confine.
|
|
232
|
+
* @throws Error when a confining backend is mounted but no policy service can name the calling session's workspace.
|
|
233
|
+
*/
|
|
234
|
+
function callFence(ctx, request) {
|
|
235
|
+
const policy = ctx.get("sandboxPolicy");
|
|
236
|
+
if (ctx.get("fs")?.sandboxMode === void 0) return void 0;
|
|
237
|
+
if (policy === void 0) throw new Error("dsh-ab-ocr: the mounted filesystem confines but ctx.sandboxPolicy is missing, so no workspace root can be resolved for this session and none of its artifacts could be written");
|
|
238
|
+
return policy.resolve(request ?? {});
|
|
239
|
+
}
|
|
240
|
+
/**
|
|
241
|
+
* Whether one thrown error is the file sandbox refusing a mutation, judged by
|
|
242
|
+
* the structured code the capability publishes rather than by its message text.
|
|
243
|
+
* @param error - the error a write threw.
|
|
244
|
+
* @returns true when a confining filesystem denied the write.
|
|
245
|
+
*/
|
|
246
|
+
function isSandboxDenial(error) {
|
|
247
|
+
return typeof error === "object" && error !== null && error.code === SANDBOX_DENIED;
|
|
248
|
+
}
|
|
249
|
+
/**
|
|
250
|
+
* Write one text artifact under the call's fence. Every text this plugin
|
|
251
|
+
* persists goes through here, so the fence is applied in one place and a
|
|
252
|
+
* refusal is restated with the path, the mode, and the workspace to move under
|
|
253
|
+
* — what the provider's own text leaves out and a caller needs.
|
|
254
|
+
* @param fs - the mounted filesystem capability every write goes through.
|
|
255
|
+
* @param target - the resolved target to write.
|
|
256
|
+
* @param content - the complete file content.
|
|
257
|
+
* @param fence - the per-call fence to stamp onto the write; absent on a bare backend.
|
|
258
|
+
* @param signal - caller cancellation, forwarded to the write.
|
|
259
|
+
* @throws Error as the backend raised it, or the same refusal restated with the remedy.
|
|
260
|
+
*/
|
|
261
|
+
async function saveText(fs, target, content, fence, signal) {
|
|
262
|
+
try {
|
|
263
|
+
await fs.writeText(target, content, void 0, signal, fence);
|
|
264
|
+
} catch (error) {
|
|
265
|
+
if (fence === void 0 || !isSandboxDenial(error)) throw error;
|
|
266
|
+
throw restated(fence, fs.processPath(target), error);
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
/**
|
|
270
|
+
* Restate one refused write as the path, the mode, and the workspace it has to
|
|
271
|
+
* move under.
|
|
272
|
+
* @param fence - the fence the refused write was stamped with.
|
|
273
|
+
* @param path - the target's path in the backend's execution world.
|
|
274
|
+
* @param cause - the refusal the backend raised.
|
|
275
|
+
* @returns the error to throw in its place, keeping the denial code a consumer keys off.
|
|
276
|
+
*/
|
|
277
|
+
function restated(fence, path, cause) {
|
|
278
|
+
const stamped = new Error(sandboxDenialMarker(fence.mode) + ": cannot write " + path + "; this session's workspace is " + fence.workspaceRoot + ". Name an `outputDir` inside it, or a source file inside it.", { cause });
|
|
279
|
+
stamped.code = SANDBOX_DENIED;
|
|
280
|
+
return stamped;
|
|
281
|
+
}
|
|
282
|
+
//#endregion
|
|
283
|
+
//#region lib/records.js
|
|
284
|
+
/**
|
|
285
|
+
* The record of one recognition, as it is stored beside the pages it came from.
|
|
286
|
+
*
|
|
287
|
+
* A recognition writes three files: `record.json` holds everything the assembly
|
|
288
|
+
* pass needs to rebuild the document —the merge options, the page geometry,
|
|
289
|
+
* and the digest that identifies them —`outline.json` holds the heading
|
|
290
|
+
* candidates the caller may correct, and `latest.json` points at the document
|
|
291
|
+
* the newest recognition produced so a later call finds it without scanning.
|
|
292
|
+
* This module owns that format for every reader and writer of it.
|
|
293
|
+
* @module @deepseek-ai/dsh-ab-ocr/records
|
|
294
|
+
*/
|
|
295
|
+
/**
|
|
296
|
+
* Ceiling on one artifact read back through the filesystem capability. This is a
|
|
297
|
+
* fixed safety invariant rather than a deployment knob: it exists so a path that
|
|
298
|
+
* is not one of this plugin's records cannot make the plugin load an arbitrarily
|
|
299
|
+
* large file into memory, and a deployment tunes how large a record can get
|
|
300
|
+
* through `maxPages` instead.
|
|
301
|
+
*/
|
|
302
|
+
const MAX_RECORD_BYTES = 67108864;
|
|
303
|
+
/**
|
|
304
|
+
* Read the positive whole numbers a stored array carries.
|
|
305
|
+
* @param value - the parsed field.
|
|
306
|
+
* @returns the numbers it carries, in order; empty for anything else.
|
|
307
|
+
*/
|
|
308
|
+
function wholeNumbers(value) {
|
|
309
|
+
if (!Array.isArray(value)) return [];
|
|
310
|
+
return value.filter((entry) => typeof entry === "number" && Number.isInteger(entry) && entry > 0);
|
|
311
|
+
}
|
|
312
|
+
/**
|
|
313
|
+
* Read one JSON artifact through the filesystem capability.
|
|
314
|
+
* @param fs - the mounted filesystem capability.
|
|
315
|
+
* @param path - the artifact's absolute path.
|
|
316
|
+
* @param signal - caller cancellation.
|
|
317
|
+
* @returns the parsed value, or undefined when the file is absent, unreadable,
|
|
318
|
+
* or not JSON.
|
|
319
|
+
*/
|
|
320
|
+
async function readJson(fs, path, signal) {
|
|
321
|
+
try {
|
|
322
|
+
const target = await fs.resolve(path);
|
|
323
|
+
const bytes = await fs.readBytes(target, signal, MAX_RECORD_BYTES);
|
|
324
|
+
return JSON.parse(Buffer.from(bytes).toString("utf8"));
|
|
325
|
+
} catch {
|
|
326
|
+
return;
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
/**
|
|
330
|
+
* Read the pointer to a document's newest recognition.
|
|
331
|
+
* @param fs - the mounted filesystem capability.
|
|
332
|
+
* @param path - the `latest.json` path.
|
|
333
|
+
* @param signal - caller cancellation.
|
|
334
|
+
* @returns the stored pointer.
|
|
335
|
+
* @throws Error when the file is missing or does not name a document and its digest.
|
|
336
|
+
*/
|
|
337
|
+
async function readLatest(fs, path, signal) {
|
|
338
|
+
const latest = await readJson(fs, path, signal);
|
|
339
|
+
if (latest === null || typeof latest !== "object") throw new Error("ocr cannot correct heading levels: no " + path + "; recognize the document first");
|
|
340
|
+
if (typeof latest.digest !== "string" || latest.digest === "" || typeof latest.output !== "string" || latest.output === "") throw new Error("ocr cannot correct heading levels: " + path + " names no document to correct");
|
|
341
|
+
return {
|
|
342
|
+
digest: latest.digest,
|
|
343
|
+
output: latest.output,
|
|
344
|
+
input: typeof latest.input === "string" ? latest.input : "",
|
|
345
|
+
pages: wholeNumbers(latest.pages),
|
|
346
|
+
totalPages: typeof latest.totalPages === "number" ? latest.totalPages : 0
|
|
347
|
+
};
|
|
348
|
+
}
|
|
349
|
+
/**
|
|
350
|
+
* Read the record one recognition stored.
|
|
351
|
+
* @param fs - the mounted filesystem capability.
|
|
352
|
+
* @param path - the `record.json` path.
|
|
353
|
+
* @param signal - caller cancellation.
|
|
354
|
+
* @returns the stored record.
|
|
355
|
+
* @throws Error when the file is missing or does not carry the merge options
|
|
356
|
+
* and page geometry the correction needs.
|
|
357
|
+
*/
|
|
358
|
+
async function readRecord(fs, path, signal) {
|
|
359
|
+
const record = await readJson(fs, path, signal);
|
|
360
|
+
if (record === null || typeof record !== "object") throw new Error("ocr cannot correct heading levels: no " + path + "; recognize the document first");
|
|
361
|
+
const merge = record.merge;
|
|
362
|
+
if (merge === void 0 || typeof merge !== "object" || merge === null) throw new Error("ocr cannot correct heading levels: " + path + " carries no merge options");
|
|
363
|
+
const geometry = record.geometry;
|
|
364
|
+
if (geometry === void 0 || typeof geometry !== "object" || geometry === null || !Array.isArray(geometry.pages)) throw new Error("ocr cannot correct heading levels: " + path + " carries no page geometry");
|
|
365
|
+
const source = record.source;
|
|
366
|
+
return {
|
|
367
|
+
digest: typeof record.digest === "string" ? record.digest : "",
|
|
368
|
+
input: typeof record.input === "string" ? record.input : "",
|
|
369
|
+
pages: wholeNumbers(record.pages),
|
|
370
|
+
totalPages: typeof record.totalPages === "number" ? record.totalPages : 0,
|
|
371
|
+
merge,
|
|
372
|
+
geometry,
|
|
373
|
+
source: {
|
|
374
|
+
size: typeof source?.size === "number" ? source.size : "",
|
|
375
|
+
mtimeMs: typeof source?.mtimeMs === "number" ? source.mtimeMs : 0
|
|
376
|
+
},
|
|
377
|
+
statUnavailable: record.statUnavailable === true
|
|
378
|
+
};
|
|
379
|
+
}
|
|
380
|
+
/**
|
|
381
|
+
* Write one JSON artifact under the call's fence.
|
|
382
|
+
* @param fs - the mounted filesystem capability.
|
|
383
|
+
* @param path - the artifact's absolute path.
|
|
384
|
+
* @param value - the value to store.
|
|
385
|
+
* @param signal - caller cancellation.
|
|
386
|
+
* @param fence - the per-call fence the confining backend judges the write by;
|
|
387
|
+
* absent on a bare backend.
|
|
388
|
+
* @returns nothing, once the write settled.
|
|
389
|
+
* @throws Error as the backend raised it, or the same refusal restated with the remedy.
|
|
390
|
+
*/
|
|
391
|
+
async function writeJson(fs, path, value, signal, fence) {
|
|
392
|
+
await saveText(fs, await fs.resolve(path), JSON.stringify(value, null, 2) + "\n", fence, signal);
|
|
393
|
+
}
|
|
394
|
+
//#endregion
|
|
395
|
+
//#region lib/documents.js
|
|
396
|
+
/**
|
|
397
|
+
* Writing one recognition's artifacts: the Markdown document the reader keeps,
|
|
398
|
+
* the record a later correction reads back, the outline sheet the model
|
|
399
|
+
* corrects, and the pointer to the newest recognition.
|
|
400
|
+
*
|
|
401
|
+
* Every write goes through the mounted filesystem capability, so a deployment's
|
|
402
|
+
* file policy sees it and its own backend decides how the bytes are published.
|
|
403
|
+
* The plain Markdown name is never overwritten with different content: when it
|
|
404
|
+
* already holds another document, the new one takes the digest name.
|
|
405
|
+
* @module @deepseek-ai/dsh-ab-ocr/documents
|
|
406
|
+
*/
|
|
407
|
+
/**
|
|
408
|
+
* Decide whether one plain Markdown path already holds this document's text.
|
|
409
|
+
* @param fs - the mounted filesystem capability.
|
|
410
|
+
* @param path - the path to read.
|
|
411
|
+
* @param markdown - the document's text, as the worker produced it.
|
|
412
|
+
* @param signal - caller cancellation.
|
|
413
|
+
* @returns true when the file holds exactly this text.
|
|
414
|
+
*/
|
|
415
|
+
async function holdsText(fs, path, markdown, signal) {
|
|
416
|
+
const expected = Buffer.from(markdown, "utf8");
|
|
417
|
+
try {
|
|
418
|
+
const target = await fs.resolve(path);
|
|
419
|
+
const bytes = await fs.readBytes(target, signal, expected.length + 1);
|
|
420
|
+
return Buffer.from(bytes).equals(expected);
|
|
421
|
+
} catch {
|
|
422
|
+
return false;
|
|
423
|
+
}
|
|
424
|
+
}
|
|
425
|
+
/**
|
|
426
|
+
* Choose the Markdown path one recognized document is written to.
|
|
427
|
+
* @param fs - the mounted filesystem capability.
|
|
428
|
+
* @param artifacts - the document's artifact paths.
|
|
429
|
+
* @param markdown - the document's text, as the worker produced it.
|
|
430
|
+
* @param signal - caller cancellation.
|
|
431
|
+
* @returns the plain path when it is free or already holds this text, and the
|
|
432
|
+
* digest path otherwise.
|
|
433
|
+
*/
|
|
434
|
+
async function chooseDocument(fs, artifacts, markdown, signal) {
|
|
435
|
+
if (await fs.stat(await fs.resolve(artifacts.document), signal) === void 0) return artifacts.document;
|
|
436
|
+
if (await holdsText(fs, artifacts.document, markdown, signal)) return artifacts.document;
|
|
437
|
+
return artifacts.digestedDocument;
|
|
438
|
+
}
|
|
439
|
+
/**
|
|
440
|
+
* Read the level corrections one recognition's stored record already carries.
|
|
441
|
+
* @param fs - the mounted filesystem capability.
|
|
442
|
+
* @param path - the record's path.
|
|
443
|
+
* @param signal - caller cancellation.
|
|
444
|
+
* @returns the corrections, or undefined when there is no record or it carries none.
|
|
445
|
+
*/
|
|
446
|
+
async function carriedLevels(fs, path, signal) {
|
|
447
|
+
let record;
|
|
448
|
+
try {
|
|
449
|
+
record = await readRecord(fs, path, signal);
|
|
450
|
+
} catch {
|
|
451
|
+
return;
|
|
452
|
+
}
|
|
453
|
+
const overrides = record.merge.levelOverrides;
|
|
454
|
+
return overrides === void 0 || Object.keys(overrides).length === 0 ? void 0 : overrides;
|
|
455
|
+
}
|
|
456
|
+
/**
|
|
457
|
+
* Record the level each candidate's line was given.
|
|
458
|
+
*
|
|
459
|
+
* The worker reports the level the merge inferred; a correction replaces it. Both
|
|
460
|
+
* are kept, because a caller re-reading the sheet after a correction has to be
|
|
461
|
+
* able to see that the correction took, and what it took the place of.
|
|
462
|
+
* @param outline - the candidates the merge collected.
|
|
463
|
+
* @param overrides - the corrections that were applied, if any.
|
|
464
|
+
* @returns the candidates, each carrying the level the document used.
|
|
465
|
+
*/
|
|
466
|
+
function appliedOutline(outline, overrides) {
|
|
467
|
+
return outline.map((entry) => {
|
|
468
|
+
const applied = overrides?.[entry.id];
|
|
469
|
+
if (applied === void 0) return {
|
|
470
|
+
...entry,
|
|
471
|
+
inferred: entry.level
|
|
472
|
+
};
|
|
473
|
+
return {
|
|
474
|
+
...entry,
|
|
475
|
+
inferred: entry.level,
|
|
476
|
+
level: applied <= 0 ? null : Math.min(applied, 6)
|
|
477
|
+
};
|
|
478
|
+
});
|
|
479
|
+
}
|
|
480
|
+
/**
|
|
481
|
+
* Write one finished document and the record an assembly pass reads back.
|
|
482
|
+
* @param fs - the mounted filesystem capability.
|
|
483
|
+
* @param artifacts - the paths this recognition's artifacts are written to.
|
|
484
|
+
* @param state - the document's state, carrying the geometry it collected.
|
|
485
|
+
* @param markdown - the document's text, after any carried correction was applied.
|
|
486
|
+
* @param outline - the heading candidates the merge collected.
|
|
487
|
+
* @param rawOutline - the event's outline field, which may carry none.
|
|
488
|
+
* @param carried - the corrections this recognition carried forward, if any.
|
|
489
|
+
* @param signal - caller cancellation.
|
|
490
|
+
* @param fence - the per-call fence this document and its records are written under; absent on a bare backend.
|
|
491
|
+
* @returns the Markdown path this call wrote.
|
|
492
|
+
*/
|
|
493
|
+
async function writeDocument(fs, artifacts, state, markdown, outline, rawOutline, carried, signal, fence) {
|
|
494
|
+
const output = await chooseDocument(fs, artifacts, markdown, signal);
|
|
495
|
+
await saveText(fs, await fs.resolve(output), markdown, fence, signal);
|
|
496
|
+
state.carried = carried;
|
|
497
|
+
const record = {
|
|
498
|
+
digest: artifacts.digest,
|
|
499
|
+
input: state.job.input,
|
|
500
|
+
pages: [...state.selection],
|
|
501
|
+
totalPages: state.totalPages,
|
|
502
|
+
merge: carried === void 0 ? state.job.job.merge : {
|
|
503
|
+
...state.job.job.merge,
|
|
504
|
+
levelOverrides: carried
|
|
505
|
+
},
|
|
506
|
+
geometry: { pages: state.geometry },
|
|
507
|
+
source: {
|
|
508
|
+
size: state.job.stamp.size,
|
|
509
|
+
mtimeMs: state.job.stamp.mtimeMs
|
|
510
|
+
},
|
|
511
|
+
statUnavailable: state.job.stamp.unavailable
|
|
512
|
+
};
|
|
513
|
+
await writeJson(fs, artifacts.record, record, signal, fence);
|
|
514
|
+
if (Array.isArray(rawOutline)) await writeJson(fs, artifacts.outline, appliedOutline(outline, carried), signal, fence);
|
|
515
|
+
const latest = {
|
|
516
|
+
digest: artifacts.digest,
|
|
517
|
+
output,
|
|
518
|
+
input: state.job.input,
|
|
519
|
+
pages: [...state.selection],
|
|
520
|
+
totalPages: state.totalPages
|
|
521
|
+
};
|
|
522
|
+
await writeJson(fs, artifacts.latest, latest, signal, fence);
|
|
523
|
+
return output;
|
|
524
|
+
}
|
|
525
|
+
/**
|
|
526
|
+
* Build the document one finished recognition reports.
|
|
527
|
+
* @param state - the document's state, carrying what its start event named.
|
|
528
|
+
* @param artifacts - the paths this recognition's artifacts were written to.
|
|
529
|
+
* @param done - the finished-document event.
|
|
530
|
+
* @param output - the Markdown path the call wrote.
|
|
531
|
+
* @param pageDir - the directory holding the per-page text, or an empty string.
|
|
532
|
+
* @param truncated - whether the heading candidate ceiling cut the outline short.
|
|
533
|
+
* @param corrections - heading-level corrections this call applied.
|
|
534
|
+
* @returns the document's canonical value.
|
|
535
|
+
*/
|
|
536
|
+
function recognizedDocument(state, artifacts, done, output, pageDir, truncated, corrections) {
|
|
537
|
+
return {
|
|
538
|
+
input: state.job.input,
|
|
539
|
+
output,
|
|
540
|
+
pages: state.selection.length > 0 ? state.selection.length : done.pages,
|
|
541
|
+
totalPages: state.totalPages > 0 ? state.totalPages : done.totalPages,
|
|
542
|
+
lines: done.lines,
|
|
543
|
+
headings: done.headings,
|
|
544
|
+
chars: done.chars,
|
|
545
|
+
droppedPageNumbers: done.droppedPageNumbers,
|
|
546
|
+
droppedRunningHeads: done.droppedRunningHeads,
|
|
547
|
+
joinedAcrossPages: done.joinedAcrossPages,
|
|
548
|
+
pageDir,
|
|
549
|
+
seconds: done.seconds,
|
|
550
|
+
digest: artifacts.digest,
|
|
551
|
+
selection: [...state.selection],
|
|
552
|
+
outlinePath: artifacts.outline,
|
|
553
|
+
outlineTruncated: truncated,
|
|
554
|
+
corrections
|
|
555
|
+
};
|
|
556
|
+
}
|
|
557
|
+
//#endregion
|
|
558
|
+
//#region lib/events.js
|
|
559
|
+
/**
|
|
560
|
+
* Interpretation of the worker's event stream, and the per-call state that
|
|
561
|
+
* interpretation accumulates.
|
|
562
|
+
*
|
|
563
|
+
* The worker is a separate process, so every value on this boundary is read
|
|
564
|
+
* defensively: a field that is absent, of the wrong type, or carries no page
|
|
565
|
+
* number is a protocol violation rather than a value to coerce. Everything an
|
|
566
|
+
* event reports about one document is kept in that document's
|
|
567
|
+
* {@link DocumentState} until it finishes, because the merge pass reads the
|
|
568
|
+
* geometry back.
|
|
569
|
+
* @module @deepseek-ai/dsh-ab-ocr/events
|
|
570
|
+
*/
|
|
571
|
+
/**
|
|
572
|
+
* Read one worker event into the value the merge-shaped result is built from.
|
|
573
|
+
* @param event - the parsed event.
|
|
574
|
+
* @returns the document's counts and body, or undefined for another event.
|
|
575
|
+
*/
|
|
576
|
+
function readDoneEvent(event) {
|
|
577
|
+
if (event.event !== "done") return void 0;
|
|
578
|
+
return {
|
|
579
|
+
id: String(event.id ?? ""),
|
|
580
|
+
pages: Number(event.pages ?? 0),
|
|
581
|
+
totalPages: Number(event.totalPages ?? 0),
|
|
582
|
+
lines: Number(event.lines ?? 0),
|
|
583
|
+
headings: Number(event.headings ?? 0),
|
|
584
|
+
chars: Number(event.chars ?? 0),
|
|
585
|
+
droppedPageNumbers: Number(event.droppedPageNumbers ?? 0),
|
|
586
|
+
droppedRunningHeads: Number(event.droppedRunningHeads ?? 0),
|
|
587
|
+
joinedAcrossPages: Number(event.joinedAcrossPages ?? 0),
|
|
588
|
+
seconds: Number(event.seconds ?? 0),
|
|
589
|
+
markdown: String(event.markdown ?? "")
|
|
590
|
+
};
|
|
591
|
+
}
|
|
592
|
+
/**
|
|
593
|
+
* Read one whole positive number off a worker event.
|
|
594
|
+
* @param value - the event field.
|
|
595
|
+
* @returns the number, or undefined when the field carries none.
|
|
596
|
+
*/
|
|
597
|
+
function wholeNumber(value) {
|
|
598
|
+
return typeof value === "number" && Number.isInteger(value) && value > 0 ? value : void 0;
|
|
599
|
+
}
|
|
600
|
+
/**
|
|
601
|
+
* Read the geometry one page event carries, which an assembly pass needs back.
|
|
602
|
+
* @param value - the event's `geometry` field.
|
|
603
|
+
* @returns the page's geometry, or undefined when the field does not carry it.
|
|
604
|
+
*/
|
|
605
|
+
function readGeometry(value) {
|
|
606
|
+
if (typeof value !== "object" || value === null) return void 0;
|
|
607
|
+
const geometry = value;
|
|
608
|
+
if (wholeNumber(geometry.index) === void 0) return void 0;
|
|
609
|
+
if (typeof geometry.width !== "number" || typeof geometry.height !== "number") return void 0;
|
|
610
|
+
if (!Array.isArray(geometry.lines)) return void 0;
|
|
611
|
+
return geometry;
|
|
612
|
+
}
|
|
613
|
+
/**
|
|
614
|
+
* Read the outline a finished document reported.
|
|
615
|
+
*
|
|
616
|
+
* The worker reports the level the merge inferred; the candidate carries that
|
|
617
|
+
* same level as its starting point until a correction replaces it.
|
|
618
|
+
* @param value - the event's `outline` field.
|
|
619
|
+
* @returns the outline entries, or an empty list when the field carries none.
|
|
620
|
+
*/
|
|
621
|
+
function readOutline(value) {
|
|
622
|
+
if (!Array.isArray(value)) return [];
|
|
623
|
+
return value.map((entry) => ({
|
|
624
|
+
...entry,
|
|
625
|
+
inferred: entry.level
|
|
626
|
+
}));
|
|
627
|
+
}
|
|
628
|
+
/**
|
|
629
|
+
* Read the failures a finished batch reported.
|
|
630
|
+
* @param value - the end event's `failures` field.
|
|
631
|
+
* @returns the failures, each naming a path and its reason.
|
|
632
|
+
*/
|
|
633
|
+
function readFailures(value) {
|
|
634
|
+
if (!Array.isArray(value)) return [];
|
|
635
|
+
return value.map((entry) => {
|
|
636
|
+
const failure = entry;
|
|
637
|
+
return {
|
|
638
|
+
id: String(failure?.id ?? ""),
|
|
639
|
+
input: String(failure?.input ?? ""),
|
|
640
|
+
error: String(failure?.error ?? "")
|
|
641
|
+
};
|
|
642
|
+
});
|
|
643
|
+
}
|
|
644
|
+
/**
|
|
645
|
+
* Start the record one document reports into.
|
|
646
|
+
* @param job - the prepared job.
|
|
647
|
+
* @returns the document's empty state.
|
|
648
|
+
*/
|
|
649
|
+
function documentState(job) {
|
|
650
|
+
return {
|
|
651
|
+
job,
|
|
652
|
+
selection: [],
|
|
653
|
+
totalPages: 0,
|
|
654
|
+
lastPage: 0,
|
|
655
|
+
geometry: [],
|
|
656
|
+
carried: void 0
|
|
657
|
+
};
|
|
658
|
+
}
|
|
659
|
+
/**
|
|
660
|
+
* Report why one page event breaks the order a document's files rely on.
|
|
661
|
+
* @param page - the page number the event carried.
|
|
662
|
+
* @param state - what the document has reported so far.
|
|
663
|
+
* @returns the problem, or undefined when the page is in order.
|
|
664
|
+
*/
|
|
665
|
+
function pageProblem(page, state) {
|
|
666
|
+
if (page === state.lastPage) return "page " + page + " arrived twice";
|
|
667
|
+
if (page < state.lastPage) return "page " + page + " arrived after page " + state.lastPage;
|
|
668
|
+
if (state.totalPages > 0 && page > state.totalPages) return "page " + page + " is past the last page of the document, " + state.totalPages;
|
|
669
|
+
}
|
|
670
|
+
/**
|
|
671
|
+
* Serialize a call's artifact writes behind one chain, so two writes never race
|
|
672
|
+
* for one path and a failed write is recorded once.
|
|
673
|
+
* @param controller - the controller whose abort stops the recognition.
|
|
674
|
+
* @returns the queue's enqueue and settle operations.
|
|
675
|
+
*/
|
|
676
|
+
function writeQueue(controller) {
|
|
677
|
+
let chain = Promise.resolve();
|
|
678
|
+
let failed;
|
|
679
|
+
return {
|
|
680
|
+
enqueue(task) {
|
|
681
|
+
chain = chain.then(task).catch((error) => {
|
|
682
|
+
if (failed !== void 0) return;
|
|
683
|
+
failed = error instanceof Error ? error : new Error(String(error));
|
|
684
|
+
controller.abort();
|
|
685
|
+
});
|
|
686
|
+
},
|
|
687
|
+
async settle() {
|
|
688
|
+
await chain;
|
|
689
|
+
return failed;
|
|
690
|
+
}
|
|
691
|
+
};
|
|
692
|
+
}
|
|
693
|
+
/**
|
|
694
|
+
* Report one document's failure once, whether it arrived as its own event or in
|
|
695
|
+
* the batch's closing list. The worker reports a failure both ways, and its two
|
|
696
|
+
* reports agree on the job identifier as well as on the path and the reason.
|
|
697
|
+
* @param call - the call collecting failures.
|
|
698
|
+
* @param failure - the failure the worker reported.
|
|
699
|
+
* @returns nothing.
|
|
700
|
+
*/
|
|
701
|
+
function reportFailure(call, failure) {
|
|
702
|
+
const keys = ["pair:" + failure.input + "\0" + failure.error];
|
|
703
|
+
if (failure.id !== "") keys.push("job:" + failure.id);
|
|
704
|
+
if (keys.some((key) => call.reported.has(key))) return;
|
|
705
|
+
for (const key of keys) call.reported.add(key);
|
|
706
|
+
call.failures.push({
|
|
707
|
+
input: failure.input,
|
|
708
|
+
error: failure.error
|
|
709
|
+
});
|
|
710
|
+
}
|
|
711
|
+
//#endregion
|
|
712
|
+
//#region lib/levels.js
|
|
713
|
+
/**
|
|
714
|
+
* The `levels` request value: heading-level corrections a caller makes after
|
|
715
|
+
* reading the outline a recognition saved.
|
|
716
|
+
*
|
|
717
|
+
* The value is compact because the model writes it: `"h12=2,h15=0,h20=3"`,
|
|
718
|
+
* where the id is an outline candidate and the number is the Markdown heading
|
|
719
|
+
* depth it should take. Zero, or anything below it, takes the line out of the
|
|
720
|
+
* heading hierarchy and leaves it as body text, which is what makes the pass
|
|
721
|
+
* able to demote a line the merge promoted. Anything else is rejected rather
|
|
722
|
+
* than half-applied.
|
|
723
|
+
* @module @deepseek-ai/dsh-ab-ocr/levels
|
|
724
|
+
*/
|
|
725
|
+
/** One correction: an outline candidate id, then the level it should take. */
|
|
726
|
+
const CORRECTION = /^h([1-9][0-9]*)=(.+)$/;
|
|
727
|
+
/** A level the worker accepts: a whole number, 0 or below meaning body text. */
|
|
728
|
+
const LEVEL = /^-?[0-9]+$/;
|
|
729
|
+
/**
|
|
730
|
+
* Parse the corrections one `levels` value carries.
|
|
731
|
+
* @param value - the model-supplied value.
|
|
732
|
+
* @returns the candidate id to heading level corrections.
|
|
733
|
+
* @throws Error when the value is empty, when a piece is not `<id>=<level>`,
|
|
734
|
+
* when the level is not a whole number, or when two pieces give one
|
|
735
|
+
* candidate different levels.
|
|
736
|
+
*/
|
|
737
|
+
function parseLevels(value) {
|
|
738
|
+
const trimmed = value.trim();
|
|
739
|
+
if (trimmed === "") throw new Error("invalid ocr: levels is empty; write corrections such as \"h12=2,h15=0\"");
|
|
740
|
+
const overrides = {};
|
|
741
|
+
for (const piece of trimmed.split(",")) {
|
|
742
|
+
const entry = piece.trim();
|
|
743
|
+
const match = CORRECTION.exec(entry);
|
|
744
|
+
if (match === null) throw new Error("invalid ocr: levels cannot read \"" + entry + "\"; write an outline id and its level such as \"h12=2\"");
|
|
745
|
+
const id = "h" + (match[1] ?? "");
|
|
746
|
+
const raw = match[2] ?? "";
|
|
747
|
+
if (!LEVEL.test(raw)) throw new Error("invalid ocr: levels gives \"" + raw + "\" to " + id + "; a level is a whole number, 0 for a line that is not a heading and 1 to 6 for a heading");
|
|
748
|
+
const level = Number(raw);
|
|
749
|
+
const previous = overrides[id];
|
|
750
|
+
if (previous !== void 0 && previous !== level) throw new Error("invalid ocr: levels gives " + id + " both " + String(previous) + " and " + String(level));
|
|
751
|
+
overrides[id] = level;
|
|
752
|
+
}
|
|
753
|
+
return overrides;
|
|
754
|
+
}
|
|
755
|
+
//#endregion
|
|
756
|
+
//#region lib/plan.js
|
|
757
|
+
/**
|
|
758
|
+
* Request planning: which files one call recognizes, where each document's
|
|
759
|
+
* artifacts go, and what the worker is told about each one.
|
|
760
|
+
*
|
|
761
|
+
* Everything here is decided before a worker process starts, so a call that
|
|
762
|
+
* names a file the worker cannot read is reported as a failure without paying
|
|
763
|
+
* for a process. The one exception is the source's own content digest, which
|
|
764
|
+
* only the worker can compute; {@link prepareJob} therefore resolves the paths
|
|
765
|
+
* that do not depend on a recognition, and the entry module names the rest when
|
|
766
|
+
* the document reports it started.
|
|
767
|
+
* @module @deepseek-ai/dsh-ab-ocr/plan
|
|
768
|
+
*/
|
|
769
|
+
/** Extensions the worker accepts as a still image or a PDF. */
|
|
770
|
+
const SUPPORTED_EXTENSIONS = [
|
|
771
|
+
".pdf",
|
|
772
|
+
".png",
|
|
773
|
+
".jpg",
|
|
774
|
+
".jpeg",
|
|
775
|
+
".jpe",
|
|
776
|
+
".bmp",
|
|
777
|
+
".dib",
|
|
778
|
+
".tif",
|
|
779
|
+
".tiff",
|
|
780
|
+
".webp",
|
|
781
|
+
".gif",
|
|
782
|
+
".jp2",
|
|
783
|
+
".pbm",
|
|
784
|
+
".pgm",
|
|
785
|
+
".ppm",
|
|
786
|
+
".tga",
|
|
787
|
+
".ico"
|
|
788
|
+
];
|
|
789
|
+
/**
|
|
790
|
+
* Resolve the Python executable that carries the OCR dependencies.
|
|
791
|
+
* @param configured - the row's pythonPath value.
|
|
792
|
+
* @param packageRoot - this package's directory.
|
|
793
|
+
* @param platform - the platform the plugin is running on.
|
|
794
|
+
* @returns the configured path, the package's own environment, or the platform's interpreter name.
|
|
795
|
+
*/
|
|
796
|
+
function resolvePython(configured, packageRoot, platform) {
|
|
797
|
+
if (configured !== "") return configured;
|
|
798
|
+
const bundled = platform === "win32" ? join(packageRoot, "python", ".venv", "Scripts", "python.exe") : join(packageRoot, "python", ".venv", "bin", "python");
|
|
799
|
+
return existsSync(bundled) ? bundled : platform === "win32" ? "python" : "python3";
|
|
800
|
+
}
|
|
801
|
+
/**
|
|
802
|
+
* Resolve the worker script this package runs.
|
|
803
|
+
* @param configured - the row's workerScript value.
|
|
804
|
+
* @param packageRoot - this package's directory.
|
|
805
|
+
* @returns the configured path, or the script shipped inside the package.
|
|
806
|
+
*/
|
|
807
|
+
function resolveWorkerScript(configured, packageRoot) {
|
|
808
|
+
return configured !== "" ? configured : join(packageRoot, "python", "ocr_worker.py");
|
|
809
|
+
}
|
|
810
|
+
/**
|
|
811
|
+
* Resolve a deployment directory value against a base.
|
|
812
|
+
* @param configured - the configured or requested directory.
|
|
813
|
+
* @param base - the directory a relative value resolves against.
|
|
814
|
+
* @returns the absolute directory, or an empty string when the value is empty.
|
|
815
|
+
*/
|
|
816
|
+
function resolveDirectory(configured, base) {
|
|
817
|
+
if (configured === "") return "";
|
|
818
|
+
return isAbsolute(configured) ? configured : resolve(base, configured);
|
|
819
|
+
}
|
|
820
|
+
/**
|
|
821
|
+
* Collect the paths one call named, in request order and without duplicates.
|
|
822
|
+
* @param args - the model-supplied call arguments.
|
|
823
|
+
* @param base - the directory a relative path resolves against.
|
|
824
|
+
* @param maxDocuments - documents one call may name.
|
|
825
|
+
* @returns the absolute paths to recognize.
|
|
826
|
+
* @throws Error when the call names no path, or more than the ceiling.
|
|
827
|
+
*/
|
|
828
|
+
function requestPaths(args, base, maxDocuments) {
|
|
829
|
+
const named = [...args.path === void 0 ? [] : [args.path], ...args.paths ?? []];
|
|
830
|
+
if (named.length === 0) throw new Error("invalid ocr: give path or paths with at least one file");
|
|
831
|
+
const paths = [];
|
|
832
|
+
for (const raw of named) {
|
|
833
|
+
const trimmed = raw.trim();
|
|
834
|
+
if (trimmed === "") throw new Error("invalid ocr: path must be a non-empty string");
|
|
835
|
+
const absolute = isAbsolute(trimmed) ? trimmed : resolve(base, trimmed);
|
|
836
|
+
if (!paths.includes(absolute)) paths.push(absolute);
|
|
837
|
+
}
|
|
838
|
+
if (paths.length > maxDocuments) throw new Error("invalid ocr: one call recognizes at most " + maxDocuments + " files");
|
|
839
|
+
return paths;
|
|
840
|
+
}
|
|
841
|
+
/**
|
|
842
|
+
* Report why a path cannot be recognized, before any process is started.
|
|
843
|
+
* @param path - the absolute path to check.
|
|
844
|
+
* @returns the reason, or undefined when the worker can read the path.
|
|
845
|
+
*/
|
|
846
|
+
function rejectPath(path) {
|
|
847
|
+
if (!existsSync(path)) return "no such file: " + path;
|
|
848
|
+
const lower = path.toLowerCase();
|
|
849
|
+
if (!SUPPORTED_EXTENSIONS.some((extension) => lower.endsWith(extension))) return "unsupported file type: neither a PDF nor a supported image";
|
|
850
|
+
}
|
|
851
|
+
/**
|
|
852
|
+
* Collect the merge switches and thresholds one call sends.
|
|
853
|
+
* @param config - the deployment's values.
|
|
854
|
+
* @returns the merge options, in the names the worker reads them under.
|
|
855
|
+
*/
|
|
856
|
+
function mergeOptions(config) {
|
|
857
|
+
return {
|
|
858
|
+
removePageNumbers: config.removePageNumbers,
|
|
859
|
+
removeRunningHeads: config.removeRunningHeads,
|
|
860
|
+
runningHeadRatio: config.runningHeadRatio,
|
|
861
|
+
runningHeadMinPages: config.runningHeadMinPages,
|
|
862
|
+
detectHeadings: config.detectHeadings,
|
|
863
|
+
headingMinRatio: config.headingMinRatio,
|
|
864
|
+
indentRatio: config.indentRatio,
|
|
865
|
+
paragraphGapRatio: config.paragraphGapRatio,
|
|
866
|
+
outlineCandidateRatio: config.outlineCandidateRatio,
|
|
867
|
+
maxOutlineCandidates: config.maxOutlineCandidates
|
|
868
|
+
};
|
|
869
|
+
}
|
|
870
|
+
/**
|
|
871
|
+
* Resolve the directory one call's artifacts are written to: the call's own
|
|
872
|
+
* request when it made one, otherwise the deployment's configured directory.
|
|
873
|
+
* @param args - the model-supplied call arguments.
|
|
874
|
+
* @param config - the deployment's values.
|
|
875
|
+
* @param base - the directory a relative value resolves against.
|
|
876
|
+
* @returns the call's output directory, or an empty string for the source's own.
|
|
877
|
+
*/
|
|
878
|
+
function requestedOutputDir(args, config, base) {
|
|
879
|
+
const requested = resolveDirectory(args.outputDir ?? "", base);
|
|
880
|
+
return requested !== "" ? requested : resolveDirectory(config.outputDir, base);
|
|
881
|
+
}
|
|
882
|
+
/**
|
|
883
|
+
* Build the worker spec and the failures that never reach the worker.
|
|
884
|
+
* @param paths - the absolute paths one call named.
|
|
885
|
+
* @param args - the model-supplied call arguments.
|
|
886
|
+
* @param config - the deployment's values.
|
|
887
|
+
* @param base - the directory a relative output directory resolves against.
|
|
888
|
+
* @returns the jobs to run and the paths rejected before running.
|
|
889
|
+
*/
|
|
890
|
+
function planJobs(paths, args, config, base) {
|
|
891
|
+
const outputDir = requestedOutputDir(args, config, base);
|
|
892
|
+
const jobs = [];
|
|
893
|
+
const failures = [];
|
|
894
|
+
for (const input of paths) {
|
|
895
|
+
const rejection = rejectPath(input);
|
|
896
|
+
if (rejection !== void 0) {
|
|
897
|
+
failures.push({
|
|
898
|
+
input,
|
|
899
|
+
error: rejection
|
|
900
|
+
});
|
|
901
|
+
continue;
|
|
902
|
+
}
|
|
903
|
+
jobs.push({
|
|
904
|
+
input,
|
|
905
|
+
base: artifactBase(input, outputDir),
|
|
906
|
+
job: {
|
|
907
|
+
mode: "recognize",
|
|
908
|
+
id: String(jobs.length + 1),
|
|
909
|
+
input,
|
|
910
|
+
pages: args.pages ?? null,
|
|
911
|
+
dpi: config.dpi,
|
|
912
|
+
maxPixels: config.maxPixels,
|
|
913
|
+
textScore: config.textScore,
|
|
914
|
+
detectColumns: config.detectColumns,
|
|
915
|
+
maxPages: config.maxPages,
|
|
916
|
+
merge: mergeOptions(config)
|
|
917
|
+
}
|
|
918
|
+
});
|
|
919
|
+
}
|
|
920
|
+
return {
|
|
921
|
+
jobs,
|
|
922
|
+
failures
|
|
923
|
+
};
|
|
924
|
+
}
|
|
925
|
+
/**
|
|
926
|
+
* Read the metadata the digest covers for one source file.
|
|
927
|
+
* @param fs - the mounted filesystem capability.
|
|
928
|
+
* @param target - the resolved source target.
|
|
929
|
+
* @param signal - caller cancellation.
|
|
930
|
+
* @returns the size and modification time the filesystem reported, or the
|
|
931
|
+
* documented fallbacks when it reported none or refused the read.
|
|
932
|
+
*/
|
|
933
|
+
async function sourceStamp(fs, target, signal) {
|
|
934
|
+
try {
|
|
935
|
+
const info = await fs.stat(target, signal);
|
|
936
|
+
if (info === void 0) return {
|
|
937
|
+
size: "",
|
|
938
|
+
mtimeMs: 0,
|
|
939
|
+
unavailable: true
|
|
940
|
+
};
|
|
941
|
+
const reported = info;
|
|
942
|
+
return {
|
|
943
|
+
size: typeof reported.size === "number" ? reported.size : "",
|
|
944
|
+
mtimeMs: typeof reported.mtimeMs === "number" ? reported.mtimeMs : 0,
|
|
945
|
+
unavailable: false
|
|
946
|
+
};
|
|
947
|
+
} catch {
|
|
948
|
+
return {
|
|
949
|
+
size: "",
|
|
950
|
+
mtimeMs: 0,
|
|
951
|
+
unavailable: true
|
|
952
|
+
};
|
|
953
|
+
}
|
|
954
|
+
}
|
|
955
|
+
/**
|
|
956
|
+
* Resolve the digest and the artifact paths of one planned job.
|
|
957
|
+
* @param fs - the mounted filesystem capability.
|
|
958
|
+
* @param planned - the job {@link planJobs} produced.
|
|
959
|
+
* @param pageDirName - the page directory's name under the base directory.
|
|
960
|
+
* @param signal - caller cancellation.
|
|
961
|
+
* @returns the job with its artifact paths and source metadata.
|
|
962
|
+
*/
|
|
963
|
+
async function prepareJob(fs, planned, pageDirName, signal) {
|
|
964
|
+
const target = await fs.resolve(planned.input);
|
|
965
|
+
const stamp = await sourceStamp(fs, target, signal);
|
|
966
|
+
planned.job.input = fs.processPath(target);
|
|
967
|
+
return {
|
|
968
|
+
...planned,
|
|
969
|
+
root: artifactRoot(planned.input, planned.base, pageDirName),
|
|
970
|
+
stamp
|
|
971
|
+
};
|
|
972
|
+
}
|
|
973
|
+
//#endregion
|
|
974
|
+
//#region lib/worker.js
|
|
975
|
+
/**
|
|
976
|
+
* Transport to the OCR worker process. The worker speaks newline-delimited
|
|
977
|
+
* JSON: one event per line on stdout, diagnostics on stderr. This module owns
|
|
978
|
+
* the process lifetime, the spec handoff, the timeout, and the cancellation, so
|
|
979
|
+
* the tool's entry module only has to interpret events.
|
|
980
|
+
* @module @deepseek-ai/dsh-ab-ocr/worker
|
|
981
|
+
*/
|
|
982
|
+
/**
|
|
983
|
+
* Split a run of text into whole lines, returning what is left of a partial one.
|
|
984
|
+
* @param buffered - text received so far, including any partial final line.
|
|
985
|
+
* @param chunk - newly received text.
|
|
986
|
+
* @returns the complete lines and the remaining partial line.
|
|
987
|
+
*/
|
|
988
|
+
function splitLines(buffered, chunk) {
|
|
989
|
+
const parts = (buffered + chunk).split("\n");
|
|
990
|
+
return {
|
|
991
|
+
lines: parts,
|
|
992
|
+
rest: parts.pop() ?? ""
|
|
993
|
+
};
|
|
994
|
+
}
|
|
995
|
+
/**
|
|
996
|
+
* Parse one event line, ignoring a line that is not a JSON object.
|
|
997
|
+
* @param line - one complete stdout line.
|
|
998
|
+
* @returns the parsed event, or undefined for a line that is not one.
|
|
999
|
+
*/
|
|
1000
|
+
function parseEvent(line) {
|
|
1001
|
+
const trimmed = line.trim();
|
|
1002
|
+
if (trimmed === "") return void 0;
|
|
1003
|
+
try {
|
|
1004
|
+
const value = JSON.parse(trimmed);
|
|
1005
|
+
if (typeof value !== "object" || value === null || typeof value.event !== "string") return;
|
|
1006
|
+
return value;
|
|
1007
|
+
} catch {
|
|
1008
|
+
return;
|
|
1009
|
+
}
|
|
1010
|
+
}
|
|
1011
|
+
/**
|
|
1012
|
+
* Run the worker once, feeding it a spec and reporting its events.
|
|
1013
|
+
* @param invocation - executable, script, and time ceiling.
|
|
1014
|
+
* @param args - extra command-line arguments.
|
|
1015
|
+
* @param input - text written to the worker's stdin, or an empty string to close it immediately.
|
|
1016
|
+
* @param onEvent - called with each parsed event as it arrives; never awaited, so a
|
|
1017
|
+
* slow consumer cannot stall the read.
|
|
1018
|
+
* @param signal - caller cancellation.
|
|
1019
|
+
* @returns the events, diagnostics, and exit state.
|
|
1020
|
+
*/
|
|
1021
|
+
function spawnWorker(invocation, args, input, onEvent, signal) {
|
|
1022
|
+
return new Promise((resolve) => {
|
|
1023
|
+
const child = spawn(invocation.python, [invocation.script, ...args], {
|
|
1024
|
+
stdio: [
|
|
1025
|
+
"pipe",
|
|
1026
|
+
"pipe",
|
|
1027
|
+
"pipe"
|
|
1028
|
+
],
|
|
1029
|
+
windowsHide: true
|
|
1030
|
+
});
|
|
1031
|
+
const events = [];
|
|
1032
|
+
let stdout = "";
|
|
1033
|
+
let stderr = "";
|
|
1034
|
+
let timedOut = false;
|
|
1035
|
+
let aborted = signal?.aborted === true;
|
|
1036
|
+
let settled = false;
|
|
1037
|
+
const timer = invocation.timeoutMs > 0 ? setTimeout(() => {
|
|
1038
|
+
timedOut = true;
|
|
1039
|
+
child.kill();
|
|
1040
|
+
}, invocation.timeoutMs) : void 0;
|
|
1041
|
+
const onAbort = () => {
|
|
1042
|
+
aborted = true;
|
|
1043
|
+
child.kill();
|
|
1044
|
+
};
|
|
1045
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
1046
|
+
const accept = (line) => {
|
|
1047
|
+
const event = parseEvent(line);
|
|
1048
|
+
if (event === void 0) return;
|
|
1049
|
+
events.push(event);
|
|
1050
|
+
onEvent(event);
|
|
1051
|
+
};
|
|
1052
|
+
const settle = (exitCode) => {
|
|
1053
|
+
if (settled) return;
|
|
1054
|
+
settled = true;
|
|
1055
|
+
if (timer !== void 0) clearTimeout(timer);
|
|
1056
|
+
signal?.removeEventListener("abort", onAbort);
|
|
1057
|
+
const tail = splitLines(stdout, "");
|
|
1058
|
+
for (const line of [...tail.lines, tail.rest]) accept(line);
|
|
1059
|
+
resolve({
|
|
1060
|
+
events,
|
|
1061
|
+
stderr,
|
|
1062
|
+
exitCode,
|
|
1063
|
+
timedOut,
|
|
1064
|
+
aborted
|
|
1065
|
+
});
|
|
1066
|
+
};
|
|
1067
|
+
child.stdout.setEncoding("utf8");
|
|
1068
|
+
child.stdout.on("data", (chunk) => {
|
|
1069
|
+
const split = splitLines(stdout, chunk);
|
|
1070
|
+
stdout = split.rest;
|
|
1071
|
+
for (const line of split.lines) accept(line);
|
|
1072
|
+
});
|
|
1073
|
+
child.stderr.setEncoding("utf8");
|
|
1074
|
+
child.stderr.on("data", (chunk) => {
|
|
1075
|
+
stderr = (stderr + chunk).slice(-4e3);
|
|
1076
|
+
});
|
|
1077
|
+
child.on("error", (error) => {
|
|
1078
|
+
stderr = (stderr + error.message).slice(-4e3);
|
|
1079
|
+
settle(null);
|
|
1080
|
+
});
|
|
1081
|
+
child.on("close", (code) => settle(code));
|
|
1082
|
+
child.stdin.on("error", () => {});
|
|
1083
|
+
child.stdin.end(input);
|
|
1084
|
+
});
|
|
1085
|
+
}
|
|
1086
|
+
/**
|
|
1087
|
+
* Ask the worker to report its environment.
|
|
1088
|
+
* @param invocation - executable, script, and the ceiling this check may take.
|
|
1089
|
+
* @param signal - caller cancellation.
|
|
1090
|
+
* @returns the report and the diagnostics, whether or not the worker started.
|
|
1091
|
+
*/
|
|
1092
|
+
async function runSelfTest(invocation, signal) {
|
|
1093
|
+
const run = await spawnWorker(invocation, ["--self-test"], "", () => {}, signal);
|
|
1094
|
+
return {
|
|
1095
|
+
report: run.events.find((event) => event.event === "ready"),
|
|
1096
|
+
stderr: run.stderr,
|
|
1097
|
+
exitCode: run.exitCode
|
|
1098
|
+
};
|
|
1099
|
+
}
|
|
1100
|
+
/**
|
|
1101
|
+
* Run one batch of documents through the worker.
|
|
1102
|
+
*
|
|
1103
|
+
* The event callback sees each event as it is parsed, which is what lets the
|
|
1104
|
+
* caller persist a finished page before the next one is recognized instead of
|
|
1105
|
+
* holding the whole document until the process exits.
|
|
1106
|
+
* @param invocation - executable, script, and time ceiling.
|
|
1107
|
+
* @param spec - the documents and options to process.
|
|
1108
|
+
* @param onEvent - called with each parsed event as it arrives.
|
|
1109
|
+
* @param signal - caller cancellation.
|
|
1110
|
+
* @returns the events, diagnostics, and exit state.
|
|
1111
|
+
*/
|
|
1112
|
+
function runSpec(invocation, spec, onEvent, signal) {
|
|
1113
|
+
return spawnWorker(invocation, [], JSON.stringify(spec), onEvent, signal);
|
|
1114
|
+
}
|
|
1115
|
+
//#endregion
|
|
1116
|
+
//#region lib/recognize.js
|
|
1117
|
+
/**
|
|
1118
|
+
* One `ocr` call: refuse an installation that cannot run, drive the worker
|
|
1119
|
+
* through the plan, write each page as it finishes, and merge the document only
|
|
1120
|
+
* after the last page.
|
|
1121
|
+
*
|
|
1122
|
+
* A later call carrying `levels` does not recognize anything: it reads the
|
|
1123
|
+
* record the recognition left behind and sends its stored geometry to the worker
|
|
1124
|
+
* with the caller's heading-level corrections, which rewrites the document at
|
|
1125
|
+
* the path the record named.
|
|
1126
|
+
*
|
|
1127
|
+
* The worker runs in its own execution world and writes only to stdout, so every
|
|
1128
|
+
* value this module persists travels back as an event and is written through the
|
|
1129
|
+
* mounted filesystem capability rather than by the worker.
|
|
1130
|
+
* @module @deepseek-ai/dsh-ab-ocr/recognize
|
|
1131
|
+
*/
|
|
1132
|
+
/** Package root, from which the worker script and its environment are resolved. */
|
|
1133
|
+
const PACKAGE_ROOT = fileURLToPath(new URL("..", import.meta.url));
|
|
1134
|
+
/** The tail of a worker's diagnostics, for an error a caller can act on. */
|
|
1135
|
+
function diagnostics(stderr) {
|
|
1136
|
+
const trimmed = stderr.trim().slice(-1200);
|
|
1137
|
+
return trimmed === "" ? "" : " Worker diagnostics: " + trimmed;
|
|
1138
|
+
}
|
|
1139
|
+
/**
|
|
1140
|
+
* Resolve the worker and refuse an installation that cannot run.
|
|
1141
|
+
*
|
|
1142
|
+
* The startup check is a separate process from the batch, so it carries its own
|
|
1143
|
+
* ceiling: an environment whose engine never loads has to be refused in the
|
|
1144
|
+
* deployment's startup window rather than held for the whole recognition
|
|
1145
|
+
* ceiling.
|
|
1146
|
+
* @param config - the deployment's values.
|
|
1147
|
+
* @param signal - caller cancellation.
|
|
1148
|
+
* @returns the invocation every call spawns.
|
|
1149
|
+
* @throws Error when the worker program is missing from the installation, or
|
|
1150
|
+
* when it starts without the OCR engine it needs.
|
|
1151
|
+
*/
|
|
1152
|
+
async function readyInvocation(config, signal) {
|
|
1153
|
+
const invocation = {
|
|
1154
|
+
python: resolvePython(config.pythonPath, PACKAGE_ROOT, process.platform),
|
|
1155
|
+
script: resolveWorkerScript(config.workerScript, PACKAGE_ROOT),
|
|
1156
|
+
timeoutMs: config.timeoutMs
|
|
1157
|
+
};
|
|
1158
|
+
if (!existsSync(invocation.script)) throw new Error("ocr cannot run: the OCR engine is not installed on this machine, because the worker program is missing from the plugin. The plugin's setup script installs it.");
|
|
1159
|
+
const selfTest = await runSelfTest({
|
|
1160
|
+
...invocation,
|
|
1161
|
+
timeoutMs: config.startupTimeoutMs
|
|
1162
|
+
}, signal);
|
|
1163
|
+
if (selfTest.report?.ready !== true) {
|
|
1164
|
+
const detail = selfTest.report === void 0 ? "its worker did not start" : "its worker started without the engine";
|
|
1165
|
+
throw new Error("ocr cannot run: the OCR engine is not installed on this machine, because " + detail + ". The plugin's setup script installs it." + diagnostics(selfTest.stderr));
|
|
1166
|
+
}
|
|
1167
|
+
return invocation;
|
|
1168
|
+
}
|
|
1169
|
+
/**
|
|
1170
|
+
* Run one batch through the worker and report what it produced.
|
|
1171
|
+
* @param invocation - the worker to run.
|
|
1172
|
+
* @param config - the deployment's engine lifetime and time ceiling.
|
|
1173
|
+
* @param jobs - the jobs to run, in order.
|
|
1174
|
+
* @param call - the documents and failures the call collects.
|
|
1175
|
+
* @param apply - called with each parsed event as it arrives.
|
|
1176
|
+
* @param signal - caller cancellation.
|
|
1177
|
+
* @returns the canonical value the call records.
|
|
1178
|
+
* @throws Error when the batch was cancelled, stopped early, lost a document, or
|
|
1179
|
+
* could not save one.
|
|
1180
|
+
*/
|
|
1181
|
+
async function runCall(invocation, config, jobs, call, apply, signal) {
|
|
1182
|
+
const forward = () => call.controller.abort();
|
|
1183
|
+
signal?.addEventListener("abort", forward, { once: true });
|
|
1184
|
+
const spec = {
|
|
1185
|
+
engineLifetime: config.engineLifetime,
|
|
1186
|
+
jobs
|
|
1187
|
+
};
|
|
1188
|
+
let run;
|
|
1189
|
+
try {
|
|
1190
|
+
run = await runSpec(invocation, spec, apply, call.controller.signal);
|
|
1191
|
+
} finally {
|
|
1192
|
+
signal?.removeEventListener("abort", forward);
|
|
1193
|
+
}
|
|
1194
|
+
const writeError = await call.queue.settle();
|
|
1195
|
+
if (call.problem !== void 0) throw call.problem;
|
|
1196
|
+
if (writeError !== void 0) throw new Error("ocr could not save its output: " + writeError.message);
|
|
1197
|
+
const end = run.events.find((event) => event.event === "end");
|
|
1198
|
+
if (end === void 0) {
|
|
1199
|
+
const fatal = run.events.find((event) => event.event === "fatal");
|
|
1200
|
+
if (fatal !== void 0) throw new Error("ocr did not start: " + String(fatal.error ?? "no reason was reported"));
|
|
1201
|
+
if (run.aborted) throw new Error("ocr was cancelled");
|
|
1202
|
+
if (run.timedOut) throw new Error("ocr timed out after " + config.timeoutMs + " ms; recognize fewer pages in one call" + diagnostics(run.stderr));
|
|
1203
|
+
throw new Error("ocr did not finish: the recognition stopped without a result" + diagnostics(run.stderr));
|
|
1204
|
+
}
|
|
1205
|
+
if (Number(end.documents) !== call.finished) throw new Error("ocr lost a document: the worker finished " + call.finished + " of the " + String(end.documents) + " it reported");
|
|
1206
|
+
for (const failure of readFailures(end.failures)) reportFailure(call, failure);
|
|
1207
|
+
return {
|
|
1208
|
+
documents: call.documents,
|
|
1209
|
+
failures: call.failures
|
|
1210
|
+
};
|
|
1211
|
+
}
|
|
1212
|
+
/**
|
|
1213
|
+
* Re-merge a stored recognition with heading-level corrections, recognizing nothing.
|
|
1214
|
+
* @param config - the deployment's worker values.
|
|
1215
|
+
* @param input - the source path the worker names in its events.
|
|
1216
|
+
* @param geometry - the stored page geometry to merge.
|
|
1217
|
+
* @param merge - the merge options, carrying the corrections to apply.
|
|
1218
|
+
* @param signal - caller cancellation.
|
|
1219
|
+
* @returns the merged document, or undefined when the worker reported a failure.
|
|
1220
|
+
*/
|
|
1221
|
+
async function assembleStored(config, input, geometry, merge, signal) {
|
|
1222
|
+
const job = {
|
|
1223
|
+
mode: "assemble",
|
|
1224
|
+
id: "1",
|
|
1225
|
+
input,
|
|
1226
|
+
record: geometry,
|
|
1227
|
+
merge
|
|
1228
|
+
};
|
|
1229
|
+
const controller = new AbortController();
|
|
1230
|
+
const call = {
|
|
1231
|
+
documents: [],
|
|
1232
|
+
failures: [],
|
|
1233
|
+
reported: /* @__PURE__ */ new Set(),
|
|
1234
|
+
controller,
|
|
1235
|
+
queue: writeQueue(controller),
|
|
1236
|
+
finished: 0
|
|
1237
|
+
};
|
|
1238
|
+
const captured = {};
|
|
1239
|
+
const collect = (event) => {
|
|
1240
|
+
const done = readDoneEvent(event);
|
|
1241
|
+
if (done === void 0) return;
|
|
1242
|
+
call.finished += 1;
|
|
1243
|
+
captured.merged = {
|
|
1244
|
+
markdown: done.markdown,
|
|
1245
|
+
outline: readOutline(event.outline),
|
|
1246
|
+
rawOutline: event.outline,
|
|
1247
|
+
truncated: event.outlineTruncated === true,
|
|
1248
|
+
done
|
|
1249
|
+
};
|
|
1250
|
+
};
|
|
1251
|
+
let assembled;
|
|
1252
|
+
try {
|
|
1253
|
+
assembled = await runCall(await readyInvocation(config, signal), config, [job], call, collect, signal);
|
|
1254
|
+
} catch {
|
|
1255
|
+
return;
|
|
1256
|
+
}
|
|
1257
|
+
if (assembled.failures.length > 0) return void 0;
|
|
1258
|
+
return captured.merged;
|
|
1259
|
+
}
|
|
1260
|
+
/**
|
|
1261
|
+
* Rewrite one recognized document with the heading levels the caller decided.
|
|
1262
|
+
*
|
|
1263
|
+
* No page is recognized again: the call reads the record the recognition left
|
|
1264
|
+
* behind and sends its stored geometry to the worker with the caller's level
|
|
1265
|
+
* corrections, then writes the assembled Markdown over the file the record
|
|
1266
|
+
* named. The digest is the recognition's, so this call finds the same record
|
|
1267
|
+
* without scanning.
|
|
1268
|
+
* @param fs - the mounted filesystem capability, or undefined when the
|
|
1269
|
+
* composition mounts none.
|
|
1270
|
+
* @param args - the model-supplied call arguments, carrying `levels`.
|
|
1271
|
+
* @param config - the deployment's worker, render, and merge values.
|
|
1272
|
+
* @param base - the directory a relative path resolves against.
|
|
1273
|
+
* @param signal - caller cancellation.
|
|
1274
|
+
* @param fence - the per-call fence a confining backend judges the rewritten document and its records by; absent on a bare backend.
|
|
1275
|
+
* @returns the canonical value the call records.
|
|
1276
|
+
* @throws Error when the request is malformed, no filesystem is mounted, the
|
|
1277
|
+
* environment is unusable, or the stored record is missing.
|
|
1278
|
+
*/
|
|
1279
|
+
async function refine(fs, args, config, base, signal, fence) {
|
|
1280
|
+
const overrides = parseLevels(args.levels ?? "");
|
|
1281
|
+
const paths = requestPaths(args, base, config.maxDocuments);
|
|
1282
|
+
const input = paths.length === 1 ? paths[0] : void 0;
|
|
1283
|
+
if (input === void 0) throw new Error("invalid ocr: levels corrects one document, so name exactly one path");
|
|
1284
|
+
if (fs === void 0) throw new Error("ocr cannot save its result: no filesystem is available to write to");
|
|
1285
|
+
const baseDir = artifactBase(input, requestedOutputDir(args, config, base));
|
|
1286
|
+
const latest = await readLatest(fs, artifactRoot(input, baseDir, config.pageDirName).latest, signal);
|
|
1287
|
+
const artifacts = artifactPaths(input, baseDir, config.pageDirName, latest.digest);
|
|
1288
|
+
const record = await readRecord(fs, artifacts.record, signal);
|
|
1289
|
+
const invocation = await readyInvocation(config, signal);
|
|
1290
|
+
const job = {
|
|
1291
|
+
mode: "assemble",
|
|
1292
|
+
id: "1",
|
|
1293
|
+
input: fs.processPath(await fs.resolve(input)),
|
|
1294
|
+
record: record.geometry,
|
|
1295
|
+
merge: {
|
|
1296
|
+
...record.merge,
|
|
1297
|
+
levelOverrides: overrides
|
|
1298
|
+
}
|
|
1299
|
+
};
|
|
1300
|
+
const controller = new AbortController();
|
|
1301
|
+
const call = {
|
|
1302
|
+
documents: [],
|
|
1303
|
+
failures: [],
|
|
1304
|
+
reported: /* @__PURE__ */ new Set(),
|
|
1305
|
+
controller,
|
|
1306
|
+
queue: writeQueue(controller),
|
|
1307
|
+
finished: 0
|
|
1308
|
+
};
|
|
1309
|
+
const corrections = Object.keys(overrides).length;
|
|
1310
|
+
const apply = (event) => {
|
|
1311
|
+
if (event.event === "error") {
|
|
1312
|
+
reportFailure(call, {
|
|
1313
|
+
id: String(event.id ?? ""),
|
|
1314
|
+
input,
|
|
1315
|
+
error: String(event.message ?? "")
|
|
1316
|
+
});
|
|
1317
|
+
return;
|
|
1318
|
+
}
|
|
1319
|
+
const done = readDoneEvent(event);
|
|
1320
|
+
if (done === void 0) return;
|
|
1321
|
+
call.finished += 1;
|
|
1322
|
+
const truncated = event.outlineTruncated === true;
|
|
1323
|
+
call.queue.enqueue(async () => {
|
|
1324
|
+
await saveText(fs, await fs.resolve(latest.output), done.markdown, fence, signal);
|
|
1325
|
+
await writeJson(fs, artifacts.record, {
|
|
1326
|
+
...record,
|
|
1327
|
+
merge: {
|
|
1328
|
+
...record.merge,
|
|
1329
|
+
levelOverrides: overrides
|
|
1330
|
+
}
|
|
1331
|
+
}, signal, fence);
|
|
1332
|
+
if (Array.isArray(event.outline)) await writeJson(fs, artifacts.outline, appliedOutline(readOutline(event.outline), overrides), signal, fence);
|
|
1333
|
+
call.documents.push({
|
|
1334
|
+
input,
|
|
1335
|
+
output: latest.output,
|
|
1336
|
+
pages: latest.pages.length > 0 ? latest.pages.length : done.pages,
|
|
1337
|
+
totalPages: done.totalPages > 0 ? done.totalPages : latest.totalPages,
|
|
1338
|
+
lines: done.lines,
|
|
1339
|
+
headings: done.headings,
|
|
1340
|
+
chars: done.chars,
|
|
1341
|
+
droppedPageNumbers: done.droppedPageNumbers,
|
|
1342
|
+
droppedRunningHeads: done.droppedRunningHeads,
|
|
1343
|
+
joinedAcrossPages: done.joinedAcrossPages,
|
|
1344
|
+
pageDir: "",
|
|
1345
|
+
seconds: done.seconds,
|
|
1346
|
+
digest: latest.digest,
|
|
1347
|
+
selection: [...latest.pages],
|
|
1348
|
+
outlinePath: artifacts.outline,
|
|
1349
|
+
outlineTruncated: truncated,
|
|
1350
|
+
corrections
|
|
1351
|
+
});
|
|
1352
|
+
});
|
|
1353
|
+
};
|
|
1354
|
+
return runCall(invocation, config, [job], call, apply, signal);
|
|
1355
|
+
}
|
|
1356
|
+
/**
|
|
1357
|
+
* Read the documents one call asked for, writing every artifact through the
|
|
1358
|
+
* filesystem capability the composition mounted.
|
|
1359
|
+
* @param fs - the mounted filesystem capability every write goes through, or
|
|
1360
|
+
* undefined when the composition mounts none.
|
|
1361
|
+
* @param args - the model-supplied call arguments.
|
|
1362
|
+
* @param config - the deployment's values.
|
|
1363
|
+
* @param base - the directory a relative path resolves against.
|
|
1364
|
+
* @param signal - caller cancellation.
|
|
1365
|
+
* @param fence - the fence every write is stamped with; absent on a bare backend.
|
|
1366
|
+
* @returns the canonical value the call records.
|
|
1367
|
+
* @throws Error when the request is malformed, no filesystem is mounted, the
|
|
1368
|
+
* environment is unusable, the deployment denies a write, the worker reports
|
|
1369
|
+
* pages out of order, or the worker fails.
|
|
1370
|
+
*/
|
|
1371
|
+
async function recognize(fs, args, config, base, signal, fence) {
|
|
1372
|
+
if (args.levels !== void 0) return refine(fs, args, config, base, signal, fence);
|
|
1373
|
+
const planned = planJobs(requestPaths(args, base, config.maxDocuments), args, config, base);
|
|
1374
|
+
if (planned.jobs.length === 0) return {
|
|
1375
|
+
documents: [],
|
|
1376
|
+
failures: planned.failures
|
|
1377
|
+
};
|
|
1378
|
+
if (fs === void 0) throw new Error("ocr cannot save its result: no filesystem is available to write to");
|
|
1379
|
+
const invocation = await readyInvocation(config, signal);
|
|
1380
|
+
const states = /* @__PURE__ */ new Map();
|
|
1381
|
+
for (const job of planned.jobs) {
|
|
1382
|
+
const prepared = await prepareJob(fs, job, config.pageDirName, signal);
|
|
1383
|
+
states.set(String(prepared.job.id), documentState(prepared));
|
|
1384
|
+
}
|
|
1385
|
+
const controller = new AbortController();
|
|
1386
|
+
const call = {
|
|
1387
|
+
documents: [],
|
|
1388
|
+
failures: [...planned.failures],
|
|
1389
|
+
reported: /* @__PURE__ */ new Set(),
|
|
1390
|
+
controller,
|
|
1391
|
+
queue: writeQueue(controller),
|
|
1392
|
+
finished: 0
|
|
1393
|
+
};
|
|
1394
|
+
/** Report one protocol violation, which aborts the rest of the batch. */
|
|
1395
|
+
const violate = (problem) => {
|
|
1396
|
+
if (call.problem === void 0) call.problem = /* @__PURE__ */ new Error("ocr failed: " + problem);
|
|
1397
|
+
controller.abort();
|
|
1398
|
+
};
|
|
1399
|
+
const apply = (event) => {
|
|
1400
|
+
const state = states.get(String(event.id));
|
|
1401
|
+
if (state === void 0) return;
|
|
1402
|
+
if (event.event === "start") {
|
|
1403
|
+
state.selection = wholeNumbers(event.pages);
|
|
1404
|
+
state.totalPages = wholeNumber(event.totalPages) ?? 0;
|
|
1405
|
+
state.artifacts = artifactPaths(state.job.input, state.job.base, config.pageDirName, digest12(state.job.input, state.job.stamp, String(event.sourceDigest ?? ""), state.job.job.pages ?? null, state.job.job.merge));
|
|
1406
|
+
return;
|
|
1407
|
+
}
|
|
1408
|
+
if (event.event === "page") {
|
|
1409
|
+
const page = wholeNumber(event.page);
|
|
1410
|
+
if (page === void 0) {
|
|
1411
|
+
violate("page " + JSON.stringify(event.page ?? null) + " is not a page number");
|
|
1412
|
+
return;
|
|
1413
|
+
}
|
|
1414
|
+
const problem = pageProblem(page, state);
|
|
1415
|
+
if (problem !== void 0) {
|
|
1416
|
+
violate(problem);
|
|
1417
|
+
return;
|
|
1418
|
+
}
|
|
1419
|
+
const geometry = readGeometry(event.geometry);
|
|
1420
|
+
if (geometry === void 0) {
|
|
1421
|
+
violate("page " + page + " arrived without the geometry its record needs");
|
|
1422
|
+
return;
|
|
1423
|
+
}
|
|
1424
|
+
const artifacts = state.artifacts;
|
|
1425
|
+
if (artifacts === void 0) {
|
|
1426
|
+
violate("page " + page + " arrived before the document named its source");
|
|
1427
|
+
return;
|
|
1428
|
+
}
|
|
1429
|
+
state.lastPage = page;
|
|
1430
|
+
state.geometry.push(geometry);
|
|
1431
|
+
if (!config.writePageFiles) return;
|
|
1432
|
+
const name = pageFileName(page);
|
|
1433
|
+
const text = String(event.text ?? "");
|
|
1434
|
+
call.queue.enqueue(async () => {
|
|
1435
|
+
await saveText(fs, await fs.resolve(join(artifacts.recordDir, name)), text === "" ? "\n" : text + "\n", fence, signal);
|
|
1436
|
+
});
|
|
1437
|
+
return;
|
|
1438
|
+
}
|
|
1439
|
+
if (event.event === "error") {
|
|
1440
|
+
reportFailure(call, {
|
|
1441
|
+
id: String(event.id ?? ""),
|
|
1442
|
+
input: state.job.input,
|
|
1443
|
+
error: String(event.message ?? "")
|
|
1444
|
+
});
|
|
1445
|
+
return;
|
|
1446
|
+
}
|
|
1447
|
+
const done = readDoneEvent(event);
|
|
1448
|
+
if (done === void 0) return;
|
|
1449
|
+
call.finished += 1;
|
|
1450
|
+
const artifacts = state.artifacts;
|
|
1451
|
+
if (artifacts === void 0) {
|
|
1452
|
+
violate("document " + state.job.input + " finished without naming its source");
|
|
1453
|
+
return;
|
|
1454
|
+
}
|
|
1455
|
+
const truncated = event.outlineTruncated === true;
|
|
1456
|
+
call.queue.enqueue(async () => {
|
|
1457
|
+
const carried = await carriedLevels(fs, artifacts.record, signal);
|
|
1458
|
+
const merged = carried === void 0 ? void 0 : await assembleStored(config, state.job.job.input, { pages: state.geometry }, {
|
|
1459
|
+
...state.job.job.merge,
|
|
1460
|
+
levelOverrides: carried
|
|
1461
|
+
}, signal);
|
|
1462
|
+
const output = await writeDocument(fs, artifacts, state, merged?.markdown ?? done.markdown, merged?.outline ?? readOutline(event.outline), merged?.rawOutline ?? event.outline, carried, signal, fence);
|
|
1463
|
+
call.documents.push(recognizedDocument(state, artifacts, merged?.done ?? done, output, config.writePageFiles ? artifacts.recordDir : "", merged?.truncated ?? truncated, carried === void 0 ? 0 : Object.keys(carried).length));
|
|
1464
|
+
});
|
|
1465
|
+
};
|
|
1466
|
+
return runCall(invocation, config, [...states.values()].map((state) => state.job.job), call, apply, signal);
|
|
1467
|
+
}
|
|
1468
|
+
//#endregion
|
|
1469
|
+
//#region lib/config.js
|
|
1470
|
+
/**
|
|
1471
|
+
* Deployment configuration of the ocr tool.
|
|
1472
|
+
*
|
|
1473
|
+
* The fields fall into two groups. The first is a bound or a cost policy the
|
|
1474
|
+
* deployment owns: the render resolution, the per-page pixel ceiling, the page,
|
|
1475
|
+
* file, call, and startup ceilings, the outline candidate ceiling, the engine
|
|
1476
|
+
* lifetime, and whether per-page text is kept at all. None of them carries a
|
|
1477
|
+
* default, because a hidden one would state an agreement the deployment never
|
|
1478
|
+
* made; a row that omits one fails to load and names the field.
|
|
1479
|
+
*
|
|
1480
|
+
* The second group is a resolution rule or a recognition heuristic — where the
|
|
1481
|
+
* interpreter, the worker, and the artifacts are found, and how a page is read
|
|
1482
|
+
* and merged. Their defaults are what a document reader wants, they are stated
|
|
1483
|
+
* here rather than computed anywhere, and a deployment overrides the ones its
|
|
1484
|
+
* documents disagree with.
|
|
1485
|
+
* @module @deepseek-ai/dsh-ab-ocr/config
|
|
1486
|
+
*/
|
|
1487
|
+
/**
|
|
1488
|
+
* Schemastery configuration for the tool. A bound or a cost policy the
|
|
1489
|
+
* deployment owns is `.required()`, so the row states what this deployment is
|
|
1490
|
+
* willing to carry; a resolution rule or a recognition heuristic keeps its
|
|
1491
|
+
* documented default.
|
|
1492
|
+
*/
|
|
1493
|
+
const Config = z.object({
|
|
1494
|
+
pythonPath: z.string().default(""),
|
|
1495
|
+
workerScript: z.string().default(""),
|
|
1496
|
+
outputDir: z.string().default(""),
|
|
1497
|
+
pageDirName: z.string().default(".ocr-pages"),
|
|
1498
|
+
writePageFiles: z.boolean().required(),
|
|
1499
|
+
dpi: z.number().step(1).min(72).max(600).required(),
|
|
1500
|
+
maxPixels: z.number().step(1).min(0).required(),
|
|
1501
|
+
textScore: z.number().min(0).max(1).default(.5),
|
|
1502
|
+
detectHeadings: z.boolean().default(true),
|
|
1503
|
+
headingMinRatio: z.number().min(1).max(4).default(1.18),
|
|
1504
|
+
indentRatio: z.number().min(0).max(4).default(1),
|
|
1505
|
+
paragraphGapRatio: z.number().min(0).max(4).default(.85),
|
|
1506
|
+
outlineCandidateRatio: z.number().min(1).max(4).default(1.05),
|
|
1507
|
+
maxOutlineCandidates: z.number().step(1).min(1).max(1e4).required(),
|
|
1508
|
+
detectColumns: z.boolean().default(true),
|
|
1509
|
+
removePageNumbers: z.boolean().default(true),
|
|
1510
|
+
removeRunningHeads: z.boolean().default(true),
|
|
1511
|
+
runningHeadRatio: z.number().min(0).max(1).default(.6),
|
|
1512
|
+
runningHeadMinPages: z.number().step(1).min(2).default(3),
|
|
1513
|
+
engineLifetime: z.union(["perDocument", "shared"]).required(),
|
|
1514
|
+
timeoutMs: z.number().step(1).min(1e3).required(),
|
|
1515
|
+
startupTimeoutMs: z.number().step(1).min(1e3).required(),
|
|
1516
|
+
maxPages: z.number().step(1).min(0).required(),
|
|
1517
|
+
maxDocuments: z.number().step(1).min(1).max(200).required()
|
|
1518
|
+
});
|
|
1519
|
+
//#endregion
|
|
1520
|
+
//#region lib/index.js
|
|
1521
|
+
/**
|
|
1522
|
+
* Model-facing `ocr` tool. One call recognizes a PDF or an image and saves the
|
|
1523
|
+
* text as a Markdown file named after the source with its spaces removed.
|
|
1524
|
+
*
|
|
1525
|
+
* The document is processed one page at a time: the worker recognizes a page,
|
|
1526
|
+
* reports its text and its geometry, and releases its pixels before rendering
|
|
1527
|
+
* the next. The page's text is written to its own file as each page arrives, so
|
|
1528
|
+
* an interrupted run keeps every page it finished. Only after the last page does
|
|
1529
|
+
* the worker merge its records —dropping the folios and running heads,
|
|
1530
|
+
* rebuilding the outline from the recognized glyph sizes and numbering, and
|
|
1531
|
+
* stitching the paragraphs a page break cut in half —and reports the one
|
|
1532
|
+
* Markdown document, which is written last, beside the record an assembly pass
|
|
1533
|
+
* reads back. A later call carrying `levels` sends that stored geometry to the
|
|
1534
|
+
* worker again with the caller's heading-level corrections and rewrites the
|
|
1535
|
+
* document without recognizing it a second time.
|
|
1536
|
+
*
|
|
1537
|
+
* This module owns the plugin's identity, its deployment configuration, its
|
|
1538
|
+
* routing section, and its tool registration. The work behind them lives in
|
|
1539
|
+
* [`plan.ts`](./plan.ts) (what one call asks for), [`recognize.ts`](./recognize.ts)
|
|
1540
|
+
* (driving the worker and writing the artifacts), [`documents.ts`](./documents.ts)
|
|
1541
|
+
* (the artifact files of one recognition), and [`events.ts`](./events.ts) (the
|
|
1542
|
+
* worker's event stream), with the pure leaf modules beside them.
|
|
1543
|
+
*
|
|
1544
|
+
* Every file is written through the filesystem capability rather than the host
|
|
1545
|
+
* filesystem, so a deployment's file policy sees the write and its own backend
|
|
1546
|
+
* decides how the bytes are published. That capability is read with `ctx.get`
|
|
1547
|
+
* when a call runs — and so is the sandbox-policy service, whose resolved
|
|
1548
|
+
* per-call policy is the only thing naming the workspace the calling session
|
|
1549
|
+
* runs in. Both are looked up rather than declared in `inject`, for the same
|
|
1550
|
+
* reason: a declared injection waits for the service, which would make mounting
|
|
1551
|
+
* this row require providers of its own. The policy is resolved once per call
|
|
1552
|
+
* and stamped onto every write, and a composition that cannot resolve one is
|
|
1553
|
+
* refused at the call rather than writing under the deployment's fallback root.
|
|
1554
|
+
*
|
|
1555
|
+
* The tool reads no session state beyond the working directory it resolves
|
|
1556
|
+
* relative paths against and returns the canonical value alone, so model-facing
|
|
1557
|
+
* text and any replay read the same recorded call.
|
|
1558
|
+
* @module @deepseek-ai/dsh-ab-ocr
|
|
1559
|
+
*/
|
|
1560
|
+
const name = "ocr";
|
|
1561
|
+
const inject = ["tools", "systemPrompt"];
|
|
1562
|
+
/** Wire name this plugin registers. */
|
|
1563
|
+
const TOOL_NAME = "ocr";
|
|
1564
|
+
/**
|
|
1565
|
+
* Name and order of the routing section this plugin contributes. Repository-owned
|
|
1566
|
+
* tool sections allocate their order centrally; an external contribution states
|
|
1567
|
+
* its own finite order and sits after the built-in tool band (TOOL_REPORT, 2900).
|
|
1568
|
+
*/
|
|
1569
|
+
const SECTION_NAME = "tool:ocr";
|
|
1570
|
+
/** Order of {@link SECTION_NAME}, after the built-in tool sections. */
|
|
1571
|
+
const SECTION_ORDER = 2975;
|
|
1572
|
+
const DESCRIPTION = [
|
|
1573
|
+
"Recognize the text of a scanned PDF or an image into one Markdown document.",
|
|
1574
|
+
"Name the file with `path` or `paths`, optionally restrict `pages` and choose `outputDir`.",
|
|
1575
|
+
"The call saves the document and an outline file, and reports both paths and the page, line, and heading counts."
|
|
1576
|
+
].join(" ");
|
|
1577
|
+
/** Model-facing routing rule: when to make a call, which a schema cannot carry. */
|
|
1578
|
+
const SECTION_TEXT = [
|
|
1579
|
+
"When the reader needs the text of a scanned PDF or an image, answer with `ocr`: it recognizes every page and saves a Markdown document plus an outline file, reporting both paths.",
|
|
1580
|
+
"When the heading levels matter, read the outline file and call `ocr` again with the same `path` and a `levels` value such as \"h12=2,h15=0\"; that rewrites the document without recognizing it again.",
|
|
1581
|
+
"Pass the document to `present` when the reader should keep it."
|
|
1582
|
+
].join(" ");
|
|
1583
|
+
/**
|
|
1584
|
+
* The pending-call presentation of one ocr call: a titled row carrying the
|
|
1585
|
+
* source path. Exported so the test drives it without a registry.
|
|
1586
|
+
* @param args - the model-supplied call arguments.
|
|
1587
|
+
* @returns the pending generic card for this call.
|
|
1588
|
+
*/
|
|
1589
|
+
function presentCall(args) {
|
|
1590
|
+
return {
|
|
1591
|
+
card: "generic",
|
|
1592
|
+
title: "Recognize document text",
|
|
1593
|
+
kind: "read",
|
|
1594
|
+
rawInput: args.path ?? (args.paths ?? []).join(", ")
|
|
1595
|
+
};
|
|
1596
|
+
}
|
|
1597
|
+
/**
|
|
1598
|
+
* Contribute the routing section and register the ocr tool. The section is
|
|
1599
|
+
* empty wherever the tool is not visible in that scope, so a restricted
|
|
1600
|
+
* composition is not told to call a hidden tool.
|
|
1601
|
+
* @param ctx - registrant context carrying the tool, system-prompt, and filesystem registries.
|
|
1602
|
+
* @param config - the deployment's worker, render, and merge values.
|
|
1603
|
+
*/
|
|
1604
|
+
function apply(ctx, config) {
|
|
1605
|
+
ctx.systemPrompt.section({
|
|
1606
|
+
name: SECTION_NAME,
|
|
1607
|
+
order: SECTION_ORDER,
|
|
1608
|
+
text: ({ scope }) => ctx.tools.get("ocr", scope) === void 0 ? "" : SECTION_TEXT
|
|
1609
|
+
});
|
|
1610
|
+
ctx.tools.register(defineTool({
|
|
1611
|
+
name: "ocr",
|
|
1612
|
+
description: DESCRIPTION,
|
|
1613
|
+
parameters: {
|
|
1614
|
+
path: {
|
|
1615
|
+
type: "string",
|
|
1616
|
+
description: "One file to recognize: a PDF, or an image such as PNG, JPEG, BMP, TIFF, or WebP."
|
|
1617
|
+
},
|
|
1618
|
+
paths: {
|
|
1619
|
+
type: "array",
|
|
1620
|
+
items: { type: "string" },
|
|
1621
|
+
description: "Several files to recognize in one call."
|
|
1622
|
+
},
|
|
1623
|
+
pages: {
|
|
1624
|
+
type: "string",
|
|
1625
|
+
description: "Pages to recognize, counted from one, as single pages and inclusive ranges, for example \"1-5,8\". Omit for every page."
|
|
1626
|
+
},
|
|
1627
|
+
outputDir: {
|
|
1628
|
+
type: "string",
|
|
1629
|
+
description: "Directory for the Markdown files, relative to the working directory. Omit to write beside each source file."
|
|
1630
|
+
},
|
|
1631
|
+
levels: {
|
|
1632
|
+
type: "string",
|
|
1633
|
+
description: "Heading levels to correct in a document an earlier call recognized: \"h<id>=<level>\" pairs from its outline file, such as \"h12=2,h15=0\". A level of 1 to 6 makes that line a heading at that depth; 0 leaves it as body text, which demotes a line the recognition promoted. The call rewrites the saved Markdown and recognizes nothing."
|
|
1634
|
+
}
|
|
1635
|
+
},
|
|
1636
|
+
output: {
|
|
1637
|
+
schema: {
|
|
1638
|
+
type: "object",
|
|
1639
|
+
additionalProperties: false,
|
|
1640
|
+
properties: {
|
|
1641
|
+
documents: {
|
|
1642
|
+
type: "array",
|
|
1643
|
+
required: true,
|
|
1644
|
+
items: {
|
|
1645
|
+
type: "object",
|
|
1646
|
+
additionalProperties: false,
|
|
1647
|
+
properties: {
|
|
1648
|
+
input: {
|
|
1649
|
+
type: "string",
|
|
1650
|
+
required: true
|
|
1651
|
+
},
|
|
1652
|
+
output: {
|
|
1653
|
+
type: "string",
|
|
1654
|
+
required: true
|
|
1655
|
+
},
|
|
1656
|
+
pages: {
|
|
1657
|
+
type: "number",
|
|
1658
|
+
required: true
|
|
1659
|
+
},
|
|
1660
|
+
totalPages: {
|
|
1661
|
+
type: "number",
|
|
1662
|
+
required: true
|
|
1663
|
+
},
|
|
1664
|
+
lines: {
|
|
1665
|
+
type: "number",
|
|
1666
|
+
required: true
|
|
1667
|
+
},
|
|
1668
|
+
headings: {
|
|
1669
|
+
type: "number",
|
|
1670
|
+
required: true
|
|
1671
|
+
},
|
|
1672
|
+
chars: {
|
|
1673
|
+
type: "number",
|
|
1674
|
+
required: true
|
|
1675
|
+
},
|
|
1676
|
+
droppedPageNumbers: {
|
|
1677
|
+
type: "number",
|
|
1678
|
+
required: true
|
|
1679
|
+
},
|
|
1680
|
+
droppedRunningHeads: {
|
|
1681
|
+
type: "number",
|
|
1682
|
+
required: true
|
|
1683
|
+
},
|
|
1684
|
+
joinedAcrossPages: {
|
|
1685
|
+
type: "number",
|
|
1686
|
+
required: true
|
|
1687
|
+
},
|
|
1688
|
+
pageDir: {
|
|
1689
|
+
type: "string",
|
|
1690
|
+
required: true
|
|
1691
|
+
},
|
|
1692
|
+
seconds: {
|
|
1693
|
+
type: "number",
|
|
1694
|
+
required: true
|
|
1695
|
+
},
|
|
1696
|
+
digest: {
|
|
1697
|
+
type: "string",
|
|
1698
|
+
required: true
|
|
1699
|
+
},
|
|
1700
|
+
selection: {
|
|
1701
|
+
type: "array",
|
|
1702
|
+
required: true,
|
|
1703
|
+
items: { type: "number" }
|
|
1704
|
+
},
|
|
1705
|
+
outlinePath: {
|
|
1706
|
+
type: "string",
|
|
1707
|
+
required: true
|
|
1708
|
+
},
|
|
1709
|
+
outlineTruncated: {
|
|
1710
|
+
type: "boolean",
|
|
1711
|
+
required: true
|
|
1712
|
+
},
|
|
1713
|
+
corrections: {
|
|
1714
|
+
type: "number",
|
|
1715
|
+
required: true
|
|
1716
|
+
}
|
|
1717
|
+
}
|
|
1718
|
+
}
|
|
1719
|
+
},
|
|
1720
|
+
failures: {
|
|
1721
|
+
type: "array",
|
|
1722
|
+
required: true,
|
|
1723
|
+
items: {
|
|
1724
|
+
type: "object",
|
|
1725
|
+
additionalProperties: false,
|
|
1726
|
+
properties: {
|
|
1727
|
+
input: {
|
|
1728
|
+
type: "string",
|
|
1729
|
+
required: true
|
|
1730
|
+
},
|
|
1731
|
+
error: {
|
|
1732
|
+
type: "string",
|
|
1733
|
+
required: true
|
|
1734
|
+
}
|
|
1735
|
+
}
|
|
1736
|
+
}
|
|
1737
|
+
}
|
|
1738
|
+
}
|
|
1739
|
+
},
|
|
1740
|
+
render: (_args, value) => [{
|
|
1741
|
+
type: "text",
|
|
1742
|
+
text: renderOcr(value)
|
|
1743
|
+
}],
|
|
1744
|
+
presentationMeta: (_args, value) => ({ documents: value.documents.map((document) => ({
|
|
1745
|
+
output: document.output,
|
|
1746
|
+
pages: document.pages,
|
|
1747
|
+
totalPages: document.totalPages,
|
|
1748
|
+
chars: document.chars,
|
|
1749
|
+
headings: document.headings,
|
|
1750
|
+
digest: document.digest
|
|
1751
|
+
})) })
|
|
1752
|
+
},
|
|
1753
|
+
execute(args, exec) {
|
|
1754
|
+
const fence = callFence(ctx, exec.agent === void 0 ? void 0 : { session: exec.agent.session });
|
|
1755
|
+
return recognize(ctx.get("fs"), args, config, fence?.workspaceRoot ?? exec.agent?.session?.header?.cwd ?? process.cwd(), exec.signal, fence);
|
|
1756
|
+
},
|
|
1757
|
+
presentCall
|
|
1758
|
+
}));
|
|
1759
|
+
}
|
|
1760
|
+
//#endregion
|
|
1761
|
+
export { Config, SECTION_NAME, SECTION_ORDER, SECTION_TEXT, SUPPORTED_EXTENSIONS, TOOL_NAME, appliedOutline, apply, artifactPaths, callFence, digest12, inject, markdownDigestFileName, markdownFileName, markdownPath, markdownStem, name, pageFileName, parseEvent, parseLevels, planJobs, presentCall, readDoneEvent, recognize, rejectPath, renderDocument, renderOcr, requestPaths, resolveDirectory, resolvePython, resolveWorkerScript, saveText, splitLines, writeDocument };
|