dsh-ab-ocr 0.1.0

package/lib/worker.js

@@ -0,0 +1,143 @@
+/**
+ * Transport to the OCR worker process. The worker speaks newline-delimited
+ * JSON: one event per line on stdout, diagnostics on stderr. This module owns
+ * the process lifetime, the spec handoff, the timeout, and the cancellation, so
+ * the tool's entry module only has to interpret events.
+ * @module @deepseek-ai/dsh-ab-ocr/worker
+ */
+import { spawn } from 'node:child_process';
+/** How much of the worker's diagnostics is kept for an error message. */
+const STDERR_TAIL = 4000;
+/**
+ * Split a run of text into whole lines, returning what is left of a partial one.
+ * @param buffered - text received so far, including any partial final line.
+ * @param chunk - newly received text.
+ * @returns the complete lines and the remaining partial line.
+ */
+export function splitLines(buffered, chunk) {
+    const combined = buffered + chunk;
+    const parts = combined.split('\n');
+    const rest = parts.pop() ?? '';
+    return { lines: parts, rest };
+}
+/**
+ * Parse one event line, ignoring a line that is not a JSON object.
+ * @param line - one complete stdout line.
+ * @returns the parsed event, or undefined for a line that is not one.
+ */
+export function parseEvent(line) {
+    const trimmed = line.trim();
+    if (trimmed === '')
+        return undefined;
+    try {
+        const value = JSON.parse(trimmed);
+        if (typeof value !== 'object' || value === null || typeof value.event !== 'string') {
+            return undefined;
+        }
+        return value;
+    }
+    catch {
+        // The worker writes only JSON on stdout, so a line that does not parse is a
+        // diagnostic that escaped the stream, not an event to act on.
+        return undefined;
+    }
+}
+/**
+ * Run the worker once, feeding it a spec and reporting its events.
+ * @param invocation - executable, script, and time ceiling.
+ * @param args - extra command-line arguments.
+ * @param input - text written to the worker's stdin, or an empty string to close it immediately.
+ * @param onEvent - called with each parsed event as it arrives; never awaited, so a
+ *   slow consumer cannot stall the read.
+ * @param signal - caller cancellation.
+ * @returns the events, diagnostics, and exit state.
+ */
+function spawnWorker(invocation, args, input, onEvent, signal) {
+    return new Promise((resolve) => {
+        const child = spawn(invocation.python, [invocation.script, ...args], {
+            stdio: ['pipe', 'pipe', 'pipe'],
+            windowsHide: true,
+        });
+        const events = [];
+        let stdout = '';
+        let stderr = '';
+        let timedOut = false;
+        let aborted = signal?.aborted === true;
+        let settled = false;
+        const timer = invocation.timeoutMs > 0
+            ? setTimeout(() => { timedOut = true; child.kill(); }, invocation.timeoutMs)
+            : undefined;
+        const onAbort = () => { aborted = true; child.kill(); };
+        signal?.addEventListener('abort', onAbort, { once: true });
+        const accept = (line) => {
+            const event = parseEvent(line);
+            if (event === undefined)
+                return;
+            events.push(event);
+            onEvent(event);
+        };
+        const settle = (exitCode) => {
+            if (settled)
+                return;
+            settled = true;
+            if (timer !== undefined)
+                clearTimeout(timer);
+            signal?.removeEventListener('abort', onAbort);
+            const tail = splitLines(stdout, '');
+            for (const line of [...tail.lines, tail.rest])
+                accept(line);
+            resolve({ events, stderr, exitCode, timedOut, aborted });
+        };
+        child.stdout.setEncoding('utf8');
+        child.stdout.on('data', (chunk) => {
+            const split = splitLines(stdout, chunk);
+            stdout = split.rest;
+            for (const line of split.lines)
+                accept(line);
+        });
+        child.stderr.setEncoding('utf8');
+        child.stderr.on('data', (chunk) => {
+            stderr = (stderr + chunk).slice(-STDERR_TAIL);
+        });
+        // A worker that cannot even start reports the failure here rather than on close.
+        child.on('error', (error) => {
+            stderr = (stderr + error.message).slice(-STDERR_TAIL);
+            settle(null);
+        });
+        child.on('close', (code) => settle(code));
+        child.stdin.on('error', () => {
+            // The worker closed its input early, which its exit code already reports.
+        });
+        child.stdin.end(input);
+    });
+}
+/**
+ * Ask the worker to report its environment.
+ * @param invocation - executable, script, and the ceiling this check may take.
+ * @param signal - caller cancellation.
+ * @returns the report and the diagnostics, whether or not the worker started.
+ */
+export async function runSelfTest(invocation, signal) {
+    const run = await spawnWorker(invocation, ['--self-test'], '', () => { }, signal);
+    return {
+        report: run.events.find(event => event.event === 'ready'),
+        stderr: run.stderr,
+        exitCode: run.exitCode,
+    };
+}
+/**
+ * Run one batch of documents through the worker.
+ *
+ * The event callback sees each event as it is parsed, which is what lets the
+ * caller persist a finished page before the next one is recognized instead of
+ * holding the whole document until the process exits.
+ * @param invocation - executable, script, and time ceiling.
+ * @param spec - the documents and options to process.
+ * @param onEvent - called with each parsed event as it arrives.
+ * @param signal - caller cancellation.
+ * @returns the events, diagnostics, and exit state.
+ */
+export function runSpec(invocation, spec, onEvent, signal) {
+    return spawnWorker(invocation, [], JSON.stringify(spec), onEvent, signal);
+}
+//# sourceMappingURL=worker.js.map

package/lib/worker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"worker.js","sourceRoot":"","sources":["../src/worker.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAA;AAuE1C,yEAAyE;AACzE,MAAM,WAAW,GAAG,IAAI,CAAA;AAExB;;;;;GAKG;AACH,MAAM,UAAU,UAAU,CAAC,QAAgB,EAAE,KAAa;IACxD,MAAM,QAAQ,GAAG,QAAQ,GAAG,KAAK,CAAA;IACjC,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IAClC,MAAM,IAAI,GAAG,KAAK,CAAC,GAAG,EAAE,IAAI,EAAE,CAAA;IAC9B,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,CAAA;AAC/B,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,UAAU,CAAC,IAAY;IACrC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAA;IAC3B,IAAI,OAAO,KAAK,EAAE;QAAE,OAAO,SAAS,CAAA;IACpC,IAAI,CAAC;QACH,MAAM,KAAK,GAAY,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAA;QAC1C,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,OAAQ,KAAqB,CAAC,KAAK,KAAK,QAAQ,EAAE,CAAC;YACpG,OAAO,SAAS,CAAA;QAClB,CAAC;QACD,OAAO,KAAoB,CAAA;IAC7B,CAAC;IAAC,MAAM,CAAC;QACP,4EAA4E;QAC5E,8DAA8D;QAC9D,OAAO,SAAS,CAAA;IAClB,CAAC;AACH,CAAC;AAED;;;;;;;;;GASG;AACH,SAAS,WAAW,CAClB,UAA4B,EAC5B,IAAc,EACd,KAAa,EACb,OAAqC,EACrC,MAA+B;IAE/B,OAAO,IAAI,OAAO,CAAY,CAAC,OAAO,EAAE,EAAE;QACxC,MAAM,KAAK,GAAG,KAAK,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,UAAU,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,EAAE;YACnE,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC;YAC/B,WAAW,EAAE,IAAI;SAClB,CAAC,CAAA;QACF,MAAM,MAAM,GAAkB,EAAE,CAAA;QAChC,IAAI,MAAM,GAAG,EAAE,CAAA;QACf,IAAI,MAAM,GAAG,EAAE,CAAA;QACf,IAAI,QAAQ,GAAG,KAAK,CAAA;QACpB,IAAI,OAAO,GAAG,MAAM,EAAE,OAAO,KAAK,IAAI,CAAA;QACtC,IAAI,OAAO,GAAG,KAAK,CAAA;QACnB,MAAM,KAAK,GAAG,UAAU,CAAC,SAAS,GAAG,CAAC;YACpC,CAAC,CAAC,UAAU,CAAC,GAAG,EAAE,GAAG,QAAQ,GAAG,IAAI,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,CAAA,CAAC,CAAC,EAAE,UAAU,CAAC,SAAS,CAAC;YAC3E,CAAC,CAAC,SAAS,CAAA;QACb,MAAM,OAAO,GAAG,GAAS,EAAE,GAAG,OAAO,GAAG,IAAI,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,CAAA,CAAC,CAAC,CAAA;QAC5D,MAAM,EAAE,gBAAgB,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAA;QAE1D,MAAM,MAAM,GAAG,CAAC,IAAY,EAAQ,EAAE;YACpC,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,CAAC,CAAA;YAC9B,IAAI,KAAK,KAAK,SAAS;gBAAE,OAAM;YAC/B,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YAClB,OAAO,CAAC,KAAK,CAAC,CAAA;QAChB,CAAC,CAAA;QAED,MAAM,MAAM,GAAG,CAAC,QAAuB,EAAQ,EAAE;YAC/C,IAAI,OAAO;gBAAE,OAAM;YACnB,OAAO,GAAG,IAAI,CAAA;YACd,IAAI,KAAK,KAAK,SAAS;gBAAE,YAAY,CAAC,KAAK,CAAC,CAAA;YAC5C,MAAM,EAAE,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAA;YAC7C,MAAM,IAAI,GAAG,UAAU,CAAC,MAAM,EAAE,EAAE,CAAC,CAAA;YACnC,KAAK,MAAM,IAAI,IAAI,CAAC,GAAG,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC;gBAAE,MAAM,CAAC,IAAI,CAAC,CAAA;YAC3D,OAAO,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAA;QAC1D,CAAC,CAAA;QAED,KAAK,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAA;QAChC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,EAAE;YACxC,MAAM,KAAK,GAAG,UAAU,CAAC,MAAM,EAAE,KAAK,CAAC,CAAA;YACvC,MAAM,GAAG,KAAK,CAAC,IAAI,CAAA;YACnB,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,KAAK;gBAAE,MAAM,CAAC,IAAI,CAAC,CAAA;QAC9C,CAAC,CAAC,CAAA;QACF,KAAK,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAA;QAChC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,EAAE;YACxC,MAAM,GAAG,CAAC,MAAM,GAAG,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,WAAW,CAAC,CAAA;QAC/C,CAAC,CAAC,CAAA;QACF,iFAAiF;QACjF,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAY,EAAE,EAAE;YACjC,MAAM,GAAG,CAAC,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,WAAW,CAAC,CAAA;YACrD,MAAM,CAAC,IAAI,CAAC,CAAA;QACd,CAAC,CAAC,CAAA;QACF,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAmB,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAA;QAExD,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;YAC3B,0EAA0E;QAC5E,CAAC,CAAC,CAAA;QACF,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;IACxB,CAAC,CAAC,CAAA;AACJ,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,UAA4B,EAC5B,MAAoB;IAEpB,MAAM,GAAG,GAAG,MAAM,WAAW,CAAC,UAAU,EAAE,CAAC,aAAa,CAAC,EAAE,EAAE,EAAE,GAAG,EAAE,GAAE,CAAC,EAAE,MAAM,CAAC,CAAA;IAChF,OAAO;QACL,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,KAAK,KAAK,OAAO,CAAC;QACzD,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,QAAQ,EAAE,GAAG,CAAC,QAAQ;KACvB,CAAA;AACH,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,OAAO,CACrB,UAA4B,EAC5B,IAAgB,EAChB,OAAqC,EACrC,MAAoB;IAEpB,OAAO,WAAW,CAAC,UAAU,EAAE,EAAE,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,CAAA;AAC3E,CAAC"}

package/package.json

@@ -0,0 +1,98 @@
+{
+  "name": "dsh-ab-ocr",
+  "version": "0.1.0",
+  "type": "module",
+  "main": "lib/index.js",
+  "types": "lib/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./lib/index.d.ts",
+      "default": "./lib/index.js"
+    },
+    "./package.json": "./package.json"
+  },
+  "files": [
+    "lib/**/*",
+    "python/ocr_worker.py",
+    "python/assemble.py",
+    "python/clean.py",
+    "python/layout.py",
+    "python/source.py",
+    "python/requirements.txt",
+    "python/README.md",
+    "scripts/setup.mjs",
+    "cordis.patch.yml",
+    "README.md",
+    "LICENSE",
+    "tsconfig.json",
+    "tsdown.config.ts"
+  ],
+  "scripts": {
+    "build": "tsc -p tsconfig.json && tsdown",
+    "clean": "rimraf lib",
+    "setup": "node scripts/setup.mjs",
+    "typecheck": "tsc -p tsconfig.json --noEmit",
+    "test": "node --test",
+    "test:python": "python -m unittest discover -s python/tests -t python",
+    "lint": "publint && attw --pack . --profile esm-only",
+    "format": "prettier --write \"src/**/*.{ts,tsx,mjs}\" \"tests/**/*.mjs\" \"*.{json,yml,md}\"",
+    "verify": "npm run typecheck && npm run test && npm run build && npm run lint",
+    "prepublishOnly": "npm run verify",
+    "prepack": "npm run build",
+    "release": "npm publish --access public --no-git-checks"
+  },
+  "dsh": {
+    "bundle": {
+      "patch": "./cordis.patch.yml"
+    }
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/deepseek-ai/deepseek-harness.git",
+    "directory": ".dsh/plugins/dsh-ab-ocr"
+  },
+  "bugs": {
+    "url": "https://github.com/deepseek-ai/deepseek-harness/issues"
+  },
+  "homepage": "https://github.com/deepseek-ai/deepseek-harness/tree/main/.dsh/plugins/dsh-ab-ocr",
+  "license": "MIT",
+  "keywords": [
+    "deepseek-harness",
+    "dsh-plugin",
+    "cordis-plugin",
+    "ocr",
+    "document-extraction",
+    "pdf",
+    "out-of-tree"
+  ],
+  "engines": {
+    "node": ">=18"
+  },
+  "dependencies": {
+    "@deepseek-ai/dsh-tools": "^0.1.7-rc.1",
+    "@deepseek-ai/schemastery": "^3.18.4"
+  },
+  "peerDependencies": {
+    "@deepseek-ai/cordis": "*"
+  },
+  "devDependencies": {
+    "@deepseek-ai/cordis": "^4.0.4",
+    "@deepseek-ai/cordis-plugin-include": "^1.0.9",
+    "@deepseek-ai/cordis-plugin-loader": "^1.0.5",
+    "@deepseek-ai/dsh-fs": "^0.1.7-rc.1",
+    "@deepseek-ai/dsh-fs-sandbox": "^0.1.7-rc.1",
+    "@deepseek-ai/dsh-sandbox": "^0.1.7-rc.1",
+    "@deepseek-ai/dsh-sandbox-policy": "^0.1.7-rc.1",
+    "@deepseek-ai/dsh-system-prompt": "^0.1.7-rc.1",
+    "@types/node": "^22.10.2",
+    "tsdown": "^0.22.2",
+    "typescript": "^6.0.3",
+    "rimraf": "^6.0.1",
+    "publint": "^0.2.0",
+    "@arethetypeswrong/cli": "^0.18.3",
+    "prettier": "^3.3.0"
+  }
+}

package/python/README.md

@@ -0,0 +1,125 @@
+# OCR worker
+
+The Python half of `ocr`. The tool spawns it once per call, hands it a JSON job
+spec on stdin, and reads newline-delimited JSON events back from stdout. It
+recognizes and merges; the tool writes every file.
+
+## Files
+
+| File | Role |
+|---|---|
+| `ocr_worker.py` | entry: reads the spec, owns the engine lifetime, reports each page, runs the merge |
+| `source.py` | opens a PDF or a still image and renders one page on demand |
+| `layout.py` | turns engine boxes into reading-order lines, finds columns, infers outline levels |
+| `clean.py` | identifies folios and running heads, and joins the lines a wrap split |
+| `assemble.py` | the merge pass: page records in, one Markdown document out |
+
+## Environment
+
+The tool looks for a virtual environment at `python/.venv` beside this
+directory and falls back to `python` on `PATH`. Override it with the
+`pythonPath` configuration field.
+
+Create the environment once. Installing into it needs permission to create
+directories with a private mode, so run this outside a restricted shell:
+
+```sh
+python -m venv python/.venv
+python/.venv/Scripts/python -m pip install -r python/requirements.txt
+```
+
+Confirm it with the worker's own check, which is what the tool runs before every
+call:
+
+```sh
+python/.venv/Scripts/python python/ocr_worker.py --self-test
+```
+
+## Protocol
+
+stdin is one JSON object:
+
+```json
+{
+  "engineLifetime": "perDocument",
+  "jobs": [
+    { "mode": "recognize", "id": "1", "input": "D:/docs/a.pdf", "pages": "1-5,8",
+      "dpi": 200, "maxPixels": 12000000, "textScore": 0.5, "detectColumns": true,
+      "maxPages": 2000, "merge": { "detectHeadings": true } },
+    { "mode": "assemble", "id": "2", "input": "D:/docs/a.pdf",
+      "record": { "pages": [ { "index": 5, "width": 595.0, "height": 842.0,
+                               "lines": [ { "text": "...", "x0": 1, "y0": 2, "x1": 3, "y1": 4, "height": 12.0 } ] } ] },
+      "merge": { "levelOverrides": { "h2": 3 } } }
+  ]
+}
+```
+
+`pages` is a comma-separated list of single pages and inclusive ranges, counted
+from one, as a person would write them in a print dialog; a range left open
+after its dash runs to the last page. It resolves to the explicit ascending list
+of page numbers to process, so `"1-5,8"` on a twelve-page document is
+`[1, 2, 3, 4, 5, 8]`. A malformed piece, a page below one, an end before its
+start, and a selection that names no page of the document are errors for that
+job.
+
+A `recognize` job renders and recognizes the document it names. An `assemble`
+job opens no document and builds no engine: it rebuilds the pages from the page
+records the caller kept and runs the merge alone.
+
+`merge` carries the merge pass's keys, all optional and in camelCase:
+`removePageNumbers`, `removeRunningHeads`, `runningHeadRatio`,
+`runningHeadMinPages`, `detectHeadings`, `headingMinRatio`, `indentRatio`,
+`paragraphGapRatio`, `outlineCandidateRatio`, `maxOutlineCandidates`, and
+`levelOverrides`.
+
+stdout is one JSON object per line:
+
+| Event | Fields |
+|---|---|
+| `start` | `id`, `input`, `pages` (the explicit list), `totalPages` |
+| `page` | `id`, `page`, `totalPages`, `lines`, `chars`, `text`, and `geometry` — the page record an assemble job rebuilds from |
+| `released` | `id`, `reason` |
+| `done` | `id`, `input`, `pages`, `totalPages`, `lines`, `headings`, `chars`, `droppedPageNumbers`, `droppedRunningHeads`, `joinedAcrossPages`, `recognized`, `seconds`, `markdown`, `outline`, `outlineTruncated` |
+| `error` | `id`, `input`, `message` |
+| `end` | `documents`, `failures` |
+
+An assemble job emits no `page` events: its `pages` and `recognized` are the
+number of pages in its record, and its `seconds` is the merge's own duration.
+
+The worker reads only the document each recognize job names. It writes no file:
+every artifact is written by the caller through the mounted filesystem
+capability, so a deployment's file policy sees the write. Diagnostics go to
+stderr.
+
+## Outline pass
+
+`done` closes with `outline`, the lines whose outline level the heuristic may
+have read wrongly, and with `outlineTruncated` when `maxOutlineCandidates` (400
+by default) cut that list short. A caller that wants a model to check the
+outline sends these candidates to the model and comes back with one `assemble`
+job per document whose `merge` carries `levelOverrides`; nothing is recognized
+twice.
+
+A kept line becomes a candidate when any of these holds: the heuristic gives it
+a level, it opens with a section number, it is at least `outlineCandidateRatio`
+(1.05 by default) times the body glyph height, or it opens with a named section
+such as an abstract or a reference list. A list item that carries no number
+never does. Ids are `h1`, `h2`, … in document order, and each candidate reports
+its `page`, its normalized `text`, the heuristic `level` (or `null`), its
+glyph-height `ratio`, and its section `numbered` when it opens with one.
+
+`levelOverrides` maps a candidate id to the heading level to use in place of the
+heuristic one: 1 to 6, with anything above 6 clamped to 6. A value below 1, like
+`null`, makes the line body text, which is how a candidate the heuristic wrongly
+called a heading is demoted. A candidate the map does not mention keeps the
+heuristic's own decision, and a value that is not an integer is an error for
+that job.
+
+## Tests
+
+The unit tests cover the pure layout, cleaning, merge, and job-protocol rules
+and need no third-party package:
+
+```sh
+python -m unittest discover -s python/tests -t python
+```