@morroc/open-ledger 0.14.1

package/dist/extract/ocr.js

@@ -0,0 +1,164 @@
+import * as z from "zod";
+import { config } from "../config.js";
+import { tryExecute } from "../lib/result.js";
+import { presetForModel } from "./presets/index.js";
+/**
+ * Nothing here is tied to a particular model: the endpoint is whatever
+ * `ocrBaseUrl` names, and the prompt, sampling, and render spec come from the
+ * preset the configured model id selects.
+ */
+/** A small local model can spend minutes on a dense page; this is a stuck-server bound, not a target. */
+export const OCR_TIMEOUT_MS = 300_000;
+// The abort covers connect and body alike, but a stalled server has been seen
+// outliving it, so a second bound races it.
+const TIMEOUT_GRACE_MS = 5_000;
+/** A liveness check, not a read: bounded independently of the per-page timeout. */
+const PROBE_TIMEOUT_MS = 5_000;
+const ERROR_BODY_CHARS = 200;
+/** The endpoint URL alone decides whether OCR is configured; `null` means cleanly unset. */
+export function resolveOcr(source = config) {
+    const baseUrl = source.ocrBaseUrl.trim().replace(/\/+$/, "");
+    if (!baseUrl)
+        return null;
+    const configured = source.ocrModel.trim();
+    const { name, preset } = presetForModel(configured);
+    return {
+        baseUrl,
+        model: configured || preset.model,
+        apiKey: source.ocrApiKey.trim(),
+        timeoutMs: OCR_TIMEOUT_MS,
+        preset: name,
+        prompt: preset.prompt,
+        params: preset.params,
+        render: preset.render,
+    };
+}
+// Exhaustive by construction: a new OcrFailure has to be classified here before
+// it compiles.
+const PAGE_LEVEL = {
+    timeout: true,
+    bad_response: true,
+    unreachable: false,
+    rejected: false,
+};
+/**
+ * Whether a failure is about this page or about the server. Page-level failures
+ * leave a placeholder and let the run finish partial; server-level ones abort —
+ * retrying 20 pages against a dead endpoint helps nobody.
+ */
+export function isServerFailure(reason) {
+    return !PAGE_LEVEL[reason];
+}
+const RESPONSE = z.object({
+    choices: z.array(z.object({ message: z.object({ content: z.string() }) })).min(1),
+});
+const MODELS = z.object({ data: z.array(z.object({ id: z.string() })) });
+function headersFor(settings) {
+    const headers = { "content-type": "application/json" };
+    // Local servers reject an empty bearer token, so the header appears only when there is a key.
+    if (settings.apiKey)
+        headers.authorization = `Bearer ${settings.apiKey}`;
+    return headers;
+}
+function requestBody(image, settings) {
+    return JSON.stringify({
+        model: settings.model,
+        messages: [
+            {
+                role: "user",
+                content: [
+                    { type: "text", text: settings.prompt },
+                    {
+                        type: "image_url",
+                        image_url: { url: `data:${image.mime};base64,${image.bytes.toString("base64")}` },
+                    },
+                ],
+            },
+        ],
+        ...settings.params,
+        stream: false,
+    });
+}
+function failed(page, reason, message) {
+    return { ok: false, page, reason, message };
+}
+function clip(body) {
+    return body.length > ERROR_BODY_CHARS ? `${body.slice(0, ERROR_BODY_CHARS)}…` : body;
+}
+const TIMED_OUT = Symbol("ocr-timeout");
+export async function ocrPage(image, settings) {
+    const controller = new AbortController();
+    const abort = setTimeout(() => controller.abort(), settings.timeoutMs);
+    const attempt = tryExecute(async () => {
+        const response = await fetch(`${settings.baseUrl}/chat/completions`, {
+            method: "POST",
+            headers: headersFor(settings),
+            body: requestBody(image, settings),
+            signal: controller.signal,
+        });
+        return { status: response.status, body: await response.text() };
+    });
+    let deadline;
+    const bound = new Promise((resolve) => {
+        deadline = setTimeout(() => resolve(TIMED_OUT), settings.timeoutMs + TIMEOUT_GRACE_MS);
+        deadline.unref();
+    });
+    const raced = await Promise.race([attempt, bound]);
+    clearTimeout(abort);
+    clearTimeout(deadline);
+    if (raced === TIMED_OUT) {
+        return failed(image.page, "timeout", `no response within ${settings.timeoutMs}ms`);
+    }
+    if (!raced.ok && controller.signal.aborted) {
+        return failed(image.page, "timeout", `aborted after ${settings.timeoutMs}ms`);
+    }
+    if (!raced.ok)
+        return failed(image.page, "unreachable", raced.error);
+    const { status, body } = raced.value;
+    if (status < 200 || status >= 300) {
+        return failed(image.page, "rejected", `HTTP ${status}: ${clip(body)}`);
+    }
+    const json = tryExecute(() => JSON.parse(body));
+    if (!json.ok)
+        return failed(image.page, "bad_response", json.error);
+    const parsed = RESPONSE.safeParse(json.value);
+    if (!parsed.success) {
+        return failed(image.page, "bad_response", z.prettifyError(parsed.error));
+    }
+    return { ok: true, page: image.page, text: parsed.data.choices[0].message.content };
+}
+/**
+ * One page per request, sequentially: the endpoint serves one model, so
+ * parallel pages queue behind each other and only eat into the per-page
+ * timeout. Stops early on a server-level failure; the caller reads the returned
+ * outcomes to see which pages made it.
+ */
+export async function ocrPages(images, settings) {
+    const outcomes = [];
+    for (const image of images) {
+        const outcome = await ocrPage(image, settings);
+        outcomes.push(outcome);
+        if (!outcome.ok && isServerFailure(outcome.reason))
+            break;
+    }
+    return outcomes;
+}
+/** The model ids the endpoint serves, for `oled doctor`. */
+export async function probeOcrEndpoint(settings) {
+    const response = await tryExecute(async () => {
+        const res = await fetch(`${settings.baseUrl}/models`, {
+            headers: headersFor(settings),
+            signal: AbortSignal.timeout(PROBE_TIMEOUT_MS),
+        });
+        if (!res.ok)
+            throw new Error(`HTTP ${res.status} ${res.statusText}`);
+        return res.json();
+    });
+    if (!response.ok)
+        return response;
+    const parsed = MODELS.safeParse(response.value);
+    if (!parsed.success) {
+        return { ok: false, error: `unexpected /models response: ${z.prettifyError(parsed.error)}` };
+    }
+    return { ok: true, value: parsed.data.data.map((model) => model.id) };
+}

package/dist/extract/pdf.js

@@ -0,0 +1,127 @@
+import { tryExecute } from "../lib/result.js";
+let mupdfPromise = null;
+// Lazy: WASM module isn't loaded until first call.
+function getMupdf() {
+    if (!mupdfPromise)
+        mupdfPromise = import("mupdf");
+    return mupdfPromise;
+}
+const PDF_MIME = "application/pdf";
+// mupdf's authenticatePassword returns 0 on a wrong password, non-zero on success.
+const MUPDF_AUTH_FAILED = 0;
+export async function isEncryptedPdf(bytes) {
+    const mupdf = await getMupdf();
+    const doc = mupdf.Document.openDocument(bytes, PDF_MIME);
+    try {
+        return doc.needsPassword();
+    }
+    finally {
+        doc.destroy();
+    }
+}
+export async function unlockPdf(bytes, password) {
+    const mupdf = await getMupdf();
+    const doc = mupdf.Document.openDocument(bytes, PDF_MIME);
+    try {
+        if (!(doc instanceof mupdf.PDFDocument)) {
+            return { ok: false, reason: "unsupported_document" };
+        }
+        if (!doc.needsPassword()) {
+            return { ok: true, decrypted: bytes };
+        }
+        if (doc.authenticatePassword(password) === MUPDF_AUTH_FAILED) {
+            return { ok: false, reason: "wrong_password" };
+        }
+        const out = doc.saveToBuffer("decrypt");
+        return { ok: true, decrypted: Buffer.from(out.asUint8Array()) };
+    }
+    finally {
+        doc.destroy();
+    }
+}
+/**
+ * `preserve-images` is what makes image blocks visible to the walker; without
+ * it a scanned page is indistinguishable from a blank one (verified against
+ * mupdf 1.27).
+ */
+const STEXT_FLAGS = "preserve-whitespace,preserve-images";
+function hasImageBlock(stext) {
+    let found = false;
+    stext.walk({
+        onImageBlock: () => {
+            found = true;
+        },
+    });
+    return found;
+}
+function probeOne(doc, index) {
+    const page = doc.loadPage(index);
+    try {
+        const stext = page.toStructuredText(STEXT_FLAGS);
+        try {
+            const text = stext.asText().replace(/\n{3,}/g, "\n\n").trim();
+            return { page: index + 1, chars: text.length, hasImage: hasImageBlock(stext), text };
+        }
+        finally {
+            stext.destroy();
+        }
+    }
+    finally {
+        page.destroy();
+    }
+}
+/**
+ * Measures and extracts the text layer in the same pass — the text-layer route
+ * has nothing left to do afterwards, so no page is ever opened twice.
+ */
+export async function probePdfPages(bytes) {
+    const mupdf = await getMupdf();
+    return tryExecute(() => {
+        const doc = mupdf.Document.openDocument(bytes, PDF_MIME);
+        try {
+            const pages = [];
+            for (let index = 0; index < doc.countPages(); index++) {
+                pages.push(probeOne(doc, index));
+            }
+            return pages;
+        }
+        finally {
+            doc.destroy();
+        }
+    });
+}
+function renderOne(mupdf, doc, index, spec) {
+    const page = doc.loadPage(index);
+    try {
+        const [x0, y0, x1, y1] = page.getBounds();
+        const longestPt = Math.max(x1 - x0, y1 - y0);
+        const scale = Math.min(spec.dpi / 72, spec.maxLongestDimPx / longestPt);
+        // Statements assume paper; alpha:false renders on white.
+        const pixmap = page.toPixmap(mupdf.Matrix.scale(scale, scale), mupdf.ColorSpace.DeviceRGB, false);
+        try {
+            return { page: index + 1, mime: "image/png", bytes: Buffer.from(pixmap.asPNG()) };
+        }
+        finally {
+            pixmap.destroy();
+        }
+    }
+    finally {
+        page.destroy();
+    }
+}
+export async function renderPdfPages(bytes, spec) {
+    const mupdf = await getMupdf();
+    return tryExecute(() => {
+        const doc = mupdf.Document.openDocument(bytes, PDF_MIME);
+        try {
+            const pages = [];
+            for (let index = 0; index < doc.countPages(); index++) {
+                pages.push(renderOne(mupdf, doc, index, spec));
+            }
+            return pages;
+        }
+        finally {
+            doc.destroy();
+        }
+    });
+}

package/dist/extract/presets/index.js

@@ -0,0 +1,18 @@
+import { lightonOcrPreset } from "./lighton-ocr.js";
+import { typhoonOcrPreset } from "./typhoon-ocr.js";
+/** Exhaustive by construction: a new PresetName breaks the build until it is listed. */
+export const PRESETS = {
+    "typhoon-ocr": typhoonOcrPreset,
+    "lighton-ocr": lightonOcrPreset,
+};
+export const PRESET_NAMES = Object.keys(PRESETS);
+/** Its prompt is a generic extract-to-markdown instruction, so it suits an unrecognized model too. */
+const HOUSE_PRESET = "typhoon-ocr";
+/**
+ * An unrecognized id — or none at all — gets the house preset, whose own model
+ * is then the one to ask for.
+ */
+export function presetForModel(modelId) {
+    const name = PRESET_NAMES.find((key) => PRESETS[key].family.test(modelId)) ?? HOUSE_PRESET;
+    return { name, preset: PRESETS[name] };
+}

package/dist/extract/presets/lighton-ocr.js

@@ -0,0 +1,12 @@
+/**
+ * LightOnOCR-2-1B: trained on 11 languages, Thai not among them. Answers in plain
+ * markdown (no wrapper to unwrap); its card prescribes 200 DPI, longest side 1540px.
+ */
+export const lightonOcrPreset = {
+    family: /lighton/i,
+    model: "lightonocr-2-1b",
+    prompt: "Convert this page to markdown. Preserve all text, numbers, and tables exactly as they appear.",
+    // The card prescribes greedy decoding, which leaves top_p at the API default and needs no seed.
+    params: { temperature: 0, top_p: 1, max_tokens: 4096 },
+    render: { dpi: 200, maxLongestDimPx: 1540 },
+};

package/dist/extract/presets/typhoon-ocr.js

@@ -0,0 +1,23 @@
+// The prompt is the model card's, kept per-preset so a card revision only touches this file.
+const PROMPT = `Extract all text from the image.
+
+Instructions:
+- Only return the clean Markdown.
+- Do not include any explanation or extra text.
+- You must include all information on the page.
+
+Formatting Rules:
+- Tables: Render tables using <table>...</table> in clean HTML format.
+- Equations: Render equations using LaTeX syntax with inline ($...$) and block ($$...$$).
+- Images/Charts/Diagrams: Wrap any clearly defined visual areas in: <figure>Describe...</figure>
+- Page Numbers: Wrap page numbers in <page_number>...</page_number>
+- Checkboxes: Use ☐ for unchecked and ☑ for checked boxes.`;
+export const typhoonOcrPreset = {
+    // Every release so far carries the family name, whatever the version suffix.
+    family: /typhoon/i,
+    model: "typhoon-ocr1.5",
+    prompt: PROMPT,
+    params: { temperature: 0.1, top_p: 0.6, max_tokens: 4096, seed: 42 },
+    // The card trains images at 1800 px on the longest side.
+    render: { dpi: 200, maxLongestDimPx: 1800 },
+};

package/dist/extract/route.js

@@ -0,0 +1,40 @@
+/** The whole routing policy.
+ *                  ocr ready     ocr unset
+ *   complete       text-layer    text-layer
+ *   partial        ocr           agent
+ *   none           ocr           agent
+ */
+const READER = {
+    "complete/ready": "text-layer",
+    "complete/unset": "text-layer",
+    "partial/ready": "ocr",
+    "partial/unset": "agent",
+    "none/ready": "ocr",
+    "none/unset": "agent",
+};
+/**
+ * File kind is deliberately not an axis: an image enters as `"none"` and routes
+ * like a scan. Availability means configured, not reachable — a dead endpoint
+ * fails the run loudly rather than degrading to images behind the caller's back.
+ */
+export function readerFor(textLayer, ocr) {
+    return READER[`${textLayer}/${ocr}`];
+}
+// Biased toward waste over loss: a scanned page with a junk text layer under this
+// bar still counts as a scan, so it gets re-read instead of silently contributing nothing.
+const PAGE_TEXT_CHARS = 400;
+export function classifyPage(page) {
+    if (page.chars >= PAGE_TEXT_CHARS)
+        return "text";
+    return page.hasImage ? "scan" : "blank";
+}
+/**
+ * Blank trailer pages don't spoil "complete"; one scanned page among text pages makes
+ * it "partial", re-reading the whole document so every source_page citation matches.
+ */
+export function verdictOf(pages) {
+    const contents = pages.map(classifyPage);
+    if (!contents.includes("text"))
+        return "none";
+    return contents.includes("scan") ? "partial" : "complete";
+}

package/dist/extract/source.js

@@ -0,0 +1,84 @@
+import { readFileSync, statSync } from "node:fs";
+import { createHash } from "node:crypto";
+import { basename, extname } from "node:path";
+import { tryExecute } from "../lib/result.js";
+/** `mime` is what `files.mime` records. */
+export const SOURCES = {
+    ".pdf": { kind: "pdf", mime: "application/pdf" },
+    ".png": { kind: "image", mime: "image/png" },
+    ".jpg": { kind: "image", mime: "image/jpeg" },
+    ".jpeg": { kind: "image", mime: "image/jpeg" },
+    ".webp": { kind: "image", mime: "image/webp" },
+};
+export const SUPPORTED_EXTS = Object.keys(SOURCES);
+export const MAX_SOURCE_BYTES = 30 * 1024 * 1024;
+// Some producers emit a preamble before the header, so the marker is searched, not at offset 0.
+const PDF_SNIFF_BYTES = 1024;
+const PNG_MAGIC = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]);
+const JPEG_MAGIC = Buffer.from([0xff, 0xd8, 0xff]);
+const SIGNATURES = [
+    {
+        kind: "pdf",
+        mime: "application/pdf",
+        matches: (bytes) => bytes.subarray(0, PDF_SNIFF_BYTES).toString("latin1").includes("%PDF-"),
+    },
+    { kind: "image", mime: "image/png", matches: (bytes) => bytes.subarray(0, 8).equals(PNG_MAGIC) },
+    { kind: "image", mime: "image/jpeg", matches: (bytes) => bytes.subarray(0, 3).equals(JPEG_MAGIC) },
+    {
+        kind: "image",
+        mime: "image/webp",
+        matches: (bytes) => bytes.subarray(0, 4).toString("latin1") === "RIFF" &&
+            bytes.subarray(8, 12).toString("latin1") === "WEBP",
+    },
+];
+export function sniffSource(bytes) {
+    const hit = SIGNATURES.find((signature) => signature.matches(bytes));
+    return hit ? { kind: hit.kind, mime: hit.mime } : null;
+}
+/** The extension declares the type and the magic bytes must agree — a mislabeled
+ *  file fails here instead of deep in mupdf or the OCR endpoint. */
+export function loadSource(path) {
+    const ext = extname(path).toLowerCase();
+    const declared = SOURCES[ext];
+    if (!declared) {
+        return {
+            ok: false,
+            reason: "unsupported_extension",
+            message: `unsupported extension ${ext || basename(path)} (accepted: ${SUPPORTED_EXTS.join(" ")})`,
+        };
+    }
+    // Size is checked before the read so an oversized file is never pulled into memory.
+    const stat = tryExecute(() => statSync(path));
+    if (!stat.ok)
+        return { ok: false, reason: "unreadable", message: stat.error };
+    if (stat.value.size > MAX_SOURCE_BYTES) {
+        return {
+            ok: false,
+            reason: "too_large",
+            message: `${stat.value.size} bytes exceeds the ${MAX_SOURCE_BYTES}-byte limit`,
+        };
+    }
+    const read = tryExecute(() => readFileSync(path));
+    if (!read.ok)
+        return { ok: false, reason: "unreadable", message: read.error };
+    const bytes = read.value;
+    const sniffed = sniffSource(bytes);
+    if (!sniffed || sniffed.mime !== declared.mime) {
+        return {
+            ok: false,
+            reason: "kind_mismatch",
+            message: `${ext} file holds ${sniffed?.mime ?? "unrecognized"} bytes`,
+        };
+    }
+    return {
+        ok: true,
+        value: {
+            path,
+            fileName: basename(path),
+            kind: declared.kind,
+            mime: declared.mime,
+            bytes,
+            hash: createHash("sha256").update(bytes).digest("hex"),
+        },
+    };
+}