macos-vision 1.2.0 → 1.3.0

package/dist/markdown/chunker.js

@@ -0,0 +1,39 @@
+import { buildPrompt } from './prompt.js';
+export function estimateTokens(text) {
+    return Math.ceil(text.length / 4);
+}
+/**
+ * Split an array of paragraphs into chunks where each chunk's estimated prompt
+ * token count stays within `chunkSizeTokens`. Paragraph boundaries are never
+ * split — chunks always break between `ParagraphGroup` objects.
+ *
+ * A paragraph whose estimated token count exceeds the budget on its own is
+ * emitted as a singleton chunk with a warning.
+ */
+export function chunkParagraphs(paragraphs, chunkSizeTokens) {
+    const result = [];
+    let currentBatch = [];
+    for (const p of paragraphs) {
+        const candidate = [...currentBatch, p];
+        const tokens = estimateTokens(buildPrompt(candidate));
+        if (currentBatch.length === 0) {
+            if (tokens > chunkSizeTokens) {
+                console.warn(`[macos-vision] Paragraph ${p.paragraphId} (page ${p.page}) ` +
+                    `exceeds chunk budget (~${tokens} est. tokens > ${chunkSizeTokens}). ` +
+                    `Processing as standalone chunk.`);
+            }
+            currentBatch = [p];
+        }
+        else if (tokens <= chunkSizeTokens) {
+            currentBatch = candidate;
+        }
+        else {
+            result.push(currentBatch);
+            currentBatch = [p];
+        }
+    }
+    if (currentBatch.length > 0) {
+        result.push(currentBatch);
+    }
+    return result;
+}

package/dist/markdown/index.d.ts

@@ -0,0 +1,61 @@
+export { OllamaUnavailableError } from './ollama.js';
+export type { ParagraphGroup } from './prompt.js';
+export interface VisionScribeOptions {
+    /**
+     * Ollama model name.
+     * @default 'mistral-nemo'
+     */
+    model?: string;
+    /**
+     * Base URL of the Ollama server.
+     * @default 'http://localhost:11434'
+     */
+    ollamaUrl?: string;
+    /**
+     * Skip the Ollama reachability check before each call.
+     * Useful in batch/eval contexts where you ping once upfront.
+     * @default false
+     */
+    skipPing?: boolean;
+    /**
+     * Maximum estimated output tokens per LLM chunk.
+     * Paragraphs are batched so that no single generate() call is expected
+     * to produce more than this many tokens. Lower values mean more (faster)
+     * chunks; higher values risk hitting the model's output token limit.
+     * @default 1800
+     */
+    chunkSizeTokens?: number;
+}
+/**
+ * Converts an image or PDF to structured Markdown using a two-stage pipeline:
+ *
+ * 1. **Apple Vision OCR** — extracts raw text blocks with bounding-box coordinates.
+ *    PDFs are automatically rasterized page-by-page.
+ * 2. **Layout inference** — groups blocks by `paragraphId` per page using spatial
+ *    heuristics (each page processed independently to avoid coordinate mixing).
+ * 3. **Chunking** — paragraphs are batched to stay within the LLM output token budget.
+ * 4. **Local LLM (Ollama)** — formats each chunk into clean Markdown without
+ *    hallucinating new content.
+ *
+ * @example
+ * ```ts
+ * const scribe = new VisionScribe({ model: 'mistral-nemo' });
+ * const markdown = await scribe.toMarkdown('invoice.png');
+ * const mdFromPdf = await scribe.toMarkdown('report.pdf');
+ * ```
+ */
+export declare class VisionScribe {
+    private readonly model;
+    private readonly ollamaUrl;
+    private readonly skipPing;
+    private readonly chunkSizeTokens;
+    constructor(options?: VisionScribeOptions);
+    /**
+     * Convert an image or PDF file to Markdown.
+     *
+     * @param imagePath Absolute or relative path to the image or PDF.
+     * @returns Markdown string. Empty string if no text was detected.
+     * @throws {OllamaUnavailableError} If the Ollama server cannot be reached.
+     */
+    toMarkdown(imagePath: string): Promise<string>;
+}

package/dist/markdown/index.js

@@ -0,0 +1,92 @@
+import { ocr, inferLayout, sortBlocksByReadingOrder } from '../index.js';
+import { ping, chat } from './ollama.js';
+import { groupByParagraph, buildUserContent, SYSTEM_PROMPT } from './prompt.js';
+import { chunkParagraphs } from './chunker.js';
+export { OllamaUnavailableError } from './ollama.js';
+/**
+ * Group raw OCR blocks by their page index.
+ * macos-vision attaches a `page` field (0-based) to blocks from PDFs.
+ * Single-image blocks have no `page` field and land in page 0.
+ *
+ * Coordinates in VisionBlock are always page-local (0–1), so blocks from
+ * different pages must NOT be passed together to inferLayout().
+ */
+function groupBlocksByPage(blocks) {
+    const pages = new Map();
+    for (const block of blocks) {
+        const page = block.page ?? 0;
+        const existing = pages.get(page) ?? [];
+        existing.push(block);
+        pages.set(page, existing);
+    }
+    return pages;
+}
+/**
+ * Converts an image or PDF to structured Markdown using a two-stage pipeline:
+ *
+ * 1. **Apple Vision OCR** — extracts raw text blocks with bounding-box coordinates.
+ *    PDFs are automatically rasterized page-by-page.
+ * 2. **Layout inference** — groups blocks by `paragraphId` per page using spatial
+ *    heuristics (each page processed independently to avoid coordinate mixing).
+ * 3. **Chunking** — paragraphs are batched to stay within the LLM output token budget.
+ * 4. **Local LLM (Ollama)** — formats each chunk into clean Markdown without
+ *    hallucinating new content.
+ *
+ * @example
+ * ```ts
+ * const scribe = new VisionScribe({ model: 'mistral-nemo' });
+ * const markdown = await scribe.toMarkdown('invoice.png');
+ * const mdFromPdf = await scribe.toMarkdown('report.pdf');
+ * ```
+ */
+export class VisionScribe {
+    model;
+    ollamaUrl;
+    skipPing;
+    chunkSizeTokens;
+    constructor(options = {}) {
+        this.model = options.model ?? 'mistral-nemo';
+        this.ollamaUrl = options.ollamaUrl ?? 'http://localhost:11434';
+        this.skipPing = options.skipPing ?? false;
+        this.chunkSizeTokens = options.chunkSizeTokens ?? 1800;
+    }
+    /**
+     * Convert an image or PDF file to Markdown.
+     *
+     * @param imagePath Absolute or relative path to the image or PDF.
+     * @returns Markdown string. Empty string if no text was detected.
+     * @throws {OllamaUnavailableError} If the Ollama server cannot be reached.
+     */
+    async toMarkdown(imagePath) {
+        // 1. Ensure Ollama is reachable before doing expensive OCR work.
+        if (!this.skipPing)
+            await ping(this.ollamaUrl);
+        // 2. Extract raw OCR blocks via Apple Vision.
+        //    For PDFs, macos-vision rasterizes each page and adds a `page` field.
+        const rawBlocks = await ocr(imagePath, { format: 'blocks' });
+        // 3. Split blocks by page — inferLayout() requires page-local coordinates.
+        const pageMap = groupBlocksByPage(rawBlocks);
+        // 4. Per page: infer layout → sort → group into paragraphs.
+        const allParagraphs = [];
+        for (const [pageIndex, pageBlocks] of [...pageMap.entries()].sort(([a], [b]) => a - b)) {
+            const layoutBlocks = inferLayout({ textBlocks: pageBlocks });
+            const sorted = sortBlocksByReadingOrder(layoutBlocks);
+            const paragraphs = groupByParagraph(sorted, pageIndex);
+            allParagraphs.push(...paragraphs);
+        }
+        if (allParagraphs.length === 0) {
+            return '';
+        }
+        // 5. Split paragraphs into chunks that fit within the output token budget.
+        const chunks = chunkParagraphs(allParagraphs, this.chunkSizeTokens);
+        // 6. Send each chunk to the LLM sequentially and join the results.
+        //    System prompt goes as role:"system", OCR text as role:"user" — this
+        //    prevents the model from treating instructions as content to summarise.
+        const parts = [];
+        for (const chunk of chunks) {
+            const part = await chat({ baseUrl: this.ollamaUrl, model: this.model }, SYSTEM_PROMPT, buildUserContent(chunk));
+            parts.push(part);
+        }
+        return parts.join('\n\n');
+    }
+}

package/dist/markdown/ollama.d.ts

@@ -0,0 +1,21 @@
+export interface OllamaOptions {
+    /** Base URL of the Ollama server. Default: `http://localhost:11434` */
+    baseUrl: string;
+    /** Model name to use for generation. Default: `mistral-nemo` */
+    model: string;
+}
+/**
+ * Verify that the Ollama server is reachable.
+ * Throws {@link OllamaUnavailableError} if the server cannot be contacted.
+ */
+export declare function ping(baseUrl: string): Promise<void>;
+/**
+ * Send a chat request to the Ollama /api/chat endpoint.
+ * Separating system and user roles significantly reduces hallucination compared
+ * to /api/generate where both are concatenated into a single completion string.
+ * Uses `stream: false` so the full response arrives in one shot.
+ */
+export declare function chat(opts: OllamaOptions, systemPrompt: string, userContent: string): Promise<string>;
+export declare class OllamaUnavailableError extends Error {
+    constructor(url: string);
+}

package/dist/markdown/ollama.js

@@ -0,0 +1,50 @@
+/**
+ * Verify that the Ollama server is reachable.
+ * Throws {@link OllamaUnavailableError} if the server cannot be contacted.
+ */
+export async function ping(baseUrl) {
+    try {
+        await fetch(`${baseUrl}/api/tags`, { signal: AbortSignal.timeout(3_000) });
+    }
+    catch {
+        throw new OllamaUnavailableError(baseUrl);
+    }
+}
+/**
+ * Send a chat request to the Ollama /api/chat endpoint.
+ * Separating system and user roles significantly reduces hallucination compared
+ * to /api/generate where both are concatenated into a single completion string.
+ * Uses `stream: false` so the full response arrives in one shot.
+ */
+export async function chat(opts, systemPrompt, userContent) {
+    const res = await fetch(`${opts.baseUrl}/api/chat`, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({
+            model: opts.model,
+            messages: [
+                { role: 'system', content: systemPrompt },
+                { role: 'user', content: userContent },
+            ],
+            stream: false,
+            options: {
+                temperature: 0, // deterministic — no creative token selection
+                top_p: 1, // full vocabulary considered; determinism comes from temperature=0
+                num_predict: -1, // no output truncation
+            },
+        }),
+        signal: AbortSignal.timeout(600_000),
+    });
+    if (!res.ok) {
+        throw new Error(`Ollama request failed: ${res.status} ${res.statusText}`);
+    }
+    const data = (await res.json());
+    return data.message.content.trim();
+}
+export class OllamaUnavailableError extends Error {
+    constructor(url) {
+        super(`Ollama is not reachable at ${url}. ` +
+            `Make sure Ollama is running (e.g. \`ollama serve\`) and the URL is correct.`);
+        this.name = 'OllamaUnavailableError';
+    }
+}

package/dist/markdown/prompt.d.ts

@@ -0,0 +1,35 @@
+import type { LayoutBlock } from '../index.js';
+/**
+ * A single paragraph: all TextBlocks sharing the same paragraphId,
+ * in reading order (top-to-bottom, left-to-right within each line).
+ */
+export interface ParagraphGroup {
+    paragraphId: number;
+    /** Average y-coordinate of the first line — used as a spatial hint. */
+    y: number;
+    lines: string[];
+    /** Zero-based page index (always 0 for single images). */
+    page: number;
+}
+/**
+ * Group sorted layout blocks by paragraphId into ParagraphGroup objects.
+ * Non-text blocks (faces, barcodes, etc.) are ignored.
+ */
+export declare function groupByParagraph(blocks: LayoutBlock[], pageIndex?: number): ParagraphGroup[];
+/**
+ * The system prompt sent as `role: "system"` in every chat request.
+ * Kept separate from user content so the model treats it as hard constraints,
+ * not as text to be summarised or analysed.
+ */
+export declare const SYSTEM_PROMPT = "ACT AS A HIGH-FIDELITY DOCUMENT PARSER. Your only goal is to reconstruct the provided OCR data into a structured Markdown document. NEVER skip text. NEVER summarize. Content must be 100% identical to the source.\n\nDO NOT SUMMARIZE.\nTranscribe every single word from the provided OCR data.\nMaintain 1:1 content fidelity. If the source has 5 paragraphs, the output must have 5 paragraphs.\n\nSTRICT OUTPUT: Output ONLY the Markdown representation. No preamble, no \"Summary of key events\", no \"Here is the result\".\n\nFORMATTING RULES:\n- Add # / ## / ### before lines that are clearly headings or titles\n- Add - before items that are clearly list entries\n- Join lines within the same paragraph into flowing prose\n- Preserve blank lines between paragraphs\n- Do NOT wrap output in code fences";
+/**
+ * Build the user-facing content block from a list of paragraphs.
+ * This is the OCR text that the model will format — no instructions included.
+ * Sent as `role: "user"` in the chat request.
+ */
+export declare function buildUserContent(paragraphs: ParagraphGroup[]): string;
+/**
+ * Build the combined string used for token estimation in the chunker.
+ * Mirrors what will be sent to the model (system + user content).
+ */
+export declare function buildPrompt(paragraphs: ParagraphGroup[]): string;

package/dist/markdown/prompt.js

@@ -0,0 +1,82 @@
+/**
+ * Group sorted layout blocks by paragraphId into ParagraphGroup objects.
+ * Non-text blocks (faces, barcodes, etc.) are ignored.
+ */
+export function groupByParagraph(blocks, pageIndex = 0) {
+    const map = new Map();
+    for (const block of blocks) {
+        if (block.kind !== 'text')
+            continue;
+        if (!map.has(block.paragraphId)) {
+            map.set(block.paragraphId, { y: block.y, lines: new Map() });
+        }
+        const para = map.get(block.paragraphId);
+        const existing = para.lines.get(block.lineId) ?? [];
+        existing.push(block.text);
+        para.lines.set(block.lineId, existing);
+    }
+    const groups = [];
+    for (const [paragraphId, { y, lines }] of map) {
+        // Join tokens within each line, then collect lines in order
+        const lineStrings = [...lines.entries()]
+            .sort(([a], [b]) => a - b)
+            .map(([, tokens]) => tokens.join(' '));
+        groups.push({ paragraphId, y, lines: lineStrings, page: pageIndex });
+    }
+    return groups;
+}
+/**
+ * The system prompt sent as `role: "system"` in every chat request.
+ * Kept separate from user content so the model treats it as hard constraints,
+ * not as text to be summarised or analysed.
+ */
+export const SYSTEM_PROMPT = `ACT AS A HIGH-FIDELITY DOCUMENT PARSER. \
+Your only goal is to reconstruct the provided OCR data into a structured \
+Markdown document. NEVER skip text. NEVER summarize. \
+Content must be 100% identical to the source.
+
+DO NOT SUMMARIZE.
+Transcribe every single word from the provided OCR data.
+Maintain 1:1 content fidelity. If the source has 5 paragraphs, the output must have 5 paragraphs.
+
+STRICT OUTPUT: Output ONLY the Markdown representation. \
+No preamble, no "Summary of key events", no "Here is the result".
+
+FORMATTING RULES:
+- Add # / ## / ### before lines that are clearly headings or titles
+- Add - before items that are clearly list entries
+- Join lines within the same paragraph into flowing prose
+- Preserve blank lines between paragraphs
+- Do NOT wrap output in code fences`;
+/**
+ * Build the user-facing content block from a list of paragraphs.
+ * This is the OCR text that the model will format — no instructions included.
+ * Sent as `role: "user"` in the chat request.
+ */
+export function buildUserContent(paragraphs) {
+    const pageNumbers = [...new Set(paragraphs.map(p => p.page))].sort((a, b) => a - b);
+    const multiPage = pageNumbers.length > 1;
+    const blocks = [];
+    for (const pageNum of pageNumbers) {
+        if (multiPage) {
+            blocks.push(`[Page ${pageNum + 1}]`);
+        }
+        const pageParagraphs = paragraphs.filter(p => p.page === pageNum);
+        for (const { paragraphId, y, lines } of pageParagraphs) {
+            const yHint = y.toFixed(2);
+            const header = `[Paragraph ${paragraphId}, y≈${yHint}]`;
+            blocks.push(`${header}\n${lines.join('\n')}`);
+        }
+    }
+    const task = 'Convert the OCR source below into Markdown. ' +
+        'Reproduce EVERY word EXACTLY. Do not respond, explain, or ask questions.\n\n' +
+        '<ocr_source>';
+    return `${task}\n\n${blocks.join('\n\n')}\n</ocr_source>`;
+}
+/**
+ * Build the combined string used for token estimation in the chunker.
+ * Mirrors what will be sent to the model (system + user content).
+ */
+export function buildPrompt(paragraphs) {
+    return `${SYSTEM_PROMPT}\n\n${buildUserContent(paragraphs)}`;
+}

package/package.json

@@ -1,15 +1,33 @@
 {
   "name": "macos-vision",
-  "version": "1.2.0",
-  "description": "Apple Vision OCR & image analysis for Node.js — native, fast, offline, no API keys",
+  "version": "1.3.0",
+  "description": "Apple Vision OCR + image/PDF analysis for Node.js, with optional Ollama-driven Markdown pipeline — native, fast, offline",
   "author": "Adrian Wolczuk",
   "license": "MIT",
   "type": "module",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
   "bin": {
-    "macos-vision": "./dist/cli.js"
+    "macos-vision": "dist/cli.js"
   },
+  "exports": {
+    ".": {
+      "import": "./dist/index.js",
+      "types": "./dist/index.d.ts"
+    },
+    "./markdown": {
+      "import": "./dist/markdown/index.js",
+      "types": "./dist/markdown/index.d.ts"
+    }
+  },
+  "files": [
+    "dist",
+    "bin",
+    "scripts/build-native.js",
+    "src/native",
+    "README.md",
+    "LICENSE"
+  ],
   "repository": {
     "type": "git",
     "url": "git+https://github.com/woladi/macos-vision.git"
@@ -25,7 +43,8 @@
     "lint": "eslint src/**/*.ts",
     "format": "prettier --write src/**/*.ts",
     "release": "release-it",
-    "release:beta": "release-it --preRelease=beta"
+    "release:beta": "release-it --preRelease=beta",
+    "eval:setup": "git clone https://github.com/opendataloader-project/opendataloader-bench eval/bench || echo 'bench already cloned'"
   },
   "keywords": [
     "ocr",
@@ -40,7 +59,13 @@
     "barcode",
     "qr-code",
     "document-detection",
-    "image-classification"
+    "image-classification",
+    "markdown",
+    "image-to-markdown",
+    "pdf-to-markdown",
+    "ollama",
+    "llm",
+    "document-pipeline"
   ],
   "lint-staged": {
     "src/**/*.ts": [

package/src/native/pdf-helper.swift

@@ -0,0 +1,122 @@
+import PDFKit
+import AppKit
+import Foundation
+
+// ─── Result struct ────────────────────────────────────────────────────────────
+
+struct PageResult: Codable {
+    let page: Int    // 0-based
+    let path: String
+}
+
+// ─── Helpers ──────────────────────────────────────────────────────────────────
+
+func fail(_ message: String) -> Never {
+    fputs("ERROR: \(message)\n", stderr)
+    exit(1)
+}
+
+func encodeJSON<T: Encodable>(_ value: T) -> String {
+    guard let data = try? JSONEncoder().encode(value),
+          let str = String(data: data, encoding: .utf8) else { return "[]" }
+    return str
+}
+
+// ─── Argument parsing ─────────────────────────────────────────────────────────
+
+let args = CommandLine.arguments
+guard args.count >= 2 else {
+    fail("Usage: pdf-helper <path-to-pdf>")
+}
+
+let pdfPath = args[1]
+let pdfURL = URL(fileURLWithPath: pdfPath)
+
+guard let pdf = PDFDocument(url: pdfURL) else {
+    fail("Cannot open PDF: \(pdfPath)")
+}
+
+let pageCount = pdf.pageCount
+guard pageCount > 0 else {
+    fail("PDF has no pages: \(pdfPath)")
+}
+
+// ─── Output directory: ~/.cache/macos-vision/{basename}-{uuid}/ ───────────────
+
+let basename = pdfURL.deletingPathExtension().lastPathComponent
+let uuid = UUID().uuidString.lowercased()
+let cacheBase = FileManager.default.homeDirectoryForCurrentUser
+    .appendingPathComponent(".cache/macos-vision")
+let outDir = cacheBase.appendingPathComponent("\(basename)-\(uuid)")
+
+do {
+    try FileManager.default.createDirectory(at: outDir, withIntermediateDirectories: true)
+} catch {
+    fail("Cannot create output directory \(outDir.path): \(error.localizedDescription)")
+}
+
+// ─── Rasterize each page at 300 DPI ──────────────────────────────────────────
+
+// PDF points are 72 pt/inch. Scale factor for 300 DPI = 300/72 ≈ 4.167
+let scale: CGFloat = 300.0 / 72.0
+
+var results: [PageResult] = []
+
+for pageIndex in 0..<pageCount {
+    guard let page = pdf.page(at: pageIndex) else {
+        fail("Cannot access page \(pageIndex) of \(pdfPath)")
+    }
+
+    let mediaBox = page.bounds(for: .mediaBox)
+    let width  = Int((mediaBox.width  * scale).rounded())
+    let height = Int((mediaBox.height * scale).rounded())
+
+    guard let bitmapRep = NSBitmapImageRep(
+        bitmapDataPlanes: nil,
+        pixelsWide: width,
+        pixelsHigh: height,
+        bitsPerSample: 8,
+        samplesPerPixel: 4,
+        hasAlpha: true,
+        isPlanar: false,
+        colorSpaceName: .calibratedRGB,
+        bytesPerRow: 0,
+        bitsPerPixel: 0
+    ) else {
+        fail("Cannot create bitmap for page \(pageIndex)")
+    }
+
+    guard let ctx = NSGraphicsContext(bitmapImageRep: bitmapRep) else {
+        fail("Cannot create graphics context for page \(pageIndex)")
+    }
+
+    // Fill white background (PDFs are transparent by default)
+    NSGraphicsContext.saveGraphicsState()
+    NSGraphicsContext.current = ctx
+    NSColor.white.setFill()
+    NSRect(x: 0, y: 0, width: width, height: height).fill()
+
+    ctx.cgContext.scaleBy(x: scale, y: scale)
+    page.draw(with: .mediaBox, to: ctx.cgContext)
+    NSGraphicsContext.restoreGraphicsState()
+
+    guard let pngData = bitmapRep.representation(using: .png, properties: [:]) else {
+        fail("Cannot encode page \(pageIndex) to PNG")
+    }
+
+    // Zero-pad page number to 3 digits: page-001.png, page-002.png, …
+    let filename = String(format: "%@-page-%03d.png", basename, pageIndex + 1)
+    let outPath = outDir.appendingPathComponent(filename)
+
+    do {
+        try pngData.write(to: outPath)
+    } catch {
+        fail("Cannot write \(outPath.path): \(error.localizedDescription)")
+    }
+
+    results.append(PageResult(page: pageIndex, path: outPath.path))
+}
+
+// ─── Output JSON ──────────────────────────────────────────────────────────────
+
+print(encodeJSON(results))