macos-vision 1.1.0 → 1.3.0

package/README.md

@@ -1,6 +1,6 @@
 # macos-vision
 
-> Apple Vision for Node.js — native, fast, offline, no API keys required.
+> Apple Vision for Node.js — native, fast, offline. Now with an optional Ollama-driven Markdown pipeline.
 
 Uses macOS's built-in [Vision framework](https://developer.apple.com/documentation/vision) via a compiled Swift binary. Works completely offline. No cloud services, no API keys, no Python, zero runtime dependencies.
 
@@ -8,11 +8,8 @@ Uses macOS's built-in [Vision framework](https://developer.apple.com/documentati
 
 - macOS 12+
 - Node.js 18+
-- Xcode Command Line Tools
-
-```bash
-xcode-select --install
-```
+- Xcode Command Line Tools (`xcode-select --install`)
+- [Ollama](https://ollama.com) running locally — only if you use the Markdown pipeline
 
 ## Installation
 
@@ -20,18 +17,18 @@ xcode-select --install
 npm install macos-vision
 ```
 
-The native Swift binary is compiled automatically on install.
-
-## What this is (and isn't)
-
-`macos-vision` gives you **raw Apple Vision results** — text, coordinates, bounding boxes, labels.
+The native Swift binaries (`vision-helper`, `pdf-helper`) are compiled automatically on install.
 
-It is **not** a document pipeline. It does not:
-- Convert PDFs or images to Markdown
-- Understand document structure (headings, tables, paragraphs)
-- Chain multiple detections into a final report
+## What you get
 
-For those use cases, use the raw output as input to an LLM or a post-processing layer of your own.
+| Capability | Engine | Network |
+|---|---|---|
+| OCR (text + bounding boxes) | Apple Vision | offline |
+| Face / barcode / rectangle / document detection | Apple Vision | offline |
+| Image classification | Apple Vision | offline |
+| Layout inference (lines, paragraphs, reading order) | heuristic in TypeScript | offline |
+| PDF rasterization | PDFKit (`pdf-helper`) | offline |
+| **Image / PDF → Markdown** | Apple Vision OCR + local LLM via Ollama | local LLM call |
 
 ---
 
@@ -44,60 +41,71 @@ npx macos-vision photo.jpg
 # Structured OCR blocks with bounding boxes
 npx macos-vision --blocks photo.jpg
 
-# Detect faces
+# Detections
 npx macos-vision --faces photo.jpg
-
-# Detect barcodes and QR codes
 npx macos-vision --barcodes photo.jpg
-
-# Detect rectangular shapes
 npx macos-vision --rectangles photo.jpg
-
-# Find document boundary
 npx macos-vision --document photo.jpg
-
-# Classify image content
 npx macos-vision --classify photo.jpg
 
 # Run all detections at once
 npx macos-vision --all photo.jpg
+
+# Image / PDF → Markdown via VisionScribe + Ollama
+npx macos-vision --markdown invoice.pdf -o notes.md
+npx macos-vision --markdown receipt.jpg --stdout
+npx macos-vision --markdown scan.png --model llama3.2
 ```
 
-Multiple flags can be combined: `npx macos-vision --blocks --faces --classify photo.jpg`
+Multiple Vision flags can be combined: `npx macos-vision --blocks --faces --classify photo.jpg`. Structured results are printed as JSON to stdout.
+
+### CLI flags
 
-Structured results are printed as JSON to stdout.
+| Flag | Description |
+|---|---|
+| `--ocr` | Plain text OCR (default when no flag is given) |
+| `--blocks` | OCR with bounding boxes (JSON) |
+| `--faces` / `--barcodes` / `--rectangles` / `--document` / `--classify` | Vision detections (JSON) |
+| `--all` | Run every Vision detection at once |
+| `--markdown` | Convert image / PDF to Markdown via VisionScribe + Ollama |
+| `--model <name>` | Ollama model (default: `mistral-nemo`). Only used with `--markdown` |
+| `--ollama-url <url>` | Ollama base URL (default: `http://localhost:11434`). Only used with `--markdown` |
+| `-o`, `--output <path>` | Write Markdown to a file. Only used with `--markdown` |
+| `--stdout` | Print Markdown to stdout instead of a file. Only used with `--markdown` |
+| `--help` | Show usage |
 
 ---
 
-## API
+## API — Vision
 
 ```js
-import { ocr, detectFaces, detectBarcodes, detectRectangles, detectDocument, classify } from 'macos-vision'
+import {
+  ocr,
+  detectFaces,
+  detectBarcodes,
+  detectRectangles,
+  detectDocument,
+  classify,
+  inferLayout,
+} from 'macos-vision';
 
 // OCR — plain text
-const text = await ocr('photo.jpg')
+const text = await ocr('photo.jpg');
 
 // OCR — structured blocks with bounding boxes
-const blocks = await ocr('photo.jpg', { format: 'blocks' })
-
-// Detect faces
-const faces = await detectFaces('photo.jpg')
-
-// Detect barcodes and QR codes
-const codes = await detectBarcodes('invoice.jpg')
-
-// Detect rectangular shapes (tables, forms, cards)
-const rects = await detectRectangles('document.jpg')
+const blocks = await ocr('photo.jpg', { format: 'blocks' });
 
-// Find document boundary in a photo
-const doc = await detectDocument('photo.jpg') // DocumentBounds | null
+// Detect faces / barcodes / rectangles / document boundary
+const faces = await detectFaces('photo.jpg');
+const codes = await detectBarcodes('invoice.jpg');
+const rects = await detectRectangles('document.jpg');
+const doc = await detectDocument('photo.jpg'); // DocumentBounds | null
 
 // Classify image content
-const labels = await classify('photo.jpg')
+const labels = await classify('photo.jpg');
 
 // Layout inference — unified reading-order-sorted representation
-const layout = inferLayout({ textBlocks: blocks, faces, barcodes: codes })
-// layout is LayoutBlock[] — ready to feed into a Markdown renderer or LLM context
+const layout = inferLayout({ textBlocks: blocks, faces, barcodes: codes });
 ```
 
 ### Layout inference
@@ -134,111 +142,161 @@ for (const block of layout) {
 
 > **Note:** Layout inference is a heuristic layer. It does not understand multi-column layouts or rotated text. Treat it as structured input for downstream tools, not as ground truth.
 
-## API
+---
 
-### `ocr(imagePath, options?)`
+## API — Markdown pipeline (VisionScribe)
 
-Extracts text from an image.
+`VisionScribe` converts an image or PDF to Markdown by combining Apple Vision OCR with a local LLM (via Ollama). The LLM never sees the image — it only formats text that Vision already extracted, which keeps the pipeline deterministic and prevents the hallucinations typical of cloud vision APIs.
 
-| Parameter | Type | Default | Description |
-|-----------|------|---------|-------------|
-| `imagePath` | `string` | — | Path to image (PNG, JPG, JPEG, WEBP) |
-| `options.format` | `'text' \| 'blocks'` | `'text'` | Plain text or structured blocks with coordinates |
-
-Returns `Promise<string>` or `Promise<VisionBlock[]>`.
+### Prerequisites
 
-```ts
-interface VisionBlock {
-  text: string
-  x: number       // 0–1 from left
-  y: number       // 0–1 from top
-  width: number   // 0–1
-  height: number  // 0–1
-}
+```bash
+brew install ollama
+ollama serve            # keep this running
+ollama pull mistral-nemo
 ```
 
----
+### Quick start
 
-### `detectFaces(imagePath)`
+```ts
+import { VisionScribe } from 'macos-vision';
+
+const scribe = new VisionScribe();
+const markdown = await scribe.toMarkdown('receipt.png');
+console.log(markdown);
+```
 
-Detects human faces and returns their bounding boxes.
+For a narrower import surface that pulls in only the markdown sub-module:
 
 ```ts
-interface Face {
-  x: number; y: number; width: number; height: number
-  confidence: number  // 0–1
-}
+import { VisionScribe } from 'macos-vision/markdown';
 ```
 
----
+### How it works
 
-### `detectBarcodes(imagePath)`
+```
+Image / PDF
+  │
+  ▼
+Apple Vision OCR          ← macOS native, deterministic, zero hallucination
+  │  VisionBlock[] per page
+  ▼
+Per-page layout inference ← each page processed independently (page-local coords)
+  │  paragraphId, lineId, y
+  ▼
+Chunker                   ← batches paragraphs to fit the LLM output window
+  │  ParagraphGroup[][]
+  ▼
+Ollama /api/chat          ← system prompt as role:"system", OCR text as role:"user"
+  │  temperature=0, top_p=1, num_predict=-1
+  ▼
+Markdown string           ← chunk results joined with blank lines
+```
 
-Detects barcodes and QR codes and decodes their payload.
+The LLM never sees the raw image; it only formats text that Apple Vision has already extracted. The system prompt instructs the model to act as a high-fidelity document parser and explicitly forbids summarising, paraphrasing, or adding content. OCR text is wrapped in `<ocr_source>` tags so the model cannot mistake it for a user asking a question. Per-page processing keeps paragraph coordinates from different pages from being mixed.
 
-```ts
-interface Barcode {
-  type: string    // e.g. 'org.iso.QRCode', 'org.gs1.EAN-13'
-  value: string   // decoded content
-  x: number; y: number; width: number; height: number
-}
-```
+### `new VisionScribe(options?)`
 
----
+| Option | Type | Default | Description |
+|---|---|---|---|
+| `model` | `string` | `'mistral-nemo'` | Ollama model name |
+| `ollamaUrl` | `string` | `'http://localhost:11434'` | Base URL of the Ollama server |
+| `skipPing` | `boolean` | `false` | Skip per-call Ollama health check (useful in batch loops) |
+| `chunkSizeTokens` | `number` | `1800` | Max estimated output tokens per LLM chunk. Lower = more chunks (safer for small models); higher = fewer calls but risks hitting model output limits |
+
+### `scribe.toMarkdown(imagePath)`
 
-### `detectRectangles(imagePath)`
+- Accepts PNG, JPEG, HEIC, HEIF, TIFF, GIF, BMP, WebP and **PDF**
+- Returns an empty string `''` if no text is detected
+- Throws `OllamaUnavailableError` if the Ollama server is not reachable (unless `skipPing: true`)
 
-Finds rectangular shapes (documents, tables, cards, forms).
+### Batch processing
 
 ```ts
-interface Rectangle {
-  topLeft: [number, number]; topRight: [number, number]
-  bottomLeft: [number, number]; bottomRight: [number, number]
-  confidence: number
+import { VisionScribe, OllamaUnavailableError } from 'macos-vision';
+
+const scribe = new VisionScribe({ skipPing: true });
+
+for (const file of files) {
+  try {
+    const md = await scribe.toMarkdown(file);
+    // …
+  } catch (e) {
+    if (e instanceof OllamaUnavailableError) {
+      console.error(e.message);
+      break;
+    }
+    throw e;
+  }
 }
 ```
 
+### Known limitations
+
+- **Local model fidelity**: small models (`mistral-nemo`, `gemma`) may occasionally summarise or paraphrase long, dense documents. Larger models (`llama3.1:70b`, `qwen2.5:32b`) produce significantly better fidelity.
+- **Tables**: multi-column table layouts are partially supported. OCR reads cells in reading order but the LLM may not always reconstruct correct Markdown table syntax.
+- **Images / charts**: non-textual content (photos, diagrams, charts) is ignored — only text blocks extracted by Apple Vision are processed.
+
 ---
 
-### `detectDocument(imagePath)`
+## Migrating from `macos-vision-md`
 
-Finds the boundary of a document in a photo (e.g. paper on a desk). Returns `null` if no document is found.
+The standalone [`macos-vision-md`](https://github.com/woladi/macos-vision-md) package has been merged into `macos-vision` as of v2.0.0. The old package will keep working as a thin re-export shim, but new projects should depend on `macos-vision` directly.
 
-```ts
-interface DocumentBounds {
-  topLeft: [number, number]; topRight: [number, number]
-  bottomLeft: [number, number]; bottomRight: [number, number]
-  confidence: number
-}
+```diff
+- import { VisionScribe } from 'macos-vision-md';
++ import { VisionScribe } from 'macos-vision';
+```
+
+```diff
+- macos-vision-md invoice.pdf -o notes.md
++ macos-vision --markdown invoice.pdf -o notes.md
 ```
 
+The `VisionScribe` API, the system prompt, and the chunking strategy are unchanged. `OllamaUnavailableError`, `VisionScribeOptions`, and `ParagraphGroup` are now exported from `macos-vision`.
+
 ---
 
-### `classify(imagePath)`
+## API reference — types
 
-Returns top image classification labels with confidence scores.
+### `ocr(imagePath, options?)`
+
+| Parameter | Type | Default | Description |
+|-----------|------|---------|-------------|
+| `imagePath` | `string` | — | Path to image (PNG, JPG, JPEG, WEBP) or PDF |
+| `options.format` | `'text' \| 'blocks'` | `'text'` | Plain text or structured blocks with coordinates |
+
+Returns `Promise<string>` or `Promise<VisionBlock[]>`.
 
 ```ts
-interface Classification {
-  identifier: string   // e.g. 'document', 'outdoor', 'animal'
-  confidence: number   // 0–1
+interface VisionBlock {
+  text: string
+  x: number       // 0–1 from left
+  y: number       // 0–1 from top
+  width: number   // 0–1
+  height: number  // 0–1
+  confidence: number
+  page?: number   // 0-based, only for PDFs
 }
 ```
 
+### `detectFaces(imagePath)` / `detectBarcodes(imagePath)` / `detectRectangles(imagePath)` / `detectDocument(imagePath)` / `classify(imagePath)`
+
+See `src/index.ts` for full type declarations.
+
 ---
 
 ## Why macos-vision?
 
 | | macos-vision | Tesseract.js | Cloud APIs |
 |---|---|---|---|
-| Offline | ✅ | ✅ | ❌ |
+| Offline OCR | ✅ | ✅ | ❌ |
+| Offline image → Markdown | ✅ (with local Ollama) | ❌ | ❌ |
 | No API key | ✅ | ✅ | ❌ |
 | Native speed | ✅ | ❌ | — |
 | Zero runtime deps | ✅ | ❌ | ❌ |
 | OCR with bounding boxes | ✅ | ✅ | ✅ |
-| Face detection | ✅ | ❌ | ✅ |
-| Barcode / QR | ✅ | ❌ | ✅ |
-| Document detection | ✅ | ❌ | ✅ |
+| Face / barcode / document detection | ✅ | ❌ | ✅ |
 | Image classification | ✅ | ❌ | ✅ |
 | macOS only | ✅ | ❌ | ❌ |
 

package/dist/cli.js

@@ -1,91 +1,154 @@
 #!/usr/bin/env node
-import { resolve } from 'path';
+import { resolve, dirname, basename, extname, join } from 'path';
+import { writeFile } from 'fs/promises';
 import { ocr, detectFaces, detectBarcodes, detectRectangles, detectDocument, classify, } from './index.js';
 const USAGE = `
-Usage: vision-cli [options] <image>
+Usage: macos-vision [options] <image-or-pdf>
 
-Options:
-  --ocr           OCR — plain text (default)
-  --blocks        OCR — structured blocks with coordinates
-  --faces         Face detection
-  --barcodes      Barcode & QR code detection
-  --rectangles    Rectangle detection
-  --document      Document boundary detection
-  --classify      Image classification
-  --all           Run all of the above
+Vision options:
+  --ocr                  OCR — plain text (default)
+  --blocks               OCR — structured blocks with coordinates
+  --faces                Face detection
+  --barcodes             Barcode & QR code detection
+  --rectangles           Rectangle detection
+  --document             Document boundary detection
+  --classify             Image classification
+  --all                  Run all of the above
 
-  --help          Show this help
+Markdown options (requires Ollama running locally):
+  --markdown             Convert image/PDF to Markdown via VisionScribe + Ollama
+  --model <name>         Ollama model name (default: mistral-nemo)
+  --ollama-url <url>     Ollama base URL (default: http://localhost:11434)
+  -o, --output <path>    Write Markdown to specified file
+  --stdout               Print Markdown to stdout instead of a file
+
+  --help                 Show this help
 
 Examples:
-  vision-cli photo.jpg
-  vision-cli --blocks --faces photo.jpg
-  vision-cli --all photo.jpg
+  macos-vision photo.jpg
+  macos-vision --blocks --faces photo.jpg
+  macos-vision --all photo.jpg
+  macos-vision --markdown invoice.pdf -o notes.md
+  macos-vision --markdown receipt.jpg --stdout
 `.trim();
 const rawArgs = process.argv.slice(2);
 if (rawArgs.includes('--help') || rawArgs.length === 0) {
     console.log(USAGE);
     process.exit(0);
 }
-const flags = new Set(rawArgs.filter((a) => a.startsWith('--')));
-const fileArgs = rawArgs.filter((a) => !a.startsWith('--'));
+// Strip value-bearing options first so the remaining tokens are either
+// boolean flags (`--something`) or positional file paths.
+function takeOpt(name, args) {
+    const i = args.indexOf(name);
+    if (i === -1)
+        return undefined;
+    const v = args[i + 1];
+    args.splice(i, 2);
+    return v;
+}
+const argv = [...rawArgs];
+const model = takeOpt('--model', argv);
+const ollamaUrl = takeOpt('--ollama-url', argv);
+const outPath = takeOpt('-o', argv) ?? takeOpt('--output', argv);
+const flags = new Set(argv.filter((a) => a.startsWith('--')));
+const fileArgs = argv.filter((a) => !a.startsWith('-'));
 if (!fileArgs[0]) {
-    console.error('Error: no image path provided.\n');
+    console.error('Error: no image or PDF path provided.\n');
     console.log(USAGE);
     process.exit(1);
 }
-const imagePath = resolve(fileArgs[0]);
-const runAll = flags.has('--all');
-const runOcr = runAll || flags.has('--ocr');
-const runBlocks = runAll || flags.has('--blocks');
-const runFaces = runAll || flags.has('--faces');
-const runBarcodes = runAll || flags.has('--barcodes');
-const runRects = runAll || flags.has('--rectangles');
-const runDoc = runAll || flags.has('--document');
-const runClassify = runAll || flags.has('--classify');
-// Default: OCR text when no feature flag is given
-const anyFeatureFlag = runAll ||
-    flags.has('--ocr') ||
-    flags.has('--blocks') ||
-    flags.has('--faces') ||
-    flags.has('--barcodes') ||
-    flags.has('--rectangles') ||
-    flags.has('--document') ||
-    flags.has('--classify');
-const useDefault = !anyFeatureFlag;
-async function main() {
-    try {
-        if (useDefault || runOcr) {
-            const text = await ocr(imagePath);
-            console.log(text);
-        }
-        if (runBlocks) {
-            const blocks = (await ocr(imagePath, { format: 'blocks' }));
-            console.log(JSON.stringify(blocks, null, 2));
-        }
-        if (runFaces) {
-            const faces = (await detectFaces(imagePath));
-            console.log(JSON.stringify(faces, null, 2));
+const inputPath = resolve(fileArgs[0]);
+// ─── Markdown pipeline ─────────────────────────────────────────────────────────────
+if (flags.has('--markdown')) {
+    const toStdout = flags.has('--stdout');
+    const opts = {};
+    if (model)
+        opts.model = model;
+    if (ollamaUrl)
+        opts.ollamaUrl = ollamaUrl;
+    (async () => {
+        const { VisionScribe, OllamaUnavailableError } = await import('./markdown/index.js');
+        const scribe = new VisionScribe(opts);
+        if (!toStdout)
+            process.stderr.write(`Converting ${fileArgs[0]}…\n`);
+        let markdown;
+        try {
+            markdown = await scribe.toMarkdown(inputPath);
         }
-        if (runBarcodes) {
-            const barcodes = (await detectBarcodes(imagePath));
-            console.log(JSON.stringify(barcodes, null, 2));
+        catch (err) {
+            if (err instanceof OllamaUnavailableError) {
+                console.error(err.message);
+                process.exit(2);
+            }
+            throw err;
         }
-        if (runRects) {
-            const rectangles = (await detectRectangles(imagePath));
-            console.log(JSON.stringify(rectangles, null, 2));
+        if (toStdout) {
+            process.stdout.write(markdown);
+            return;
         }
-        if (runDoc) {
-            const doc = (await detectDocument(imagePath));
-            console.log(JSON.stringify(doc, null, 2));
+        const finalPath = outPath ??
+            join(dirname(inputPath), basename(inputPath, extname(inputPath)) + '.md');
+        await writeFile(finalPath, markdown, 'utf8');
+        process.stderr.write(`Saved: ${finalPath}\n`);
+    })().catch((err) => {
+        console.error(err instanceof Error ? err.message : String(err));
+        process.exit(1);
+    });
+}
+else {
+    // ─── Vision pipeline (OCR / detections / classification) ───────────────────────
+    const runAll = flags.has('--all');
+    const runOcr = runAll || flags.has('--ocr');
+    const runBlocks = runAll || flags.has('--blocks');
+    const runFaces = runAll || flags.has('--faces');
+    const runBarcodes = runAll || flags.has('--barcodes');
+    const runRects = runAll || flags.has('--rectangles');
+    const runDoc = runAll || flags.has('--document');
+    const runClassify = runAll || flags.has('--classify');
+    // Default: OCR text when no feature flag is given
+    const anyFeatureFlag = runAll ||
+        flags.has('--ocr') ||
+        flags.has('--blocks') ||
+        flags.has('--faces') ||
+        flags.has('--barcodes') ||
+        flags.has('--rectangles') ||
+        flags.has('--document') ||
+        flags.has('--classify');
+    const useDefault = !anyFeatureFlag;
+    (async () => {
+        try {
+            if (useDefault || runOcr) {
+                const text = await ocr(inputPath);
+                console.log(text);
+            }
+            if (runBlocks) {
+                const blocks = (await ocr(inputPath, { format: 'blocks' }));
+                console.log(JSON.stringify(blocks, null, 2));
+            }
+            if (runFaces) {
+                const faces = (await detectFaces(inputPath));
+                console.log(JSON.stringify(faces, null, 2));
+            }
+            if (runBarcodes) {
+                const barcodes = (await detectBarcodes(inputPath));
+                console.log(JSON.stringify(barcodes, null, 2));
+            }
+            if (runRects) {
+                const rectangles = (await detectRectangles(inputPath));
+                console.log(JSON.stringify(rectangles, null, 2));
+            }
+            if (runDoc) {
+                const doc = (await detectDocument(inputPath));
+                console.log(JSON.stringify(doc, null, 2));
+            }
+            if (runClassify) {
+                const labels = (await classify(inputPath));
+                console.log(JSON.stringify(labels, null, 2));
+            }
         }
-        if (runClassify) {
-            const labels = (await classify(imagePath));
-            console.log(JSON.stringify(labels, null, 2));
+        catch (error) {
+            console.error('Error:', error);
+            process.exit(1);
         }
-    }
-    catch (error) {
-        console.error('Error:', error);
-        process.exit(1);
-    }
+    })();
 }
-main();

package/dist/index.d.ts

@@ -1,3 +1,24 @@
+export interface PdfPage {
+    /** 0-based page index */
+    page: number;
+    /** Absolute path to the rasterized PNG file */
+    path: string;
+}
+export interface PdfRasterizeResult {
+    /** Pages in document order */
+    pages: PdfPage[];
+    /** Directory containing all rasterized PNGs */
+    cacheDir: string;
+}
+/**
+ * Rasterizes a PDF to 300 DPI PNG files using the native `pdf-helper` binary
+ * (PDFKit-based). Files are saved persistently to `~/.cache/macos-vision/`
+ * so they can be reused by downstream tools — **caller is responsible for cleanup**.
+ *
+ * @param pdfPath - Absolute or relative path to the PDF file.
+ * @returns An object with `pages` (sorted array of `{page, path}`) and `cacheDir`.
+ */
+export declare function rasterizePdf(pdfPath: string): Promise<PdfRasterizeResult>;
 export interface VisionBlock {
     /** Recognized text */
     text: string;
@@ -91,3 +112,5 @@ export interface Classification {
 export declare function classify(imagePath: string): Promise<Classification[]>;
 export type { BlockKind, BaseBlock, TextBlock, FaceBlock, BarcodeBlock, RectangleBlock, DocumentBlock, LayoutBlock, InferLayoutInput, } from './layout.js';
 export { inferLayout, sortBlocksByReadingOrder } from './layout.js';
+export { VisionScribe, OllamaUnavailableError } from './markdown/index.js';
+export type { VisionScribeOptions, ParagraphGroup } from './markdown/index.js';