@mlx-node/vlm 0.0.0

package/README.md

@@ -0,0 +1,266 @@
+# @mlx-node/vlm
+
+Vision-language models and document processing pipelines for Node.js on Apple Silicon. Extract text, tables, and structure from documents and images using PaddleOCR-VL and the PP-StructureV3 pipeline — all running locally on Metal GPU.
+
+## Requirements
+
+- macOS with Apple Silicon (M1 or later)
+- Node.js 18+
+
+## Installation
+
+```bash
+npm install @mlx-node/vlm
+```
+
+## Quick Start
+
+### Document Structure Analysis
+
+Use `StructureV3Pipeline` for layout detection + OCR without a VLM:
+
+```typescript
+import { StructureV3Pipeline } from '@mlx-node/vlm';
+
+const pipeline = await StructureV3Pipeline.load({
+  layoutModelPath: './models/PP-DocLayoutV3',
+  textDetModelPath: './models/PP-OCRv5-det',
+  textRecModelPath: './models/PP-OCRv5-rec',
+  dictPath: './models/PP-OCRv5-rec/en_dict.txt',
+});
+
+const result = await pipeline.analyze('./document.png');
+console.log(result.markdown);
+```
+
+### VLM-based OCR
+
+Use `VLModel` for higher-quality OCR with document understanding:
+
+```typescript
+import { VLModel, parsePaddleResponse } from '@mlx-node/vlm';
+
+const model = await VLModel.load('./models/PaddleOCR-VL-1.5-mlx');
+
+const result = model.chat([{ role: 'user', content: 'Read the text in this image.' }], {
+  images: [imageBuffer],
+  maxNewTokens: 2048,
+});
+
+const formatted = parsePaddleResponse(result.text, { format: 'markdown' });
+console.log(formatted);
+```
+
+## StructureV3Pipeline
+
+The PP-StructureV3 pipeline combines multiple specialized models for fast, accurate document processing:
+
+```
+Input Image
+    │
+    ▼ (optional)
+Orientation Correction → Dewarping
+    │
+    ▼
+Layout Detection (25 categories)
+    │
+    ▼
+Per-element Cropping
+    │
+    ▼
+Text Detection → Text Recognition
+    │
+    ▼
+Markdown Assembly
+```
+
+### Pipeline Options
+
+```typescript
+const result = await pipeline.analyze(imageBuffer, {
+  layoutThreshold: 0.5, // Layout detection confidence threshold
+  textDetThreshold: 0.3, // Text detection confidence threshold
+  includeDetails: true, // Include per-element bounding boxes
+  useDocOrientationClassify: true, // Auto-correct rotation (requires DocOrientationModel)
+  useDocUnwarping: true, // Auto-correct perspective (requires DocUnwarpModel)
+});
+
+// result.elements — array of StructuredElement with label, bbox, text lines
+// result.markdown — assembled markdown document
+```
+
+### OCR Without Layout
+
+Run OCR directly on a pre-cropped image:
+
+```typescript
+const lines = await pipeline.ocrImage(croppedImageBuffer);
+for (const line of lines) {
+  console.log(`${line.text} (confidence: ${line.score.toFixed(2)})`);
+}
+```
+
+## Individual Models
+
+Each pipeline model can be used independently:
+
+### DocLayoutModel — Document Layout Detection
+
+PP-DocLayoutV3 with RT-DETR architecture, 25 layout categories (title, text, table, figure, formula, header, footer, etc.):
+
+```typescript
+import { DocLayoutModel } from '@mlx-node/vlm';
+
+const layout = await DocLayoutModel.load('./models/PP-DocLayoutV3');
+const elements = layout.detect(imageBuffer, 0.5); // threshold
+
+for (const el of elements) {
+  console.log(`${el.labelName} (${el.score.toFixed(2)}) at [${el.bbox}]`);
+}
+```
+
+### TextDetModel — Text Line Detection
+
+PP-OCRv5 DBNet with PPHGNetV2 backbone:
+
+```typescript
+import { TextDetModel } from '@mlx-node/vlm';
+
+const detector = await TextDetModel.load('./models/PP-OCRv5-det');
+const boxes = detector.detect(imageBuffer);
+```
+
+### TextRecModel — Text Recognition
+
+PP-OCRv5 SVTR neck + CTC head:
+
+```typescript
+import { TextRecModel } from '@mlx-node/vlm';
+
+const recognizer = await TextRecModel.load('./models/PP-OCRv5-rec');
+const results = recognizer.recognizeBatch(croppedImages);
+// [{ text: "Hello world", score: 0.98 }, ...]
+```
+
+### DocOrientationModel — Orientation Classification
+
+Classifies document rotation (0/90/180/270 degrees):
+
+```typescript
+import { DocOrientationModel } from '@mlx-node/vlm';
+
+const classifier = await DocOrientationModel.load('./models/PP-LCNet_x1_0_doc_ori-mlx');
+const { angle, score } = classifier.classify(imageBuffer);
+const { image } = classifier.classifyAndRotate(imageBuffer); // auto-correct
+```
+
+### DocUnwarpModel — Document Dewarping
+
+UVDocNet-based perspective correction:
+
+```typescript
+import { DocUnwarpModel } from '@mlx-node/vlm';
+
+const unwarper = await DocUnwarpModel.load('./models/UVDoc-mlx');
+const { image } = unwarper.unwarp(imageBuffer);
+```
+
+## VLModel — Vision-Language Model
+
+PaddleOCR-VL architecture (ERNIE language model + vision encoder):
+
+```typescript
+import { VLModel } from '@mlx-node/vlm';
+
+const model = await VLModel.load('./models/PaddleOCR-VL-1.5-mlx');
+
+// Single image chat
+const result = model.chat(messages, {
+  images: [imageBuffer],
+  maxNewTokens: 2048,
+  temperature: 0.1,
+});
+
+// Simple OCR
+const text = model.ocr(imageBuffer);
+
+// Batch OCR (multiple images)
+const results = model.ocrBatch([image1, image2, image3]);
+
+// Batch chat (different prompts per image)
+const batchResults = model.batch([
+  { messages: tableMessages, images: [tableImage] },
+  { messages: formulaMessages, images: [formulaImage] },
+]);
+```
+
+## Output Parsing
+
+Parse VLM output into structured documents:
+
+```typescript
+import { parseVlmOutput, formatDocument, parsePaddleResponse, OutputFormat } from '@mlx-node/vlm';
+
+// Two-step: parse then format
+const doc = parseVlmOutput(result.text);
+const markdown = formatDocument(doc, { format: OutputFormat.Markdown });
+
+// One-step: parse and format
+const html = parsePaddleResponse(result.text, { format: 'html' });
+```
+
+### XLSX Export
+
+```typescript
+import { saveToXlsx, documentToXlsx } from '@mlx-node/vlm';
+
+// Direct save
+saveToXlsx(result.text, './output.xlsx');
+
+// Or get buffer
+const doc = parseVlmOutput(result.text);
+const buffer = documentToXlsx(doc);
+```
+
+## API Reference
+
+### Pipeline
+
+| Class/Function                         | Description                                                       |
+| -------------------------------------- | ----------------------------------------------------------------- |
+| `StructureV3Pipeline.load(config)`     | Load pipeline with layout, text detection, and recognition models |
+| `pipeline.analyze(image, options?)`    | Full document analysis returning `StructuredDocument`             |
+| `pipeline.ocrImage(image, threshold?)` | OCR on a single cropped image                                     |
+
+### Models
+
+| Class                 | Description                                                                     |
+| --------------------- | ------------------------------------------------------------------------------- |
+| `VLModel`             | PaddleOCR-VL vision-language model — `chat()`, `ocr()`, `ocrBatch()`, `batch()` |
+| `DocLayoutModel`      | PP-DocLayoutV3 layout detection — `detect()`                                    |
+| `TextDetModel`        | PP-OCRv5 text detection — `detect()`, `detectCrop()`                            |
+| `TextRecModel`        | PP-OCRv5 text recognition — `recognize()`, `recognizeBatch()`                   |
+| `DocOrientationModel` | Orientation classifier — `classify()`, `classifyAndRotate()`                    |
+| `DocUnwarpModel`      | Document dewarping — `unwarp()`                                                 |
+
+### Output
+
+| Function                             | Description                              |
+| ------------------------------------ | ---------------------------------------- |
+| `parseVlmOutput(text)`               | Parse raw VLM text to `ParsedDocument`   |
+| `formatDocument(doc, config?)`       | Format to markdown, HTML, plain, or JSON |
+| `parsePaddleResponse(text, config?)` | Parse and format in one step             |
+| `documentToXlsx(doc)`                | Convert parsed document to XLSX buffer   |
+| `saveToXlsx(text, path)`             | Parse and save to XLSX file              |
+
+### Configs
+
+| Export                      | Description                                      |
+| --------------------------- | ------------------------------------------------ |
+| `PADDLEOCR_VL_CONFIGS`      | Pre-defined PaddleOCR-VL 1.5 config              |
+| `createPaddleocrVlConfig()` | Default config factory                           |
+| `OutputFormat`              | Enum: `Raw`, `Plain`, `Markdown`, `Html`, `Json` |
+
+## License
+
+[MIT](https://github.com/mlx-node/mlx-node/blob/main/LICENSE)

package/dist/index.d.ts

@@ -0,0 +1,41 @@
+/**
+ * @mlx-node/vlm - Vision Language Model support for MLX-Node
+ *
+ * This package provides VLM capabilities including:
+ * - VLModel for OCR and document understanding tasks
+ *
+ * @example
+ * ```typescript
+ * import { VLModel } from '@mlx-node/vlm';
+ *
+ * // Load a model
+ * const model = await VLModel.load('./models/paddleocr-vl');
+ *
+ * // Chat with images
+ * const imageBuffer = readFileSync('./photo.jpg');
+ * const result = await model.chat(
+ *   [{ role: 'user', content: 'What is in this image?' }],
+ *   { images: [imageBuffer] }
+ * );
+ * console.log(result.text);
+ *
+ * // Simple OCR
+ * const text = await model.ocr(readFileSync('./document.jpg'));
+ *
+ * // Batch OCR (multiple images)
+ * const texts = await model.ocrBatch([readFileSync('page1.jpg'), readFileSync('page2.jpg')]);
+ * ```
+ */
+export { VLModel, createPaddleocrVlConfig } from '@mlx-node/core';
+export { DocLayoutModel, type LayoutElement } from '@mlx-node/core';
+export { TextDetModel, type TextBox, TextRecModel, type RecResult } from '@mlx-node/core';
+export { DocOrientationModel, type OrientationResult, type ClassifyRotateResult, DocUnwarpModel, type UnwarpResult, } from '@mlx-node/core';
+export { StructureV3Pipeline, type StructureV3Config, type AnalyzeOptions, type StructuredElement, type StructuredDocument, type TextLine, } from './pipeline/structure-v3.js';
+export type { VisionConfig, TextConfig, ModelConfig, VlmChatConfig, VlmChatMessage, VlmBatchItem, } from '@mlx-node/core';
+export { PADDLEOCR_VL_CONFIGS, type PaddleOCRVLConfig } from './models/paddleocr-vl-configs.js';
+export { VlmChatResult, type VLMChatResult } from '@mlx-node/core';
+export { parsePaddleResponse, parseVlmOutput, formatDocument, type ParsedDocument, type DocumentElement, type Table, type TableRow, type TableCell, type Paragraph, type ParserConfig, OutputFormat, } from '@mlx-node/core';
+export { documentToXlsx, saveToXlsx } from '@mlx-node/core';
+export { Qwen3Tokenizer as Tokenizer } from '@mlx-node/lm';
+export { MxArray, type DType } from '@mlx-node/core';
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAKH,OAAO,EAAE,OAAO,EAAE,uBAAuB,EAAE,MAAM,gBAAgB,CAAC;AAGlE,OAAO,EAAE,cAAc,EAAE,KAAK,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAGpE,OAAO,EAAE,YAAY,EAAE,KAAK,OAAO,EAAE,YAAY,EAAE,KAAK,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAG1F,OAAO,EACL,mBAAmB,EACnB,KAAK,iBAAiB,EACtB,KAAK,oBAAoB,EACzB,cAAc,EACd,KAAK,YAAY,GAClB,MAAM,gBAAgB,CAAC;AAGxB,OAAO,EACL,mBAAmB,EACnB,KAAK,iBAAiB,EACtB,KAAK,cAAc,EACnB,KAAK,iBAAiB,EACtB,KAAK,kBAAkB,EACvB,KAAK,QAAQ,GACd,MAAM,4BAA4B,CAAC;AAGpC,YAAY,EACV,YAAY,EACZ,UAAU,EACV,WAAW,EACX,aAAa,EACb,cAAc,EACd,YAAY,GACb,MAAM,gBAAgB,CAAC;AAGxB,OAAO,EAAE,oBAAoB,EAAE,KAAK,iBAAiB,EAAE,MAAM,kCAAkC,CAAC;AAGhG,OAAO,EAAE,aAAa,EAAE,KAAK,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAGnE,OAAO,EACL,mBAAmB,EACnB,cAAc,EACd,cAAc,EACd,KAAK,cAAc,EACnB,KAAK,eAAe,EACpB,KAAK,KAAK,EACV,KAAK,QAAQ,EACb,KAAK,SAAS,EACd,KAAK,SAAS,EACd,KAAK,YAAY,EACjB,YAAY,GACb,MAAM,gBAAgB,CAAC;AAGxB,OAAO,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAG5D,OAAO,EAAE,cAAc,IAAI,SAAS,EAAE,MAAM,cAAc,CAAC;AAC3D,OAAO,EAAE,OAAO,EAAE,KAAK,KAAK,EAAE,MAAM,gBAAgB,CAAC"}

package/dist/index.js

@@ -0,0 +1,50 @@
+/**
+ * @mlx-node/vlm - Vision Language Model support for MLX-Node
+ *
+ * This package provides VLM capabilities including:
+ * - VLModel for OCR and document understanding tasks
+ *
+ * @example
+ * ```typescript
+ * import { VLModel } from '@mlx-node/vlm';
+ *
+ * // Load a model
+ * const model = await VLModel.load('./models/paddleocr-vl');
+ *
+ * // Chat with images
+ * const imageBuffer = readFileSync('./photo.jpg');
+ * const result = await model.chat(
+ *   [{ role: 'user', content: 'What is in this image?' }],
+ *   { images: [imageBuffer] }
+ * );
+ * console.log(result.text);
+ *
+ * // Simple OCR
+ * const text = await model.ocr(readFileSync('./document.jpg'));
+ *
+ * // Batch OCR (multiple images)
+ * const texts = await model.ocrBatch([readFileSync('page1.jpg'), readFileSync('page2.jpg')]);
+ * ```
+ */
+// ============== PUBLIC API ==============
+// Main model class and factory functions (exposed directly from Rust)
+export { VLModel, createPaddleocrVlConfig } from '@mlx-node/core';
+// Document layout analysis model
+export { DocLayoutModel } from '@mlx-node/core';
+// Text detection and recognition models (PP-OCRv5)
+export { TextDetModel, TextRecModel } from '@mlx-node/core';
+// Document preprocessing models
+export { DocOrientationModel, DocUnwarpModel, } from '@mlx-node/core';
+// Document understanding pipeline (PP-StructureV3)
+export { StructureV3Pipeline, } from './pipeline/structure-v3.js';
+// Model-specific configs
+export { PADDLEOCR_VL_CONFIGS } from './models/paddleocr-vl-configs.js';
+// Chat result type
+export { VlmChatResult } from '@mlx-node/core';
+// Output parsing and formatting (Rust implementation)
+export { parsePaddleResponse, parseVlmOutput, formatDocument, OutputFormat, } from '@mlx-node/core';
+// XLSX export (Rust implementation)
+export { documentToXlsx, saveToXlsx } from '@mlx-node/core';
+// Re-export shared utilities
+export { Qwen3Tokenizer as Tokenizer } from '@mlx-node/lm';
+export { MxArray } from '@mlx-node/core';

package/dist/models/index.d.ts

@@ -0,0 +1,5 @@
+/**
+ * Model exports
+ */
+export { PADDLEOCR_VL_CONFIGS, type PaddleOCRVLConfig } from './paddleocr-vl-configs.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/models/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/models/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,OAAO,EAAE,oBAAoB,EAAE,KAAK,iBAAiB,EAAE,MAAM,2BAA2B,CAAC"}

package/dist/models/index.js

@@ -0,0 +1,4 @@
+/**
+ * Model exports
+ */
+export { PADDLEOCR_VL_CONFIGS } from './paddleocr-vl-configs.js';

package/dist/models/paddleocr-vl-configs.d.ts

@@ -0,0 +1,53 @@
+/**
+ * PaddleOCR-VL Model Configurations
+ *
+ * Based on the PaddleOCR-VL-1.5 model architecture.
+ */
+export interface VisionConfig {
+    modelType: 'paddleocr_vl';
+    hiddenSize: number;
+    intermediateSize: number;
+    numHiddenLayers: number;
+    numAttentionHeads: number;
+    numChannels: number;
+    imageSize: number;
+    patchSize: number;
+    hiddenAct: string;
+    layerNormEps: number;
+    attentionDropout: number;
+    spatialMergeSize: number;
+}
+export interface TextConfig {
+    modelType: 'paddleocr_vl';
+    hiddenSize: number;
+    numHiddenLayers: number;
+    intermediateSize: number;
+    numAttentionHeads: number;
+    rmsNormEps: number;
+    vocabSize: number;
+    numKeyValueHeads: number;
+    maxPositionEmbeddings: number;
+    ropeTheta: number;
+    ropeTraditional: boolean;
+    useBias: boolean;
+    headDim: number;
+    mropeSection: [number, number, number];
+}
+export interface PaddleOCRVLConfig {
+    visionConfig: VisionConfig;
+    textConfig: TextConfig;
+    modelType: 'paddleocr_vl';
+    ignoreIndex: number;
+    imageTokenId: number;
+    videoTokenId: number;
+    visionStartTokenId: number;
+    visionEndTokenId: number;
+    eosTokenId: number;
+}
+/**
+ * Available PaddleOCR-VL configurations
+ */
+export declare const PADDLEOCR_VL_CONFIGS: {
+    readonly 'paddleocr-vl-1.5': PaddleOCRVLConfig;
+};
+//# sourceMappingURL=paddleocr-vl-configs.d.ts.map

package/dist/models/paddleocr-vl-configs.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"paddleocr-vl-configs.d.ts","sourceRoot":"","sources":["../../src/models/paddleocr-vl-configs.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,MAAM,WAAW,YAAY;IAC3B,SAAS,EAAE,cAAc,CAAC;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,gBAAgB,EAAE,MAAM,CAAC;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,gBAAgB,EAAE,MAAM,CAAC;IACzB,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,UAAU;IACzB,SAAS,EAAE,cAAc,CAAC;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,eAAe,EAAE,MAAM,CAAC;IACxB,gBAAgB,EAAE,MAAM,CAAC;IACzB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,CAAC;IACzB,qBAAqB,EAAE,MAAM,CAAC;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,eAAe,EAAE,OAAO,CAAC;IACzB,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;CACxC;AAED,MAAM,WAAW,iBAAiB;IAChC,YAAY,EAAE,YAAY,CAAC;IAC3B,UAAU,EAAE,UAAU,CAAC;IACvB,SAAS,EAAE,cAAc,CAAC;IAC1B,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,gBAAgB,EAAE,MAAM,CAAC;IACzB,UAAU,EAAE,MAAM,CAAC;CACpB;AA6CD;;GAEG;AACH,eAAO,MAAM,oBAAoB;;CAEvB,CAAC"}

package/dist/models/paddleocr-vl-configs.js

@@ -0,0 +1,53 @@
+/**
+ * PaddleOCR-VL Model Configurations
+ *
+ * Based on the PaddleOCR-VL-1.5 model architecture.
+ */
+/**
+ * PaddleOCR-VL-1.5 default configuration
+ */
+const PADDLEOCR_VL_1_5_CONFIG = {
+    modelType: 'paddleocr_vl',
+    ignoreIndex: -100,
+    imageTokenId: 100295,
+    videoTokenId: 100296,
+    visionStartTokenId: 101305,
+    visionEndTokenId: 101306,
+    eosTokenId: 2,
+    visionConfig: {
+        modelType: 'paddleocr_vl',
+        hiddenSize: 1152,
+        intermediateSize: 4304,
+        numHiddenLayers: 27,
+        numAttentionHeads: 16,
+        numChannels: 3,
+        imageSize: 384,
+        patchSize: 14,
+        hiddenAct: 'gelu_pytorch_tanh',
+        layerNormEps: 1e-6,
+        attentionDropout: 0.0,
+        spatialMergeSize: 2,
+    },
+    textConfig: {
+        modelType: 'paddleocr_vl',
+        hiddenSize: 1024,
+        numHiddenLayers: 18,
+        intermediateSize: 3072,
+        numAttentionHeads: 16,
+        rmsNormEps: 1e-5,
+        vocabSize: 103424,
+        numKeyValueHeads: 2,
+        maxPositionEmbeddings: 131072,
+        ropeTheta: 500000.0,
+        ropeTraditional: false,
+        useBias: false,
+        headDim: 128,
+        mropeSection: [16, 24, 24],
+    },
+};
+/**
+ * Available PaddleOCR-VL configurations
+ */
+export const PADDLEOCR_VL_CONFIGS = {
+    'paddleocr-vl-1.5': PADDLEOCR_VL_1_5_CONFIG,
+};

package/dist/pipeline/structure-v3.d.ts

@@ -0,0 +1,130 @@
+/**
+ * PP-StructureV3 Document Understanding Pipeline
+ *
+ * Combines PP-DocLayoutV3 (layout detection) with PP-OCRv5 (text detection + recognition)
+ * for fast, accurate document understanding without a VLM.
+ *
+ * Pipeline:
+ *   1. DocLayoutModel detects layout elements (titles, text, tables, figures...)
+ *   2. For text/title/list elements: TextDetModel detects text lines → TextRecModel recognizes text
+ *   3. For table/formula/chart: VLM fallback (optional)
+ *   4. Results assembled into structured markdown in reading order
+ *
+ * @example
+ * ```typescript
+ * import { StructureV3Pipeline } from '@mlx-node/vlm';
+ *
+ * const pipeline = StructureV3Pipeline.load({
+ *   layoutModelPath: './models/PP-DocLayoutV3',
+ *   textDetModelPath: './models/PP-OCRv5_server_det',
+ *   textRecModelPath: './models/PP-OCRv5_server_rec',
+ *   dictPath: './models/PP-OCRv5_server_rec/ppocr_keys_v1.txt',
+ * });
+ *
+ * const result = pipeline.analyze('./document.png');
+ * console.log(result.markdown);
+ * ```
+ */
+/** Configuration for loading the StructureV3 pipeline. */
+export interface StructureV3Config {
+    /** Path to PP-DocLayoutV3 model directory */
+    layoutModelPath: string;
+    /** Path to PP-OCRv5 text detection model directory */
+    textDetModelPath: string;
+    /** Path to PP-OCRv5 text recognition model directory */
+    textRecModelPath: string;
+    /** Path to character dictionary file (e.g., ppocr_keys_v1.txt) */
+    dictPath: string;
+    /** Path to doc orientation classification model directory (optional) */
+    docOrientationModelPath?: string;
+    /** Path to doc unwarping model directory (optional) */
+    docUnwarpModelPath?: string;
+}
+/** Options for document analysis. */
+export interface AnalyzeOptions {
+    /** Layout detection confidence threshold (default: 0.5) */
+    layoutThreshold?: number;
+    /** Text detection confidence threshold (default: 0.3) */
+    textDetThreshold?: number;
+    /** Whether to include element-level details in output (default: false) */
+    includeDetails?: boolean;
+    /** Whether to run document orientation classification (default: true if model loaded) */
+    useDocOrientationClassify?: boolean;
+    /** Whether to run document unwarping (default: true if model loaded) */
+    useDocUnwarping?: boolean;
+}
+/** A recognized text line within a layout element. */
+export interface TextLine {
+    /** Bounding box [x1, y1, x2, y2] relative to the element crop */
+    bbox: number[];
+    /** Recognized text */
+    text: string;
+    /** Recognition confidence */
+    score: number;
+}
+/** A structured document element with recognized content. */
+export interface StructuredElement {
+    /** Element type from layout detection */
+    label: string;
+    /** Detection confidence */
+    score: number;
+    /** Bounding box [x1, y1, x2, y2] in original image coordinates */
+    bbox: number[];
+    /** Reading order index */
+    order: number;
+    /** Recognized text content */
+    text: string;
+    /** Individual text lines (if includeDetails is true) */
+    lines?: TextLine[];
+}
+/** Result of document analysis. */
+export interface StructuredDocument {
+    /** Structured elements in reading order */
+    elements: StructuredElement[];
+    /** Assembled markdown output */
+    markdown: string;
+}
+/**
+ * PP-StructureV3 document understanding pipeline.
+ *
+ * Uses dedicated OCR models (TextDet + TextRec) instead of a VLM,
+ * providing ~4-5x faster text extraction with ~6x lower memory usage.
+ */
+export declare class StructureV3Pipeline {
+    private layout;
+    private textDet;
+    private textRec;
+    private docOrientation;
+    private docUnwarp;
+    private constructor();
+    /**
+     * Load all models and create the pipeline.
+     */
+    static load(config: StructureV3Config): StructureV3Pipeline;
+    /**
+     * Analyze a document image and extract structured content.
+     *
+     * @param imageData - Buffer with encoded image bytes, or a file path string
+     * @param options - Analysis options
+     * @returns Structured document with elements and markdown
+     */
+    analyze(imageData: Buffer | string, options?: AnalyzeOptions): Promise<StructuredDocument>;
+    /**
+     * Run text detection + recognition on a single image (no layout detection).
+     *
+     * Useful for processing pre-cropped text regions.
+     */
+    ocrImage(imageData: Buffer, textDetThreshold?: number): Promise<TextLine[]>;
+    /**
+     * Detect text lines and recognize text in a cropped region.
+     *
+     * For each detected text line bounding box, sub-crops that line from the
+     * crop image and passes the individual line image to text recognition.
+     */
+    private ocrRegion;
+    /**
+     * Crop a layout element from the source image and return PNG bytes.
+     */
+    private cropElement;
+}
+//# sourceMappingURL=structure-v3.d.ts.map

package/dist/pipeline/structure-v3.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"structure-v3.d.ts","sourceRoot":"","sources":["../../src/pipeline/structure-v3.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAgBH,0DAA0D;AAC1D,MAAM,WAAW,iBAAiB;IAChC,6CAA6C;IAC7C,eAAe,EAAE,MAAM,CAAC;IACxB,sDAAsD;IACtD,gBAAgB,EAAE,MAAM,CAAC;IACzB,wDAAwD;IACxD,gBAAgB,EAAE,MAAM,CAAC;IACzB,kEAAkE;IAClE,QAAQ,EAAE,MAAM,CAAC;IACjB,wEAAwE;IACxE,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,uDAAuD;IACvD,kBAAkB,CAAC,EAAE,MAAM,CAAC;CAC7B;AAED,qCAAqC;AACrC,MAAM,WAAW,cAAc;IAC7B,2DAA2D;IAC3D,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,yDAAyD;IACzD,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,0EAA0E;IAC1E,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,yFAAyF;IACzF,yBAAyB,CAAC,EAAE,OAAO,CAAC;IACpC,wEAAwE;IACxE,eAAe,CAAC,EAAE,OAAO,CAAC;CAC3B;AAED,sDAAsD;AACtD,MAAM,WAAW,QAAQ;IACvB,iEAAiE;IACjE,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,sBAAsB;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,6BAA6B;IAC7B,KAAK,EAAE,MAAM,CAAC;CACf;AAED,6DAA6D;AAC7D,MAAM,WAAW,iBAAiB;IAChC,yCAAyC;IACzC,KAAK,EAAE,MAAM,CAAC;IACd,2BAA2B;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,kEAAkE;IAClE,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,0BAA0B;IAC1B,KAAK,EAAE,MAAM,CAAC;IACd,8BAA8B;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,wDAAwD;IACxD,KAAK,CAAC,EAAE,QAAQ,EAAE,CAAC;CACpB;AAED,mCAAmC;AACnC,MAAM,WAAW,kBAAkB;IACjC,2CAA2C;IAC3C,QAAQ,EAAE,iBAAiB,EAAE,CAAC;IAC9B,gCAAgC;IAChC,QAAQ,EAAE,MAAM,CAAC;CAClB;AAqCD;;;;;GAKG;AACH,qBAAa,mBAAmB;IAC9B,OAAO,CAAC,MAAM,CAAiB;IAC/B,OAAO,CAAC,OAAO,CAAe;IAC9B,OAAO,CAAC,OAAO,CAAe;IAC9B,OAAO,CAAC,cAAc,CAA6B;IACnD,OAAO,CAAC,SAAS,CAAwB;IAEzC,OAAO;IAcP;;OAEG;IACH,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,iBAAiB,GAAG,mBAAmB;IAa3D;;;;;;OAMG;IACG,OAAO,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,EAAE,OAAO,GAAE,cAAmB,GAAG,OAAO,CAAC,kBAAkB,CAAC;IAgGpG;;;;OAIG;IACG,QAAQ,CAAC,SAAS,EAAE,MAAM,EAAE,gBAAgB,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;IAIjF;;;;;OAKG;YACW,SAAS;IAwDvB;;OAEG;YACW,WAAW;CAW1B"}

package/dist/pipeline/structure-v3.js

@@ -0,0 +1,314 @@
+/**
+ * PP-StructureV3 Document Understanding Pipeline
+ *
+ * Combines PP-DocLayoutV3 (layout detection) with PP-OCRv5 (text detection + recognition)
+ * for fast, accurate document understanding without a VLM.
+ *
+ * Pipeline:
+ *   1. DocLayoutModel detects layout elements (titles, text, tables, figures...)
+ *   2. For text/title/list elements: TextDetModel detects text lines → TextRecModel recognizes text
+ *   3. For table/formula/chart: VLM fallback (optional)
+ *   4. Results assembled into structured markdown in reading order
+ *
+ * @example
+ * ```typescript
+ * import { StructureV3Pipeline } from '@mlx-node/vlm';
+ *
+ * const pipeline = StructureV3Pipeline.load({
+ *   layoutModelPath: './models/PP-DocLayoutV3',
+ *   textDetModelPath: './models/PP-OCRv5_server_det',
+ *   textRecModelPath: './models/PP-OCRv5_server_rec',
+ *   dictPath: './models/PP-OCRv5_server_rec/ppocr_keys_v1.txt',
+ * });
+ *
+ * const result = pipeline.analyze('./document.png');
+ * console.log(result.markdown);
+ * ```
+ */
+import { readFileSync } from 'node:fs';
+import { DocLayoutModel, TextDetModel, TextRecModel, DocOrientationModel, DocUnwarpModel, } from '@mlx-node/core';
+// ============================================================================
+// Element type sets
+// ============================================================================
+/** Elements that contain text and should be processed with OCR */
+const TEXT_LABELS = new Set([
+    'title',
+    'doc_title',
+    'paragraph_title',
+    'text',
+    'abstract',
+    'list',
+    'table_caption',
+    'table_footnote',
+    'figure_caption',
+    'chart_caption',
+    'formula_caption',
+    'code_txt',
+    'header',
+    'footer',
+    'footnote',
+    'margin_note',
+    'reference',
+    'content',
+    'index',
+    'handwriting',
+]);
+/** Elements that are non-content (skipped) */
+const SKIP_LABELS = new Set(['abandon']);
+// ============================================================================
+// Pipeline
+// ============================================================================
+/**
+ * PP-StructureV3 document understanding pipeline.
+ *
+ * Uses dedicated OCR models (TextDet + TextRec) instead of a VLM,
+ * providing ~4-5x faster text extraction with ~6x lower memory usage.
+ */
+export class StructureV3Pipeline {
+    layout;
+    textDet;
+    textRec;
+    docOrientation;
+    docUnwarp;
+    constructor(layout, textDet, textRec, docOrientation, docUnwarp) {
+        this.layout = layout;
+        this.textDet = textDet;
+        this.textRec = textRec;
+        this.docOrientation = docOrientation;
+        this.docUnwarp = docUnwarp;
+    }
+    /**
+     * Load all models and create the pipeline.
+     */
+    static load(config) {
+        const layout = DocLayoutModel.load(config.layoutModelPath);
+        const textDet = TextDetModel.load(config.textDetModelPath);
+        const textRec = TextRecModel.load(config.textRecModelPath, config.dictPath);
+        const docOrientation = config.docOrientationModelPath
+            ? DocOrientationModel.load(config.docOrientationModelPath)
+            : null;
+        const docUnwarp = config.docUnwarpModelPath ? DocUnwarpModel.load(config.docUnwarpModelPath) : null;
+        return new StructureV3Pipeline(layout, textDet, textRec, docOrientation, docUnwarp);
+    }
+    /**
+     * Analyze a document image and extract structured content.
+     *
+     * @param imageData - Buffer with encoded image bytes, or a file path string
+     * @param options - Analysis options
+     * @returns Structured document with elements and markdown
+     */
+    async analyze(imageData, options = {}) {
+        const { layoutThreshold = 0.5, textDetThreshold, includeDetails = false } = options;
+        let imageBuffer = typeof imageData === 'string' ? readFileSync(imageData) : imageData;
+        // Step 0a: Document orientation correction
+        if (this.docOrientation && (options.useDocOrientationClassify ?? true)) {
+            const rotateResult = this.docOrientation.classifyAndRotate(imageBuffer);
+            if (rotateResult.angle !== 0) {
+                imageBuffer = Buffer.from(rotateResult.image);
+            }
+        }
+        // Step 0b: Document unwarping
+        if (this.docUnwarp && (options.useDocUnwarping ?? true)) {
+            const unwarpResult = this.docUnwarp.unwarp(imageBuffer);
+            imageBuffer = Buffer.from(unwarpResult.image);
+        }
+        // Step 1: Layout detection (on preprocessed image)
+        const layoutElements = this.layout.detect(imageBuffer, layoutThreshold);
+        if (layoutElements.length === 0) {
+            return { elements: [], markdown: '' };
+        }
+        // Step 2: Process each element
+        const elements = [];
+        for (const el of layoutElements) {
+            const label = el.labelName;
+            if (SKIP_LABELS.has(label)) {
+                continue;
+            }
+            if (TEXT_LABELS.has(label)) {
+                // OCR path: detect text lines then recognize
+                const cropBuffer = await this.cropElement(imageBuffer, el);
+                const textLines = await this.ocrRegion(cropBuffer, textDetThreshold);
+                const fullText = textLines.map((l) => l.text).join('\n');
+                elements.push({
+                    label,
+                    score: el.score,
+                    bbox: el.bbox,
+                    order: el.order,
+                    text: fullText,
+                    lines: includeDetails ? textLines : undefined,
+                });
+            }
+            else if (label === 'table') {
+                // Table: detect text lines in each cell (simplified — no cell-level structure yet)
+                const cropBuffer = await this.cropElement(imageBuffer, el);
+                const textLines = await this.ocrRegion(cropBuffer, textDetThreshold);
+                const fullText = textLines.map((l) => l.text).join('\n');
+                elements.push({
+                    label,
+                    score: el.score,
+                    bbox: el.bbox,
+                    order: el.order,
+                    text: fullText,
+                    lines: includeDetails ? textLines : undefined,
+                });
+            }
+            else if (label === 'isolate_formula') {
+                // Formula: basic OCR (no LaTeX recognition yet)
+                const cropBuffer = await this.cropElement(imageBuffer, el);
+                const textLines = await this.ocrRegion(cropBuffer, textDetThreshold);
+                const fullText = textLines.map((l) => l.text).join(' ');
+                elements.push({
+                    label,
+                    score: el.score,
+                    bbox: el.bbox,
+                    order: el.order,
+                    text: fullText,
+                });
+            }
+            else {
+                // figure, chart, seal, etc. - placeholder
+                elements.push({
+                    label,
+                    score: el.score,
+                    bbox: el.bbox,
+                    order: el.order,
+                    text: '',
+                });
+            }
+        }
+        // Step 3: Assemble markdown
+        const markdown = assembleMarkdown(elements);
+        return { elements, markdown };
+    }
+    /**
+     * Run text detection + recognition on a single image (no layout detection).
+     *
+     * Useful for processing pre-cropped text regions.
+     */
+    async ocrImage(imageData, textDetThreshold) {
+        return this.ocrRegion(imageData, textDetThreshold);
+    }
+    /**
+     * Detect text lines and recognize text in a cropped region.
+     *
+     * For each detected text line bounding box, sub-crops that line from the
+     * crop image and passes the individual line image to text recognition.
+     */
+    async ocrRegion(imageData, textDetThreshold) {
+        // Detect text lines within the crop
+        const textBoxes = this.textDet.detect(imageData, textDetThreshold);
+        if (textBoxes.length === 0) {
+            // Fall back to recognizing the entire crop as one text line
+            const result = this.textRec.recognize(imageData);
+            if (result.text.trim()) {
+                return [{ bbox: [0, 0, 0, 0], text: result.text, score: result.score }];
+            }
+            return [];
+        }
+        // Sort text boxes by vertical position (top to bottom, left to right)
+        const sorted = [...textBoxes].sort((a, b) => {
+            const yDiff = a.bbox[1] - b.bbox[1];
+            if (Math.abs(yDiff) > 10)
+                return yDiff;
+            return a.bbox[0] - b.bbox[0];
+        });
+        // Single detected line: recognize the full crop directly (no sub-crop needed)
+        if (sorted.length === 1) {
+            const result = this.textRec.recognize(imageData);
+            return [
+                {
+                    bbox: sorted[0].bbox,
+                    text: result.text,
+                    score: result.score,
+                },
+            ];
+        }
+        // Multiple detected lines: sub-crop each line from the crop image
+        const { Transformer } = await import('@napi-rs/image');
+        const lineBuffers = [];
+        for (let i = 0; i < sorted.length; i++) {
+            const [x1, y1, x2, y2] = sorted[i].bbox;
+            const x = Math.max(0, Math.round(x1));
+            const y = Math.max(0, Math.round(y1));
+            const w = Math.max(1, Math.round(x2 - x1));
+            const h = Math.max(1, Math.round(y2 - y1));
+            const linePng = await new Transformer(imageData).crop(x, y, w, h).png();
+            lineBuffers.push(Buffer.from(linePng));
+        }
+        const results = this.textRec.recognizeBatch(lineBuffers);
+        return sorted.map((box, i) => ({
+            bbox: box.bbox,
+            text: results[i]?.text ?? '',
+            score: results[i]?.score ?? 0,
+        }));
+    }
+    /**
+     * Crop a layout element from the source image and return PNG bytes.
+     */
+    async cropElement(imageBuffer, el) {
+        const [x1, y1, x2, y2] = el.bbox;
+        const x = Math.max(0, Math.round(x1));
+        const y = Math.max(0, Math.round(y1));
+        const w = Math.max(1, Math.round(x2 - x1));
+        const h = Math.max(1, Math.round(y2 - y1));
+        const { Transformer } = await import('@napi-rs/image');
+        const cropped = await new Transformer(imageBuffer).crop(x, y, w, h).png();
+        return Buffer.from(cropped);
+    }
+}
+// ============================================================================
+// Markdown assembly
+// ============================================================================
+/** Format a single element as markdown. */
+function formatElement(label, text, order) {
+    const trimmed = text.trim();
+    if (!trimmed)
+        return '';
+    switch (label) {
+        case 'doc_title':
+            return `# ${trimmed}\n`;
+        case 'title':
+            return `## ${trimmed}\n`;
+        case 'paragraph_title':
+            return `### ${trimmed}\n`;
+        case 'abstract':
+            return `> ${trimmed}\n`;
+        case 'table':
+            return `${trimmed}\n`;
+        case 'table_caption':
+        case 'figure_caption':
+        case 'chart_caption':
+        case 'formula_caption':
+            return `*${trimmed}*\n`;
+        case 'isolate_formula':
+            return `$$\n${trimmed}\n$$\n`;
+        case 'code_txt':
+            return `\`\`\`\n${trimmed}\n\`\`\`\n`;
+        case 'figure':
+        case 'chart':
+            return trimmed ? `[${label}: ${trimmed}]\n` : `[${label}]\n`;
+        case 'header':
+        case 'footer':
+            return `<!-- ${label}: ${trimmed} -->\n`;
+        case 'footnote':
+        case 'table_footnote':
+            return `[^note-${order}]: ${trimmed}\n`;
+        case 'list':
+            return `${trimmed}\n`;
+        case 'seal':
+            return `[seal: ${trimmed}]\n`;
+        default:
+            return `${trimmed}\n`;
+    }
+}
+/** Assemble structured elements into markdown. */
+function assembleMarkdown(elements) {
+    const parts = [];
+    for (const el of elements) {
+        const formatted = formatElement(el.label, el.text, el.order);
+        if (formatted) {
+            parts.push(formatted);
+        }
+    }
+    return parts.join('\n');
+}

package/package.json

@@ -0,0 +1,35 @@
+{
+  "name": "@mlx-node/vlm",
+  "version": "0.0.0",
+  "homepage": "https://github.com/mlx-node/mlx-node",
+  "bugs": {
+    "url": "https://github.com/mlx-node/mlx-node/issues"
+  },
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/mlx-node/mlx-node.git",
+    "directory": "packages/vlm"
+  },
+  "files": [
+    "dist"
+  ],
+  "type": "module",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  },
+  "scripts": {
+    "build": "tsc -b",
+    "test": "vite test run"
+  },
+  "dependencies": {
+    "@mlx-node/core": "workspace:*",
+    "@mlx-node/lm": "workspace:*",
+    "@napi-rs/image": "^1.12.0"
+  }
+}