@mlx-node/vlm 0.0.6 → 0.0.7

package/README.md

@@ -1,6 +1,6 @@
 # @mlx-node/vlm
 
-Vision-language models and document processing pipelines for Node.js on Apple Silicon. Extract text, tables, and structure from documents and images using PaddleOCR-VL and the PP-StructureV3 pipeline — all running locally on Metal GPU.
+Vision-language models and document processing pipelines for Node.js on Apple Silicon. Extract text, tables, and structure from documents and images using PaddleOCR-VL, Qianfan-OCR (InternVL), and the PP-StructureV3 pipeline — all running locally on Metal GPU.
 
 ## Requirements
 
@@ -13,6 +13,39 @@ Vision-language models and document processing pipelines for Node.js on Apple Si
 npm install @mlx-node/vlm
 ```
 
+## Multi-turn chat with VLMs
+
+`QianfanOCRModel` conforms to the same `ChatSession<M>` surface as the language models in `@mlx-node/lm`, so a single session handle drives both single-shot and multi-turn VLM conversations:
+
+```typescript
+import { ChatSession } from '@mlx-node/lm';
+import { QianfanOCRModel } from '@mlx-node/vlm';
+import { readFileSync } from 'node:fs';
+
+const model = await QianfanOCRModel.load('./models/Qianfan-VL');
+const session = new ChatSession(model, { system: 'You read documents precisely.' });
+
+// First turn with an image.
+const r1 = await session.send('Extract the text from this receipt.', {
+  images: [readFileSync('./receipt.jpg')],
+});
+console.log(r1.text);
+
+// Text-only follow-up reuses the same KV cache against the same image.
+const r2 = await session.send('What is the total price?');
+console.log(r2.text);
+
+// Swapping the image mid-session forcibly restarts the cache with the new image.
+const r3 = await session.send('And this one?', {
+  images: [readFileSync('./other-receipt.jpg')],
+});
+console.log(r3.text);
+```
+
+`loadSession()` from `@mlx-node/lm` cannot load Qianfan-OCR (that would introduce a circular package dependency), so construct the wrapper here with `QianfanOCRModel.load()` and pass it to `ChatSession` directly.
+
+The one-shot `VLModel.chat()` API shown below for PaddleOCR-VL is a single-turn OCR entry point and is intentionally kept out of the session surface.
+
 ## Quick Start
 
 ### Document Structure Analysis

package/dist/index.d.ts

@@ -33,6 +33,9 @@ export { DocOrientationModel, type OrientationResult, type ClassifyRotateResult,
 export { StructureV3Pipeline, type StructureV3Config, type AnalyzeOptions, type StructuredElement, type StructuredDocument, type TextLine, } from './pipeline/structure-v3.js';
 export type { VisionConfig, TextConfig, ModelConfig, VlmChatConfig, VlmChatMessage, VlmBatchItem, } from '@mlx-node/core';
 export { PADDLEOCR_VL_CONFIGS, type PaddleOCRVLConfig } from './models/paddleocr-vl-configs.js';
+export { QianfanOCRModel } from './models/qianfan-ocr.js';
+export { createQianfanOcrConfig } from '@mlx-node/core';
+export type { QianfanOcrConfig, InternVisionConfig, Qwen3LmConfig } from '@mlx-node/core';
 export { VlmChatResult, type VLMChatResult } from '@mlx-node/core';
 export { parsePaddleResponse, parseVlmOutput, formatDocument, type ParsedDocument, type DocumentElement, type Table, type TableRow, type TableCell, type Paragraph, type ParserConfig, OutputFormat, } from '@mlx-node/core';
 export { documentToXlsx, saveToXlsx } from '@mlx-node/core';

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAKH,OAAO,EAAE,OAAO,EAAE,uBAAuB,EAAE,MAAM,gBAAgB,CAAC;AAGlE,OAAO,EAAE,cAAc,EAAE,KAAK,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAGpE,OAAO,EAAE,YAAY,EAAE,KAAK,OAAO,EAAE,YAAY,EAAE,KAAK,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAG1F,OAAO,EACL,mBAAmB,EACnB,KAAK,iBAAiB,EACtB,KAAK,oBAAoB,EACzB,cAAc,EACd,KAAK,YAAY,GAClB,MAAM,gBAAgB,CAAC;AAGxB,OAAO,EACL,mBAAmB,EACnB,KAAK,iBAAiB,EACtB,KAAK,cAAc,EACnB,KAAK,iBAAiB,EACtB,KAAK,kBAAkB,EACvB,KAAK,QAAQ,GACd,MAAM,4BAA4B,CAAC;AAGpC,YAAY,EACV,YAAY,EACZ,UAAU,EACV,WAAW,EACX,aAAa,EACb,cAAc,EACd,YAAY,GACb,MAAM,gBAAgB,CAAC;AAGxB,OAAO,EAAE,oBAAoB,EAAE,KAAK,iBAAiB,EAAE,MAAM,kCAAkC,CAAC;AAGhG,OAAO,EAAE,aAAa,EAAE,KAAK,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAGnE,OAAO,EACL,mBAAmB,EACnB,cAAc,EACd,cAAc,EACd,KAAK,cAAc,EACnB,KAAK,eAAe,EACpB,KAAK,KAAK,EACV,KAAK,QAAQ,EACb,KAAK,SAAS,EACd,KAAK,SAAS,EACd,KAAK,YAAY,EACjB,YAAY,GACb,MAAM,gBAAgB,CAAC;AAGxB,OAAO,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAG5D,OAAO,EAAE,cAAc,IAAI,SAAS,EAAE,MAAM,cAAc,CAAC;AAC3D,OAAO,EAAE,OAAO,EAAE,KAAK,KAAK,EAAE,MAAM,gBAAgB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAKH,OAAO,EAAE,OAAO,EAAE,uBAAuB,EAAE,MAAM,gBAAgB,CAAC;AAGlE,OAAO,EAAE,cAAc,EAAE,KAAK,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAGpE,OAAO,EAAE,YAAY,EAAE,KAAK,OAAO,EAAE,YAAY,EAAE,KAAK,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAG1F,OAAO,EACL,mBAAmB,EACnB,KAAK,iBAAiB,EACtB,KAAK,oBAAoB,EACzB,cAAc,EACd,KAAK,YAAY,GAClB,MAAM,gBAAgB,CAAC;AAGxB,OAAO,EACL,mBAAmB,EACnB,KAAK,iBAAiB,EACtB,KAAK,cAAc,EACnB,KAAK,iBAAiB,EACtB,KAAK,kBAAkB,EACvB,KAAK,QAAQ,GACd,MAAM,4BAA4B,CAAC;AAGpC,YAAY,EACV,YAAY,EACZ,UAAU,EACV,WAAW,EACX,aAAa,EACb,cAAc,EACd,YAAY,GACb,MAAM,gBAAgB,CAAC;AAGxB,OAAO,EAAE,oBAAoB,EAAE,KAAK,iBAAiB,EAAE,MAAM,kCAAkC,CAAC;AAGhG,OAAO,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAC1D,OAAO,EAAE,sBAAsB,EAAE,MAAM,gBAAgB,CAAC;AACxD,YAAY,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAG1F,OAAO,EAAE,aAAa,EAAE,KAAK,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAGnE,OAAO,EACL,mBAAmB,EACnB,cAAc,EACd,cAAc,EACd,KAAK,cAAc,EACnB,KAAK,eAAe,EACpB,KAAK,KAAK,EACV,KAAK,QAAQ,EACb,KAAK,SAAS,EACd,KAAK,SAAS,EACd,KAAK,YAAY,EACjB,YAAY,GACb,MAAM,gBAAgB,CAAC;AAGxB,OAAO,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAG5D,OAAO,EAAE,cAAc,IAAI,SAAS,EAAE,MAAM,cAAc,CAAC;AAC3D,OAAO,EAAE,OAAO,EAAE,KAAK,KAAK,EAAE,MAAM,gBAAgB,CAAC"}

package/dist/index.js

@@ -39,6 +39,9 @@ export { DocOrientationModel, DocUnwarpModel, } from '@mlx-node/core';
 export { StructureV3Pipeline, } from './pipeline/structure-v3.js';
 // Model-specific configs
 export { PADDLEOCR_VL_CONFIGS } from './models/paddleocr-vl-configs.js';
+// Qianfan-OCR model (InternVL architecture)
+export { QianfanOCRModel } from './models/qianfan-ocr.js';
+export { createQianfanOcrConfig } from '@mlx-node/core';
 // Chat result type
 export { VlmChatResult } from '@mlx-node/core';
 // Output parsing and formatting (Rust implementation)

package/dist/models/qianfan-ocr.d.ts

@@ -0,0 +1,27 @@
+import { QianfanOCRModel as QianfanOCRModelNative } from '@mlx-node/core';
+import type { ChatConfig, ChatMessage } from '@mlx-node/core';
+import type { ChatStreamEvent } from '@mlx-node/lm';
+/**
+ * Qianfan-OCR Vision-Language Model wrapper.
+ *
+ * Streaming is driven through the `ChatSession` API — the overrides
+ * below adapt the callback-based native methods to
+ * `AsyncGenerator<ChatStreamEvent>` so the wrapper structurally
+ * satisfies `SessionCapableModel` from `@mlx-node/lm`.
+ *
+ * Qianfan-OCR is a VLM (InternViT + Qwen3 language model). The
+ * continue path cannot splice new vision features into a live KV
+ * cache — image changes always require a fresh session start, which
+ * the high-level `ChatSession` wrapper handles via its
+ * `lastImagesKey` check.
+ */
+export declare class QianfanOCRModel extends QianfanOCRModelNative {
+    static load(modelPath: string): Promise<QianfanOCRModel>;
+    /** Streaming variant of {@link QianfanOCRModel#chatSessionStart}. */
+    chatStreamSessionStart(messages: ChatMessage[], config?: ChatConfig | null, signal?: AbortSignal): AsyncGenerator<ChatStreamEvent>;
+    /** Streaming variant of {@link QianfanOCRModel#chatSessionContinue}. */
+    chatStreamSessionContinue(userMessage: string, images: Uint8Array[] | null, config?: ChatConfig | null, signal?: AbortSignal): AsyncGenerator<ChatStreamEvent>;
+    /** Streaming variant of {@link QianfanOCRModel#chatSessionContinueTool}. */
+    chatStreamSessionContinueTool(toolCallId: string, content: string, config?: ChatConfig | null, signal?: AbortSignal): AsyncGenerator<ChatStreamEvent>;
+}
+//# sourceMappingURL=qianfan-ocr.d.ts.map

package/dist/models/qianfan-ocr.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"qianfan-ocr.d.ts","sourceRoot":"","sources":["../../src/models/qianfan-ocr.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,IAAI,qBAAqB,EAAE,MAAM,gBAAgB,CAAC;AAC1E,OAAO,KAAK,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE9D,OAAO,KAAK,EAAE,eAAe,EAAuB,MAAM,cAAc,CAAC;AAezE;;;;;;;;;;;;;GAaG;AACH,qBAAa,eAAgB,SAAQ,qBAAqB;WAClC,IAAI,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC;IAMvE,qEAAqE;IAE9D,sBAAsB,CAC3B,QAAQ,EAAE,WAAW,EAAE,EACvB,MAAM,CAAC,EAAE,UAAU,GAAG,IAAI,EAC1B,MAAM,CAAC,EAAE,WAAW,GACnB,cAAc,CAAC,eAAe,CAAC;IAOlC,wEAAwE;IAEjE,yBAAyB,CAC9B,WAAW,EAAE,MAAM,EACnB,MAAM,EAAE,UAAU,EAAE,GAAG,IAAI,EAC3B,MAAM,CAAC,EAAE,UAAU,GAAG,IAAI,EAC1B,MAAM,CAAC,EAAE,WAAW,GACnB,cAAc,CAAC,eAAe,CAAC;IAQlC,4EAA4E;IAErE,6BAA6B,CAClC,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,MAAM,EACf,MAAM,CAAC,EAAE,UAAU,GAAG,IAAI,EAC1B,MAAM,CAAC,EAAE,WAAW,GACnB,cAAc,CAAC,eAAe,CAAC;CAOnC"}

package/dist/models/qianfan-ocr.js

@@ -0,0 +1,63 @@
+import { QianfanOCRModel as QianfanOCRModelNative } from '@mlx-node/core';
+import { _runChatStream } from '@mlx-node/lm';
+// Save references to the native callback-based session streaming methods
+// before we override them. Captured at module load time so the subclass
+// overrides can delegate without recursing into themselves. Each wrapper
+// method below bridges the callback API to `AsyncGenerator<ChatStreamEvent>`
+// so the wrapper structurally satisfies `SessionCapableModel` and can be
+// passed to `ChatSession<QianfanOCRModel>` from `@mlx-node/lm`.
+// oxlint-disable-next-line @typescript-eslint/unbound-method
+const _nativeQianfanOcrChatStreamSessionStart = QianfanOCRModelNative.prototype.chatStreamSessionStart;
+// oxlint-disable-next-line @typescript-eslint/unbound-method
+const _nativeQianfanOcrChatStreamSessionContinue = QianfanOCRModelNative.prototype.chatStreamSessionContinue;
+// oxlint-disable-next-line @typescript-eslint/unbound-method
+const _nativeQianfanOcrChatStreamSessionContinueTool = QianfanOCRModelNative.prototype.chatStreamSessionContinueTool;
+/**
+ * Qianfan-OCR Vision-Language Model wrapper.
+ *
+ * Streaming is driven through the `ChatSession` API — the overrides
+ * below adapt the callback-based native methods to
+ * `AsyncGenerator<ChatStreamEvent>` so the wrapper structurally
+ * satisfies `SessionCapableModel` from `@mlx-node/lm`.
+ *
+ * Qianfan-OCR is a VLM (InternViT + Qwen3 language model). The
+ * continue path cannot splice new vision features into a live KV
+ * cache — image changes always require a fresh session start, which
+ * the high-level `ChatSession` wrapper handles via its
+ * `lastImagesKey` check.
+ */
+export class QianfanOCRModel extends QianfanOCRModelNative {
+    static async load(modelPath) {
+        const instance = await QianfanOCRModelNative.load(modelPath);
+        Object.setPrototypeOf(instance, QianfanOCRModel.prototype);
+        return instance;
+    }
+    /** Streaming variant of {@link QianfanOCRModel#chatSessionStart}. */
+    // @ts-expect-error — override callback-based native method with AsyncGenerator
+    async *chatStreamSessionStart(messages, config, signal) {
+        yield* _runChatStream((callback) => _nativeQianfanOcrChatStreamSessionStart.call(this, messages, config ?? null, callback), signal);
+    }
+    /** Streaming variant of {@link QianfanOCRModel#chatSessionContinue}. */
+    // @ts-expect-error — override callback-based native method with AsyncGenerator
+    async *chatStreamSessionContinue(userMessage, images, config, signal) {
+        yield* _runChatStream((callback) => _nativeQianfanOcrChatStreamSessionContinue.call(this, userMessage, images, config ?? null, callback), signal);
+    }
+    /** Streaming variant of {@link QianfanOCRModel#chatSessionContinueTool}. */
+    // @ts-expect-error — override callback-based native method with AsyncGenerator
+    async *chatStreamSessionContinueTool(toolCallId, content, config, signal) {
+        yield* _runChatStream((callback) => _nativeQianfanOcrChatStreamSessionContinueTool.call(this, toolCallId, content, config ?? null, callback), signal);
+    }
+}
+// -------------------------------------------------------------------
+// Compile-time conformance check
+// -------------------------------------------------------------------
+//
+// Ensures the wrapper structurally satisfies `SessionCapableModel` so
+// `ChatSession<QianfanOCRModel>` will type-check in downstream code.
+// The assignment is compile-only — the `null as unknown as T`
+// placeholder never runs.
+function _assertSessionCapable() {
+    const _qianfan = null;
+    void _qianfan;
+}
+void _assertSessionCapable;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mlx-node/vlm",
-  "version": "0.0.6",
+  "version": "0.0.7",
   "homepage": "https://github.com/mlx-node/mlx-node",
   "bugs": {
     "url": "https://github.com/mlx-node/mlx-node/issues"
@@ -28,11 +28,11 @@
     "test": "vite test run"
   },
   "dependencies": {
-    "@mlx-node/core": "0.0.6",
-    "@mlx-node/lm": "0.0.6",
+    "@mlx-node/core": "0.0.7",
+    "@mlx-node/lm": "0.0.7",
     "@napi-rs/image": "^1.12.0"
   },
   "devDependencies": {
-    "@types/node": "@types/node@25.5.0"
+    "@types/node": "@types/node@24.12.2"
   }
 }