@mlx-node/vlm 0.0.6 → 0.0.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +34 -2
- package/dist/index.d.ts +3 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -5
- package/dist/models/qianfan-ocr.d.ts +34 -0
- package/dist/models/qianfan-ocr.d.ts.map +1 -0
- package/dist/models/qianfan-ocr.js +38 -0
- package/dist/pipeline/structure-v3.d.ts.map +1 -1
- package/package.json +5 -5
- package/dist/models/index.d.ts +0 -5
- package/dist/models/index.d.ts.map +0 -1
- package/dist/models/index.js +0 -4
- package/dist/models/paddleocr-vl-configs.d.ts +0 -53
- package/dist/models/paddleocr-vl-configs.d.ts.map +0 -1
- package/dist/models/paddleocr-vl-configs.js +0 -53
package/README.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# @mlx-node/vlm
|
|
2
2
|
|
|
3
|
-
Vision-language models and document processing pipelines for Node.js on Apple Silicon. Extract text, tables, and structure from documents and images using PaddleOCR-VL and the PP-StructureV3 pipeline — all running locally on Metal GPU.
|
|
3
|
+
Vision-language models and document processing pipelines for Node.js on Apple Silicon. Extract text, tables, and structure from documents and images using PaddleOCR-VL, Qianfan-OCR (InternVL), and the PP-StructureV3 pipeline — all running locally on Metal GPU.
|
|
4
4
|
|
|
5
5
|
## Requirements
|
|
6
6
|
|
|
@@ -13,6 +13,39 @@ Vision-language models and document processing pipelines for Node.js on Apple Si
|
|
|
13
13
|
npm install @mlx-node/vlm
|
|
14
14
|
```
|
|
15
15
|
|
|
16
|
+
## Multi-turn chat with VLMs
|
|
17
|
+
|
|
18
|
+
`QianfanOCRModel` conforms to the same `ChatSession<M>` surface as the language models in `@mlx-node/lm`, so a single session handle drives both single-shot and multi-turn VLM conversations:
|
|
19
|
+
|
|
20
|
+
```typescript
|
|
21
|
+
import { ChatSession } from '@mlx-node/lm';
|
|
22
|
+
import { QianfanOCRModel } from '@mlx-node/vlm';
|
|
23
|
+
import { readFileSync } from 'node:fs';
|
|
24
|
+
|
|
25
|
+
const model = await QianfanOCRModel.load('./models/Qianfan-VL');
|
|
26
|
+
const session = new ChatSession(model, { system: 'You read documents precisely.' });
|
|
27
|
+
|
|
28
|
+
// First turn with an image.
|
|
29
|
+
const r1 = await session.send('Extract the text from this receipt.', {
|
|
30
|
+
images: [readFileSync('./receipt.jpg')],
|
|
31
|
+
});
|
|
32
|
+
console.log(r1.text);
|
|
33
|
+
|
|
34
|
+
// Text-only follow-up reuses the same KV cache against the same image.
|
|
35
|
+
const r2 = await session.send('What is the total price?');
|
|
36
|
+
console.log(r2.text);
|
|
37
|
+
|
|
38
|
+
// Swapping the image mid-session forcibly restarts the cache with the new image.
|
|
39
|
+
const r3 = await session.send('And this one?', {
|
|
40
|
+
images: [readFileSync('./other-receipt.jpg')],
|
|
41
|
+
});
|
|
42
|
+
console.log(r3.text);
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
`loadSession()` from `@mlx-node/lm` cannot load Qianfan-OCR (that would introduce a circular package dependency), so construct the wrapper here with `QianfanOCRModel.load()` and pass it to `ChatSession` directly.
|
|
46
|
+
|
|
47
|
+
The one-shot `VLModel.chat()` API shown below for PaddleOCR-VL is a single-turn OCR entry point and is intentionally kept out of the session surface.
|
|
48
|
+
|
|
16
49
|
## Quick Start
|
|
17
50
|
|
|
18
51
|
### Document Structure Analysis
|
|
@@ -257,7 +290,6 @@ const buffer = documentToXlsx(doc);
|
|
|
257
290
|
|
|
258
291
|
| Export | Description |
|
|
259
292
|
| --------------------------- | ------------------------------------------------ |
|
|
260
|
-
| `PADDLEOCR_VL_CONFIGS` | Pre-defined PaddleOCR-VL 1.5 config |
|
|
261
293
|
| `createPaddleocrVlConfig()` | Default config factory |
|
|
262
294
|
| `OutputFormat` | Enum: `Raw`, `Plain`, `Markdown`, `Html`, `Json` |
|
|
263
295
|
|
package/dist/index.d.ts
CHANGED
|
@@ -32,10 +32,10 @@ export { TextDetModel, type TextBox, TextRecModel, type RecResult } from '@mlx-n
|
|
|
32
32
|
export { DocOrientationModel, type OrientationResult, type ClassifyRotateResult, DocUnwarpModel, type UnwarpResult, } from '@mlx-node/core';
|
|
33
33
|
export { StructureV3Pipeline, type StructureV3Config, type AnalyzeOptions, type StructuredElement, type StructuredDocument, type TextLine, } from './pipeline/structure-v3.js';
|
|
34
34
|
export type { VisionConfig, TextConfig, ModelConfig, VlmChatConfig, VlmChatMessage, VlmBatchItem, } from '@mlx-node/core';
|
|
35
|
-
export {
|
|
35
|
+
export { QianfanOCRModel } from './models/qianfan-ocr.js';
|
|
36
|
+
export { createQianfanOcrConfig } from '@mlx-node/core';
|
|
37
|
+
export type { QianfanOcrConfig, InternVisionConfig, Qwen3LmConfig } from '@mlx-node/core';
|
|
36
38
|
export { VlmChatResult, type VLMChatResult } from '@mlx-node/core';
|
|
37
39
|
export { parsePaddleResponse, parseVlmOutput, formatDocument, type ParsedDocument, type DocumentElement, type Table, type TableRow, type TableCell, type Paragraph, type ParserConfig, OutputFormat, } from '@mlx-node/core';
|
|
38
40
|
export { documentToXlsx, saveToXlsx } from '@mlx-node/core';
|
|
39
|
-
export { Qwen3Tokenizer as Tokenizer } from '@mlx-node/lm';
|
|
40
|
-
export { MxArray, type DType } from '@mlx-node/core';
|
|
41
41
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAKH,OAAO,EAAE,OAAO,EAAE,uBAAuB,EAAE,MAAM,gBAAgB,CAAC;AAGlE,OAAO,EAAE,cAAc,EAAE,KAAK,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAGpE,OAAO,EAAE,YAAY,EAAE,KAAK,OAAO,EAAE,YAAY,EAAE,KAAK,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAG1F,OAAO,EACL,mBAAmB,EACnB,KAAK,iBAAiB,EACtB,KAAK,oBAAoB,EACzB,cAAc,EACd,KAAK,YAAY,GAClB,MAAM,gBAAgB,CAAC;AAGxB,OAAO,EACL,mBAAmB,EACnB,KAAK,iBAAiB,EACtB,KAAK,cAAc,EACnB,KAAK,iBAAiB,EACtB,KAAK,kBAAkB,EACvB,KAAK,QAAQ,GACd,MAAM,4BAA4B,CAAC;AAGpC,YAAY,EACV,YAAY,EACZ,UAAU,EACV,WAAW,EACX,aAAa,EACb,cAAc,EACd,YAAY,GACb,MAAM,gBAAgB,CAAC;AAGxB,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAKH,OAAO,EAAE,OAAO,EAAE,uBAAuB,EAAE,MAAM,gBAAgB,CAAC;AAGlE,OAAO,EAAE,cAAc,EAAE,KAAK,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAGpE,OAAO,EAAE,YAAY,EAAE,KAAK,OAAO,EAAE,YAAY,EAAE,KAAK,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAG1F,OAAO,EACL,mBAAmB,EACnB,KAAK,iBAAiB,EACtB,KAAK,oBAAoB,EACzB,cAAc,EACd,KAAK,YAAY,GAClB,MAAM,gBAAgB,CAAC;AAGxB,OAAO,EACL,mBAAmB,EACnB,KAAK,iBAAiB,EACtB,KAAK,cAAc,EACnB,KAAK,iBAAiB,EACtB,KAAK,kBAAkB,EACvB,KAAK,QAAQ,GACd,MAAM,4BAA4B,CAAC;AAGpC,YAAY,EACV,YAAY,EACZ,UAAU,EACV,WAAW,EACX,aAAa,EACb,cAAc,EACd,YAAY,GACb,MAAM,gBAAgB,CAAC;AAGxB,OAAO,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAC1D,OAAO,EAAE,sBAAsB,EAAE,MAAM,gBAAgB,CAAC;AACxD,YAAY,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAG1F,OAAO,EAAE,aAAa,EAAE,KAAK,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAGnE,OAAO,EACL,mBAAmB,EACnB,cAAc,EACd,cAAc,EACd,KAAK,cAAc,EACnB,KAAK,eAAe,EACpB,KAAK,KAAK,EACV,KAAK,QAAQ,EACb,KAAK,SAAS,EACd,KAAK,SAAS,EACd,KAAK,YAAY,EACjB,YAAY,GACb,MAAM,gBAAgB,CAAC;AAGxB,OAAO,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -37,14 +37,12 @@ export { TextDetModel, TextRecModel } from '@mlx-node/core';
|
|
|
37
37
|
export { DocOrientationModel, DocUnwarpModel, } from '@mlx-node/core';
|
|
38
38
|
// Document understanding pipeline (PP-StructureV3)
|
|
39
39
|
export { StructureV3Pipeline, } from './pipeline/structure-v3.js';
|
|
40
|
-
//
|
|
41
|
-
export {
|
|
40
|
+
// Qianfan-OCR model (InternVL architecture)
|
|
41
|
+
export { QianfanOCRModel } from './models/qianfan-ocr.js';
|
|
42
|
+
export { createQianfanOcrConfig } from '@mlx-node/core';
|
|
42
43
|
// Chat result type
|
|
43
44
|
export { VlmChatResult } from '@mlx-node/core';
|
|
44
45
|
// Output parsing and formatting (Rust implementation)
|
|
45
46
|
export { parsePaddleResponse, parseVlmOutput, formatDocument, OutputFormat, } from '@mlx-node/core';
|
|
46
47
|
// XLSX export (Rust implementation)
|
|
47
48
|
export { documentToXlsx, saveToXlsx } from '@mlx-node/core';
|
|
48
|
-
// Re-export shared utilities
|
|
49
|
-
export { Qwen3Tokenizer as Tokenizer } from '@mlx-node/lm';
|
|
50
|
-
export { MxArray } from '@mlx-node/core';
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { QianfanOCRModel as QianfanOCRModelNative } from '@mlx-node/core';
|
|
2
|
+
declare const QianfanOCRModel_base: {
|
|
3
|
+
new (config: import("@mlx-node/core").QianfanOcrConfig): import("@mlx-node/lm").StreamingInstance<typeof QianfanOCRModelNative, {
|
|
4
|
+
readonly recordModelPath: false;
|
|
5
|
+
}>;
|
|
6
|
+
load(modelPath: string): Promise<import("@mlx-node/lm").StreamingInstance<typeof QianfanOCRModelNative, {
|
|
7
|
+
readonly recordModelPath: false;
|
|
8
|
+
}>>;
|
|
9
|
+
};
|
|
10
|
+
/**
|
|
11
|
+
* Qianfan-OCR Vision-Language Model wrapper.
|
|
12
|
+
*
|
|
13
|
+
* Built from the shared {@link makeStreamingModel} factory in
|
|
14
|
+
* `@mlx-node/lm` — the empty `extends` inherits the AsyncGenerator
|
|
15
|
+
* session-streaming overrides (`chatStreamSessionStart` /
|
|
16
|
+
* `chatStreamSessionContinue` / `chatStreamSessionContinueTool`) so the
|
|
17
|
+
* wrapper structurally satisfies `SessionCapableModel` and can be
|
|
18
|
+
* passed to `ChatSession<QianfanOCRModel>`. Importing the factory from
|
|
19
|
+
* `@mlx-node/lm` is one-directional (vlm → lm), matching the existing
|
|
20
|
+
* `_runChatStream` dependency, so it introduces no circular dependency.
|
|
21
|
+
*
|
|
22
|
+
* Qianfan-OCR is a VLM (InternViT + Qwen3 language model). The continue
|
|
23
|
+
* path cannot splice new vision features into a live KV cache — image
|
|
24
|
+
* changes always require a fresh session start, which the high-level
|
|
25
|
+
* `ChatSession` wrapper handles via its `lastImagesKey` check.
|
|
26
|
+
*
|
|
27
|
+
* Unlike the LM families, Qianfan-OCR does NOT record its model path
|
|
28
|
+
* (`recordModelPath: false`) and therefore exposes no
|
|
29
|
+
* `applyChatTemplate` — preserving its historical surface.
|
|
30
|
+
*/
|
|
31
|
+
export declare class QianfanOCRModel extends QianfanOCRModel_base {
|
|
32
|
+
}
|
|
33
|
+
export {};
|
|
34
|
+
//# sourceMappingURL=qianfan-ocr.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"qianfan-ocr.d.ts","sourceRoot":"","sources":["../../src/models/qianfan-ocr.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,IAAI,qBAAqB,EAAE,MAAM,gBAAgB,CAAC;;;;;;;;;AAI1E;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,qBAAa,eAAgB,SAAQ,oBAAqE;CAAG"}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import { QianfanOCRModel as QianfanOCRModelNative } from '@mlx-node/core';
|
|
2
|
+
import { makeStreamingModel } from '@mlx-node/lm';
|
|
3
|
+
/**
|
|
4
|
+
* Qianfan-OCR Vision-Language Model wrapper.
|
|
5
|
+
*
|
|
6
|
+
* Built from the shared {@link makeStreamingModel} factory in
|
|
7
|
+
* `@mlx-node/lm` — the empty `extends` inherits the AsyncGenerator
|
|
8
|
+
* session-streaming overrides (`chatStreamSessionStart` /
|
|
9
|
+
* `chatStreamSessionContinue` / `chatStreamSessionContinueTool`) so the
|
|
10
|
+
* wrapper structurally satisfies `SessionCapableModel` and can be
|
|
11
|
+
* passed to `ChatSession<QianfanOCRModel>`. Importing the factory from
|
|
12
|
+
* `@mlx-node/lm` is one-directional (vlm → lm), matching the existing
|
|
13
|
+
* `_runChatStream` dependency, so it introduces no circular dependency.
|
|
14
|
+
*
|
|
15
|
+
* Qianfan-OCR is a VLM (InternViT + Qwen3 language model). The continue
|
|
16
|
+
* path cannot splice new vision features into a live KV cache — image
|
|
17
|
+
* changes always require a fresh session start, which the high-level
|
|
18
|
+
* `ChatSession` wrapper handles via its `lastImagesKey` check.
|
|
19
|
+
*
|
|
20
|
+
* Unlike the LM families, Qianfan-OCR does NOT record its model path
|
|
21
|
+
* (`recordModelPath: false`) and therefore exposes no
|
|
22
|
+
* `applyChatTemplate` — preserving its historical surface.
|
|
23
|
+
*/
|
|
24
|
+
export class QianfanOCRModel extends makeStreamingModel(QianfanOCRModelNative, { recordModelPath: false }) {
|
|
25
|
+
}
|
|
26
|
+
// -------------------------------------------------------------------
|
|
27
|
+
// Compile-time conformance check
|
|
28
|
+
// -------------------------------------------------------------------
|
|
29
|
+
//
|
|
30
|
+
// Ensures the wrapper structurally satisfies `SessionCapableModel` so
|
|
31
|
+
// `ChatSession<QianfanOCRModel>` will type-check in downstream code.
|
|
32
|
+
// The assignment is compile-only — the `null as unknown as T`
|
|
33
|
+
// placeholder never runs.
|
|
34
|
+
function _assertSessionCapable() {
|
|
35
|
+
const _qianfan = null;
|
|
36
|
+
void _qianfan;
|
|
37
|
+
}
|
|
38
|
+
void _assertSessionCapable;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"structure-v3.d.ts","sourceRoot":"","sources":["../../src/pipeline/structure-v3.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAiBH,0DAA0D;AAC1D,MAAM,WAAW,iBAAiB;IAChC,6CAA6C;IAC7C,eAAe,EAAE,MAAM,CAAC;IACxB,sDAAsD;IACtD,gBAAgB,EAAE,MAAM,CAAC;IACzB,wDAAwD;IACxD,gBAAgB,EAAE,MAAM,CAAC;IACzB,kEAAkE;IAClE,QAAQ,EAAE,MAAM,CAAC;IACjB,wEAAwE;IACxE,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,uDAAuD;IACvD,kBAAkB,CAAC,EAAE,MAAM,CAAC;CAC7B;AAED,qCAAqC;AACrC,MAAM,WAAW,cAAc;IAC7B,2DAA2D;IAC3D,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,yDAAyD;IACzD,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,0EAA0E;IAC1E,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,yFAAyF;IACzF,yBAAyB,CAAC,EAAE,OAAO,CAAC;IACpC,wEAAwE;IACxE,eAAe,CAAC,EAAE,OAAO,CAAC;CAC3B;AAED,sDAAsD;AACtD,MAAM,WAAW,QAAQ;IACvB,iEAAiE;IACjE,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,sBAAsB;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,6BAA6B;IAC7B,KAAK,EAAE,MAAM,CAAC;CACf;AAED,6DAA6D;AAC7D,MAAM,WAAW,iBAAiB;IAChC,yCAAyC;IACzC,KAAK,EAAE,MAAM,CAAC;IACd,2BAA2B;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,kEAAkE;IAClE,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,0BAA0B;IAC1B,KAAK,EAAE,MAAM,CAAC;IACd,8BAA8B;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,wDAAwD;IACxD,KAAK,CAAC,EAAE,QAAQ,EAAE,CAAC;CACpB;AAED,mCAAmC;AACnC,MAAM,WAAW,kBAAkB;IACjC,2CAA2C;IAC3C,QAAQ,EAAE,iBAAiB,EAAE,CAAC;IAC9B,gCAAgC;IAChC,QAAQ,EAAE,MAAM,CAAC;CAClB;AAqCD;;;;;GAKG;AACH,qBAAa,mBAAmB;IAC9B,OAAO,CAAC,MAAM,CAAiB;IAC/B,OAAO,CAAC,OAAO,CAAe;IAC9B,OAAO,CAAC,OAAO,CAAe;IAC9B,OAAO,CAAC,cAAc,CAA6B;IACnD,OAAO,CAAC,SAAS,CAAwB;IAEzC,OAAO;
|
|
1
|
+
{"version":3,"file":"structure-v3.d.ts","sourceRoot":"","sources":["../../src/pipeline/structure-v3.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAiBH,0DAA0D;AAC1D,MAAM,WAAW,iBAAiB;IAChC,6CAA6C;IAC7C,eAAe,EAAE,MAAM,CAAC;IACxB,sDAAsD;IACtD,gBAAgB,EAAE,MAAM,CAAC;IACzB,wDAAwD;IACxD,gBAAgB,EAAE,MAAM,CAAC;IACzB,kEAAkE;IAClE,QAAQ,EAAE,MAAM,CAAC;IACjB,wEAAwE;IACxE,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,uDAAuD;IACvD,kBAAkB,CAAC,EAAE,MAAM,CAAC;CAC7B;AAED,qCAAqC;AACrC,MAAM,WAAW,cAAc;IAC7B,2DAA2D;IAC3D,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,yDAAyD;IACzD,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,0EAA0E;IAC1E,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,yFAAyF;IACzF,yBAAyB,CAAC,EAAE,OAAO,CAAC;IACpC,wEAAwE;IACxE,eAAe,CAAC,EAAE,OAAO,CAAC;CAC3B;AAED,sDAAsD;AACtD,MAAM,WAAW,QAAQ;IACvB,iEAAiE;IACjE,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,sBAAsB;IACtB,IAAI,EAAE,MAAM,CAAC;IACb,6BAA6B;IAC7B,KAAK,EAAE,MAAM,CAAC;CACf;AAED,6DAA6D;AAC7D,MAAM,WAAW,iBAAiB;IAChC,yCAAyC;IACzC,KAAK,EAAE,MAAM,CAAC;IACd,2BAA2B;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,kEAAkE;IAClE,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,0BAA0B;IAC1B,KAAK,EAAE,MAAM,CAAC;IACd,8BAA8B;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,wDAAwD;IACxD,KAAK,CAAC,EAAE,QAAQ,EAAE,CAAC;CACpB;AAED,mCAAmC;AACnC,MAAM,WAAW,kBAAkB;IACjC,2CAA2C;IAC3C,QAAQ,EAAE,iBAAiB,EAAE,CAAC;IAC9B,gCAAgC;IAChC,QAAQ,EAAE,MAAM,CAAC;CAClB;AAqCD;;;;;GAKG;AACH,qBAAa,mBAAmB;IAC9B,OAAO,CAAC,MAAM,CAAiB;IAC/B,OAAO,CAAC,OAAO,CAAe;IAC9B,OAAO,CAAC,OAAO,CAAe;IAC9B,OAAO,CAAC,cAAc,CAA6B;IACnD,OAAO,CAAC,SAAS,CAAwB;IAEzC,OAAO,eAYN;IAED;;OAEG;IACH,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,iBAAiB,GAAG,mBAAmB,CAW1D;IAED;;;;;;OAMG;IACG,OAAO,CAAC,SAAS,EAAE,UAAU,GAAG,MAAM,EAAE,OAAO,GAAE,cAAmB,GAAG,OAAO,CAAC,kBAAkB,CAAC,CA8FvG;IAED;;;;OAIG;IACG,QAAQ,CAAC,SAAS,EAAE,MAAM,EAAE,gBAAgB,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,CAEhF;IAED;;;;;OAKG;YACW,SAAS;IAwDvB;;OAEG;YACW,WAAW;CAW1B"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mlx-node/vlm",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.8",
|
|
4
4
|
"homepage": "https://github.com/mlx-node/mlx-node",
|
|
5
5
|
"bugs": {
|
|
6
6
|
"url": "https://github.com/mlx-node/mlx-node/issues"
|
|
@@ -28,11 +28,11 @@
|
|
|
28
28
|
"test": "vite test run"
|
|
29
29
|
},
|
|
30
30
|
"dependencies": {
|
|
31
|
-
"@mlx-node/core": "0.0.
|
|
32
|
-
"@mlx-node/lm": "0.0.
|
|
33
|
-
"@napi-rs/image": "^1.
|
|
31
|
+
"@mlx-node/core": "0.0.8",
|
|
32
|
+
"@mlx-node/lm": "0.0.8",
|
|
33
|
+
"@napi-rs/image": "^1.13.0"
|
|
34
34
|
},
|
|
35
35
|
"devDependencies": {
|
|
36
|
-
"@types/node": "@types/node@
|
|
36
|
+
"@types/node": "@types/node@24.12.2"
|
|
37
37
|
}
|
|
38
38
|
}
|
package/dist/models/index.d.ts
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/models/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,OAAO,EAAE,oBAAoB,EAAE,KAAK,iBAAiB,EAAE,MAAM,2BAA2B,CAAC"}
|
package/dist/models/index.js
DELETED
|
@@ -1,53 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* PaddleOCR-VL Model Configurations
|
|
3
|
-
*
|
|
4
|
-
* Based on the PaddleOCR-VL-1.5 model architecture.
|
|
5
|
-
*/
|
|
6
|
-
export interface VisionConfig {
|
|
7
|
-
modelType: 'paddleocr_vl';
|
|
8
|
-
hiddenSize: number;
|
|
9
|
-
intermediateSize: number;
|
|
10
|
-
numHiddenLayers: number;
|
|
11
|
-
numAttentionHeads: number;
|
|
12
|
-
numChannels: number;
|
|
13
|
-
imageSize: number;
|
|
14
|
-
patchSize: number;
|
|
15
|
-
hiddenAct: string;
|
|
16
|
-
layerNormEps: number;
|
|
17
|
-
attentionDropout: number;
|
|
18
|
-
spatialMergeSize: number;
|
|
19
|
-
}
|
|
20
|
-
export interface TextConfig {
|
|
21
|
-
modelType: 'paddleocr_vl';
|
|
22
|
-
hiddenSize: number;
|
|
23
|
-
numHiddenLayers: number;
|
|
24
|
-
intermediateSize: number;
|
|
25
|
-
numAttentionHeads: number;
|
|
26
|
-
rmsNormEps: number;
|
|
27
|
-
vocabSize: number;
|
|
28
|
-
numKeyValueHeads: number;
|
|
29
|
-
maxPositionEmbeddings: number;
|
|
30
|
-
ropeTheta: number;
|
|
31
|
-
ropeTraditional: boolean;
|
|
32
|
-
useBias: boolean;
|
|
33
|
-
headDim: number;
|
|
34
|
-
mropeSection: [number, number, number];
|
|
35
|
-
}
|
|
36
|
-
export interface PaddleOCRVLConfig {
|
|
37
|
-
visionConfig: VisionConfig;
|
|
38
|
-
textConfig: TextConfig;
|
|
39
|
-
modelType: 'paddleocr_vl';
|
|
40
|
-
ignoreIndex: number;
|
|
41
|
-
imageTokenId: number;
|
|
42
|
-
videoTokenId: number;
|
|
43
|
-
visionStartTokenId: number;
|
|
44
|
-
visionEndTokenId: number;
|
|
45
|
-
eosTokenId: number;
|
|
46
|
-
}
|
|
47
|
-
/**
|
|
48
|
-
* Available PaddleOCR-VL configurations
|
|
49
|
-
*/
|
|
50
|
-
export declare const PADDLEOCR_VL_CONFIGS: {
|
|
51
|
-
readonly 'paddleocr-vl-1.5': PaddleOCRVLConfig;
|
|
52
|
-
};
|
|
53
|
-
//# sourceMappingURL=paddleocr-vl-configs.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"paddleocr-vl-configs.d.ts","sourceRoot":"","sources":["../../src/models/paddleocr-vl-configs.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,MAAM,WAAW,YAAY;IAC3B,SAAS,EAAE,cAAc,CAAC;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,gBAAgB,EAAE,MAAM,CAAC;IACzB,eAAe,EAAE,MAAM,CAAC;IACxB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,gBAAgB,EAAE,MAAM,CAAC;IACzB,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,UAAU;IACzB,SAAS,EAAE,cAAc,CAAC;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,eAAe,EAAE,MAAM,CAAC;IACxB,gBAAgB,EAAE,MAAM,CAAC;IACzB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,gBAAgB,EAAE,MAAM,CAAC;IACzB,qBAAqB,EAAE,MAAM,CAAC;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,eAAe,EAAE,OAAO,CAAC;IACzB,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;CACxC;AAED,MAAM,WAAW,iBAAiB;IAChC,YAAY,EAAE,YAAY,CAAC;IAC3B,UAAU,EAAE,UAAU,CAAC;IACvB,SAAS,EAAE,cAAc,CAAC;IAC1B,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,gBAAgB,EAAE,MAAM,CAAC;IACzB,UAAU,EAAE,MAAM,CAAC;CACpB;AA6CD;;GAEG;AACH,eAAO,MAAM,oBAAoB;;CAEvB,CAAC"}
|
|
@@ -1,53 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* PaddleOCR-VL Model Configurations
|
|
3
|
-
*
|
|
4
|
-
* Based on the PaddleOCR-VL-1.5 model architecture.
|
|
5
|
-
*/
|
|
6
|
-
/**
|
|
7
|
-
* PaddleOCR-VL-1.5 default configuration
|
|
8
|
-
*/
|
|
9
|
-
const PADDLEOCR_VL_1_5_CONFIG = {
|
|
10
|
-
modelType: 'paddleocr_vl',
|
|
11
|
-
ignoreIndex: -100,
|
|
12
|
-
imageTokenId: 100295,
|
|
13
|
-
videoTokenId: 100296,
|
|
14
|
-
visionStartTokenId: 101305,
|
|
15
|
-
visionEndTokenId: 101306,
|
|
16
|
-
eosTokenId: 2,
|
|
17
|
-
visionConfig: {
|
|
18
|
-
modelType: 'paddleocr_vl',
|
|
19
|
-
hiddenSize: 1152,
|
|
20
|
-
intermediateSize: 4304,
|
|
21
|
-
numHiddenLayers: 27,
|
|
22
|
-
numAttentionHeads: 16,
|
|
23
|
-
numChannels: 3,
|
|
24
|
-
imageSize: 384,
|
|
25
|
-
patchSize: 14,
|
|
26
|
-
hiddenAct: 'gelu_pytorch_tanh',
|
|
27
|
-
layerNormEps: 1e-6,
|
|
28
|
-
attentionDropout: 0.0,
|
|
29
|
-
spatialMergeSize: 2,
|
|
30
|
-
},
|
|
31
|
-
textConfig: {
|
|
32
|
-
modelType: 'paddleocr_vl',
|
|
33
|
-
hiddenSize: 1024,
|
|
34
|
-
numHiddenLayers: 18,
|
|
35
|
-
intermediateSize: 3072,
|
|
36
|
-
numAttentionHeads: 16,
|
|
37
|
-
rmsNormEps: 1e-5,
|
|
38
|
-
vocabSize: 103424,
|
|
39
|
-
numKeyValueHeads: 2,
|
|
40
|
-
maxPositionEmbeddings: 131072,
|
|
41
|
-
ropeTheta: 500000.0,
|
|
42
|
-
ropeTraditional: false,
|
|
43
|
-
useBias: false,
|
|
44
|
-
headDim: 128,
|
|
45
|
-
mropeSection: [16, 24, 24],
|
|
46
|
-
},
|
|
47
|
-
};
|
|
48
|
-
/**
|
|
49
|
-
* Available PaddleOCR-VL configurations
|
|
50
|
-
*/
|
|
51
|
-
export const PADDLEOCR_VL_CONFIGS = {
|
|
52
|
-
'paddleocr-vl-1.5': PADDLEOCR_VL_1_5_CONFIG,
|
|
53
|
-
};
|