docling.rs 0.33.0 → 0.37.0

package/README.md

@@ -93,6 +93,62 @@ for await (const chunk of streamFileMarkdown('paper.pdf')) {
 }
 ```
 
+### Chunking (docling's chunkers, for RAG)
+
+`chunkFile` / `chunk` / `chunkDocument` (each with an `…Async` variant) run
+docling's chunkers over a converted document and return embedding-ready
+records. The default is the structure-driven **hierarchical** chunker (one
+chunk per document item — whole lists, triplet-serialized tables — with its
+heading path); pass `chunker: 'hybrid'` to refine against a token budget
+(split oversized chunks, merge undersized same-heading neighbours), matching
+docling's `HybridChunker`. The hybrid token counts come from a HuggingFace
+`tokenizer.json`: pass a path via `tokenizer`, or omit it to use
+`models/chunk/tokenizer.json` (all-MiniLM-L6-v2's — fetched by
+`scripts/download_dependencies.sh` alongside the ML models, resolved through
+the same install-home logic).
+
+```js
+import { chunkFileAsync, Pipeline, chunkDocumentAsync } from 'docling.rs'
+
+const chunks = await chunkFileAsync('report.docx', {
+  chunker: 'hybrid',
+  tokenizer: 'tokenizer.json', // e.g. all-MiniLM-L6-v2's
+  maxTokens: 256,
+})
+for (const c of chunks) {
+  await embed(c.contextualized) // heading path + text, ready for the embedder
+}
+
+// Chunk something you already converted (no re-conversion), e.g. a PDF
+// that went through the warm Pipeline:
+const { content } = new Pipeline().convertFile('paper.pdf', { to: 'json' })
+const pdfChunks = await chunkDocumentAsync(content)
+```
+
+Each `Chunk` is `{ text, headings?, docItems, contextualized }` — `docItems`
+holds the source items' JSON-pointer refs (`"#/texts/12"`), `contextualized`
+is docling's `contextualize()` rendering to feed the embedding model.
+
+#### Streaming chunks
+
+`streamFileChunks` / `streamChunks` / `streamDocumentChunks` are the streaming
+counterparts: async generators that yield each chunk **as the chunkers produce
+it** — the first chunk is ready for embedding while the rest of the document
+is still being chunked, and no all-chunks array is materialized. Abandoning
+the generator early (`break`) cancels the background chunking.
+
+```js
+import { streamFileChunks } from 'docling.rs'
+
+for await (const c of streamFileChunks('report.docx', {
+  chunker: 'hybrid',
+  tokenizer: 'tokenizer.json',
+  maxTokens: 256,
+})) {
+  await embed(c.contextualized) // embedding overlaps the remaining chunking
+}
+```
+
 ### PDF / images: getting the ML models
 
 Declarative formats (Markdown, HTML, DOCX, XLSX, …) are pure Rust and need
@@ -204,6 +260,11 @@ JSON output always embeds extracted images as data URIs.
 | `convertFileAsync(path, options?)` | `Promise<ConvertResult>` | Off the event loop. |
 | `convertAsync(input, options?)` | `Promise<ConvertResult>` | Off the event loop. |
 | `streamFileMarkdown(path, options?)` | `AsyncGenerator<string>` | Markdown chunks in document order. |
+| `chunkFile(path, options?)` | `Chunk[]` | Convert + run docling's hierarchical/hybrid chunker. |
+| `chunk(input, options?)` | `Chunk[]` | Same, over in-memory bytes. |
+| `chunkDocument(documentJson, options?)` | `Chunk[]` | Chunk an already-converted docling JSON document. |
+| `chunkFileAsync` / `chunkAsync` / `chunkDocumentAsync` | `Promise<Chunk[]>` | Off the event loop. |
+| `streamFileChunks` / `streamChunks` / `streamDocumentChunks` | `AsyncGenerator<Chunk>` | Chunks yielded as produced; `break` cancels. |
 | `supportedFormats()` | `string[]` | Supported input format ids. |
 | `formatFromName(name)` | `string \| null` | Detect a format id from a filename/extension. |
 | `checkDependencies(options?)` | `DependencyStatus` | Report which PDF/image deps are present. |

package/deps.js

@@ -92,9 +92,22 @@ function resolvePaths(dir) {
     tfDecoder:
       process.env.DOCLING_TABLEFORMER_DECODER || path.join(models, 'tableformer', 'decoder.onnx'),
     tfBbox: process.env.DOCLING_TABLEFORMER_BBOX || path.join(models, 'tableformer', 'bbox.onnx'),
+    chunkTokenizer:
+      process.env.DOCLING_CHUNK_TOKENIZER || path.join(models, 'chunk', 'tokenizer.json'),
   }
 }
 
+/**
+ * The hybrid chunker's default tokenizer (all-MiniLM-L6-v2's tokenizer.json,
+ * fetched by `scripts/download_dependencies.sh` into `models/chunk/`), resolved
+ * through the same install-home logic as the ML models. Returns `null` when not
+ * installed — the native side then reports a clear error with the download hint.
+ */
+function defaultChunkTokenizer(dir) {
+  const p = resolvePaths(dir)
+  return fs.existsSync(p.chunkTokenizer) ? p.chunkTokenizer : null
+}
+
 /**
  * Report which dependencies are present on disk. `ready` is true when the
  * minimum for PDF (pdfium + layout) is present.
@@ -108,6 +121,7 @@ function checkDependencies(options = {}) {
     layout: has(p.layout),
     ocr: has(p.ocrRec) && has(p.ocrDict),
     tableformer: has(p.tfEncoder) && has(p.tfDecoder) && has(p.tfBbox),
+    chunkTokenizer: has(p.chunkTokenizer),
   }
   status.ready = status.pdfium && status.layout
   status.missing = [
@@ -185,4 +199,5 @@ module.exports = {
   assertMlReady,
   resolvePaths,
   exportEnv,
+  defaultChunkTokenizer,
 }

package/index.d.ts

@@ -8,9 +8,19 @@ import type {
   ConvertOptions,
   ConvertInput,
   ConvertResult,
+  ChunkOptions,
+  Chunk,
 } from './native'
 
-export type { ConverterOptions, OutputOptions, ConvertOptions, ConvertInput, ConvertResult }
+export type {
+  ConverterOptions,
+  OutputOptions,
+  ConvertOptions,
+  ConvertInput,
+  ConvertResult,
+  ChunkOptions,
+  Chunk,
+}
 
 // Format helpers pass straight through from the native binding.
 export { supportedFormats, formatFromName } from './native'
@@ -27,6 +37,27 @@ export declare function convertFileAsync(path: string, options?: ConvertOptions
 /** Async (Promise) bytes conversion, off the event loop. */
 export declare function convertAsync(input: ConvertInput, options?: ConvertOptions | null): Promise<ConvertResult>
 
+/**
+ * Chunk a file with docling's chunkers: convert it, then run the hierarchical
+ * (default) or hybrid (`chunker: 'hybrid'` + `tokenizer`) chunker over the
+ * document. Throws for PDF/image/METS if deps aren't installed.
+ */
+export declare function chunkFile(path: string, options?: ChunkOptions | null): Array<Chunk>
+/** Async (Promise) {@link chunkFile}; conversion + chunking run off the event loop. */
+export declare function chunkFileAsync(path: string, options?: ChunkOptions | null): Promise<Array<Chunk>>
+/** Chunk in-memory bytes (same input contract as {@link convert}). */
+export declare function chunk(input: ConvertInput, options?: ChunkOptions | null): Array<Chunk>
+/** Async (Promise) {@link chunk}. */
+export declare function chunkAsync(input: ConvertInput, options?: ChunkOptions | null): Promise<Array<Chunk>>
+/**
+ * Chunk an already-converted document, passed as docling-core JSON (the
+ * `content` of a `convert*` call with `to: 'json'`) — so a document converted
+ * once (e.g. through the warm PDF {@link Pipeline}) chunks without re-converting.
+ */
+export declare function chunkDocument(documentJson: string, options?: ChunkOptions | null): Array<Chunk>
+/** Async (Promise) {@link chunkDocument}. */
+export declare function chunkDocumentAsync(documentJson: string, options?: ChunkOptions | null): Promise<Array<Chunk>>
+
 /** A reusable converter holding config (strict / fetchImages / allowedFormats). */
 export declare class DocumentConverter {
   constructor(options?: ConverterOptions | null)
@@ -87,6 +118,8 @@ export interface DependencyStatus {
   ocr: boolean
   /** TableFormer encoder/decoder/bbox present. */
   tableformer: boolean
+  /** Hybrid-chunker tokenizer (models/chunk/tokenizer.json) present. */
+  chunkTokenizer: boolean
   /** True when the minimum for PDF (pdfium + layout) is present. */
   ready: boolean
   /** Human-readable list of the missing required assets. */
@@ -120,3 +153,32 @@ export declare function streamFileMarkdown(
   filePath: string,
   options?: StreamOptions,
 ): AsyncGenerator<string, void, unknown>
+
+/**
+ * Stream a file's chunks as the chunkers produce them — the streaming
+ * counterpart of {@link chunkFile}. The first chunk is ready (e.g. for
+ * embedding) while the rest of the document is still being chunked, and no
+ * all-chunks array is materialized. Abandoning the generator early (`break`)
+ * cancels the background chunking. Throws for PDF/image/METS if deps aren't
+ * installed.
+ */
+export declare function streamFileChunks(
+  filePath: string,
+  options?: ChunkOptions | null,
+): AsyncGenerator<Chunk, void, unknown>
+
+/** Streaming counterpart of {@link chunk} (same contract as {@link streamFileChunks}). */
+export declare function streamChunks(
+  input: ConvertInput,
+  options?: ChunkOptions | null,
+): AsyncGenerator<Chunk, void, unknown>
+
+/**
+ * Streaming counterpart of {@link chunkDocument}: chunk an already-converted
+ * docling-core JSON document (same contract as {@link streamFileChunks}).
+ * Touches no ML models.
+ */
+export declare function streamDocumentChunks(
+  documentJson: string,
+  options?: ChunkOptions | null,
+): AsyncGenerator<Chunk, void, unknown>

package/index.js

@@ -13,7 +13,7 @@
 'use strict'
 
 const native = require('./native.js')
-const { checkDependencies, assertMlReady } = require('./deps.js')
+const { checkDependencies, assertMlReady, defaultChunkTokenizer } = require('./deps.js')
 
 // Resolve the format id of an input for the ML guard. Uses the native
 // extension→format map; falls back to an explicitly-passed format string.
@@ -89,6 +89,48 @@ async function convertAsync(input, options) {
   return native.convertAsync(input, options)
 }
 
+// --- guarded chunking functions ---------------------------------------------
+
+// For the hybrid chunker with no explicit tokenizer, resolve the default one
+// (models/chunk/tokenizer.json) through the same install-home logic as the ML
+// models — so DOCLING_RS_HOME / ~/.cache installs work, not only ./models. The
+// native side keeps its own CWD-relative fallback as a backstop.
+function withDefaultTokenizer(options) {
+  if (!options || options.tokenizer) return options
+  if (String(options.chunker || '').toLowerCase() !== 'hybrid') return options
+  const tokenizer = defaultChunkTokenizer()
+  return tokenizer ? { ...options, tokenizer } : options
+}
+
+function chunkFile(path, options) {
+  assertMlReady(mlFormatOf(path))
+  return native.chunkFile(path, withDefaultTokenizer(options))
+}
+
+function chunk(input, options) {
+  assertMlReady(mlFormatOf(input && input.name, input && input.format))
+  return native.chunk(input, withDefaultTokenizer(options))
+}
+
+// async so a guard failure surfaces as a rejected promise, not a sync throw.
+async function chunkFileAsync(path, options) {
+  assertMlReady(mlFormatOf(path))
+  return native.chunkFileAsync(path, withDefaultTokenizer(options))
+}
+
+async function chunkAsync(input, options) {
+  assertMlReady(mlFormatOf(input && input.name, input && input.format))
+  return native.chunkAsync(input, withDefaultTokenizer(options))
+}
+
+function chunkDocument(documentJson, options) {
+  return native.chunkDocument(documentJson, withDefaultTokenizer(options))
+}
+
+async function chunkDocumentAsync(documentJson, options) {
+  return native.chunkDocumentAsync(documentJson, withDefaultTokenizer(options))
+}
+
 // --- guarded classes --------------------------------------------------------
 
 class DocumentConverter {
@@ -196,15 +238,70 @@ async function* streamFileMarkdown(filePath, options = {}) {
   )
 }
 
+/**
+ * Stream a file's chunks as the chunkers produce them — the streaming
+ * counterpart of `chunkFile`. The first chunk is ready (e.g. for embedding)
+ * while the rest of the document is still being chunked, and no all-chunks
+ * array is materialized. Abandoning the generator early (`break`) cancels the
+ * background chunking.
+ *
+ * @param {string} filePath
+ * @param {object} [options] same `ChunkOptions` as `chunkFile`
+ * @returns {AsyncGenerator<Chunk, void, unknown>}
+ */
+async function* streamFileChunks(filePath, options = {}) {
+  assertMlReady(mlFormatOf(filePath))
+  const opts = withDefaultTokenizer(options)
+  yield* chunkStream((callback) => native.chunkFileStreaming(filePath, callback, opts))
+}
+
+/**
+ * Streaming counterpart of `chunk`: chunk in-memory bytes, yielding each chunk
+ * as it is produced (same contract as {@link streamFileChunks}).
+ *
+ * @param {object} input same `ConvertInput` as `chunk`
+ * @param {object} [options] same `ChunkOptions` as `chunk`
+ * @returns {AsyncGenerator<Chunk, void, unknown>}
+ */
+async function* streamChunks(input, options = {}) {
+  assertMlReady(mlFormatOf(input && input.name, input && input.format))
+  const opts = withDefaultTokenizer(options)
+  yield* chunkStream((callback) => native.chunkStreaming(input, callback, opts))
+}
+
+/**
+ * Streaming counterpart of `chunkDocument`: chunk an already-converted
+ * docling-core JSON document, yielding each chunk as it is produced (same
+ * contract as {@link streamFileChunks}). Touches no ML models — unguarded.
+ *
+ * @param {string} documentJson
+ * @param {object} [options] same `ChunkOptions` as `chunkDocument`
+ * @returns {AsyncGenerator<Chunk, void, unknown>}
+ */
+async function* streamDocumentChunks(documentJson, options = {}) {
+  const opts = withDefaultTokenizer(options)
+  yield* chunkStream((callback) => native.chunkDocumentStreaming(documentJson, callback, opts))
+}
+
 // --- exports (explicit, so ESM named imports work in Node and Bun) ----------
 
 module.exports.convert = convert
 module.exports.convertFile = convertFile
 module.exports.convertAsync = convertAsync
 module.exports.convertFileAsync = convertFileAsync
+module.exports.chunk = chunk
+module.exports.chunkFile = chunkFile
+module.exports.chunkAsync = chunkAsync
+module.exports.chunkFileAsync = chunkFileAsync
+// Chunking an already-converted JSON document touches no ML models — unguarded.
+module.exports.chunkDocument = chunkDocument
+module.exports.chunkDocumentAsync = chunkDocumentAsync
 module.exports.DocumentConverter = DocumentConverter
 module.exports.Pipeline = Pipeline
 module.exports.streamFileMarkdown = streamFileMarkdown
+module.exports.streamFileChunks = streamFileChunks
+module.exports.streamChunks = streamChunks
+module.exports.streamDocumentChunks = streamDocumentChunks
 module.exports.checkDependencies = checkDependencies
 module.exports.supportedFormats = native.supportedFormats
 module.exports.formatFromName = native.formatFromName

package/native.d.ts

@@ -98,6 +98,100 @@ export declare function convert(input: ConvertInput, options?: ConvertOptions |
 export declare function convertFileAsync(path: string, options?: ConvertOptions | undefined | null): Promise<ConvertResult>
 /** Async (Promise-returning) [`convert`]. */
 export declare function convertAsync(input: ConvertInput, options?: ConvertOptions | undefined | null): Promise<ConvertResult>
+/** Options for the chunk* functions. */
+export interface ChunkOptions {
+  /**
+   * `"hierarchical"` (default): one chunk per document item, docling's
+   * structure-driven chunker. `"hybrid"`: tokenization-aware refinement —
+   * splits oversized chunks and merges undersized same-heading neighbours;
+   * requires `tokenizer`.
+   */
+  chunker?: string
+  /**
+   * Path to a HuggingFace `tokenizer.json` (e.g. all-MiniLM-L6-v2's) for the
+   * hybrid chunker's token counts. When omitted, falls back to
+   * `models/chunk/tokenizer.json` (populated by
+   * `scripts/install/download_dependencies.sh`).
+   */
+  tokenizer?: string
+  /**
+   * The hybrid chunker's token budget per chunk. Default `256` (docling's
+   * default for the MiniLM embedding model).
+   */
+  maxTokens?: number
+  /**
+   * Merge undersized peer chunks with the same headings (hybrid only).
+   * Default `true`, matching docling.
+   */
+  mergePeers?: boolean
+}
+/** One chunk record — the analogue of docling's `DocChunk`. */
+export interface Chunk {
+  /** The chunk body (markdown-flavoured text, same as docling's `DocChunk.text`). */
+  text: string
+  /**
+   * The heading path above the chunk, outermost first; absent for content
+   * above any heading.
+   */
+  headings?: Array<string>
+  /**
+   * JSON-pointer refs of the document items the chunk was built from
+   * (`"#/texts/12"`, `"#/tables/0"`, …).
+   */
+  docItems: Array<string>
+  /**
+   * The embedding-ready rendering: heading path + text, newline-joined
+   * (docling's `chunker.contextualize(chunk)`).
+   */
+  contextualized: string
+}
+/**
+ * Chunk a file on disk with docling's chunkers: convert it, then run the
+ * hierarchical (default) or hybrid chunker over the document.
+ */
+export declare function chunkFile(path: string, options?: ChunkOptions | undefined | null): Array<Chunk>
+/**
+ * Async (Promise-returning) [`chunk_file`]; conversion + chunking run on the
+ * libuv thread pool.
+ */
+export declare function chunkFileAsync(path: string, options?: ChunkOptions | undefined | null): Promise<Array<Chunk>>
+/** Chunk in-memory bytes (same contract as [`convert`], then chunk). */
+export declare function chunk(input: ConvertInput, options?: ChunkOptions | undefined | null): Array<Chunk>
+/** Async (Promise-returning) [`chunk`]. */
+export declare function chunkAsync(input: ConvertInput, options?: ChunkOptions | undefined | null): Promise<Array<Chunk>>
+/**
+ * Chunk an already-converted document, passed as docling-core JSON (the
+ * `content` of a `convert*` call with `to: "json"`) — so a document converted
+ * once (e.g. through the warm PDF `Pipeline`) can be chunked without
+ * re-converting.
+ */
+export declare function chunkDocument(documentJson: string, options?: ChunkOptions | undefined | null): Array<Chunk>
+/** Async (Promise-returning) [`chunk_document`]. */
+export declare function chunkDocumentAsync(documentJson: string, options?: ChunkOptions | undefined | null): Promise<Array<Chunk>>
+/**
+ * Chunk a file and stream each chunk as the chunkers produce it — no
+ * all-chunks array is materialized, and the first chunk reaches JS while the
+ * rest of the document is still being chunked.
+ *
+ * `callback` is invoked as `(err, chunk)`: once per chunk with `chunk` a
+ * `Chunk`, once with `chunk === null` at the end, or once with a non-null
+ * `err` on failure. Prefer the `streamFileChunks` async-generator wrapper in
+ * JS over calling this directly.
+ */
+export declare function chunkFileStreaming(path: string, callback: (err: Error | null, arg: Chunk | undefined | null) => any, options?: ChunkOptions | undefined | null): void
+/**
+ * Streaming [`chunk`]: chunk in-memory bytes, pushing each chunk through the
+ * callback (same contract as [`chunk_file_streaming`]). Prefer the
+ * `streamChunks` async-generator wrapper in JS.
+ */
+export declare function chunkStreaming(input: ConvertInput, callback: (err: Error | null, arg: Chunk | undefined | null) => any, options?: ChunkOptions | undefined | null): void
+/**
+ * Streaming [`chunk_document`]: chunk an already-converted docling-core JSON
+ * document, pushing each chunk through the callback (same contract as
+ * [`chunk_file_streaming`]). Prefer the `streamDocumentChunks`
+ * async-generator wrapper in JS.
+ */
+export declare function chunkDocumentStreaming(documentJson: string, callback: (err: Error | null, arg: Chunk | undefined | null) => any, options?: ChunkOptions | undefined | null): void
 /** The list of supported input format ids. */
 export declare function supportedFormats(): Array<string>
 /**

package/native.js

@@ -310,7 +310,7 @@ if (!nativeBinding) {
   throw new Error(`Failed to load native binding`)
 }
 
-const { convertFile, convert, convertFileAsync, convertAsync, DocumentConverter, Pipeline, supportedFormats, formatFromName } = nativeBinding
+const { convertFile, convert, convertFileAsync, convertAsync, DocumentConverter, Pipeline, chunkFile, chunkFileAsync, chunk, chunkAsync, chunkDocument, chunkDocumentAsync, chunkFileStreaming, chunkStreaming, chunkDocumentStreaming, supportedFormats, formatFromName } = nativeBinding
 
 module.exports.convertFile = convertFile
 module.exports.convert = convert
@@ -318,5 +318,14 @@ module.exports.convertFileAsync = convertFileAsync
 module.exports.convertAsync = convertAsync
 module.exports.DocumentConverter = DocumentConverter
 module.exports.Pipeline = Pipeline
+module.exports.chunkFile = chunkFile
+module.exports.chunkFileAsync = chunkFileAsync
+module.exports.chunk = chunk
+module.exports.chunkAsync = chunkAsync
+module.exports.chunkDocument = chunkDocument
+module.exports.chunkDocumentAsync = chunkDocumentAsync
+module.exports.chunkFileStreaming = chunkFileStreaming
+module.exports.chunkStreaming = chunkStreaming
+module.exports.chunkDocumentStreaming = chunkDocumentStreaming
 module.exports.supportedFormats = supportedFormats
 module.exports.formatFromName = formatFromName

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "docling.rs",
-  "version": "0.33.0",
+  "version": "0.37.0",
   "description": "Node.js / Bun bindings for docling.rs — a Rust port of docling. Convert Markdown, HTML, DOCX, PPTX, XLSX, PDF, images and more into a unified DoclingDocument (Markdown or docling-core JSON).",
   "keywords": [
     "docling",
@@ -64,8 +64,8 @@
     "@napi-rs/cli": "^2.18.4"
   },
   "optionalDependencies": {
-    "docling.rs-linux-x64-gnu": "0.33.0",
-    "docling.rs-linux-arm64-gnu": "0.33.0",
-    "docling.rs-win32-x64-msvc": "0.33.0"
+    "docling.rs-linux-x64-gnu": "0.37.0",
+    "docling.rs-linux-arm64-gnu": "0.37.0",
+    "docling.rs-win32-x64-msvc": "0.37.0"
   }
 }