docling.rs 0.32.0 → 0.37.0

package/README.md

@@ -93,6 +93,62 @@ for await (const chunk of streamFileMarkdown('paper.pdf')) {
 }
 ```
 
+### Chunking (docling's chunkers, for RAG)
+
+`chunkFile` / `chunk` / `chunkDocument` (each with an `…Async` variant) run
+docling's chunkers over a converted document and return embedding-ready
+records. The default is the structure-driven **hierarchical** chunker (one
+chunk per document item — whole lists, triplet-serialized tables — with its
+heading path); pass `chunker: 'hybrid'` to refine against a token budget
+(split oversized chunks, merge undersized same-heading neighbours), matching
+docling's `HybridChunker`. The hybrid token counts come from a HuggingFace
+`tokenizer.json`: pass a path via `tokenizer`, or omit it to use
+`models/chunk/tokenizer.json` (all-MiniLM-L6-v2's — fetched by
+`scripts/download_dependencies.sh` alongside the ML models, resolved through
+the same install-home logic).
+
+```js
+import { chunkFileAsync, Pipeline, chunkDocumentAsync } from 'docling.rs'
+
+const chunks = await chunkFileAsync('report.docx', {
+  chunker: 'hybrid',
+  tokenizer: 'tokenizer.json', // e.g. all-MiniLM-L6-v2's
+  maxTokens: 256,
+})
+for (const c of chunks) {
+  await embed(c.contextualized) // heading path + text, ready for the embedder
+}
+
+// Chunk something you already converted (no re-conversion), e.g. a PDF
+// that went through the warm Pipeline:
+const { content } = new Pipeline().convertFile('paper.pdf', { to: 'json' })
+const pdfChunks = await chunkDocumentAsync(content)
+```
+
+Each `Chunk` is `{ text, headings?, docItems, contextualized }` — `docItems`
+holds the source items' JSON-pointer refs (`"#/texts/12"`), `contextualized`
+is docling's `contextualize()` rendering to feed the embedding model.
+
+#### Streaming chunks
+
+`streamFileChunks` / `streamChunks` / `streamDocumentChunks` are the streaming
+counterparts: async generators that yield each chunk **as the chunkers produce
+it** — the first chunk is ready for embedding while the rest of the document
+is still being chunked, and no all-chunks array is materialized. Abandoning
+the generator early (`break`) cancels the background chunking.
+
+```js
+import { streamFileChunks } from 'docling.rs'
+
+for await (const c of streamFileChunks('report.docx', {
+  chunker: 'hybrid',
+  tokenizer: 'tokenizer.json',
+  maxTokens: 256,
+})) {
+  await embed(c.contextualized) // embedding overlaps the remaining chunking
+}
+```
+
 ### PDF / images: getting the ML models
 
 Declarative formats (Markdown, HTML, DOCX, XLSX, …) are pure Rust and need
@@ -159,12 +215,22 @@ import { Pipeline } from 'docling.rs'
 
 const pipeline = new Pipeline({ strict: true })
 for (const path of pdfPaths) {
-  const { content } = pipeline.convertFile(path, { to: 'markdown' }) // warm models
+  const { content } = await pipeline.convertFileAsync(path, { to: 'json' }) // warm models, off the event loop
+}
+
+// Or stream a PDF's Markdown as pages finish converting:
+for await (const chunk of pipeline.streamFileMarkdown('paper.pdf')) {
+  process.stdout.write(chunk)
 }
 ```
 
-`Pipeline` handles `pdf` and `image` inputs (the ML pipeline) and is synchronous
-— reuse one instance behind a job queue.
+`Pipeline` handles `pdf` and `image` inputs (the ML pipeline). The sync
+`convertFile` / `convert` block the event loop; the `*Async` variants run on the
+libuv thread pool, and `streamFileMarkdown` yields Markdown chunks in document
+order as pages finish. Conversions on one instance run one at a time (the
+models are mutable sessions) — overlapping `*Async` calls queue in submission
+order, so batch throughput comes from keeping the models warm, not from
+parallel calls.
 
 ### Images
 
@@ -194,12 +260,18 @@ JSON output always embeds extracted images as data URIs.
 | `convertFileAsync(path, options?)` | `Promise<ConvertResult>` | Off the event loop. |
 | `convertAsync(input, options?)` | `Promise<ConvertResult>` | Off the event loop. |
 | `streamFileMarkdown(path, options?)` | `AsyncGenerator<string>` | Markdown chunks in document order. |
+| `chunkFile(path, options?)` | `Chunk[]` | Convert + run docling's hierarchical/hybrid chunker. |
+| `chunk(input, options?)` | `Chunk[]` | Same, over in-memory bytes. |
+| `chunkDocument(documentJson, options?)` | `Chunk[]` | Chunk an already-converted docling JSON document. |
+| `chunkFileAsync` / `chunkAsync` / `chunkDocumentAsync` | `Promise<Chunk[]>` | Off the event loop. |
+| `streamFileChunks` / `streamChunks` / `streamDocumentChunks` | `AsyncGenerator<Chunk>` | Chunks yielded as produced; `break` cancels. |
 | `supportedFormats()` | `string[]` | Supported input format ids. |
 | `formatFromName(name)` | `string \| null` | Detect a format id from a filename/extension. |
 | `checkDependencies(options?)` | `DependencyStatus` | Report which PDF/image deps are present. |
 
 `Pipeline` is the reusable warm PDF/image converter: `new Pipeline(converterOptions)`
-then `convertFile` / `convert`.
+then `convertFile` / `convert` / `convertFileAsync` / `convertAsync` /
+`convertFileStreaming` / `streamFileMarkdown`.
 
 `DocumentConverter` is the reusable form: `new DocumentConverter(converterOptions)`
 then `convert` / `convertFile` / `convertFileAsync` / `convertAsync` /

package/deps.js

@@ -92,9 +92,22 @@ function resolvePaths(dir) {
     tfDecoder:
       process.env.DOCLING_TABLEFORMER_DECODER || path.join(models, 'tableformer', 'decoder.onnx'),
     tfBbox: process.env.DOCLING_TABLEFORMER_BBOX || path.join(models, 'tableformer', 'bbox.onnx'),
+    chunkTokenizer:
+      process.env.DOCLING_CHUNK_TOKENIZER || path.join(models, 'chunk', 'tokenizer.json'),
   }
 }
 
+/**
+ * The hybrid chunker's default tokenizer (all-MiniLM-L6-v2's tokenizer.json,
+ * fetched by `scripts/download_dependencies.sh` into `models/chunk/`), resolved
+ * through the same install-home logic as the ML models. Returns `null` when not
+ * installed — the native side then reports a clear error with the download hint.
+ */
+function defaultChunkTokenizer(dir) {
+  const p = resolvePaths(dir)
+  return fs.existsSync(p.chunkTokenizer) ? p.chunkTokenizer : null
+}
+
 /**
  * Report which dependencies are present on disk. `ready` is true when the
  * minimum for PDF (pdfium + layout) is present.
@@ -108,6 +121,7 @@ function checkDependencies(options = {}) {
     layout: has(p.layout),
     ocr: has(p.ocrRec) && has(p.ocrDict),
     tableformer: has(p.tfEncoder) && has(p.tfDecoder) && has(p.tfBbox),
+    chunkTokenizer: has(p.chunkTokenizer),
   }
   status.ready = status.pdfium && status.layout
   status.missing = [
@@ -185,4 +199,5 @@ module.exports = {
   assertMlReady,
   resolvePaths,
   exportEnv,
+  defaultChunkTokenizer,
 }

package/index.d.ts

@@ -8,9 +8,19 @@ import type {
   ConvertOptions,
   ConvertInput,
   ConvertResult,
+  ChunkOptions,
+  Chunk,
 } from './native'
 
-export type { ConverterOptions, OutputOptions, ConvertOptions, ConvertInput, ConvertResult }
+export type {
+  ConverterOptions,
+  OutputOptions,
+  ConvertOptions,
+  ConvertInput,
+  ConvertResult,
+  ChunkOptions,
+  Chunk,
+}
 
 // Format helpers pass straight through from the native binding.
 export { supportedFormats, formatFromName } from './native'
@@ -27,6 +37,27 @@ export declare function convertFileAsync(path: string, options?: ConvertOptions
 /** Async (Promise) bytes conversion, off the event loop. */
 export declare function convertAsync(input: ConvertInput, options?: ConvertOptions | null): Promise<ConvertResult>
 
+/**
+ * Chunk a file with docling's chunkers: convert it, then run the hierarchical
+ * (default) or hybrid (`chunker: 'hybrid'` + `tokenizer`) chunker over the
+ * document. Throws for PDF/image/METS if deps aren't installed.
+ */
+export declare function chunkFile(path: string, options?: ChunkOptions | null): Array<Chunk>
+/** Async (Promise) {@link chunkFile}; conversion + chunking run off the event loop. */
+export declare function chunkFileAsync(path: string, options?: ChunkOptions | null): Promise<Array<Chunk>>
+/** Chunk in-memory bytes (same input contract as {@link convert}). */
+export declare function chunk(input: ConvertInput, options?: ChunkOptions | null): Array<Chunk>
+/** Async (Promise) {@link chunk}. */
+export declare function chunkAsync(input: ConvertInput, options?: ChunkOptions | null): Promise<Array<Chunk>>
+/**
+ * Chunk an already-converted document, passed as docling-core JSON (the
+ * `content` of a `convert*` call with `to: 'json'`) — so a document converted
+ * once (e.g. through the warm PDF {@link Pipeline}) chunks without re-converting.
+ */
+export declare function chunkDocument(documentJson: string, options?: ChunkOptions | null): Array<Chunk>
+/** Async (Promise) {@link chunkDocument}. */
+export declare function chunkDocumentAsync(documentJson: string, options?: ChunkOptions | null): Promise<Array<Chunk>>
+
 /** A reusable converter holding config (strict / fetchImages / allowedFormats). */
 export declare class DocumentConverter {
   constructor(options?: ConverterOptions | null)
@@ -37,15 +68,40 @@ export declare class DocumentConverter {
   convertFileStreaming(path: string, callback: StreamCallback, options?: OutputOptions | null): void
 }
 
+/** Output options for {@link Pipeline.streamFileMarkdown} (streamable modes only). */
+export interface PipelineStreamOptions {
+  imageMode?: 'placeholder' | 'embedded'
+  artifactsDir?: string
+}
+
 /**
  * A reusable PDF/image pipeline that keeps the ONNX models loaded across calls.
  * Use instead of the per-call functions when converting many PDFs/images — the
  * one-shot path reloads every model each call. Handles `pdf` and `image` inputs.
+ *
+ * The `*Async` variants run the conversion off the event loop; overlapping
+ * calls on one instance queue (the models are mutable sessions), so batch
+ * throughput comes from keeping the models warm, not from parallel calls.
  */
 export declare class Pipeline {
   constructor(options?: ConverterOptions | null)
   convertFile(path: string, options?: OutputOptions | null): ConvertResult
   convert(input: ConvertInput, options?: OutputOptions | null): ConvertResult
+  /** Async (Promise) file conversion on the warm pipeline, off the event loop. */
+  convertFileAsync(path: string, options?: OutputOptions | null): Promise<ConvertResult>
+  /** Async (Promise) bytes conversion on the warm pipeline, off the event loop. */
+  convertAsync(input: ConvertInput, options?: OutputOptions | null): Promise<ConvertResult>
+  /** Callback-form streaming (prefer {@link Pipeline.streamFileMarkdown}). */
+  convertFileStreaming(path: string, callback: StreamCallback, options?: OutputOptions | null): void
+  /**
+   * Stream a PDF's Markdown in chunks through the warm pipeline, in document
+   * order, as pages finish converting (an image arrives as a single chunk).
+   * Concatenating the chunks reproduces the buffered Markdown byte-for-byte.
+   */
+  streamFileMarkdown(
+    filePath: string,
+    options?: PipelineStreamOptions,
+  ): AsyncGenerator<string, void, unknown>
 }
 
 // --- dependency provisioning (PDF/image ML pipeline) -----------------------
@@ -62,6 +118,8 @@ export interface DependencyStatus {
   ocr: boolean
   /** TableFormer encoder/decoder/bbox present. */
   tableformer: boolean
+  /** Hybrid-chunker tokenizer (models/chunk/tokenizer.json) present. */
+  chunkTokenizer: boolean
   /** True when the minimum for PDF (pdfium + layout) is present. */
   ready: boolean
   /** Human-readable list of the missing required assets. */
@@ -95,3 +153,32 @@ export declare function streamFileMarkdown(
   filePath: string,
   options?: StreamOptions,
 ): AsyncGenerator<string, void, unknown>
+
+/**
+ * Stream a file's chunks as the chunkers produce them — the streaming
+ * counterpart of {@link chunkFile}. The first chunk is ready (e.g. for
+ * embedding) while the rest of the document is still being chunked, and no
+ * all-chunks array is materialized. Abandoning the generator early (`break`)
+ * cancels the background chunking. Throws for PDF/image/METS if deps aren't
+ * installed.
+ */
+export declare function streamFileChunks(
+  filePath: string,
+  options?: ChunkOptions | null,
+): AsyncGenerator<Chunk, void, unknown>
+
+/** Streaming counterpart of {@link chunk} (same contract as {@link streamFileChunks}). */
+export declare function streamChunks(
+  input: ConvertInput,
+  options?: ChunkOptions | null,
+): AsyncGenerator<Chunk, void, unknown>
+
+/**
+ * Streaming counterpart of {@link chunkDocument}: chunk an already-converted
+ * docling-core JSON document (same contract as {@link streamFileChunks}).
+ * Touches no ML models.
+ */
+export declare function streamDocumentChunks(
+  documentJson: string,
+  options?: ChunkOptions | null,
+): AsyncGenerator<Chunk, void, unknown>

package/index.js

@@ -5,14 +5,15 @@
 //   1. dependency guards — converting a PDF/image/METS input throws a clear
 //      error unless the ML models + pdfium are on disk (see
 //      scripts/download_dependencies.sh);
-//   2. a `streamFileMarkdown` async generator over Markdown chunks.
+//   2. `streamFileMarkdown` async generators over Markdown chunks (module-level
+//      and on the warm `Pipeline`).
 //
 // Works in Node.js and Bun (Bun implements N-API).
 
 'use strict'
 
 const native = require('./native.js')
-const { checkDependencies, assertMlReady } = require('./deps.js')
+const { checkDependencies, assertMlReady, defaultChunkTokenizer } = require('./deps.js')
 
 // Resolve the format id of an input for the ML guard. Uses the native
 // extension→format map; falls back to an explicitly-passed format string.
@@ -23,6 +24,48 @@ function mlFormatOf(name, format) {
   return native.formatFromName(name || '') || ''
 }
 
+// Bridge a native (err, chunk) streaming callback into an async generator.
+// `start` receives the callback and kicks off the native conversion. Chunks
+// are delivered on the event loop (via a threadsafe function); a null chunk
+// ends the stream, a non-null err ends it with a throw.
+async function* chunkStream(start) {
+  const queue = []
+  let done = false
+  let failure = null
+  let notify = null
+  const wake = () => {
+    if (notify) {
+      const n = notify
+      notify = null
+      n()
+    }
+  }
+
+  start((err, chunk) => {
+    if (err) {
+      failure = err
+      done = true
+    } else if (chunk === null || chunk === undefined) {
+      done = true
+    } else {
+      queue.push(chunk)
+    }
+    wake()
+  })
+
+  while (true) {
+    if (queue.length > 0) {
+      yield queue.shift()
+      continue
+    }
+    if (failure) throw failure
+    if (done) return
+    await new Promise((resolve) => {
+      notify = resolve
+    })
+  }
+}
+
 // --- guarded one-shot functions --------------------------------------------
 
 function convertFile(path, options) {
@@ -46,6 +89,48 @@ async function convertAsync(input, options) {
   return native.convertAsync(input, options)
 }
 
+// --- guarded chunking functions ---------------------------------------------
+
+// For the hybrid chunker with no explicit tokenizer, resolve the default one
+// (models/chunk/tokenizer.json) through the same install-home logic as the ML
+// models — so DOCLING_RS_HOME / ~/.cache installs work, not only ./models. The
+// native side keeps its own CWD-relative fallback as a backstop.
+function withDefaultTokenizer(options) {
+  if (!options || options.tokenizer) return options
+  if (String(options.chunker || '').toLowerCase() !== 'hybrid') return options
+  const tokenizer = defaultChunkTokenizer()
+  return tokenizer ? { ...options, tokenizer } : options
+}
+
+function chunkFile(path, options) {
+  assertMlReady(mlFormatOf(path))
+  return native.chunkFile(path, withDefaultTokenizer(options))
+}
+
+function chunk(input, options) {
+  assertMlReady(mlFormatOf(input && input.name, input && input.format))
+  return native.chunk(input, withDefaultTokenizer(options))
+}
+
+// async so a guard failure surfaces as a rejected promise, not a sync throw.
+async function chunkFileAsync(path, options) {
+  assertMlReady(mlFormatOf(path))
+  return native.chunkFileAsync(path, withDefaultTokenizer(options))
+}
+
+async function chunkAsync(input, options) {
+  assertMlReady(mlFormatOf(input && input.name, input && input.format))
+  return native.chunkAsync(input, withDefaultTokenizer(options))
+}
+
+function chunkDocument(documentJson, options) {
+  return native.chunkDocument(documentJson, withDefaultTokenizer(options))
+}
+
+async function chunkDocumentAsync(documentJson, options) {
+  return native.chunkDocumentAsync(documentJson, withDefaultTokenizer(options))
+}
+
 // --- guarded classes --------------------------------------------------------
 
 class DocumentConverter {
@@ -94,6 +179,41 @@ class Pipeline {
     assertMlReady(mlFormatOf(input && input.name, input && input.format))
     return this._inner.convert(input, options)
   }
+
+  // async so a guard failure surfaces as a rejected promise, not a sync throw.
+  async convertFileAsync(path, options) {
+    assertMlReady(mlFormatOf(path))
+    return this._inner.convertFileAsync(path, options)
+  }
+
+  async convertAsync(input, options) {
+    assertMlReady(mlFormatOf(input && input.name, input && input.format))
+    return this._inner.convertAsync(input, options)
+  }
+
+  convertFileStreaming(path, callback, options) {
+    assertMlReady(mlFormatOf(path))
+    return this._inner.convertFileStreaming(path, callback, options)
+  }
+
+  /**
+   * Stream a PDF's Markdown in chunks through the warm pipeline, in document
+   * order, as pages finish converting (an image arrives as a single chunk).
+   * Same contract as the module-level `streamFileMarkdown`, but reusing this
+   * instance's loaded models — no per-call model reload.
+   *
+   * @param {string} filePath
+   * @param {object} [options] output options (`imageMode`: `placeholder` or
+   *   `embedded`; `referenced` is rejected)
+   * @returns {AsyncGenerator<string, void, unknown>}
+   */
+  async *streamFileMarkdown(filePath, options = {}) {
+    assertMlReady(mlFormatOf(filePath))
+    const { imageMode, artifactsDir } = options
+    yield* chunkStream((callback) =>
+      this._inner.convertFileStreaming(filePath, callback, { imageMode, artifactsDir }),
+    )
+  }
 }
 
 // --- streaming --------------------------------------------------------------
@@ -113,49 +233,54 @@ async function* streamFileMarkdown(filePath, options = {}) {
   assertMlReady(mlFormatOf(filePath))
   const { strict, fetchImages, allowedFormats, imageMode, artifactsDir } = options
   const converter = new native.DocumentConverter({ strict, fetchImages, allowedFormats })
+  yield* chunkStream((callback) =>
+    converter.convertFileStreaming(filePath, callback, { imageMode, artifactsDir }),
+  )
+}
 
-  // Bridge the native (err, chunk) callback into an async generator. Chunks are
-  // delivered on the event loop (via a threadsafe function); a null chunk ends
-  // the stream, a non-null err ends it with a throw.
-  const queue = []
-  let done = false
-  let failure = null
-  let notify = null
-  const wake = () => {
-    if (notify) {
-      const n = notify
-      notify = null
-      n()
-    }
-  }
+/**
+ * Stream a file's chunks as the chunkers produce them — the streaming
+ * counterpart of `chunkFile`. The first chunk is ready (e.g. for embedding)
+ * while the rest of the document is still being chunked, and no all-chunks
+ * array is materialized. Abandoning the generator early (`break`) cancels the
+ * background chunking.
+ *
+ * @param {string} filePath
+ * @param {object} [options] same `ChunkOptions` as `chunkFile`
+ * @returns {AsyncGenerator<Chunk, void, unknown>}
+ */
+async function* streamFileChunks(filePath, options = {}) {
+  assertMlReady(mlFormatOf(filePath))
+  const opts = withDefaultTokenizer(options)
+  yield* chunkStream((callback) => native.chunkFileStreaming(filePath, callback, opts))
+}
 
-  converter.convertFileStreaming(
-    filePath,
-    (err, chunk) => {
-      if (err) {
-        failure = err
-        done = true
-      } else if (chunk === null || chunk === undefined) {
-        done = true
-      } else {
-        queue.push(chunk)
-      }
-      wake()
-    },
-    { imageMode, artifactsDir },
-  )
+/**
+ * Streaming counterpart of `chunk`: chunk in-memory bytes, yielding each chunk
+ * as it is produced (same contract as {@link streamFileChunks}).
+ *
+ * @param {object} input same `ConvertInput` as `chunk`
+ * @param {object} [options] same `ChunkOptions` as `chunk`
+ * @returns {AsyncGenerator<Chunk, void, unknown>}
+ */
+async function* streamChunks(input, options = {}) {
+  assertMlReady(mlFormatOf(input && input.name, input && input.format))
+  const opts = withDefaultTokenizer(options)
+  yield* chunkStream((callback) => native.chunkStreaming(input, callback, opts))
+}
 
-  while (true) {
-    if (queue.length > 0) {
-      yield queue.shift()
-      continue
-    }
-    if (failure) throw failure
-    if (done) return
-    await new Promise((resolve) => {
-      notify = resolve
-    })
-  }
+/**
+ * Streaming counterpart of `chunkDocument`: chunk an already-converted
+ * docling-core JSON document, yielding each chunk as it is produced (same
+ * contract as {@link streamFileChunks}). Touches no ML models — unguarded.
+ *
+ * @param {string} documentJson
+ * @param {object} [options] same `ChunkOptions` as `chunkDocument`
+ * @returns {AsyncGenerator<Chunk, void, unknown>}
+ */
+async function* streamDocumentChunks(documentJson, options = {}) {
+  const opts = withDefaultTokenizer(options)
+  yield* chunkStream((callback) => native.chunkDocumentStreaming(documentJson, callback, opts))
 }
 
 // --- exports (explicit, so ESM named imports work in Node and Bun) ----------
@@ -164,9 +289,19 @@ module.exports.convert = convert
 module.exports.convertFile = convertFile
 module.exports.convertAsync = convertAsync
 module.exports.convertFileAsync = convertFileAsync
+module.exports.chunk = chunk
+module.exports.chunkFile = chunkFile
+module.exports.chunkAsync = chunkAsync
+module.exports.chunkFileAsync = chunkFileAsync
+// Chunking an already-converted JSON document touches no ML models — unguarded.
+module.exports.chunkDocument = chunkDocument
+module.exports.chunkDocumentAsync = chunkDocumentAsync
 module.exports.DocumentConverter = DocumentConverter
 module.exports.Pipeline = Pipeline
 module.exports.streamFileMarkdown = streamFileMarkdown
+module.exports.streamFileChunks = streamFileChunks
+module.exports.streamChunks = streamChunks
+module.exports.streamDocumentChunks = streamDocumentChunks
 module.exports.checkDependencies = checkDependencies
 module.exports.supportedFormats = native.supportedFormats
 module.exports.formatFromName = native.formatFromName

package/native.d.ts

@@ -98,6 +98,100 @@ export declare function convert(input: ConvertInput, options?: ConvertOptions |
 export declare function convertFileAsync(path: string, options?: ConvertOptions | undefined | null): Promise<ConvertResult>
 /** Async (Promise-returning) [`convert`]. */
 export declare function convertAsync(input: ConvertInput, options?: ConvertOptions | undefined | null): Promise<ConvertResult>
+/** Options for the chunk* functions. */
+export interface ChunkOptions {
+  /**
+   * `"hierarchical"` (default): one chunk per document item, docling's
+   * structure-driven chunker. `"hybrid"`: tokenization-aware refinement —
+   * splits oversized chunks and merges undersized same-heading neighbours;
+   * requires `tokenizer`.
+   */
+  chunker?: string
+  /**
+   * Path to a HuggingFace `tokenizer.json` (e.g. all-MiniLM-L6-v2's) for the
+   * hybrid chunker's token counts. When omitted, falls back to
+   * `models/chunk/tokenizer.json` (populated by
+   * `scripts/install/download_dependencies.sh`).
+   */
+  tokenizer?: string
+  /**
+   * The hybrid chunker's token budget per chunk. Default `256` (docling's
+   * default for the MiniLM embedding model).
+   */
+  maxTokens?: number
+  /**
+   * Merge undersized peer chunks with the same headings (hybrid only).
+   * Default `true`, matching docling.
+   */
+  mergePeers?: boolean
+}
+/** One chunk record — the analogue of docling's `DocChunk`. */
+export interface Chunk {
+  /** The chunk body (markdown-flavoured text, same as docling's `DocChunk.text`). */
+  text: string
+  /**
+   * The heading path above the chunk, outermost first; absent for content
+   * above any heading.
+   */
+  headings?: Array<string>
+  /**
+   * JSON-pointer refs of the document items the chunk was built from
+   * (`"#/texts/12"`, `"#/tables/0"`, …).
+   */
+  docItems: Array<string>
+  /**
+   * The embedding-ready rendering: heading path + text, newline-joined
+   * (docling's `chunker.contextualize(chunk)`).
+   */
+  contextualized: string
+}
+/**
+ * Chunk a file on disk with docling's chunkers: convert it, then run the
+ * hierarchical (default) or hybrid chunker over the document.
+ */
+export declare function chunkFile(path: string, options?: ChunkOptions | undefined | null): Array<Chunk>
+/**
+ * Async (Promise-returning) [`chunk_file`]; conversion + chunking run on the
+ * libuv thread pool.
+ */
+export declare function chunkFileAsync(path: string, options?: ChunkOptions | undefined | null): Promise<Array<Chunk>>
+/** Chunk in-memory bytes (same contract as [`convert`], then chunk). */
+export declare function chunk(input: ConvertInput, options?: ChunkOptions | undefined | null): Array<Chunk>
+/** Async (Promise-returning) [`chunk`]. */
+export declare function chunkAsync(input: ConvertInput, options?: ChunkOptions | undefined | null): Promise<Array<Chunk>>
+/**
+ * Chunk an already-converted document, passed as docling-core JSON (the
+ * `content` of a `convert*` call with `to: "json"`) — so a document converted
+ * once (e.g. through the warm PDF `Pipeline`) can be chunked without
+ * re-converting.
+ */
+export declare function chunkDocument(documentJson: string, options?: ChunkOptions | undefined | null): Array<Chunk>
+/** Async (Promise-returning) [`chunk_document`]. */
+export declare function chunkDocumentAsync(documentJson: string, options?: ChunkOptions | undefined | null): Promise<Array<Chunk>>
+/**
+ * Chunk a file and stream each chunk as the chunkers produce it — no
+ * all-chunks array is materialized, and the first chunk reaches JS while the
+ * rest of the document is still being chunked.
+ *
+ * `callback` is invoked as `(err, chunk)`: once per chunk with `chunk` a
+ * `Chunk`, once with `chunk === null` at the end, or once with a non-null
+ * `err` on failure. Prefer the `streamFileChunks` async-generator wrapper in
+ * JS over calling this directly.
+ */
+export declare function chunkFileStreaming(path: string, callback: (err: Error | null, arg: Chunk | undefined | null) => any, options?: ChunkOptions | undefined | null): void
+/**
+ * Streaming [`chunk`]: chunk in-memory bytes, pushing each chunk through the
+ * callback (same contract as [`chunk_file_streaming`]). Prefer the
+ * `streamChunks` async-generator wrapper in JS.
+ */
+export declare function chunkStreaming(input: ConvertInput, callback: (err: Error | null, arg: Chunk | undefined | null) => any, options?: ChunkOptions | undefined | null): void
+/**
+ * Streaming [`chunk_document`]: chunk an already-converted docling-core JSON
+ * document, pushing each chunk through the callback (same contract as
+ * [`chunk_file_streaming`]). Prefer the `streamDocumentChunks`
+ * async-generator wrapper in JS.
+ */
+export declare function chunkDocumentStreaming(documentJson: string, callback: (err: Error | null, arg: Chunk | undefined | null) => any, options?: ChunkOptions | undefined | null): void
 /** The list of supported input format ids. */
 export declare function supportedFormats(): Array<string>
 /**
@@ -154,4 +248,25 @@ export declare class Pipeline {
   convertFile(path: string, options?: OutputOptions | undefined | null): ConvertResult
   /** Convert PDF or image bytes, reusing the warm models. */
   convert(input: ConvertInput, options?: OutputOptions | undefined | null): ConvertResult
+  /**
+   * Async (Promise-returning) file conversion on the warm pipeline. The
+   * CPU-bound work runs on the libuv thread pool, keeping the event loop
+   * free; calls on the same instance run one at a time (the models are
+   * mutable sessions), so overlapping Promises queue in submission order.
+   */
+  convertFileAsync(path: string, options?: OutputOptions | undefined | null): Promise<ConvertResult>
+  /** Async (Promise-returning) bytes conversion on the warm pipeline. */
+  convertAsync(input: ConvertInput, options?: OutputOptions | undefined | null): Promise<ConvertResult>
+  /**
+   * Stream a PDF's Markdown in chunks through the warm pipeline, in document
+   * order, as pages finish converting (an image converts in one step and
+   * arrives as a single chunk).
+   *
+   * `callback` is invoked as `(err, chunk)`: once per Markdown chunk with
+   * `chunk` a string, once with `chunk === null` at the end, or once with a
+   * non-null `err` on failure. Only `placeholder` / `embedded` image modes
+   * stream; `referenced` is rejected. Prefer the `streamFileMarkdown`
+   * async-generator wrapper in JS over calling this directly.
+   */
+  convertFileStreaming(path: string, callback: (err: Error | null, arg: string | undefined | null) => any, options?: OutputOptions | undefined | null): void
 }

package/native.js

@@ -310,7 +310,7 @@ if (!nativeBinding) {
   throw new Error(`Failed to load native binding`)
 }
 
-const { convertFile, convert, convertFileAsync, convertAsync, DocumentConverter, Pipeline, supportedFormats, formatFromName } = nativeBinding
+const { convertFile, convert, convertFileAsync, convertAsync, DocumentConverter, Pipeline, chunkFile, chunkFileAsync, chunk, chunkAsync, chunkDocument, chunkDocumentAsync, chunkFileStreaming, chunkStreaming, chunkDocumentStreaming, supportedFormats, formatFromName } = nativeBinding
 
 module.exports.convertFile = convertFile
 module.exports.convert = convert
@@ -318,5 +318,14 @@ module.exports.convertFileAsync = convertFileAsync
 module.exports.convertAsync = convertAsync
 module.exports.DocumentConverter = DocumentConverter
 module.exports.Pipeline = Pipeline
+module.exports.chunkFile = chunkFile
+module.exports.chunkFileAsync = chunkFileAsync
+module.exports.chunk = chunk
+module.exports.chunkAsync = chunkAsync
+module.exports.chunkDocument = chunkDocument
+module.exports.chunkDocumentAsync = chunkDocumentAsync
+module.exports.chunkFileStreaming = chunkFileStreaming
+module.exports.chunkStreaming = chunkStreaming
+module.exports.chunkDocumentStreaming = chunkDocumentStreaming
 module.exports.supportedFormats = supportedFormats
 module.exports.formatFromName = formatFromName

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "docling.rs",
-  "version": "0.32.0",
+  "version": "0.37.0",
   "description": "Node.js / Bun bindings for docling.rs — a Rust port of docling. Convert Markdown, HTML, DOCX, PPTX, XLSX, PDF, images and more into a unified DoclingDocument (Markdown or docling-core JSON).",
   "keywords": [
     "docling",
@@ -64,8 +64,8 @@
     "@napi-rs/cli": "^2.18.4"
   },
   "optionalDependencies": {
-    "docling.rs-linux-x64-gnu": "0.32.0",
-    "docling.rs-linux-arm64-gnu": "0.32.0",
-    "docling.rs-win32-x64-msvc": "0.32.0"
+    "docling.rs-linux-x64-gnu": "0.37.0",
+    "docling.rs-linux-arm64-gnu": "0.37.0",
+    "docling.rs-win32-x64-msvc": "0.37.0"
   }
 }