@markitdownjs/chunking 0.1.1 → 0.2.0

package/dist/__tests__/chunker.test.js

@@ -1,42 +1,44 @@
-import { describe, it, expect } from 'vitest';
-import { DocumentChunker } from '../chunker.js';
-import { createNode } from '@markitdownjs/shared';
-describe('DocumentChunker', () => {
+import { describe, it, expect } from "vitest";
+import { DocumentChunker } from "../chunker.js";
+import { createNode } from "@markitdownjs/shared";
+describe("DocumentChunker", () => {
     const chunker = new DocumentChunker();
     const ast = createNode({
-        type: 'document',
+        type: "document",
         children: [
             createNode({
-                type: 'heading',
+                type: "heading",
                 level: 1,
-                children: [createNode({ type: 'text', value: 'Introduction' })],
+                children: [createNode({ type: "text", value: "Introduction" })],
             }),
             createNode({
-                type: 'paragraph',
-                children: [createNode({ type: 'text', value: 'This is the introduction paragraph.' })],
+                type: "paragraph",
+                children: [
+                    createNode({ type: "text", value: "This is the introduction paragraph." }),
+                ],
             }),
             createNode({
-                type: 'heading',
+                type: "heading",
                 level: 1,
-                children: [createNode({ type: 'text', value: 'Main Content' })],
+                children: [createNode({ type: "text", value: "Main Content" })],
             }),
             createNode({
-                type: 'paragraph',
-                children: [createNode({ type: 'text', value: 'This is the main content.' })],
+                type: "paragraph",
+                children: [createNode({ type: "text", value: "This is the main content." })],
             }),
         ],
     });
-    it('should chunk by heading', () => {
-        const chunks = chunker.chunk(ast, { strategy: 'heading' });
+    it("should chunk by heading", () => {
+        const chunks = chunker.chunk(ast, { strategy: "heading" });
         expect(chunks.length).toBeGreaterThanOrEqual(2);
-        expect(chunks[0].metadata.headingPath).toContain('Introduction');
+        expect(chunks[0].metadata.headingPath).toContain("Introduction");
     });
-    it('should chunk by fixed size', () => {
-        const chunks = chunker.chunk(ast, { strategy: 'fixed', maxTokens: 5, overlap: 2 });
+    it("should chunk by fixed size", () => {
+        const chunks = chunker.chunk(ast, { strategy: "fixed", maxTokens: 5, overlap: 2 });
         expect(chunks.length).toBeGreaterThanOrEqual(1);
     });
-    it('should generate chunk IDs', () => {
-        const chunks = chunker.chunk(ast, { strategy: 'heading' });
+    it("should generate chunk IDs", () => {
+        const chunks = chunker.chunk(ast, { strategy: "heading" });
         for (const chunk of chunks) {
             expect(chunk.id).toBeDefined();
             expect(chunk.id.length).toBeGreaterThan(0);

package/dist/__tests__/chunker.test.js.map

@@ -1 +1 @@
-{"version":3,"file":"chunker.test.js","sourceRoot":"","sources":["../../src/__tests__/chunker.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAChD,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAGlD,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;IAC/B,MAAM,OAAO,GAAG,IAAI,eAAe,EAAE,CAAC;IAEtC,MAAM,GAAG,GAAG,UAAU,CAAe;QACnC,IAAI,EAAE,UAAU;QAChB,QAAQ,EAAE;YACR,UAAU,CAAc;gBACtB,IAAI,EAAE,SAAS;gBACf,KAAK,EAAE,CAAC;gBACR,QAAQ,EAAE,CAAC,UAAU,CAAW,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC,CAAC;aAC1E,CAAC;YACF,UAAU,CAAgB;gBACxB,IAAI,EAAE,WAAW;gBACjB,QAAQ,EAAE,CAAC,UAAU,CAAW,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,qCAAqC,EAAE,CAAC,CAAC;aACjG,CAAC;YACF,UAAU,CAAc;gBACtB,IAAI,EAAE,SAAS;gBACf,KAAK,EAAE,CAAC;gBACR,QAAQ,EAAE,CAAC,UAAU,CAAW,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC,CAAC;aAC1E,CAAC;YACF,UAAU,CAAgB;gBACxB,IAAI,EAAE,WAAW;gBACjB,QAAQ,EAAE,CAAC,UAAU,CAAW,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,2BAA2B,EAAE,CAAC,CAAC;aACvF,CAAC;SACH;KACF,CAAC,CAAC;IAEH,EAAE,CAAC,yBAAyB,EAAE,GAAG,EAAE;QACjC,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC,CAAC;QAC3D,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC;QAChD,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC;IACpE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4BAA4B,EAAE,GAAG,EAAE;QACpC,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC;QACnF,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC;IAClD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2BAA2B,EAAE,GAAG,EAAE;QACnC,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC,CAAC;QAC3D,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;YAC/B,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QAC7C,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
+{"version":3,"file":"chunker.test.js","sourceRoot":"","sources":["../../src/__tests__/chunker.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAChD,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAGlD,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;IAC/B,MAAM,OAAO,GAAG,IAAI,eAAe,EAAE,CAAC;IAEtC,MAAM,GAAG,GAAG,UAAU,CAAe;QACnC,IAAI,EAAE,UAAU;QAChB,QAAQ,EAAE;YACR,UAAU,CAAc;gBACtB,IAAI,EAAE,SAAS;gBACf,KAAK,EAAE,CAAC;gBACR,QAAQ,EAAE,CAAC,UAAU,CAAW,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC,CAAC;aAC1E,CAAC;YACF,UAAU,CAAgB;gBACxB,IAAI,EAAE,WAAW;gBACjB,QAAQ,EAAE;oBACR,UAAU,CAAW,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,qCAAqC,EAAE,CAAC;iBACrF;aACF,CAAC;YACF,UAAU,CAAc;gBACtB,IAAI,EAAE,SAAS;gBACf,KAAK,EAAE,CAAC;gBACR,QAAQ,EAAE,CAAC,UAAU,CAAW,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC,CAAC;aAC1E,CAAC;YACF,UAAU,CAAgB;gBACxB,IAAI,EAAE,WAAW;gBACjB,QAAQ,EAAE,CAAC,UAAU,CAAW,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,2BAA2B,EAAE,CAAC,CAAC;aACvF,CAAC;SACH;KACF,CAAC,CAAC;IAEH,EAAE,CAAC,yBAAyB,EAAE,GAAG,EAAE;QACjC,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC,CAAC;QAC3D,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC;QAChD,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC;IACpE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4BAA4B,EAAE,GAAG,EAAE;QACpC,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC;QACnF,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC;IAClD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2BAA2B,EAAE,GAAG,EAAE;QACnC,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC,CAAC;QAC3D,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;YAC/B,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QAC7C,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/chunker.d.ts

@@ -18,7 +18,7 @@ export declare class DocumentChunker {
     getStrategy(name: string): ChunkingStrategy | undefined;
     /** List all registered strategy names */
     listStrategies(): string[];
-    /** Chunk an AST using the specified strategy */
+    /** Chunk an AST using the specified strategy, then classify content types */
     chunk(ast: AnyNode, options: ChunkingOptions): DocumentChunk[];
 }
 //# sourceMappingURL=chunker.d.ts.map

package/dist/chunker.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"chunker.d.ts","sourceRoot":"","sources":["../src/chunker.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,OAAO,EACP,aAAa,EAEb,eAAe,EAChB,MAAM,sBAAsB,CAAC;AAO9B,sDAAsD;AACtD,MAAM,WAAW,gBAAgB;IAC/B,+BAA+B;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,wCAAwC;IACxC,KAAK,CAAC,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,eAAe,GAAG,aAAa,EAAE,CAAC;CAChE;AAED,iEAAiE;AACjE,wBAAgB,WAAW,CACzB,KAAK,EAAE,OAAO,EAAE,EAChB,WAAW,EAAE,MAAM,EAAE,EACrB,OAAO,EAAE,eAAe,EACxB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,EAChB,IAAI,CAAC,EAAE,MAAM,GACZ,aAAa,GAAG,IAAI,CAsBtB;AAED,oEAAoE;AACpE,qBAAa,eAAe;IAC1B,OAAO,CAAC,UAAU,CAAgC;;IAUlD,0CAA0C;IAC1C,gBAAgB,CAAC,QAAQ,EAAE,gBAAgB,GAAG,IAAI;IAIlD,wCAAwC;IACxC,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,gBAAgB,GAAG,SAAS;IAIvD,yCAAyC;IACzC,cAAc,IAAI,MAAM,EAAE;IAI1B,gDAAgD;IAChD,KAAK,CAAC,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,eAAe,GAAG,aAAa,EAAE;CAQ/D"}
+{"version":3,"file":"chunker.d.ts","sourceRoot":"","sources":["../src/chunker.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,aAAa,EAAiB,eAAe,EAAE,MAAM,sBAAsB,CAAC;AASnG,sDAAsD;AACtD,MAAM,WAAW,gBAAgB;IAC/B,+BAA+B;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,wCAAwC;IACxC,KAAK,CAAC,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,eAAe,GAAG,aAAa,EAAE,CAAC;CAChE;AAoBD,iEAAiE;AACjE,wBAAgB,WAAW,CACzB,KAAK,EAAE,OAAO,EAAE,EAChB,WAAW,EAAE,MAAM,EAAE,EACrB,OAAO,EAAE,eAAe,EACxB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,EAChB,IAAI,CAAC,EAAE,MAAM,GACZ,aAAa,GAAG,IAAI,CAoBtB;AAED,oEAAoE;AACpE,qBAAa,eAAe;IAC1B,OAAO,CAAC,UAAU,CAAgC;;IAUlD,0CAA0C;IAC1C,gBAAgB,CAAC,QAAQ,EAAE,gBAAgB,GAAG,IAAI;IAIlD,wCAAwC;IACxC,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,gBAAgB,GAAG,SAAS;IAIvD,yCAAyC;IACzC,cAAc,IAAI,MAAM,EAAE;IAI1B,6EAA6E;IAC7E,KAAK,CAAC,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,eAAe,GAAG,aAAa,EAAE;CAS/D"}

package/dist/chunker.js

@@ -3,15 +3,32 @@ import { HeadingChunkingStrategy } from "./strategies/heading-strategy.js";
 import { PageChunkingStrategy } from "./strategies/page-strategy.js";
 import { SemanticChunkingStrategy } from "./strategies/semantic-strategy.js";
 import { FixedChunkingStrategy } from "./strategies/fixed-strategy.js";
+import { getDefaultTokenizerRegistry } from "./tokenizers/index.js";
+import { classifyChunks } from "./classifier.js";
+/** Resolves the tokenizer to use for the given options */
+function resolveTokenizer(options) {
+    // Custom function takes priority.
+    if (options.tokenizerFn)
+        return options.tokenizerFn;
+    // Look up by name/model in the registry.
+    if (options.tokenizer) {
+        const registry = getDefaultTokenizerRegistry();
+        const tok = registry.resolve(options.tokenizer);
+        return (text) => tok.countTokens(text);
+    }
+    // Default: fast-bpe fallback.
+    const registry = getDefaultTokenizerRegistry();
+    const tok = registry.resolve("fast-bpe");
+    return (text) => tok.countTokens(text);
+}
 /** Creates a document chunk from collected nodes and metadata */
 export function createChunk(nodes, headingPath, options, startIndex, endIndex, page) {
     if (nodes.length === 0)
         return null;
     const content = nodes.map((n) => getNodeText(n)).join("\n\n");
-    const tokenCount = content.split(/\s+/).filter(Boolean).length;
-    const root = nodes.length === 1
-        ? nodes[0]
-        : { type: "document", children: nodes };
+    const countTokens = resolveTokenizer(options);
+    const tokenCount = countTokens(content);
+    const root = nodes.length === 1 ? nodes[0] : { type: "document", children: nodes };
     const metadata = {
         chunkId: generateId(),
         headingPath,
@@ -45,14 +62,15 @@ export class DocumentChunker {
     listStrategies() {
         return Array.from(this.strategies.keys());
     }
-    /** Chunk an AST using the specified strategy */
+    /** Chunk an AST using the specified strategy, then classify content types */
     chunk(ast, options) {
         const strategyName = options.strategy ?? "heading";
         const strategy = this.strategies.get(strategyName);
         if (!strategy) {
             throw new Error(`Unknown chunking strategy: ${strategyName}`);
         }
-        return strategy.chunk(ast, options);
+        const rawChunks = strategy.chunk(ast, options);
+        return classifyChunks(rawChunks);
     }
 }
 //# sourceMappingURL=chunker.js.map

package/dist/chunker.js.map

@@ -1 +1 @@
-{"version":3,"file":"chunker.js","sourceRoot":"","sources":["../src/chunker.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAC/D,OAAO,EAAE,uBAAuB,EAAE,MAAM,kCAAkC,CAAC;AAC3E,OAAO,EAAE,oBAAoB,EAAE,MAAM,+BAA+B,CAAC;AACrE,OAAO,EAAE,wBAAwB,EAAE,MAAM,mCAAmC,CAAC;AAC7E,OAAO,EAAE,qBAAqB,EAAE,MAAM,gCAAgC,CAAC;AAUvE,iEAAiE;AACjE,MAAM,UAAU,WAAW,CACzB,KAAgB,EAChB,WAAqB,EACrB,OAAwB,EACxB,UAAkB,EAClB,QAAgB,EAChB,IAAa;IAEb,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAEpC,MAAM,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC9D,MAAM,UAAU,GAAG,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC;IAE/D,MAAM,IAAI,GACR,KAAK,CAAC,MAAM,KAAK,CAAC;QAChB,CAAC,CAAC,KAAK,CAAC,CAAC,CAAE;QACX,CAAC,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;IAE5C,MAAM,QAAQ,GAAkB;QAC9B,OAAO,EAAE,UAAU,EAAE;QACrB,WAAW;QACX,UAAU;QACV,UAAU,EAAE,OAAO,CAAC,UAAU;QAC9B,UAAU;QACV,QAAQ;QACR,IAAI;KACL,CAAC;IAEF,OAAO,EAAE,EAAE,EAAE,QAAQ,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC;AAChE,CAAC;AAED,oEAAoE;AACpE,MAAM,OAAO,eAAe;IAClB,UAAU,CAAgC;IAElD;QACE,IAAI,CAAC,UAAU,GAAG,IAAI,GAAG,EAAE,CAAC;QAC5B,IAAI,CAAC,gBAAgB,CAAC,IAAI,uBAAuB,EAAE,CAAC,CAAC;QACrD,IAAI,CAAC,gBAAgB,CAAC,IAAI,oBAAoB,EAAE,CAAC,CAAC;QAClD,IAAI,CAAC,gBAAgB,CAAC,IAAI,wBAAwB,EAAE,CAAC,CAAC;QACtD,IAAI,CAAC,gBAAgB,CAAC,IAAI,qBAAqB,EAAE,CAAC,CAAC;IACrD,CAAC;IAED,0CAA0C;IAC1C,gBAAgB,CAAC,QAA0B;QACzC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IAC/C,CAAC;IAED,wCAAwC;IACxC,WAAW,CAAC,IAAY;QACtB,OAAO,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACnC,CAAC;IAED,yCAAyC;IACzC,cAAc;QACZ,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC;IAC5C,CAAC;IAED,gDAAgD;IAChD,KAAK,CAAC,GAAY,EAAE,OAAwB;QAC1C,MAAM,YAAY,GAAG,OAAO,CAAC,QAAQ,IAAI,SAAS,CAAC;QACnD,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;QACnD,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,IAAI,KAAK,CAAC,8BAA8B,YAAY,EAAE,CAAC,CAAC;QAChE,CAAC;QACD,OAAO,QAAQ,CAAC,KAAK,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IACtC,CAAC;CACF"}
+{"version":3,"file":"chunker.js","sourceRoot":"","sources":["../src/chunker.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAC/D,OAAO,EAAE,uBAAuB,EAAE,MAAM,kCAAkC,CAAC;AAC3E,OAAO,EAAE,oBAAoB,EAAE,MAAM,+BAA+B,CAAC;AACrE,OAAO,EAAE,wBAAwB,EAAE,MAAM,mCAAmC,CAAC;AAC7E,OAAO,EAAE,qBAAqB,EAAE,MAAM,gCAAgC,CAAC;AACvE,OAAO,EAAE,2BAA2B,EAAE,MAAM,uBAAuB,CAAC;AACpE,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAUjD,0DAA0D;AAC1D,SAAS,gBAAgB,CAAC,OAAwB;IAChD,kCAAkC;IAClC,IAAI,OAAO,CAAC,WAAW;QAAE,OAAO,OAAO,CAAC,WAAW,CAAC;IAEpD,yCAAyC;IACzC,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;QACtB,MAAM,QAAQ,GAAG,2BAA2B,EAAE,CAAC;QAC/C,MAAM,GAAG,GAAG,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAChD,OAAO,CAAC,IAAY,EAAE,EAAE,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;IACjD,CAAC;IAED,8BAA8B;IAC9B,MAAM,QAAQ,GAAG,2BAA2B,EAAE,CAAC;IAC/C,MAAM,GAAG,GAAG,QAAQ,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IACzC,OAAO,CAAC,IAAY,EAAE,EAAE,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;AACjD,CAAC;AAED,iEAAiE;AACjE,MAAM,UAAU,WAAW,CACzB,KAAgB,EAChB,WAAqB,EACrB,OAAwB,EACxB,UAAkB,EAClB,QAAgB,EAChB,IAAa;IAEb,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAEpC,MAAM,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC9D,MAAM,WAAW,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAC9C,MAAM,UAAU,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;IAExC,MAAM,IAAI,GAAY,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;IAE7F,MAAM,QAAQ,GAAkB;QAC9B,OAAO,EAAE,UAAU,EAAE;QACrB,WAAW;QACX,UAAU;QACV,UAAU,EAAE,OAAO,CAAC,UAAU;QAC9B,UAAU;QACV,QAAQ;QACR,IAAI;KACL,CAAC;IAEF,OAAO,EAAE,EAAE,EAAE,QAAQ,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC;AAChE,CAAC;AAED,oEAAoE;AACpE,MAAM,OAAO,eAAe;IAClB,UAAU,CAAgC;IAElD;QACE,IAAI,CAAC,UAAU,GAAG,IAAI,GAAG,EAAE,CAAC;QAC5B,IAAI,CAAC,gBAAgB,CAAC,IAAI,uBAAuB,EAAE,CAAC,CAAC;QACrD,IAAI,CAAC,gBAAgB,CAAC,IAAI,oBAAoB,EAAE,CAAC,CAAC;QAClD,IAAI,CAAC,gBAAgB,CAAC,IAAI,wBAAwB,EAAE,CAAC,CAAC;QACtD,IAAI,CAAC,gBAAgB,CAAC,IAAI,qBAAqB,EAAE,CAAC,CAAC;IACrD,CAAC;IAED,0CAA0C;IAC1C,gBAAgB,CAAC,QAA0B;QACzC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IAC/C,CAAC;IAED,wCAAwC;IACxC,WAAW,CAAC,IAAY;QACtB,OAAO,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACnC,CAAC;IAED,yCAAyC;IACzC,cAAc;QACZ,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC;IAC5C,CAAC;IAED,6EAA6E;IAC7E,KAAK,CAAC,GAAY,EAAE,OAAwB;QAC1C,MAAM,YAAY,GAAG,OAAO,CAAC,QAAQ,IAAI,SAAS,CAAC;QACnD,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;QACnD,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,IAAI,KAAK,CAAC,8BAA8B,YAAY,EAAE,CAAC,CAAC;QAChE,CAAC;QACD,MAAM,SAAS,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QAC/C,OAAO,cAAc,CAAC,SAAS,CAAC,CAAC;IACnC,CAAC;CACF"}

package/dist/classifier.d.ts

@@ -0,0 +1,20 @@
+import type { DocumentChunk } from "@markitdownjs/shared";
+/** Content type classification for chunks */
+export type ContentType = "narrative" | "table" | "code" | "list" | "callout" | "heading-only" | "mixed";
+/**
+ * Classify a chunk's content type based on its AST children.
+ * Different content types benefit from different embedding strategies:
+ * - "table": structured data → may need special table-aware embeddings
+ * - "code": source code → code-specific embeddings
+ * - "narrative": prose paragraphs → general text embeddings
+ * - "list": enumerated items → may need list-aware handling
+ * - "callout": notes/warnings → distinct semantic treatment
+ * - "heading-only": pure structural markers → low embedding value
+ * - "mixed": combination of types
+ */
+export declare function classifyChunkContentType(chunk: DocumentChunk): ContentType;
+/**
+ * Classify all chunks in an array and populate their contentType metadata.
+ */
+export declare function classifyChunks(chunks: DocumentChunk[]): DocumentChunk[];
+//# sourceMappingURL=classifier.d.ts.map

package/dist/classifier.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"classifier.d.ts","sourceRoot":"","sources":["../src/classifier.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAW,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAEnE,6CAA6C;AAC7C,MAAM,MAAM,WAAW,GAAG,WAAW,GAAG,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,SAAS,GAAG,cAAc,GAAG,OAAO,CAAC;AAEzG;;;;;;;;;;GAUG;AACH,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,aAAa,GAAG,WAAW,CAuB1E;AAqCD;;GAEG;AACH,wBAAgB,cAAc,CAAC,MAAM,EAAE,aAAa,EAAE,GAAG,aAAa,EAAE,CAWvE"}

package/dist/classifier.js

@@ -0,0 +1,79 @@
+/**
+ * Classify a chunk's content type based on its AST children.
+ * Different content types benefit from different embedding strategies:
+ * - "table": structured data → may need special table-aware embeddings
+ * - "code": source code → code-specific embeddings
+ * - "narrative": prose paragraphs → general text embeddings
+ * - "list": enumerated items → may need list-aware handling
+ * - "callout": notes/warnings → distinct semantic treatment
+ * - "heading-only": pure structural markers → low embedding value
+ * - "mixed": combination of types
+ */
+export function classifyChunkContentType(chunk) {
+    const types = new Set();
+    collectNodeTypes(chunk.ast, types);
+    // Remove "narrative" from the set if other specific types are present,
+    // since narrative is the fallback/default.
+    if (types.size > 1) {
+        types.delete("narrative");
+    }
+    // Single type → return it directly.
+    if (types.size === 1) {
+        return Array.from(types)[0];
+    }
+    // Multiple types → "mixed".
+    if (types.size > 1) {
+        return "mixed";
+    }
+    // No types found → "narrative" as default.
+    return "narrative";
+}
+/**
+ * Recursively collect content types from AST nodes.
+ */
+function collectNodeTypes(node, types) {
+    switch (node.type) {
+        case "table":
+            types.add("table");
+            break;
+        case "code":
+            types.add("code");
+            break;
+        case "list":
+        case "list-item":
+            types.add("list");
+            break;
+        case "callout":
+            types.add("callout");
+            break;
+        case "heading":
+            types.add("heading-only");
+            break;
+        case "paragraph":
+        case "text":
+            types.add("narrative");
+            break;
+    }
+    // Recurse into children.
+    if ("children" in node && Array.isArray(node.children)) {
+        for (const child of node.children) {
+            collectNodeTypes(child, types);
+        }
+    }
+}
+/**
+ * Classify all chunks in an array and populate their contentType metadata.
+ */
+export function classifyChunks(chunks) {
+    return chunks.map((chunk) => {
+        const contentType = classifyChunkContentType(chunk);
+        return {
+            ...chunk,
+            metadata: {
+                ...chunk.metadata,
+                contentType,
+            },
+        };
+    });
+}
+//# sourceMappingURL=classifier.js.map

package/dist/classifier.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"classifier.js","sourceRoot":"","sources":["../src/classifier.ts"],"names":[],"mappings":"AAKA;;;;;;;;;;GAUG;AACH,MAAM,UAAU,wBAAwB,CAAC,KAAoB;IAC3D,MAAM,KAAK,GAAG,IAAI,GAAG,EAAe,CAAC;IAErC,gBAAgB,CAAC,KAAK,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IAEnC,uEAAuE;IACvE,2CAA2C;IAC3C,IAAI,KAAK,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;QACnB,KAAK,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;IAC5B,CAAC;IAED,oCAAoC;IACpC,IAAI,KAAK,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;QACrB,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAE,CAAC;IAC/B,CAAC;IAED,4BAA4B;IAC5B,IAAI,KAAK,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;QACnB,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,2CAA2C;IAC3C,OAAO,WAAW,CAAC;AACrB,CAAC;AAED;;GAEG;AACH,SAAS,gBAAgB,CAAC,IAAa,EAAE,KAAuB;IAC9D,QAAQ,IAAI,CAAC,IAAI,EAAE,CAAC;QAClB,KAAK,OAAO;YACV,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YACnB,MAAM;QACR,KAAK,MAAM;YACT,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YAClB,MAAM;QACR,KAAK,MAAM,CAAC;QACZ,KAAK,WAAW;YACd,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YAClB,MAAM;QACR,KAAK,SAAS;YACZ,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YACrB,MAAM;QACR,KAAK,SAAS;YACZ,KAAK,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;YAC1B,MAAM;QACR,KAAK,WAAW,CAAC;QACjB,KAAK,MAAM;YACT,KAAK,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;YACvB,MAAM;IACV,CAAC;IAED,yBAAyB;IACzB,IAAI,UAAU,IAAI,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;QACvD,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClC,gBAAgB,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QACjC,CAAC;IACH,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,MAAuB;IACpD,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;QAC1B,MAAM,WAAW,GAAG,wBAAwB,CAAC,KAAK,CAAC,CAAC;QACpD,OAAO;YACL,GAAG,KAAK;YACR,QAAQ,EAAE;gBACR,GAAG,KAAK,CAAC,QAAQ;gBACjB,WAAW;aACZ;SACF,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC"}

package/dist/fingerprint.d.ts

@@ -0,0 +1,35 @@
+import type { AnyNode, DocumentChunk } from "@markitdownjs/shared";
+/** Fingerprint for deduplication */
+export interface ChunkFingerprint {
+    /** Hash of normalized content (lowercase, whitespace-collapsed) */
+    contentHash: string;
+    /** Hash of AST structure (node types only, no content) */
+    structureHash: string;
+}
+/**
+ * Compute a stable content hash for a chunk.
+ * Normalizes the content by lowercasing and collapsing whitespace before hashing.
+ */
+export declare function computeContentHash(text: string): string;
+/**
+ * Compute a structure hash for an AST node.
+ * Hashes only the node types (not content), so reformatting doesn't change the hash.
+ */
+export declare function computeStructureHash(node: AnyNode): string;
+/**
+ * Compute fingerprints for all chunks.
+ * Returns the chunks with fingerprint metadata attached.
+ */
+export declare function fingerprintChunks(chunks: DocumentChunk[]): (DocumentChunk & {
+    fingerprint: ChunkFingerprint;
+})[];
+/**
+ * Detect which chunks have changed between two versions.
+ * Returns the indices of changed chunks.
+ */
+export declare function detectChangedChunks(oldChunks: {
+    fingerprint: ChunkFingerprint;
+}[], newChunks: {
+    fingerprint: ChunkFingerprint;
+}[]): number[];
+//# sourceMappingURL=fingerprint.d.ts.map

package/dist/fingerprint.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fingerprint.d.ts","sourceRoot":"","sources":["../src/fingerprint.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAEnE,oCAAoC;AACpC,MAAM,WAAW,gBAAgB;IAC/B,mEAAmE;IACnE,WAAW,EAAE,MAAM,CAAC;IACpB,0DAA0D;IAC1D,aAAa,EAAE,MAAM,CAAC;CACvB;AAcD;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAMvD;AAED;;;GAGG;AACH,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,OAAO,GAAG,MAAM,CAI1D;AAeD;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,aAAa,EAAE,GAAG,CAAC,aAAa,GAAG;IAAE,WAAW,EAAE,gBAAgB,CAAA;CAAE,CAAC,EAAE,CAQhH;AAED;;;GAGG;AACH,wBAAgB,mBAAmB,CACjC,SAAS,EAAE;IAAE,WAAW,EAAE,gBAAgB,CAAA;CAAE,EAAE,EAC9C,SAAS,EAAE;IAAE,WAAW,EAAE,gBAAgB,CAAA;CAAE,EAAE,GAC7C,MAAM,EAAE,CAgBV"}

package/dist/fingerprint.js

@@ -0,0 +1,76 @@
+/**
+ * Simple deterministic hash using DJB2 algorithm.
+ * Produces a 32-bit hex string — sufficient for deduplication fingerprints.
+ */
+function djb2Hash(str) {
+    let hash = 5381;
+    for (let i = 0; i < str.length; i++) {
+        hash = ((hash << 5) + hash + str.charCodeAt(i)) | 0;
+    }
+    return Math.abs(hash).toString(16).padStart(8, "0");
+}
+/**
+ * Compute a stable content hash for a chunk.
+ * Normalizes the content by lowercasing and collapsing whitespace before hashing.
+ */
+export function computeContentHash(text) {
+    const normalized = text
+        .toLowerCase()
+        .replace(/\s+/g, " ")
+        .trim();
+    return `djb2:${djb2Hash(normalized)}`;
+}
+/**
+ * Compute a structure hash for an AST node.
+ * Hashes only the node types (not content), so reformatting doesn't change the hash.
+ */
+export function computeStructureHash(node) {
+    const types = collectNodeTypes(node);
+    const typeString = types.join(",");
+    return `djb2:${djb2Hash(typeString)}`;
+}
+/**
+ * Collect all node types in order from an AST tree.
+ */
+function collectNodeTypes(node) {
+    const types = [node.type];
+    if ("children" in node && Array.isArray(node.children)) {
+        for (const child of node.children) {
+            types.push(...collectNodeTypes(child));
+        }
+    }
+    return types;
+}
+/**
+ * Compute fingerprints for all chunks.
+ * Returns the chunks with fingerprint metadata attached.
+ */
+export function fingerprintChunks(chunks) {
+    return chunks.map((chunk) => ({
+        ...chunk,
+        fingerprint: {
+            contentHash: computeContentHash(chunk.content),
+            structureHash: computeStructureHash(chunk.ast),
+        },
+    }));
+}
+/**
+ * Detect which chunks have changed between two versions.
+ * Returns the indices of changed chunks.
+ */
+export function detectChangedChunks(oldChunks, newChunks) {
+    const changed = [];
+    const maxLen = Math.max(oldChunks.length, newChunks.length);
+    for (let i = 0; i < maxLen; i++) {
+        const oldFp = oldChunks[i]?.fingerprint;
+        const newFp = newChunks[i]?.fingerprint;
+        if (!oldFp || !newFp) {
+            changed.push(i);
+        }
+        else if (oldFp.contentHash !== newFp.contentHash || oldFp.structureHash !== newFp.structureHash) {
+            changed.push(i);
+        }
+    }
+    return changed;
+}
+//# sourceMappingURL=fingerprint.js.map

package/dist/fingerprint.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fingerprint.js","sourceRoot":"","sources":["../src/fingerprint.ts"],"names":[],"mappings":"AAUA;;;GAGG;AACH,SAAS,QAAQ,CAAC,GAAW;IAC3B,IAAI,IAAI,GAAG,IAAI,CAAC;IAChB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACpC,IAAI,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,GAAG,IAAI,GAAG,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IACtD,CAAC;IACD,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;AACtD,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,kBAAkB,CAAC,IAAY;IAC7C,MAAM,UAAU,GAAG,IAAI;SACpB,WAAW,EAAE;SACb,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;SACpB,IAAI,EAAE,CAAC;IACV,OAAO,QAAQ,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;AACxC,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,oBAAoB,CAAC,IAAa;IAChD,MAAM,KAAK,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAAC;IACrC,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACnC,OAAO,QAAQ,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;AACxC,CAAC;AAED;;GAEG;AACH,SAAS,gBAAgB,CAAC,IAAa;IACrC,MAAM,KAAK,GAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACpC,IAAI,UAAU,IAAI,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;QACvD,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClC,KAAK,CAAC,IAAI,CAAC,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAC,CAAC;QACzC,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,iBAAiB,CAAC,MAAuB;IACvD,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QAC5B,GAAG,KAAK;QACR,WAAW,EAAE;YACX,WAAW,EAAE,kBAAkB,CAAC,KAAK,CAAC,OAAO,CAAC;YAC9C,aAAa,EAAE,oBAAoB,CAAC,KAAK,CAAC,GAAG,CAAC;SAC/C;KACF,CAAC,CAAC,CAAC;AACN,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,mBAAmB,CACjC,SAA8C,EAC9C,SAA8C;IAE9C,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,MAAM,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;IAE5D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAChC,MAAM,KAAK,GAAG,SAAS,CAAC,CAAC,CAAC,EAAE,WAAW,CAAC;QACxC,MAAM,KAAK,GAAG,SAAS,CAAC,CAAC,CAAC,EAAE,WAAW,CAAC;QAExC,IAAI,CAAC,KAAK,IAAI,CAAC,KAAK,EAAE,CAAC;YACrB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;aAAM,IAAI,KAAK,CAAC,WAAW,KAAK,KAAK,CAAC,WAAW,IAAI,KAAK,CAAC,aAAa,KAAK,KAAK,CAAC,aAAa,EAAE,CAAC;YAClG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC"}

package/dist/index.d.ts

@@ -4,4 +4,12 @@ export { HeadingChunkingStrategy } from "./strategies/heading-strategy.js";
 export { PageChunkingStrategy } from "./strategies/page-strategy.js";
 export { SemanticChunkingStrategy } from "./strategies/semantic-strategy.js";
 export { FixedChunkingStrategy } from "./strategies/fixed-strategy.js";
+export { classifyChunkContentType, classifyChunks } from "./classifier.js";
+export type { ContentType } from "./classifier.js";
+export { computeContentHash, computeStructureHash, fingerprintChunks, detectChangedChunks } from "./fingerprint.js";
+export type { ChunkFingerprint } from "./fingerprint.js";
+export type { Tokenizer } from "./tokenizers/types.js";
+export { FastBPETokenizer } from "./tokenizers/fast-bpe.js";
+export { TikTokenizer, Cl100kTokenizer, O200kTokenizer } from "./tokenizers/tiktoken.js";
+export { TokenizerRegistry, getDefaultTokenizerRegistry } from "./tokenizers/registry.js";
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAC5D,YAAY,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,EAAE,uBAAuB,EAAE,MAAM,kCAAkC,CAAC;AAC3E,OAAO,EAAE,oBAAoB,EAAE,MAAM,+BAA+B,CAAC;AACrE,OAAO,EAAE,wBAAwB,EAAE,MAAM,mCAAmC,CAAC;AAC7E,OAAO,EAAE,qBAAqB,EAAE,MAAM,gCAAgC,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAC5D,YAAY,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,EAAE,uBAAuB,EAAE,MAAM,kCAAkC,CAAC;AAC3E,OAAO,EAAE,oBAAoB,EAAE,MAAM,+BAA+B,CAAC;AACrE,OAAO,EAAE,wBAAwB,EAAE,MAAM,mCAAmC,CAAC;AAC7E,OAAO,EAAE,qBAAqB,EAAE,MAAM,gCAAgC,CAAC;AAGvE,OAAO,EAAE,wBAAwB,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAC3E,YAAY,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAGnD,OAAO,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,iBAAiB,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAC;AACpH,YAAY,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAGzD,YAAY,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAC5D,OAAO,EAAE,YAAY,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AACzF,OAAO,EAAE,iBAAiB,EAAE,2BAA2B,EAAE,MAAM,0BAA0B,CAAC"}

package/dist/index.js

@@ -3,4 +3,11 @@ export { HeadingChunkingStrategy } from "./strategies/heading-strategy.js";
 export { PageChunkingStrategy } from "./strategies/page-strategy.js";
 export { SemanticChunkingStrategy } from "./strategies/semantic-strategy.js";
 export { FixedChunkingStrategy } from "./strategies/fixed-strategy.js";
+// Content type classification
+export { classifyChunkContentType, classifyChunks } from "./classifier.js";
+// Deduplication fingerprinting
+export { computeContentHash, computeStructureHash, fingerprintChunks, detectChangedChunks } from "./fingerprint.js";
+export { FastBPETokenizer } from "./tokenizers/fast-bpe.js";
+export { TikTokenizer, Cl100kTokenizer, O200kTokenizer } from "./tokenizers/tiktoken.js";
+export { TokenizerRegistry, getDefaultTokenizerRegistry } from "./tokenizers/registry.js";
 //# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAE5D,OAAO,EAAE,uBAAuB,EAAE,MAAM,kCAAkC,CAAC;AAC3E,OAAO,EAAE,oBAAoB,EAAE,MAAM,+BAA+B,CAAC;AACrE,OAAO,EAAE,wBAAwB,EAAE,MAAM,mCAAmC,CAAC;AAC7E,OAAO,EAAE,qBAAqB,EAAE,MAAM,gCAAgC,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAE5D,OAAO,EAAE,uBAAuB,EAAE,MAAM,kCAAkC,CAAC;AAC3E,OAAO,EAAE,oBAAoB,EAAE,MAAM,+BAA+B,CAAC;AACrE,OAAO,EAAE,wBAAwB,EAAE,MAAM,mCAAmC,CAAC;AAC7E,OAAO,EAAE,qBAAqB,EAAE,MAAM,gCAAgC,CAAC;AAEvE,8BAA8B;AAC9B,OAAO,EAAE,wBAAwB,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAG3E,+BAA+B;AAC/B,OAAO,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,iBAAiB,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAC;AAKpH,OAAO,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAC5D,OAAO,EAAE,YAAY,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AACzF,OAAO,EAAE,iBAAiB,EAAE,2BAA2B,EAAE,MAAM,0BAA0B,CAAC"}

package/dist/strategies/heading-strategy.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"heading-strategy.d.ts","sourceRoot":"","sources":["../../src/strategies/heading-strategy.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,OAAO,EACP,aAAa,EACb,eAAe,EAEhB,MAAM,sBAAsB,CAAC;AAM9B,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAKtD,8CAA8C;AAC9C,qBAAa,uBAAwB,YAAW,gBAAgB;IAC9D,IAAI,SAAa;IAEjB,KAAK,CAAC,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,eAAe,GAAG,aAAa,EAAE;CA+D/D"}
+{"version":3,"file":"heading-strategy.d.ts","sourceRoot":"","sources":["../../src/strategies/heading-strategy.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,aAAa,EAAE,eAAe,EAAe,MAAM,sBAAsB,CAAC;AAEjG,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAWtD,8CAA8C;AAC9C,qBAAa,uBAAwB,YAAW,gBAAgB;IAC9D,IAAI,SAAa;IAEjB,KAAK,CAAC,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,eAAe,GAAG,aAAa,EAAE;CAuF/D"}

package/dist/strategies/heading-strategy.js

@@ -1,28 +1,52 @@
-import { getNodeText, isNodeOfType, countTokens, } from "@markitdownjs/shared";
+import { getNodeText, isNodeOfType } from "@markitdownjs/shared";
 import { createChunk } from "../chunker.js";
 const DEFAULT_MAX_TOKENS = 512;
+/** Resolve the tokenizer function from options (cached per chunk call) */
+function resolveCountTokens(options) {
+    if (options.tokenizerFn)
+        return options.tokenizerFn;
+    return (text) => text.split(/\s+/).filter(Boolean).length;
+}
 /** Splits a document by heading boundaries */
 export class HeadingChunkingStrategy {
     name = "heading";
     chunk(ast, options) {
         const maxTokens = options.maxTokens ?? DEFAULT_MAX_TOKENS;
+        const overlapTokens = options.overlap ?? 0;
+        const countTokens = resolveCountTokens(options);
         const headingPath = [];
         const chunks = [];
         let currentNodes = [];
         let startIndex = 0;
         let globalOffset = 0;
+        // Overlap state: trailing text from the previous chunk.
+        let overlapText = "";
         const flushChunk = () => {
             if (currentNodes.length === 0)
                 return;
-            const chunk = createChunk([...currentNodes], [...headingPath], options, startIndex, globalOffset);
+            // Prepend overlap from previous chunk if configured.
+            let nodesToChunk = [...currentNodes];
+            if (overlapText && overlapTokens > 0) {
+                const overlapNode = { type: "paragraph", children: [{ type: "text", value: overlapText }] };
+                nodesToChunk = [overlapNode, ...nodesToChunk];
+            }
+            const chunk = createChunk(nodesToChunk, [...headingPath], options, startIndex, globalOffset);
             if (chunk) {
-                if (countTokens(chunk.ast) > maxTokens) {
-                    const split = splitLargeChunk(currentNodes, headingPath, options, maxTokens, startIndex);
+                // Set overlap metadata.
+                if (overlapText && overlapTokens > 0) {
+                    chunk.metadata.overlapEnd = overlapText.length;
+                }
+                if (chunk.metadata.tokenCount > maxTokens) {
+                    const split = splitLargeChunk(currentNodes, headingPath, options, maxTokens, startIndex, countTokens);
                     chunks.push(...split);
                 }
                 else {
                     chunks.push(chunk);
                 }
+                // Compute trailing overlap for the next chunk.
+                if (overlapTokens > 0) {
+                    overlapText = extractTrailingText(currentNodes, overlapTokens, countTokens);
+                }
             }
             currentNodes = [];
             startIndex = globalOffset;
@@ -41,29 +65,59 @@ export class HeadingChunkingStrategy {
                     walk(child, node);
                 }
             }
-            const nodeTokens = countTokens(node);
+            const nodeTokens = countTokens(getNodeText(node));
             if (node.type !== "heading" &&
                 node.type !== "document" &&
                 !(parent && isNodeOfType(parent, "heading"))) {
                 currentNodes.push(node);
                 globalOffset += nodeTokens;
             }
-            if (nodeTokens > 0 && countTokensArray(currentNodes) >= maxTokens) {
+            if (nodeTokens > 0 && countNodesTokens(currentNodes, countTokens) >= maxTokens) {
                 flushChunk();
             }
         }
     }
 }
-function countTokensArray(nodes) {
-    return nodes.reduce((sum, n) => sum + countTokens(n), 0);
+/**
+ * Extract trailing text from nodes for overlap.
+ * Returns the last `overlapTokens` worth of text.
+ */
+function extractTrailingText(nodes, overlapTokens, countTokens) {
+    if (nodes.length === 0)
+        return "";
+    // Walk nodes from the end, collecting text until we reach the overlap budget.
+    const parts = [];
+    let tokenBudget = overlapTokens;
+    for (let i = nodes.length - 1; i >= 0; i--) {
+        const text = getNodeText(nodes[i]);
+        const tokens = countTokens(text);
+        if (tokens > tokenBudget) {
+            // Partial node — take trailing words.
+            const words = text.split(/\s+/).filter(Boolean);
+            const taken = [];
+            let takenTokens = 0;
+            for (let j = words.length - 1; j >= 0 && takenTokens < tokenBudget; j--) {
+                taken.unshift(words[j]);
+                takenTokens++;
+            }
+            parts.unshift(taken.join(" "));
+            break;
+        }
+        parts.unshift(text);
+        tokenBudget -= tokens;
+    }
+    return parts.join("\n\n");
+}
+function countNodesTokens(nodes, countTokens) {
+    return nodes.reduce((sum, n) => sum + countTokens(getNodeText(n)), 0);
 }
-function splitLargeChunk(nodes, headingPath, options, maxTokens, startIndex) {
+function splitLargeChunk(nodes, headingPath, options, maxTokens, startIndex, countTokens) {
     const chunks = [];
     let buffer = [];
     let offset = startIndex;
     for (const node of nodes) {
-        const nodeTokens = countTokens(node);
-        const bufferTokens = countTokensArray(buffer);
+        const nodeTokens = countTokens(getNodeText(node));
+        const bufferTokens = countNodesTokens(buffer, countTokens);
         if (bufferTokens + nodeTokens > maxTokens && buffer.length > 0) {
             const chunk = createChunk(buffer, headingPath, options, offset, offset + bufferTokens);
             if (chunk)
@@ -72,7 +126,7 @@ function splitLargeChunk(nodes, headingPath, options, maxTokens, startIndex) {
             buffer = [];
         }
         if (nodeTokens > maxTokens && node.type === "paragraph" && "children" in node) {
-            const subChunks = splitParagraph(node, headingPath, options, maxTokens, offset);
+            const subChunks = splitParagraph(node, headingPath, options, maxTokens, offset, countTokens);
             chunks.push(...subChunks);
             offset += nodeTokens;
         }
@@ -81,36 +135,57 @@ function splitLargeChunk(nodes, headingPath, options, maxTokens, startIndex) {
         }
     }
     if (buffer.length > 0) {
-        const bufferTokens = countTokensArray(buffer);
+        const bufferTokens = countNodesTokens(buffer, countTokens);
         const chunk = createChunk(buffer, headingPath, options, offset, offset + bufferTokens);
         if (chunk)
             chunks.push(chunk);
     }
     return chunks;
 }
-function splitParagraph(node, headingPath, options, maxTokens, startIndex) {
+function splitParagraph(node, headingPath, options, maxTokens, startIndex, countTokens) {
     const text = getNodeText(node);
-    const words = text.split(/\s+/).filter(Boolean);
+    const sentences = text.split(/(?<=[.!?])\s+/).filter(Boolean);
     const chunks = [];
     let offset = startIndex;
-    for (let i = 0; i < words.length; i += maxTokens) {
-        const slice = words.slice(i, i + maxTokens);
-        const value = slice.join(" ");
-        const tokenCount = slice.length;
+    let buffer = "";
+    let bufferTokens = 0;
+    for (const sentence of sentences) {
+        const sentenceTokens = countTokens(sentence);
+        if (bufferTokens + sentenceTokens > maxTokens && buffer) {
+            chunks.push({
+                id: crypto.randomUUID(),
+                content: buffer,
+                metadata: {
+                    chunkId: crypto.randomUUID(),
+                    headingPath: [...headingPath],
+                    tokenCount: bufferTokens,
+                    sourceFile: options.sourceFile,
+                    startIndex: offset,
+                    endIndex: offset + bufferTokens,
+                },
+                ast: { type: "paragraph", children: [{ type: "text", value: buffer }] },
+            });
+            offset += bufferTokens;
+            buffer = "";
+            bufferTokens = 0;
+        }
+        buffer = buffer ? `${buffer} ${sentence}` : sentence;
+        bufferTokens += sentenceTokens;
+    }
+    if (buffer) {
         chunks.push({
             id: crypto.randomUUID(),
-            content: value,
+            content: buffer,
             metadata: {
                 chunkId: crypto.randomUUID(),
                 headingPath: [...headingPath],
-                tokenCount,
+                tokenCount: bufferTokens,
                 sourceFile: options.sourceFile,
                 startIndex: offset,
-                endIndex: offset + tokenCount,
+                endIndex: offset + bufferTokens,
             },
-            ast: { type: "paragraph", children: [{ type: "text", value }] },
+            ast: { type: "paragraph", children: [{ type: "text", value: buffer }] },
         });
-        offset += tokenCount;
     }
     return chunks;
 }

package/dist/strategies/heading-strategy.js.map

@@ -1 +1 @@
-{"version":3,"file":"heading-strategy.js","sourceRoot":"","sources":["../../src/strategies/heading-strategy.ts"],"names":[],"mappings":"AAMA,OAAO,EACL,WAAW,EACX,YAAY,EACZ,WAAW,GACZ,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAE5C,MAAM,kBAAkB,GAAG,GAAG,CAAC;AAE/B,8CAA8C;AAC9C,MAAM,OAAO,uBAAuB;IAClC,IAAI,GAAG,SAAS,CAAC;IAEjB,KAAK,CAAC,GAAY,EAAE,OAAwB;QAC1C,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,kBAAkB,CAAC;QAC1D,MAAM,WAAW,GAAa,EAAE,CAAC;QACjC,MAAM,MAAM,GAAoB,EAAE,CAAC;QACnC,IAAI,YAAY,GAAc,EAAE,CAAC;QACjC,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,IAAI,YAAY,GAAG,CAAC,CAAC;QAErB,MAAM,UAAU,GAAG,GAAS,EAAE;YAC5B,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO;YACtC,MAAM,KAAK,GAAG,WAAW,CACvB,CAAC,GAAG,YAAY,CAAC,EACjB,CAAC,GAAG,WAAW,CAAC,EAChB,OAAO,EACP,UAAU,EACV,YAAY,CACb,CAAC;YACF,IAAI,KAAK,EAAE,CAAC;gBACV,IAAI,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,SAAS,EAAE,CAAC;oBACvC,MAAM,KAAK,GAAG,eAAe,CAAC,YAAY,EAAE,WAAW,EAAE,OAAO,EAAE,SAAS,EAAE,UAAU,CAAC,CAAC;oBACzF,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC;gBACxB,CAAC;qBAAM,CAAC;oBACN,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACrB,CAAC;YACH,CAAC;YACD,YAAY,GAAG,EAAE,CAAC;YAClB,UAAU,GAAG,YAAY,CAAC;QAC5B,CAAC,CAAC;QAEF,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QAEhB,UAAU,EAAE,CAAC;QAEb,OAAO,MAAM,CAAC;QAEd,SAAS,IAAI,CAAC,IAAa,EAAE,MAAsB;YACjD,IAAI,YAAY,CAAc,IAAI,EAAE,SAAS,CAAC,EAAE,CAAC;gBAC/C,UAAU,EAAE,CAAC;gBACb,WAAW,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC;gBACpC,WAAW,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;YAClD,CAAC;YAED,IAAI,UAAU,IAAI,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACvD,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;oBAClC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;gBACpB,CAAC;YACH,CAAC;YAED,MAAM,UAAU,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;YACrC,IACE,IAAI,CAAC,IAAI,KAAK,SAAS;gBACvB,IAAI,CAAC,IAAI,KAAK,UAAU;gBACxB,CAAC,CAAC,MAAM,IAAI,YAAY,CAAc,MAAM,EAAE,SAAS,CAAC,CAAC,EACzD,CAAC;gBACD,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACxB,YAAY,IAAI,UAAU,CAAC;YAC7B,CAAC;YAED,IAAI,UAAU,GAAG,CAAC,IAAI,gBAAgB,CAAC,YAAY,CAAC,IAAI,SAAS,EAAE,CAAC;gBAClE,UAAU,EAAE,CAAC;YACf,CAAC;QACH,CAAC;IACH,CAAC;CACF;AAED,SAAS,gBAAgB,CAAC,KAAgB;IACxC,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,WAAW,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC;AAED,SAAS,eAAe,CACtB,KAAgB,EAChB,WAAqB,EACrB,OAAwB,EACxB,SAAiB,EACjB,UAAkB;IAElB,MAAM,MAAM,GAAoB,EAAE,CAAC;IACnC,IAAI,MAAM,GAAc,EAAE,CAAC;IAC3B,IAAI,MAAM,GAAG,UAAU,CAAC;IAExB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,UAAU,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;QACrC,MAAM,YAAY,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC;QAE9C,IAAI,YAAY,GAAG,UAAU,GAAG,SAAS,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC/D,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,EAAE,WAAW,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,GAAG,YAAY,CAAC,CAAC;YACvF,IAAI,KAAK;gBAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC9B,MAAM,IAAI,YAAY,CAAC;YACvB,MAAM,GAAG,EAAE,CAAC;QACd,CAAC;QAED,IAAI,UAAU,GAAG,SAAS,IAAI,IAAI,CAAC,IAAI,KAAK,WAAW,IAAI,UAAU,IAAI,IAAI,EAAE,CAAC;YAC9E,MAAM,SAAS,GAAG,cAAc,CAAC,IAAe,EAAE,WAAW,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;YAC3F,MAAM,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;YAC1B,MAAM,IAAI,UAAU,CAAC;QACvB,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpB,CAAC;IACH,CAAC;IAED,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtB,MAAM,YAAY,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC;QAC9C,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,EAAE,WAAW,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,GAAG,YAAY,CAAC,CAAC;QACvF,IAAI,KAAK;YAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAChC,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,cAAc,CACrB,IAAa,EACb,WAAqB,EACrB,OAAwB,EACxB,SAAiB,EACjB,UAAkB;IAElB,MAAM,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;IAC/B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAChD,MAAM,MAAM,GAAoB,EAAE,CAAC;IACnC,IAAI,MAAM,GAAG,UAAU,CAAC;IAExB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;QACjD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC;QAC5C,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC9B,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC;QAChC,MAAM,CAAC,IAAI,CAAC;YACV,EAAE,EAAE,MAAM,CAAC,UAAU,EAAE;YACvB,OAAO,EAAE,KAAK;YACd,QAAQ,EAAE;gBACR,OAAO,EAAE,MAAM,CAAC,UAAU,EAAE;gBAC5B,WAAW,EAAE,CAAC,GAAG,WAAW,CAAC;gBAC7B,UAAU;gBACV,UAAU,EAAE,OAAO,CAAC,UAAU;gBAC9B,UAAU,EAAE,MAAM;gBAClB,QAAQ,EAAE,MAAM,GAAG,UAAU;aAC9B;YACD,GAAG,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE;SAChE,CAAC,CAAC;QACH,MAAM,IAAI,UAAU,CAAC;IACvB,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}
+{"version":3,"file":"heading-strategy.js","sourceRoot":"","sources":["../../src/strategies/heading-strategy.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAEjE,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAE5C,MAAM,kBAAkB,GAAG,GAAG,CAAC;AAE/B,0EAA0E;AAC1E,SAAS,kBAAkB,CAAC,OAAwB;IAClD,IAAI,OAAO,CAAC,WAAW;QAAE,OAAO,OAAO,CAAC,WAAW,CAAC;IACpD,OAAO,CAAC,IAAY,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC;AACpE,CAAC;AAED,8CAA8C;AAC9C,MAAM,OAAO,uBAAuB;IAClC,IAAI,GAAG,SAAS,CAAC;IAEjB,KAAK,CAAC,GAAY,EAAE,OAAwB;QAC1C,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,kBAAkB,CAAC;QAC1D,MAAM,aAAa,GAAG,OAAO,CAAC,OAAO,IAAI,CAAC,CAAC;QAC3C,MAAM,WAAW,GAAG,kBAAkB,CAAC,OAAO,CAAC,CAAC;QAChD,MAAM,WAAW,GAAa,EAAE,CAAC;QACjC,MAAM,MAAM,GAAoB,EAAE,CAAC;QACnC,IAAI,YAAY,GAAc,EAAE,CAAC;QACjC,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,IAAI,YAAY,GAAG,CAAC,CAAC;QAErB,wDAAwD;QACxD,IAAI,WAAW,GAAG,EAAE,CAAC;QAErB,MAAM,UAAU,GAAG,GAAS,EAAE;YAC5B,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO;YAEtC,qDAAqD;YACrD,IAAI,YAAY,GAAG,CAAC,GAAG,YAAY,CAAC,CAAC;YACrC,IAAI,WAAW,IAAI,aAAa,GAAG,CAAC,EAAE,CAAC;gBACrC,MAAM,WAAW,GAAY,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC,EAAE,CAAC;gBACrG,YAAY,GAAG,CAAC,WAAW,EAAE,GAAG,YAAY,CAAC,CAAC;YAChD,CAAC;YAED,MAAM,KAAK,GAAG,WAAW,CACvB,YAAY,EACZ,CAAC,GAAG,WAAW,CAAC,EAChB,OAAO,EACP,UAAU,EACV,YAAY,CACb,CAAC;YACF,IAAI,KAAK,EAAE,CAAC;gBACV,wBAAwB;gBACxB,IAAI,WAAW,IAAI,aAAa,GAAG,CAAC,EAAE,CAAC;oBACpC,KAAK,CAAC,QAAgB,CAAC,UAAU,GAAG,WAAW,CAAC,MAAM,CAAC;gBAC1D,CAAC;gBAED,IAAI,KAAK,CAAC,QAAQ,CAAC,UAAU,GAAG,SAAS,EAAE,CAAC;oBAC1C,MAAM,KAAK,GAAG,eAAe,CAAC,YAAY,EAAE,WAAW,EAAE,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,WAAW,CAAC,CAAC;oBACtG,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC;gBACxB,CAAC;qBAAM,CAAC;oBACN,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACrB,CAAC;gBAED,+CAA+C;gBAC/C,IAAI,aAAa,GAAG,CAAC,EAAE,CAAC;oBACtB,WAAW,GAAG,mBAAmB,CAAC,YAAY,EAAE,aAAa,EAAE,WAAW,CAAC,CAAC;gBAC9E,CAAC;YACH,CAAC;YAED,YAAY,GAAG,EAAE,CAAC;YAClB,UAAU,GAAG,YAAY,CAAC;QAC5B,CAAC,CAAC;QAEF,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QAEhB,UAAU,EAAE,CAAC;QAEb,OAAO,MAAM,CAAC;QAEd,SAAS,IAAI,CAAC,IAAa,EAAE,MAAsB;YACjD,IAAI,YAAY,CAAc,IAAI,EAAE,SAAS,CAAC,EAAE,CAAC;gBAC/C,UAAU,EAAE,CAAC;gBACb,WAAW,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC;gBACpC,WAAW,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;YAClD,CAAC;YAED,IAAI,UAAU,IAAI,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACvD,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;oBAClC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;gBACpB,CAAC;YACH,CAAC;YAED,MAAM,UAAU,GAAG,WAAW,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC;YAClD,IACE,IAAI,CAAC,IAAI,KAAK,SAAS;gBACvB,IAAI,CAAC,IAAI,KAAK,UAAU;gBACxB,CAAC,CAAC,MAAM,IAAI,YAAY,CAAc,MAAM,EAAE,SAAS,CAAC,CAAC,EACzD,CAAC;gBACD,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACxB,YAAY,IAAI,UAAU,CAAC;YAC7B,CAAC;YAED,IAAI,UAAU,GAAG,CAAC,IAAI,gBAAgB,CAAC,YAAY,EAAE,WAAW,CAAC,IAAI,SAAS,EAAE,CAAC;gBAC/E,UAAU,EAAE,CAAC;YACf,CAAC;QACH,CAAC;IACH,CAAC;CACF;AAED;;;GAGG;AACH,SAAS,mBAAmB,CAC1B,KAAgB,EAChB,aAAqB,EACrB,WAAqC;IAErC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAElC,8EAA8E;IAC9E,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,WAAW,GAAG,aAAa,CAAC;IAEhC,KAAK,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3C,MAAM,IAAI,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,CAAC;QACpC,MAAM,MAAM,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;QACjC,IAAI,MAAM,GAAG,WAAW,EAAE,CAAC;YACzB,sCAAsC;YACtC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAChD,MAAM,KAAK,GAAa,EAAE,CAAC;YAC3B,IAAI,WAAW,GAAG,CAAC,CAAC;YACpB,KAAK,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,WAAW,GAAG,WAAW,EAAE,CAAC,EAAE,EAAE,CAAC;gBACxE,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,CAAC;gBACzB,WAAW,EAAE,CAAC;YAChB,CAAC;YACD,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;YAC/B,MAAM;QACR,CAAC;QACD,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACpB,WAAW,IAAI,MAAM,CAAC;IACxB,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AAC5B,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAgB,EAAE,WAAqC;IAC/E,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,WAAW,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AACxE,CAAC;AAED,SAAS,eAAe,CACtB,KAAgB,EAChB,WAAqB,EACrB,OAAwB,EACxB,SAAiB,EACjB,UAAkB,EAClB,WAAqC;IAErC,MAAM,MAAM,GAAoB,EAAE,CAAC;IACnC,IAAI,MAAM,GAAc,EAAE,CAAC;IAC3B,IAAI,MAAM,GAAG,UAAU,CAAC;IAExB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,UAAU,GAAG,WAAW,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC;QAClD,MAAM,YAAY,GAAG,gBAAgB,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;QAE3D,IAAI,YAAY,GAAG,UAAU,GAAG,SAAS,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC/D,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,EAAE,WAAW,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,GAAG,YAAY,CAAC,CAAC;YACvF,IAAI,KAAK;gBAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC9B,MAAM,IAAI,YAAY,CAAC;YACvB,MAAM,GAAG,EAAE,CAAC;QACd,CAAC;QAED,IAAI,UAAU,GAAG,SAAS,IAAI,IAAI,CAAC,IAAI,KAAK,WAAW,IAAI,UAAU,IAAI,IAAI,EAAE,CAAC;YAC9E,MAAM,SAAS,GAAG,cAAc,CAAC,IAAe,EAAE,WAAW,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,WAAW,CAAC,CAAC;YACxG,MAAM,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;YAC1B,MAAM,IAAI,UAAU,CAAC;QACvB,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpB,CAAC;IACH,CAAC;IAED,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtB,MAAM,YAAY,GAAG,gBAAgB,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;QAC3D,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,EAAE,WAAW,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,GAAG,YAAY,CAAC,CAAC;QACvF,IAAI,KAAK;YAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAChC,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,cAAc,CACrB,IAAa,EACb,WAAqB,EACrB,OAAwB,EACxB,SAAiB,EACjB,UAAkB,EAClB,WAAqC;IAErC,MAAM,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;IAC/B,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAC9D,MAAM,MAAM,GAAoB,EAAE,CAAC;IACnC,IAAI,MAAM,GAAG,UAAU,CAAC;IACxB,IAAI,MAAM,GAAG,EAAE,CAAC;IAChB,IAAI,YAAY,GAAG,CAAC,CAAC;IAErB,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,MAAM,cAAc,GAAG,WAAW,CAAC,QAAQ,CAAC,CAAC;QAC7C,IAAI,YAAY,GAAG,cAAc,GAAG,SAAS,IAAI,MAAM,EAAE,CAAC;YACxD,MAAM,CAAC,IAAI,CAAC;gBACV,EAAE,EAAE,MAAM,CAAC,UAAU,EAAE;gBACvB,OAAO,EAAE,MAAM;gBACf,QAAQ,EAAE;oBACR,OAAO,EAAE,MAAM,CAAC,UAAU,EAAE;oBAC5B,WAAW,EAAE,CAAC,GAAG,WAAW,CAAC;oBAC7B,UAAU,EAAE,YAAY;oBACxB,UAAU,EAAE,OAAO,CAAC,UAAU;oBAC9B,UAAU,EAAE,MAAM;oBAClB,QAAQ,EAAE,MAAM,GAAG,YAAY;iBAChC;gBACD,GAAG,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,EAAE;aACxE,CAAC,CAAC;YACH,MAAM,IAAI,YAAY,CAAC;YACvB,MAAM,GAAG,EAAE,CAAC;YACZ,YAAY,GAAG,CAAC,CAAC;QACnB,CAAC;QACD,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,IAAI,QAAQ,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC;QACrD,YAAY,IAAI,cAAc,CAAC;IACjC,CAAC;IAED,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,CAAC,IAAI,CAAC;YACV,EAAE,EAAE,MAAM,CAAC,UAAU,EAAE;YACvB,OAAO,EAAE,MAAM;YACf,QAAQ,EAAE;gBACR,OAAO,EAAE,MAAM,CAAC,UAAU,EAAE;gBAC5B,WAAW,EAAE,CAAC,GAAG,WAAW,CAAC;gBAC7B,UAAU,EAAE,YAAY;gBACxB,UAAU,EAAE,OAAO,CAAC,UAAU;gBAC9B,UAAU,EAAE,MAAM;gBAClB,QAAQ,EAAE,MAAM,GAAG,YAAY;aAChC;YACD,GAAG,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,EAAE;SACxE,CAAC,CAAC;IACL,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/strategies/page-strategy.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"page-strategy.d.ts","sourceRoot":"","sources":["../../src/strategies/page-strategy.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,OAAO,EACP,aAAa,EACb,eAAe,EAEhB,MAAM,sBAAsB,CAAC;AAM9B,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAKtD,iDAAiD;AACjD,qBAAa,oBAAqB,YAAW,gBAAgB;IAC3D,IAAI,SAAU;IAEd,KAAK,CAAC,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,eAAe,GAAG,aAAa,EAAE;CAsE/D"}
+{"version":3,"file":"page-strategy.d.ts","sourceRoot":"","sources":["../../src/strategies/page-strategy.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,aAAa,EAAE,eAAe,EAAiB,MAAM,sBAAsB,CAAC;AAEnG,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAKtD,iDAAiD;AACjD,qBAAa,oBAAqB,YAAW,gBAAgB;IAC3D,IAAI,SAAU;IAEd,KAAK,CAAC,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,eAAe,GAAG,aAAa,EAAE;CA+D/D"}

package/dist/strategies/page-strategy.js

@@ -1,4 +1,4 @@
-import { walkAst, isNodeOfType, countTokens, } from "@markitdownjs/shared";
+import { walkAst, isNodeOfType, countTokens } from "@markitdownjs/shared";
 import { createChunk } from "../chunker.js";
 const DEFAULT_MAX_TOKENS = 512;
 /** Splits a document by page break boundaries */

package/dist/strategies/page-strategy.js.map

@@ -1 +1 @@
-{"version":3,"file":"page-strategy.js","sourceRoot":"","sources":["../../src/strategies/page-strategy.ts"],"names":[],"mappings":"AAMA,OAAO,EACL,OAAO,EACP,YAAY,EACZ,WAAW,GACZ,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAE5C,MAAM,kBAAkB,GAAG,GAAG,CAAC;AAE/B,iDAAiD;AACjD,MAAM,OAAO,oBAAoB;IAC/B,IAAI,GAAG,MAAM,CAAC;IAEd,KAAK,CAAC,GAAY,EAAE,OAAwB;QAC1C,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,kBAAkB,CAAC;QAC1D,MAAM,MAAM,GAAoB,EAAE,CAAC;QACnC,IAAI,YAAY,GAAc,EAAE,CAAC;QACjC,IAAI,WAA+B,CAAC;QACpC,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,IAAI,aAAa,GAAG,KAAK,CAAC;QAE1B,MAAM,UAAU,GAAG,CAAC,IAAa,EAAQ,EAAE;YACzC,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO;YACtC,MAAM,KAAK,GAAG,WAAW,CACvB,CAAC,GAAG,YAAY,CAAC,EACjB,EAAE,EACF,OAAO,EACP,UAAU,EACV,YAAY,EACZ,IAAI,CACL,CAAC;YACF,IAAI,KAAK,EAAE,CAAC;gBACV,IAAI,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,SAAS,EAAE,CAAC;oBACvC,MAAM,KAAK,GAAG,eAAe,CAAC,YAAY,EAAE,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,IAAI,CAAC,CAAC;oBAClF,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC;gBACxB,CAAC;qBAAM,CAAC;oBACN,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACrB,CAAC;YACH,CAAC;YACD,YAAY,GAAG,EAAE,CAAC;YAClB,UAAU,GAAG,YAAY,CAAC;QAC5B,CAAC,CAAC;QAEF,OAAO,CAAC,GAAG,EAAE,CAAC,IAAI,EAAE,OAAO,EAAE,EAAE;YAC7B,IAAI,YAAY,CAAgB,IAAI,EAAE,YAAY,CAAC,EAAE,CAAC;gBACpD,aAAa,GAAG,IAAI,CAAC;gBACrB,UAAU,CAAC,WAAW,CAAC,CAAC;gBACxB,WAAW,GAAG,IAAI,CAAC,UAAU,CAAC;gBAC9B,OAAO;YACT,CAAC;YAED,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACxB,MAAM,MAAM,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;YACjC,YAAY,IAAI,MAAM,CAAC;YAEvB,IAAI,MAAM,GAAG,CAAC,IAAI,gBAAgB,CAAC,YAAY,CAAC,IAAI,SAAS,EAAE,CAAC;gBAC9D,UAAU,CAAC,WAAW,CAAC,CAAC;YAC1B,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,UAAU,CAAC,WAAW,CAAC,CAAC;QAExB,IAAI,CAAC,aAAa,EAAE,CAAC;YACnB,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO,EAAE,CAAC;YACnC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACxB,MAAM,MAAM,GAAG,MAAM,CAAC,CAAC,CAAE,CAAC;gBAC1B,MAAM,WAAW,GAAG,WAAW,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBAC5C,IAAI,WAAW,GAAG,SAAS,EAAE,CAAC;oBAC5B,OAAO,eAAe,CACpB,CAAC,MAAM,CAAC,GAAG,CAAC,EACZ,OAAO,EACP,SAAS,EACT,MAAM,CAAC,QAAQ,CAAC,UAAU,EAC1B,MAAM,CAAC,QAAQ,CAAC,IAAI,CACrB,CAAC;gBACJ,CAAC;gBACD,OAAO,MAAM,CAAC;YAChB,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;CACF;AAED,SAAS,gBAAgB,CAAC,KAAgB;IACxC,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,WAAW,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC;AAED,SAAS,eAAe,CACtB,KAAgB,EAChB,OAAwB,EACxB,SAAiB,EACjB,UAAkB,EAClB,IAAa;IAEb,MAAM,MAAM,GAAoB,EAAE,CAAC;IACnC,IAAI,MAAM,GAAc,EAAE,CAAC;IAC3B,IAAI,MAAM,GAAG,UAAU,CAAC;IAExB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,UAAU,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;QACrC,MAAM,YAAY,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC;QAE9C,IAAI,YAAY,GAAG,UAAU,GAAG,SAAS,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC/D,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,EAAE,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,GAAG,YAAY,EAAE,IAAI,CAAC,CAAC;YACpF,IAAI,KAAK;gBAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC9B,MAAM,IAAI,YAAY,CAAC;YACvB,MAAM,GAAG,EAAE,CAAC;QACd,CAAC;QAED,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACpB,CAAC;IAED,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtB,MAAM,YAAY,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC;QAC9C,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,EAAE,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,GAAG,YAAY,EAAE,IAAI,CAAC,CAAC;QACpF,IAAI,KAAK;YAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAChC,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}
+{"version":3,"file":"page-strategy.js","sourceRoot":"","sources":["../../src/strategies/page-strategy.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AAE1E,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAE5C,MAAM,kBAAkB,GAAG,GAAG,CAAC;AAE/B,iDAAiD;AACjD,MAAM,OAAO,oBAAoB;IAC/B,IAAI,GAAG,MAAM,CAAC;IAEd,KAAK,CAAC,GAAY,EAAE,OAAwB;QAC1C,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,kBAAkB,CAAC;QAC1D,MAAM,MAAM,GAAoB,EAAE,CAAC;QACnC,IAAI,YAAY,GAAc,EAAE,CAAC;QACjC,IAAI,WAA+B,CAAC;QACpC,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,IAAI,aAAa,GAAG,KAAK,CAAC;QAE1B,MAAM,UAAU,GAAG,CAAC,IAAa,EAAQ,EAAE;YACzC,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO;YACtC,MAAM,KAAK,GAAG,WAAW,CAAC,CAAC,GAAG,YAAY,CAAC,EAAE,EAAE,EAAE,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,IAAI,CAAC,CAAC;YAC1F,IAAI,KAAK,EAAE,CAAC;gBACV,IAAI,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,SAAS,EAAE,CAAC;oBACvC,MAAM,KAAK,GAAG,eAAe,CAAC,YAAY,EAAE,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,IAAI,CAAC,CAAC;oBAClF,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC;gBACxB,CAAC;qBAAM,CAAC;oBACN,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACrB,CAAC;YACH,CAAC;YACD,YAAY,GAAG,EAAE,CAAC;YAClB,UAAU,GAAG,YAAY,CAAC;QAC5B,CAAC,CAAC;QAEF,OAAO,CAAC,GAAG,EAAE,CAAC,IAAI,EAAE,OAAO,EAAE,EAAE;YAC7B,IAAI,YAAY,CAAgB,IAAI,EAAE,YAAY,CAAC,EAAE,CAAC;gBACpD,aAAa,GAAG,IAAI,CAAC;gBACrB,UAAU,CAAC,WAAW,CAAC,CAAC;gBACxB,WAAW,GAAG,IAAI,CAAC,UAAU,CAAC;gBAC9B,OAAO;YACT,CAAC;YAED,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACxB,MAAM,MAAM,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;YACjC,YAAY,IAAI,MAAM,CAAC;YAEvB,IAAI,MAAM,GAAG,CAAC,IAAI,gBAAgB,CAAC,YAAY,CAAC,IAAI,SAAS,EAAE,CAAC;gBAC9D,UAAU,CAAC,WAAW,CAAC,CAAC;YAC1B,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,UAAU,CAAC,WAAW,CAAC,CAAC;QAExB,IAAI,CAAC,aAAa,EAAE,CAAC;YACnB,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO,EAAE,CAAC;YACnC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACxB,MAAM,MAAM,GAAG,MAAM,CAAC,CAAC,CAAE,CAAC;gBAC1B,MAAM,WAAW,GAAG,WAAW,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBAC5C,IAAI,WAAW,GAAG,SAAS,EAAE,CAAC;oBAC5B,OAAO,eAAe,CACpB,CAAC,MAAM,CAAC,GAAG,CAAC,EACZ,OAAO,EACP,SAAS,EACT,MAAM,CAAC,QAAQ,CAAC,UAAU,EAC1B,MAAM,CAAC,QAAQ,CAAC,IAAI,CACrB,CAAC;gBACJ,CAAC;gBACD,OAAO,MAAM,CAAC;YAChB,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;CACF;AAED,SAAS,gBAAgB,CAAC,KAAgB;IACxC,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,WAAW,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC;AAED,SAAS,eAAe,CACtB,KAAgB,EAChB,OAAwB,EACxB,SAAiB,EACjB,UAAkB,EAClB,IAAa;IAEb,MAAM,MAAM,GAAoB,EAAE,CAAC;IACnC,IAAI,MAAM,GAAc,EAAE,CAAC;IAC3B,IAAI,MAAM,GAAG,UAAU,CAAC;IAExB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,UAAU,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;QACrC,MAAM,YAAY,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC;QAE9C,IAAI,YAAY,GAAG,UAAU,GAAG,SAAS,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC/D,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,EAAE,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,GAAG,YAAY,EAAE,IAAI,CAAC,CAAC;YACpF,IAAI,KAAK;gBAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC9B,MAAM,IAAI,YAAY,CAAC;YACvB,MAAM,GAAG,EAAE,CAAC;QACd,CAAC;QAED,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACpB,CAAC;IAED,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtB,MAAM,YAAY,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC;QAC9C,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,EAAE,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,GAAG,YAAY,EAAE,IAAI,CAAC,CAAC;QACpF,IAAI,KAAK;YAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAChC,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/strategies/semantic-strategy.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"semantic-strategy.d.ts","sourceRoot":"","sources":["../../src/strategies/semantic-strategy.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,OAAO,EACP,aAAa,EACb,eAAe,EAGhB,MAAM,sBAAsB,CAAC;AAO9B,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAMtD,yEAAyE;AACzE,qBAAa,wBAAyB,YAAW,gBAAgB;IAC/D,IAAI,SAAc;IAElB,KAAK,CAAC,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,eAAe,GAAG,aAAa,EAAE;CA2D/D"}
+{"version":3,"file":"semantic-strategy.d.ts","sourceRoot":"","sources":["../../src/strategies/semantic-strategy.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,OAAO,EACP,aAAa,EACb,eAAe,EAGhB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAMtD,yEAAyE;AACzE,qBAAa,wBAAyB,YAAW,gBAAgB;IAC/D,IAAI,SAAc;IAElB,KAAK,CAAC,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,eAAe,GAAG,aAAa,EAAE;CA2D/D"}

package/dist/strategies/semantic-strategy.js

@@ -1,4 +1,4 @@
-import { walkAst, getNodeText, isNodeOfType, countTokens, } from "@markitdownjs/shared";
+import { walkAst, getNodeText, isNodeOfType, countTokens } from "@markitdownjs/shared";
 import { createChunk } from "../chunker.js";
 const DEFAULT_MAX_TOKENS = 512;
 const DEFAULT_OVERLAP = 50;
@@ -70,8 +70,7 @@ function groupBySemanticBoundary(nodes) {
             }
             currentGroup.push(node);
         }
-        else if (node.type === "thematic-break" ||
-            node.type === "horizontal-rule") {
+        else if (node.type === "thematic-break" || node.type === "horizontal-rule") {
             if (currentGroup.length > 0) {
                 groups.push(currentGroup);
                 currentGroup = [];

package/dist/strategies/semantic-strategy.js.map

@@ -1 +1 @@
-{"version":3,"file":"semantic-strategy.js","sourceRoot":"","sources":["../../src/strategies/semantic-strategy.ts"],"names":[],"mappings":"AAOA,OAAO,EACL,OAAO,EACP,WAAW,EACX,YAAY,EACZ,WAAW,GACZ,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAE5C,MAAM,kBAAkB,GAAG,GAAG,CAAC;AAC/B,MAAM,eAAe,GAAG,EAAE,CAAC;AAE3B,yEAAyE;AACzE,MAAM,OAAO,wBAAwB;IACnC,IAAI,GAAG,UAAU,CAAC;IAElB,KAAK,CAAC,GAAY,EAAE,OAAwB;QAC1C,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,kBAAkB,CAAC;QAC1D,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,eAAe,CAAC;QACnD,MAAM,KAAK,GAAc,EAAE,CAAC;QAE5B,OAAO,CAAC,GAAG,EAAE,CAAC,IAAI,EAAE,OAAO,EAAE,EAAE;YAC7B,IACE,IAAI,CAAC,IAAI,KAAK,WAAW;gBACzB,IAAI,CAAC,IAAI,KAAK,SAAS;gBACvB,IAAI,CAAC,IAAI,KAAK,MAAM;gBACpB,IAAI,CAAC,IAAI,KAAK,OAAO;gBACrB,IAAI,CAAC,IAAI,KAAK,MAAM;gBACpB,IAAI,CAAC,IAAI,KAAK,YAAY;gBAC1B,IAAI,CAAC,IAAI,KAAK,gBAAgB;gBAC9B,IAAI,CAAC,IAAI,KAAK,iBAAiB,EAC/B,CAAC;gBACD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACnB,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAElC,MAAM,cAAc,GAAG,uBAAuB,CAAC,KAAK,CAAC,CAAC;QACtD,MAAM,MAAM,GAAoB,EAAE,CAAC;QACnC,IAAI,WAAW,GAAG,EAAE,CAAC;QAErB,KAAK,MAAM,KAAK,IAAI,cAAc,EAAE,CAAC;YACnC,MAAM,WAAW,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,WAAW,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YAEtE,IAAI,WAAW,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;gBAC/B,MAAM,YAAY,GAAG,WAAW,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC;gBAC9E,MAAM,YAAY,GAAG,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAC5C,IAAI,YAAY,EAAE,CAAC;oBACjB,KAAK,CAAC,OAAO,CAAC;wBACZ,IAAI,EAAE,WAAW;wBACjB,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,YAAY,EAAE,CAAC;qBACjC,CAAC,CAAC;gBACtB,CAAC;YACH,CAAC;YAED,IAAI,WAAW,IAAI,SAAS,EAAE,CAAC;gBAC7B,MAAM,KAAK,GAAG,WAAW,CAAC,KAAK,EAAE,kBAAkB,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,WAAW,CAAC,CAAC;gBACrF,IAAI,KAAK;oBAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAC9B,MAAM,QAAQ,GAAG,WAAW,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAE,CAAC,CAAC;gBACvD,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;gBACpD,WAAW,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAChD,CAAC;iBAAM,CAAC;gBACN,MAAM,SAAS,GAAG,kBAAkB,CAAC,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;gBACzE,MAAM,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;gBAC1B,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACzB,MAAM,SAAS,GAAG,SAAS,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAE,CAAC;oBACnD,MAAM,KAAK,GAAG,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;oBAC7D,WAAW,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAChD,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;CACF;AAED,SAAS,uBAAuB,CAAC,KAAgB;IAC/C,MAAM,MAAM,GAAgB,EAAE,CAAC;IAC/B,IAAI,YAAY,GAAc,EAAE,CAAC;IAEjC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,YAAY,CAAc,IAAI,EAAE,SAAS,CAAC,EAAE,CAAC;YAC/C,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC5B,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;gBAC1B,YAAY,GAAG,EAAE,CAAC;YACpB,CAAC;YACD,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1B,CAAC;aAAM,IACL,IAAI,CAAC,IAAI,KAAK,gBAAgB;YAC9B,IAAI,CAAC,IAAI,KAAK,iBAAiB,EAC/B,CAAC;YACD,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC5B,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;gBAC1B,YAAY,GAAG,EAAE,CAAC;YACpB,CAAC;QACH,CAAC;aAAM,CAAC;YACN,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5B,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IAC5B,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAgB;IAC1C,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,YAAY,CAAc,IAAI,EAAE,SAAS,CAAC,EAAE,CAAC;YAC/C,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC;YAC7B,IAAI,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;QAC3C,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,kBAAkB,CACzB,KAAgB,EAChB,SAAiB,EACjB,OAAe,EACf,OAAwB;IAExB,MAAM,MAAM,GAAoB,EAAE,CAAC;IACnC,IAAI,MAAM,GAAc,EAAE,CAAC;IAC3B,IAAI,YAAY,GAAG,CAAC,CAAC;IAErB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,UAAU,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;QAErC,IAAI,YAAY,GAAG,UAAU,GAAG,SAAS,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC/D,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,EAAE,kBAAkB,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,YAAY,CAAC,CAAC;YACxF,IAAI,KAAK;gBAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAE9B,MAAM,QAAQ,GAAG,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAE,CAAC,CAAC;YACzD,MAAM,YAAY,GAAG,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC;YAC3E,MAAM,YAAY,GAAG,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAE5C,MAAM,GAAG,EAAE,CAAC;YACZ,YAAY,GAAG,CAAC,CAAC;YAEjB,IAAI,YAAY,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;gBAChC,MAAM,CAAC,IAAI,CAAC;oBACV,IAAI,EAAE,WAAW;oBACjB,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,YAAY,EAAE,CAAC;iBACjC,CAAC,CAAC;gBACpB,YAAY,GAAG,OAAO,CAAC;YACzB,CAAC;QACH,CAAC;QAED,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAClB,YAAY,IAAI,UAAU,CAAC;IAC7B,CAAC;IAED,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtB,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,EAAE,kBAAkB,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,YAAY,CAAC,CAAC;QACxF,IAAI,KAAK;YAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAChC,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}
+{"version":3,"file":"semantic-strategy.js","sourceRoot":"","sources":["../../src/strategies/semantic-strategy.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AAEvF,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAE5C,MAAM,kBAAkB,GAAG,GAAG,CAAC;AAC/B,MAAM,eAAe,GAAG,EAAE,CAAC;AAE3B,yEAAyE;AACzE,MAAM,OAAO,wBAAwB;IACnC,IAAI,GAAG,UAAU,CAAC;IAElB,KAAK,CAAC,GAAY,EAAE,OAAwB;QAC1C,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,kBAAkB,CAAC;QAC1D,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,eAAe,CAAC;QACnD,MAAM,KAAK,GAAc,EAAE,CAAC;QAE5B,OAAO,CAAC,GAAG,EAAE,CAAC,IAAI,EAAE,OAAO,EAAE,EAAE;YAC7B,IACE,IAAI,CAAC,IAAI,KAAK,WAAW;gBACzB,IAAI,CAAC,IAAI,KAAK,SAAS;gBACvB,IAAI,CAAC,IAAI,KAAK,MAAM;gBACpB,IAAI,CAAC,IAAI,KAAK,OAAO;gBACrB,IAAI,CAAC,IAAI,KAAK,MAAM;gBACpB,IAAI,CAAC,IAAI,KAAK,YAAY;gBAC1B,IAAI,CAAC,IAAI,KAAK,gBAAgB;gBAC9B,IAAI,CAAC,IAAI,KAAK,iBAAiB,EAC/B,CAAC;gBACD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACnB,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAElC,MAAM,cAAc,GAAG,uBAAuB,CAAC,KAAK,CAAC,CAAC;QACtD,MAAM,MAAM,GAAoB,EAAE,CAAC;QACnC,IAAI,WAAW,GAAG,EAAE,CAAC;QAErB,KAAK,MAAM,KAAK,IAAI,cAAc,EAAE,CAAC;YACnC,MAAM,WAAW,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,WAAW,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YAEtE,IAAI,WAAW,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;gBAC/B,MAAM,YAAY,GAAG,WAAW,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC;gBAC9E,MAAM,YAAY,GAAG,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAC5C,IAAI,YAAY,EAAE,CAAC;oBACjB,KAAK,CAAC,OAAO,CAAC;wBACZ,IAAI,EAAE,WAAW;wBACjB,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,YAAY,EAAE,CAAC;qBACjC,CAAC,CAAC;gBACtB,CAAC;YACH,CAAC;YAED,IAAI,WAAW,IAAI,SAAS,EAAE,CAAC;gBAC7B,MAAM,KAAK,GAAG,WAAW,CAAC,KAAK,EAAE,kBAAkB,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,WAAW,CAAC,CAAC;gBACrF,IAAI,KAAK;oBAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAC9B,MAAM,QAAQ,GAAG,WAAW,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAE,CAAC,CAAC;gBACvD,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;gBACpD,WAAW,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAChD,CAAC;iBAAM,CAAC;gBACN,MAAM,SAAS,GAAG,kBAAkB,CAAC,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;gBACzE,MAAM,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;gBAC1B,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACzB,MAAM,SAAS,GAAG,SAAS,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAE,CAAC;oBACnD,MAAM,KAAK,GAAG,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;oBAC7D,WAAW,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAChD,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;CACF;AAED,SAAS,uBAAuB,CAAC,KAAgB;IAC/C,MAAM,MAAM,GAAgB,EAAE,CAAC;IAC/B,IAAI,YAAY,GAAc,EAAE,CAAC;IAEjC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,YAAY,CAAc,IAAI,EAAE,SAAS,CAAC,EAAE,CAAC;YAC/C,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC5B,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;gBAC1B,YAAY,GAAG,EAAE,CAAC;YACpB,CAAC;YACD,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1B,CAAC;aAAM,IAAI,IAAI,CAAC,IAAI,KAAK,gBAAgB,IAAI,IAAI,CAAC,IAAI,KAAK,iBAAiB,EAAE,CAAC;YAC7E,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC5B,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;gBAC1B,YAAY,GAAG,EAAE,CAAC;YACpB,CAAC;QACH,CAAC;aAAM,CAAC;YACN,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5B,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IAC5B,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAgB;IAC1C,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,YAAY,CAAc,IAAI,EAAE,SAAS,CAAC,EAAE,CAAC;YAC/C,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC;YAC7B,IAAI,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;QAC3C,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,kBAAkB,CACzB,KAAgB,EAChB,SAAiB,EACjB,OAAe,EACf,OAAwB;IAExB,MAAM,MAAM,GAAoB,EAAE,CAAC;IACnC,IAAI,MAAM,GAAc,EAAE,CAAC;IAC3B,IAAI,YAAY,GAAG,CAAC,CAAC;IAErB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,UAAU,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;QAErC,IAAI,YAAY,GAAG,UAAU,GAAG,SAAS,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC/D,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,EAAE,kBAAkB,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,YAAY,CAAC,CAAC;YACxF,IAAI,KAAK;gBAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAE9B,MAAM,QAAQ,GAAG,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAE,CAAC,CAAC;YACzD,MAAM,YAAY,GAAG,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC;YAC3E,MAAM,YAAY,GAAG,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAE5C,MAAM,GAAG,EAAE,CAAC;YACZ,YAAY,GAAG,CAAC,CAAC;YAEjB,IAAI,YAAY,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;gBAChC,MAAM,CAAC,IAAI,CAAC;oBACV,IAAI,EAAE,WAAW;oBACjB,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,YAAY,EAAE,CAAC;iBACjC,CAAC,CAAC;gBACpB,YAAY,GAAG,OAAO,CAAC;YACzB,CAAC;QACH,CAAC;QAED,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAClB,YAAY,IAAI,UAAU,CAAC;IAC7B,CAAC;IAED,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtB,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,EAAE,kBAAkB,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,YAAY,CAAC,CAAC;QACxF,IAAI,KAAK;YAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAChC,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/tokenizers/fast-bpe.d.ts

@@ -0,0 +1,14 @@
+import type { Tokenizer } from "./types.js";
+/**
+ * Fast BPE tokenizer — built-in fallback with ~5% error margin.
+ * Uses whitespace splitting + heuristic subword estimation.
+ * No external dependencies. Good enough for approximate chunk sizing.
+ */
+export declare class FastBPETokenizer implements Tokenizer {
+    readonly name = "fast-bpe";
+    readonly models: string[];
+    encode(text: string): number[];
+    decode(_tokens: number[]): string;
+    countTokens(text: string): number;
+}
+//# sourceMappingURL=fast-bpe.d.ts.map

package/dist/tokenizers/fast-bpe.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fast-bpe.d.ts","sourceRoot":"","sources":["../../src/tokenizers/fast-bpe.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAE5C;;;;GAIG;AACH,qBAAa,gBAAiB,YAAW,SAAS;IAChD,QAAQ,CAAC,IAAI,cAAc;IAC3B,QAAQ,CAAC,MAAM,WAA2B;IAE1C,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE;IAiB9B,MAAM,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,MAAM;IAKjC,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM;CAalC"}

package/dist/tokenizers/fast-bpe.js

@@ -0,0 +1,45 @@
+/**
+ * Fast BPE tokenizer — built-in fallback with ~5% error margin.
+ * Uses whitespace splitting + heuristic subword estimation.
+ * No external dependencies. Good enough for approximate chunk sizing.
+ */
+export class FastBPETokenizer {
+    name = "fast-bpe";
+    models = ["fast-bpe", "default"];
+    encode(text) {
+        // Approximate: split on whitespace and punctuation, each segment ≈ 1 token.
+        const tokens = [];
+        const regex = /[\w']+|[^\w\s]+|\s+/g;
+        let match;
+        while ((match = regex.exec(text)) !== null) {
+            const segment = match[0];
+            if (segment.trim() === "")
+                continue;
+            // Longer segments need multiple tokens (approx 4 chars per token for CJK, 1 for Latin).
+            const estimated = Math.max(1, Math.ceil(segment.length / 3.5));
+            for (let i = 0; i < estimated; i++) {
+                tokens.push(i);
+            }
+        }
+        return tokens;
+    }
+    decode(_tokens) {
+        // Approximate decode — not accurate for fast-bpe but sufficient for counting.
+        return "";
+    }
+    countTokens(text) {
+        if (!text || text.trim() === "")
+            return 0;
+        let count = 0;
+        const regex = /[\w']+|[^\w\s]+|\s+/g;
+        let match;
+        while ((match = regex.exec(text)) !== null) {
+            const segment = match[0];
+            if (segment.trim() === "")
+                continue;
+            count += Math.max(1, Math.ceil(segment.length / 3.5));
+        }
+        return count;
+    }
+}
+//# sourceMappingURL=fast-bpe.js.map

package/dist/tokenizers/fast-bpe.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fast-bpe.js","sourceRoot":"","sources":["../../src/tokenizers/fast-bpe.ts"],"names":[],"mappings":"AAEA;;;;GAIG;AACH,MAAM,OAAO,gBAAgB;IAClB,IAAI,GAAG,UAAU,CAAC;IAClB,MAAM,GAAG,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;IAE1C,MAAM,CAAC,IAAY;QACjB,4EAA4E;QAC5E,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,MAAM,KAAK,GAAG,sBAAsB,CAAC;QACrC,IAAI,KAAK,CAAC;QACV,OAAO,CAAC,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YAC3C,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACzB,IAAI,OAAO,CAAC,IAAI,EAAE,KAAK,EAAE;gBAAE,SAAS;YACpC,wFAAwF;YACxF,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC;YAC/D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,EAAE,CAAC,EAAE,EAAE,CAAC;gBACnC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACjB,CAAC;QACH,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,MAAM,CAAC,OAAiB;QACtB,8EAA8E;QAC9E,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,WAAW,CAAC,IAAY;QACtB,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE;YAAE,OAAO,CAAC,CAAC;QAE1C,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,MAAM,KAAK,GAAG,sBAAsB,CAAC;QACrC,IAAI,KAAK,CAAC;QACV,OAAO,CAAC,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YAC3C,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACzB,IAAI,OAAO,CAAC,IAAI,EAAE,KAAK,EAAE;gBAAE,SAAS;YACpC,KAAK,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC;QACxD,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;CACF"}

package/dist/tokenizers/index.d.ts

@@ -0,0 +1,5 @@
+export type { Tokenizer } from "./types.js";
+export { FastBPETokenizer } from "./fast-bpe.js";
+export { TikTokenizer, Cl100kTokenizer, O200kTokenizer } from "./tiktoken.js";
+export { TokenizerRegistry, getDefaultTokenizerRegistry } from "./registry.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/tokenizers/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/tokenizers/index.ts"],"names":[],"mappings":"AAAA,YAAY,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAC5C,OAAO,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AACjD,OAAO,EAAE,YAAY,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAC9E,OAAO,EAAE,iBAAiB,EAAE,2BAA2B,EAAE,MAAM,eAAe,CAAC"}

package/dist/tokenizers/index.js

@@ -0,0 +1,4 @@
+export { FastBPETokenizer } from "./fast-bpe.js";
+export { TikTokenizer, Cl100kTokenizer, O200kTokenizer } from "./tiktoken.js";
+export { TokenizerRegistry, getDefaultTokenizerRegistry } from "./registry.js";
+//# sourceMappingURL=index.js.map

package/dist/tokenizers/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/tokenizers/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AACjD,OAAO,EAAE,YAAY,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAC9E,OAAO,EAAE,iBAAiB,EAAE,2BAA2B,EAAE,MAAM,eAAe,CAAC"}

package/dist/tokenizers/registry.d.ts

@@ -0,0 +1,22 @@
+import type { Tokenizer } from "./types.js";
+/**
+ * Registry for looking up tokenizers by name or model.
+ * Ships with three built-in tokenizers; users can register custom ones.
+ */
+export declare class TokenizerRegistry {
+    private tokenizers;
+    private modelMap;
+    constructor();
+    /** Register a tokenizer. Overwrites if name already exists. */
+    register(tokenizer: Tokenizer): void;
+    /** Get a tokenizer by name (e.g., "cl100k_base", "o200k_base", "fast-bpe") */
+    getByName(name: string): Tokenizer | undefined;
+    /** Get a tokenizer by model name (e.g., "gpt-4", "gpt-4o", "claude-3-opus") */
+    getByModel(model: string): Tokenizer | undefined;
+    /** Get a tokenizer, trying name first then model. Falls back to fast-bpe. */
+    resolve(nameOrModel: string): Tokenizer;
+    /** List all registered tokenizer names */
+    list(): string[];
+}
+export declare function getDefaultTokenizerRegistry(): TokenizerRegistry;
+//# sourceMappingURL=registry.d.ts.map

package/dist/tokenizers/registry.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"registry.d.ts","sourceRoot":"","sources":["../../src/tokenizers/registry.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAK5C;;;GAGG;AACH,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,UAAU,CAAgC;IAClD,OAAO,CAAC,QAAQ,CAAgC;;IAahD,+DAA+D;IAC/D,QAAQ,CAAC,SAAS,EAAE,SAAS,GAAG,IAAI;IAOpC,8EAA8E;IAC9E,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,GAAG,SAAS;IAI9C,+EAA+E;IAC/E,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,GAAG,SAAS;IAIhD,6EAA6E;IAC7E,OAAO,CAAC,WAAW,EAAE,MAAM,GAAG,SAAS;IAQvC,0CAA0C;IAC1C,IAAI,IAAI,MAAM,EAAE;CAGjB;AAKD,wBAAgB,2BAA2B,IAAI,iBAAiB,CAK/D"}

package/dist/tokenizers/registry.js

@@ -0,0 +1,54 @@
+import { FastBPETokenizer } from "./fast-bpe.js";
+import { Cl100kTokenizer } from "./tiktoken.js";
+import { O200kTokenizer } from "./tiktoken.js";
+/**
+ * Registry for looking up tokenizers by name or model.
+ * Ships with three built-in tokenizers; users can register custom ones.
+ */
+export class TokenizerRegistry {
+    tokenizers = new Map();
+    modelMap = new Map();
+    constructor() {
+        // Register built-in tokenizers.
+        const fastBpe = new FastBPETokenizer();
+        const cl100k = new Cl100kTokenizer();
+        const o200k = new O200kTokenizer();
+        this.register(fastBpe);
+        this.register(cl100k);
+        this.register(o200k);
+    }
+    /** Register a tokenizer. Overwrites if name already exists. */
+    register(tokenizer) {
+        this.tokenizers.set(tokenizer.name, tokenizer);
+        for (const model of tokenizer.models) {
+            this.modelMap.set(model.toLowerCase(), tokenizer);
+        }
+    }
+    /** Get a tokenizer by name (e.g., "cl100k_base", "o200k_base", "fast-bpe") */
+    getByName(name) {
+        return this.tokenizers.get(name);
+    }
+    /** Get a tokenizer by model name (e.g., "gpt-4", "gpt-4o", "claude-3-opus") */
+    getByModel(model) {
+        return this.modelMap.get(model.toLowerCase());
+    }
+    /** Get a tokenizer, trying name first then model. Falls back to fast-bpe. */
+    resolve(nameOrModel) {
+        return (this.getByName(nameOrModel) ??
+            this.getByModel(nameOrModel) ??
+            this.tokenizers.get("fast-bpe"));
+    }
+    /** List all registered tokenizer names */
+    list() {
+        return Array.from(this.tokenizers.keys());
+    }
+}
+/** Default singleton registry */
+let defaultRegistry = null;
+export function getDefaultTokenizerRegistry() {
+    if (!defaultRegistry) {
+        defaultRegistry = new TokenizerRegistry();
+    }
+    return defaultRegistry;
+}
+//# sourceMappingURL=registry.js.map

package/dist/tokenizers/registry.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"registry.js","sourceRoot":"","sources":["../../src/tokenizers/registry.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AACjD,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAChD,OAAO,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAE/C;;;GAGG;AACH,MAAM,OAAO,iBAAiB;IACpB,UAAU,GAAG,IAAI,GAAG,EAAqB,CAAC;IAC1C,QAAQ,GAAG,IAAI,GAAG,EAAqB,CAAC;IAEhD;QACE,gCAAgC;QAChC,MAAM,OAAO,GAAG,IAAI,gBAAgB,EAAE,CAAC;QACvC,MAAM,MAAM,GAAG,IAAI,eAAe,EAAE,CAAC;QACrC,MAAM,KAAK,GAAG,IAAI,cAAc,EAAE,CAAC;QAEnC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;QACvB,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACtB,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IACvB,CAAC;IAED,+DAA+D;IAC/D,QAAQ,CAAC,SAAoB;QAC3B,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;QAC/C,KAAK,MAAM,KAAK,IAAI,SAAS,CAAC,MAAM,EAAE,CAAC;YACrC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,WAAW,EAAE,EAAE,SAAS,CAAC,CAAC;QACpD,CAAC;IACH,CAAC;IAED,8EAA8E;IAC9E,SAAS,CAAC,IAAY;QACpB,OAAO,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACnC,CAAC;IAED,+EAA+E;IAC/E,UAAU,CAAC,KAAa;QACtB,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC;IAChD,CAAC;IAED,6EAA6E;IAC7E,OAAO,CAAC,WAAmB;QACzB,OAAO,CACL,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC;YAC3B,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC;YAC5B,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,UAAU,CAAE,CACjC,CAAC;IACJ,CAAC;IAED,0CAA0C;IAC1C,IAAI;QACF,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC;IAC5C,CAAC;CACF;AAED,iCAAiC;AACjC,IAAI,eAAe,GAA6B,IAAI,CAAC;AAErD,MAAM,UAAU,2BAA2B;IACzC,IAAI,CAAC,eAAe,EAAE,CAAC;QACrB,eAAe,GAAG,IAAI,iBAAiB,EAAE,CAAC;IAC5C,CAAC;IACD,OAAO,eAAe,CAAC;AACzB,CAAC"}

package/dist/tokenizers/tiktoken.d.ts

@@ -0,0 +1,26 @@
+import type { Tokenizer } from "./types.js";
+/**
+ * Base class for tiktoken-based tokenizers.
+ * Uses the `tiktoken` npm package (optional dependency) for accurate tokenization.
+ * Falls back to character-based approximation when tiktoken is not installed.
+ */
+export declare class TikTokenizer implements Tokenizer {
+    readonly name: string;
+    readonly models: string[];
+    constructor(name: string, _encodingName: string, models: string[]);
+    encode(text: string): number[];
+    private _approximateEncode;
+    decode(_tokens: number[]): string;
+    countTokens(text: string): number;
+    /** Character-based approximate token count */
+    private approximateCount;
+}
+/** GPT-4, text-embedding-ada-002 tokenizer */
+export declare class Cl100kTokenizer extends TikTokenizer {
+    constructor();
+}
+/** GPT-4o, GPT-4o-mini tokenizer (newer encoding) */
+export declare class O200kTokenizer extends TikTokenizer {
+    constructor();
+}
+//# sourceMappingURL=tiktoken.d.ts.map

package/dist/tokenizers/tiktoken.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tiktoken.d.ts","sourceRoot":"","sources":["../../src/tokenizers/tiktoken.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAE5C;;;;GAIG;AACH,qBAAa,YAAa,YAAW,SAAS;IAC5C,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC;gBAEd,IAAI,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE;IAKjE,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE;IAK9B,OAAO,CAAC,kBAAkB;IAgB1B,MAAM,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,MAAM;IAIjC,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM;IAKjC,8CAA8C;IAC9C,OAAO,CAAC,gBAAgB;CAgBzB;AAED,8CAA8C;AAC9C,qBAAa,eAAgB,SAAQ,YAAY;;CAYhD;AAED,qDAAqD;AACrD,qBAAa,cAAe,SAAQ,YAAY;;CAS/C"}

package/dist/tokenizers/tiktoken.js

@@ -0,0 +1,87 @@
+/**
+ * Base class for tiktoken-based tokenizers.
+ * Uses the `tiktoken` npm package (optional dependency) for accurate tokenization.
+ * Falls back to character-based approximation when tiktoken is not installed.
+ */
+export class TikTokenizer {
+    name;
+    models;
+    constructor(name, _encodingName, models) {
+        this.name = name;
+        this.models = models;
+    }
+    encode(text) {
+        if (!text)
+            return [];
+        return this._approximateEncode(text);
+    }
+    _approximateEncode(text) {
+        const tokens = [];
+        const regex = /[\w']+|[^\w\s]+|\s+/g;
+        let match;
+        while ((match = regex.exec(text)) !== null) {
+            const segment = match[0];
+            if (segment.trim() === "")
+                continue;
+            const isCjk = /[\u4e00-\u9fff\u3400-\u4dbf]/.test(segment);
+            const estimated = isCjk ? segment.length : Math.max(1, Math.ceil(segment.length / 4));
+            for (let i = 0; i < estimated; i++) {
+                tokens.push(i);
+            }
+        }
+        return tokens;
+    }
+    decode(_tokens) {
+        return "";
+    }
+    countTokens(text) {
+        if (!text || text.trim() === "")
+            return 0;
+        return this.approximateCount(text);
+    }
+    /** Character-based approximate token count */
+    approximateCount(text) {
+        let count = 0;
+        const regex = /[\w']+|[^\w\s]+|\s+/g;
+        let match;
+        while ((match = regex.exec(text)) !== null) {
+            const segment = match[0];
+            if (segment.trim() === "")
+                continue;
+            const isCjk = /[\u4e00-\u9fff\u3400-\u4dbf]/.test(segment);
+            if (isCjk) {
+                count += segment.length;
+            }
+            else {
+                count += Math.max(1, Math.ceil(segment.length / 4));
+            }
+        }
+        return Math.max(1, count);
+    }
+}
+/** GPT-4, text-embedding-ada-002 tokenizer */
+export class Cl100kTokenizer extends TikTokenizer {
+    constructor() {
+        super("cl100k_base", "cl100k_base", [
+            "gpt-4",
+            "gpt-4-turbo",
+            "gpt-4o",
+            "gpt-4o-mini",
+            "text-embedding-ada-002",
+            "text-embedding-3-small",
+            "text-embedding-3-large",
+        ]);
+    }
+}
+/** GPT-4o, GPT-4o-mini tokenizer (newer encoding) */
+export class O200kTokenizer extends TikTokenizer {
+    constructor() {
+        super("o200k_base", "o200k_base", [
+            "gpt-4o",
+            "gpt-4o-mini",
+            "gpt-4.1",
+            "gpt-4.1-mini",
+        ]);
+    }
+}
+//# sourceMappingURL=tiktoken.js.map

package/dist/tokenizers/tiktoken.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tiktoken.js","sourceRoot":"","sources":["../../src/tokenizers/tiktoken.ts"],"names":[],"mappings":"AAEA;;;;GAIG;AACH,MAAM,OAAO,YAAY;IACd,IAAI,CAAS;IACb,MAAM,CAAW;IAE1B,YAAY,IAAY,EAAE,aAAqB,EAAE,MAAgB;QAC/D,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;IAED,MAAM,CAAC,IAAY;QACjB,IAAI,CAAC,IAAI;YAAE,OAAO,EAAE,CAAC;QACrB,OAAO,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC;IACvC,CAAC;IAEO,kBAAkB,CAAC,IAAY;QACrC,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,MAAM,KAAK,GAAG,sBAAsB,CAAC;QACrC,IAAI,KAAK,CAAC;QACV,OAAO,CAAC,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YAC3C,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACzB,IAAI,OAAO,CAAC,IAAI,EAAE,KAAK,EAAE;gBAAE,SAAS;YACpC,MAAM,KAAK,GAAG,8BAA8B,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAC3D,MAAM,SAAS,GAAG,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;YACtF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,EAAE,CAAC,EAAE,EAAE,CAAC;gBACnC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACjB,CAAC;QACH,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,MAAM,CAAC,OAAiB;QACtB,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,WAAW,CAAC,IAAY;QACtB,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE;YAAE,OAAO,CAAC,CAAC;QAC1C,OAAO,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;IACrC,CAAC;IAED,8CAA8C;IACtC,gBAAgB,CAAC,IAAY;QACnC,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,MAAM,KAAK,GAAG,sBAAsB,CAAC;QACrC,IAAI,KAAK,CAAC;QACV,OAAO,CAAC,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YAC3C,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACzB,IAAI,OAAO,CAAC,IAAI,EAAE,KAAK,EAAE;gBAAE,SAAS;YACpC,MAAM,KAAK,GAAG,8BAA8B,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAC3D,IAAI,KAAK,EAAE,CAAC;gBACV,KAAK,IAAI,OAAO,CAAC,MAAM,CAAC;YAC1B,CAAC;iBAAM,CAAC;gBACN,KAAK,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;YACtD,CAAC;QACH,CAAC;QACD,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IAC5B,CAAC;CACF;AAED,8CAA8C;AAC9C,MAAM,OAAO,eAAgB,SAAQ,YAAY;IAC/C;QACE,KAAK,CAAC,aAAa,EAAE,aAAa,EAAE;YAClC,OAAO;YACP,aAAa;YACb,QAAQ;YACR,aAAa;YACb,wBAAwB;YACxB,wBAAwB;YACxB,wBAAwB;SACzB,CAAC,CAAC;IACL,CAAC;CACF;AAED,qDAAqD;AACrD,MAAM,OAAO,cAAe,SAAQ,YAAY;IAC9C;QACE,KAAK,CAAC,YAAY,EAAE,YAAY,EAAE;YAChC,QAAQ;YACR,aAAa;YACb,SAAS;YACT,cAAc;SACf,CAAC,CAAC;IACL,CAAC;CACF"}

package/dist/tokenizers/types.d.ts

@@ -0,0 +1,14 @@
+/** Tokenizer interface for accurate token counting across different LLM models */
+export interface Tokenizer {
+    /** Human-readable name (e.g., "cl100k_base", "o200k_base") */
+    readonly name: string;
+    /** Model families this tokenizer supports */
+    readonly models: string[];
+    /** Encode text into token IDs */
+    encode(text: string): number[];
+    /** Decode token IDs back to text */
+    decode(tokens: number[]): string;
+    /** Count tokens in text (fast path, no need to materialize IDs) */
+    countTokens(text: string): number;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/tokenizers/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/tokenizers/types.ts"],"names":[],"mappings":"AAAA,kFAAkF;AAClF,MAAM,WAAW,SAAS;IACxB,8DAA8D;IAC9D,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,6CAA6C;IAC7C,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC;IAC1B,iCAAiC;IACjC,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAC/B,oCAAoC;IACpC,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,CAAC;IACjC,mEAAmE;IACnE,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC;CACnC"}

package/dist/tokenizers/types.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=types.js.map

package/dist/tokenizers/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/tokenizers/types.ts"],"names":[],"mappings":""}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@markitdownjs/chunking",
-  "version": "0.1.1",
+  "version": "0.2.0",
   "description": "AI-ready document chunking strategies for MarkItDownJS",
   "license": "MIT",
   "type": "module",
@@ -17,7 +17,7 @@
     "dist"
   ],
   "dependencies": {
-    "@markitdownjs/shared": "0.1.1"
+    "@markitdownjs/shared": "0.3.0"
   },
   "devDependencies": {
     "typescript": "^5.5.0"