@markitdownjs/chunking 0.1.0

package/dist/__tests__/chunker.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=chunker.test.d.ts.map

package/dist/__tests__/chunker.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunker.test.d.ts","sourceRoot":"","sources":["../../src/__tests__/chunker.test.ts"],"names":[],"mappings":""}

package/dist/__tests__/chunker.test.js

@@ -0,0 +1,46 @@
+import { describe, it, expect } from 'vitest';
+import { DocumentChunker } from '../chunker.js';
+import { createNode } from '@markitdownjs/shared';
+describe('DocumentChunker', () => {
+    const chunker = new DocumentChunker();
+    const ast = createNode({
+        type: 'document',
+        children: [
+            createNode({
+                type: 'heading',
+                level: 1,
+                children: [createNode({ type: 'text', value: 'Introduction' })],
+            }),
+            createNode({
+                type: 'paragraph',
+                children: [createNode({ type: 'text', value: 'This is the introduction paragraph.' })],
+            }),
+            createNode({
+                type: 'heading',
+                level: 1,
+                children: [createNode({ type: 'text', value: 'Main Content' })],
+            }),
+            createNode({
+                type: 'paragraph',
+                children: [createNode({ type: 'text', value: 'This is the main content.' })],
+            }),
+        ],
+    });
+    it('should chunk by heading', () => {
+        const chunks = chunker.chunk(ast, { strategy: 'heading' });
+        expect(chunks.length).toBeGreaterThanOrEqual(2);
+        expect(chunks[0].metadata.headingPath).toContain('Introduction');
+    });
+    it('should chunk by fixed size', () => {
+        const chunks = chunker.chunk(ast, { strategy: 'fixed', maxTokens: 5, overlap: 2 });
+        expect(chunks.length).toBeGreaterThanOrEqual(1);
+    });
+    it('should generate chunk IDs', () => {
+        const chunks = chunker.chunk(ast, { strategy: 'heading' });
+        for (const chunk of chunks) {
+            expect(chunk.id).toBeDefined();
+            expect(chunk.id.length).toBeGreaterThan(0);
+        }
+    });
+});
+//# sourceMappingURL=chunker.test.js.map

package/dist/__tests__/chunker.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunker.test.js","sourceRoot":"","sources":["../../src/__tests__/chunker.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAChD,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAGlD,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;IAC/B,MAAM,OAAO,GAAG,IAAI,eAAe,EAAE,CAAC;IAEtC,MAAM,GAAG,GAAG,UAAU,CAAe;QACnC,IAAI,EAAE,UAAU;QAChB,QAAQ,EAAE;YACR,UAAU,CAAc;gBACtB,IAAI,EAAE,SAAS;gBACf,KAAK,EAAE,CAAC;gBACR,QAAQ,EAAE,CAAC,UAAU,CAAW,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC,CAAC;aAC1E,CAAC;YACF,UAAU,CAAgB;gBACxB,IAAI,EAAE,WAAW;gBACjB,QAAQ,EAAE,CAAC,UAAU,CAAW,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,qCAAqC,EAAE,CAAC,CAAC;aACjG,CAAC;YACF,UAAU,CAAc;gBACtB,IAAI,EAAE,SAAS;gBACf,KAAK,EAAE,CAAC;gBACR,QAAQ,EAAE,CAAC,UAAU,CAAW,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC,CAAC;aAC1E,CAAC;YACF,UAAU,CAAgB;gBACxB,IAAI,EAAE,WAAW;gBACjB,QAAQ,EAAE,CAAC,UAAU,CAAW,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,2BAA2B,EAAE,CAAC,CAAC;aACvF,CAAC;SACH;KACF,CAAC,CAAC;IAEH,EAAE,CAAC,yBAAyB,EAAE,GAAG,EAAE;QACjC,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC,CAAC;QAC3D,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC;QAChD,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC;IACpE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4BAA4B,EAAE,GAAG,EAAE;QACpC,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC;QACnF,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC;IAClD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2BAA2B,EAAE,GAAG,EAAE;QACnC,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC,CAAC;QAC3D,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;YAC/B,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QAC7C,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/chunker.d.ts

@@ -0,0 +1,24 @@
+import type { AnyNode, DocumentChunk, ChunkingOptions } from "@markitdownjs/shared";
+/** Interface for chunking strategy implementations */
+export interface ChunkingStrategy {
+    /** Strategy name identifier */
+    name: string;
+    /** Split an AST into document chunks */
+    chunk(ast: AnyNode, options: ChunkingOptions): DocumentChunk[];
+}
+/** Creates a document chunk from collected nodes and metadata */
+export declare function createChunk(nodes: AnyNode[], headingPath: string[], options: ChunkingOptions, startIndex: number, endIndex: number, page?: number): DocumentChunk | null;
+/** Main document chunker that delegates to registered strategies */
+export declare class DocumentChunker {
+    private strategies;
+    constructor();
+    /** Register a custom chunking strategy */
+    registerStrategy(strategy: ChunkingStrategy): void;
+    /** Get a registered strategy by name */
+    getStrategy(name: string): ChunkingStrategy | undefined;
+    /** List all registered strategy names */
+    listStrategies(): string[];
+    /** Chunk an AST using the specified strategy */
+    chunk(ast: AnyNode, options: ChunkingOptions): DocumentChunk[];
+}
+//# sourceMappingURL=chunker.d.ts.map

package/dist/chunker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunker.d.ts","sourceRoot":"","sources":["../src/chunker.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,OAAO,EACP,aAAa,EAEb,eAAe,EAChB,MAAM,sBAAsB,CAAC;AAO9B,sDAAsD;AACtD,MAAM,WAAW,gBAAgB;IAC/B,+BAA+B;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,wCAAwC;IACxC,KAAK,CAAC,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,eAAe,GAAG,aAAa,EAAE,CAAC;CAChE;AAED,iEAAiE;AACjE,wBAAgB,WAAW,CACzB,KAAK,EAAE,OAAO,EAAE,EAChB,WAAW,EAAE,MAAM,EAAE,EACrB,OAAO,EAAE,eAAe,EACxB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,EAChB,IAAI,CAAC,EAAE,MAAM,GACZ,aAAa,GAAG,IAAI,CAsBtB;AAED,oEAAoE;AACpE,qBAAa,eAAe;IAC1B,OAAO,CAAC,UAAU,CAAgC;;IAUlD,0CAA0C;IAC1C,gBAAgB,CAAC,QAAQ,EAAE,gBAAgB,GAAG,IAAI;IAIlD,wCAAwC;IACxC,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,gBAAgB,GAAG,SAAS;IAIvD,yCAAyC;IACzC,cAAc,IAAI,MAAM,EAAE;IAI1B,gDAAgD;IAChD,KAAK,CAAC,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,eAAe,GAAG,aAAa,EAAE;CAQ/D"}

package/dist/chunker.js

@@ -0,0 +1,58 @@
+import { getNodeText, generateId } from "@markitdownjs/shared";
+import { HeadingChunkingStrategy } from "./strategies/heading-strategy.js";
+import { PageChunkingStrategy } from "./strategies/page-strategy.js";
+import { SemanticChunkingStrategy } from "./strategies/semantic-strategy.js";
+import { FixedChunkingStrategy } from "./strategies/fixed-strategy.js";
+/** Creates a document chunk from collected nodes and metadata */
+export function createChunk(nodes, headingPath, options, startIndex, endIndex, page) {
+    if (nodes.length === 0)
+        return null;
+    const content = nodes.map((n) => getNodeText(n)).join("\n\n");
+    const tokenCount = content.split(/\s+/).filter(Boolean).length;
+    const root = nodes.length === 1
+        ? nodes[0]
+        : { type: "document", children: nodes };
+    const metadata = {
+        chunkId: generateId(),
+        headingPath,
+        tokenCount,
+        sourceFile: options.sourceFile,
+        startIndex,
+        endIndex,
+        page,
+    };
+    return { id: metadata.chunkId, content, metadata, ast: root };
+}
+/** Main document chunker that delegates to registered strategies */
+export class DocumentChunker {
+    strategies;
+    constructor() {
+        this.strategies = new Map();
+        this.registerStrategy(new HeadingChunkingStrategy());
+        this.registerStrategy(new PageChunkingStrategy());
+        this.registerStrategy(new SemanticChunkingStrategy());
+        this.registerStrategy(new FixedChunkingStrategy());
+    }
+    /** Register a custom chunking strategy */
+    registerStrategy(strategy) {
+        this.strategies.set(strategy.name, strategy);
+    }
+    /** Get a registered strategy by name */
+    getStrategy(name) {
+        return this.strategies.get(name);
+    }
+    /** List all registered strategy names */
+    listStrategies() {
+        return Array.from(this.strategies.keys());
+    }
+    /** Chunk an AST using the specified strategy */
+    chunk(ast, options) {
+        const strategyName = options.strategy ?? "heading";
+        const strategy = this.strategies.get(strategyName);
+        if (!strategy) {
+            throw new Error(`Unknown chunking strategy: ${strategyName}`);
+        }
+        return strategy.chunk(ast, options);
+    }
+}
+//# sourceMappingURL=chunker.js.map

package/dist/chunker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunker.js","sourceRoot":"","sources":["../src/chunker.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAC/D,OAAO,EAAE,uBAAuB,EAAE,MAAM,kCAAkC,CAAC;AAC3E,OAAO,EAAE,oBAAoB,EAAE,MAAM,+BAA+B,CAAC;AACrE,OAAO,EAAE,wBAAwB,EAAE,MAAM,mCAAmC,CAAC;AAC7E,OAAO,EAAE,qBAAqB,EAAE,MAAM,gCAAgC,CAAC;AAUvE,iEAAiE;AACjE,MAAM,UAAU,WAAW,CACzB,KAAgB,EAChB,WAAqB,EACrB,OAAwB,EACxB,UAAkB,EAClB,QAAgB,EAChB,IAAa;IAEb,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAEpC,MAAM,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC9D,MAAM,UAAU,GAAG,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC;IAE/D,MAAM,IAAI,GACR,KAAK,CAAC,MAAM,KAAK,CAAC;QAChB,CAAC,CAAC,KAAK,CAAC,CAAC,CAAE;QACX,CAAC,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;IAE5C,MAAM,QAAQ,GAAkB;QAC9B,OAAO,EAAE,UAAU,EAAE;QACrB,WAAW;QACX,UAAU;QACV,UAAU,EAAE,OAAO,CAAC,UAAU;QAC9B,UAAU;QACV,QAAQ;QACR,IAAI;KACL,CAAC;IAEF,OAAO,EAAE,EAAE,EAAE,QAAQ,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC;AAChE,CAAC;AAED,oEAAoE;AACpE,MAAM,OAAO,eAAe;IAClB,UAAU,CAAgC;IAElD;QACE,IAAI,CAAC,UAAU,GAAG,IAAI,GAAG,EAAE,CAAC;QAC5B,IAAI,CAAC,gBAAgB,CAAC,IAAI,uBAAuB,EAAE,CAAC,CAAC;QACrD,IAAI,CAAC,gBAAgB,CAAC,IAAI,oBAAoB,EAAE,CAAC,CAAC;QAClD,IAAI,CAAC,gBAAgB,CAAC,IAAI,wBAAwB,EAAE,CAAC,CAAC;QACtD,IAAI,CAAC,gBAAgB,CAAC,IAAI,qBAAqB,EAAE,CAAC,CAAC;IACrD,CAAC;IAED,0CAA0C;IAC1C,gBAAgB,CAAC,QAA0B;QACzC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IAC/C,CAAC;IAED,wCAAwC;IACxC,WAAW,CAAC,IAAY;QACtB,OAAO,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACnC,CAAC;IAED,yCAAyC;IACzC,cAAc;QACZ,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC;IAC5C,CAAC;IAED,gDAAgD;IAChD,KAAK,CAAC,GAAY,EAAE,OAAwB;QAC1C,MAAM,YAAY,GAAG,OAAO,CAAC,QAAQ,IAAI,SAAS,CAAC;QACnD,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;QACnD,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,IAAI,KAAK,CAAC,8BAA8B,YAAY,EAAE,CAAC,CAAC;QAChE,CAAC;QACD,OAAO,QAAQ,CAAC,KAAK,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IACtC,CAAC;CACF"}

package/dist/index.d.ts

@@ -0,0 +1,7 @@
+export { DocumentChunker, createChunk } from "./chunker.js";
+export type { ChunkingStrategy } from "./chunker.js";
+export { HeadingChunkingStrategy } from "./strategies/heading-strategy.js";
+export { PageChunkingStrategy } from "./strategies/page-strategy.js";
+export { SemanticChunkingStrategy } from "./strategies/semantic-strategy.js";
+export { FixedChunkingStrategy } from "./strategies/fixed-strategy.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAC5D,YAAY,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,EAAE,uBAAuB,EAAE,MAAM,kCAAkC,CAAC;AAC3E,OAAO,EAAE,oBAAoB,EAAE,MAAM,+BAA+B,CAAC;AACrE,OAAO,EAAE,wBAAwB,EAAE,MAAM,mCAAmC,CAAC;AAC7E,OAAO,EAAE,qBAAqB,EAAE,MAAM,gCAAgC,CAAC"}

package/dist/index.js

@@ -0,0 +1,6 @@
+export { DocumentChunker, createChunk } from "./chunker.js";
+export { HeadingChunkingStrategy } from "./strategies/heading-strategy.js";
+export { PageChunkingStrategy } from "./strategies/page-strategy.js";
+export { SemanticChunkingStrategy } from "./strategies/semantic-strategy.js";
+export { FixedChunkingStrategy } from "./strategies/fixed-strategy.js";
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAE5D,OAAO,EAAE,uBAAuB,EAAE,MAAM,kCAAkC,CAAC;AAC3E,OAAO,EAAE,oBAAoB,EAAE,MAAM,+BAA+B,CAAC;AACrE,OAAO,EAAE,wBAAwB,EAAE,MAAM,mCAAmC,CAAC;AAC7E,OAAO,EAAE,qBAAqB,EAAE,MAAM,gCAAgC,CAAC"}

package/dist/strategies/fixed-strategy.d.ts

@@ -0,0 +1,8 @@
+import type { AnyNode, DocumentChunk, ChunkingOptions } from "@markitdownjs/shared";
+import type { ChunkingStrategy } from "../chunker.js";
+/** Splits a document into fixed-size chunks with overlap */
+export declare class FixedChunkingStrategy implements ChunkingStrategy {
+    name: string;
+    chunk(ast: AnyNode, options: ChunkingOptions): DocumentChunk[];
+}
+//# sourceMappingURL=fixed-strategy.d.ts.map

package/dist/strategies/fixed-strategy.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fixed-strategy.d.ts","sourceRoot":"","sources":["../../src/strategies/fixed-strategy.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,OAAO,EACP,aAAa,EACb,eAAe,EAGhB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAKtD,4DAA4D;AAC5D,qBAAa,qBAAsB,YAAW,gBAAgB;IAC5D,IAAI,SAAW;IAEf,KAAK,CAAC,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,eAAe,GAAG,aAAa,EAAE;CA+C/D"}

package/dist/strategies/fixed-strategy.js

@@ -0,0 +1,55 @@
+import { getNodeText } from "@markitdownjs/shared";
+const DEFAULT_MAX_TOKENS = 512;
+const DEFAULT_OVERLAP = 50;
+/** Splits a document into fixed-size chunks with overlap */
+export class FixedChunkingStrategy {
+    name = "fixed";
+    chunk(ast, options) {
+        const maxTokens = options.maxTokens ?? DEFAULT_MAX_TOKENS;
+        const overlap = options.overlap ?? DEFAULT_OVERLAP;
+        const text = getNodeText(ast);
+        const words = text.split(/\s+/).filter(Boolean);
+        if (words.length === 0)
+            return [];
+        const chunks = [];
+        let startIdx = 0;
+        let globalOffset = 0;
+        while (startIdx < words.length) {
+            const endIdx = Math.min(startIdx + maxTokens, words.length);
+            const chunkWords = words.slice(startIdx, endIdx);
+            const content = chunkWords.join(" ");
+            const tokenCount = chunkWords.length;
+            const astNode = buildChunkAst(content);
+            const chunk = {
+                id: crypto.randomUUID(),
+                content,
+                metadata: {
+                    chunkId: crypto.randomUUID(),
+                    headingPath: [],
+                    tokenCount,
+                    sourceFile: options.sourceFile,
+                    startIndex: globalOffset,
+                    endIndex: globalOffset + tokenCount,
+                },
+                ast: astNode,
+            };
+            chunks.push(chunk);
+            globalOffset += tokenCount;
+            startIdx = endIdx - overlap;
+            if (startIdx >= endIdx)
+                break;
+            if (startIdx >= words.length)
+                break;
+            if (endIdx >= words.length)
+                break;
+        }
+        return chunks;
+    }
+}
+function buildChunkAst(text) {
+    return {
+        type: "paragraph",
+        children: [{ type: "text", value: text }],
+    };
+}
+//# sourceMappingURL=fixed-strategy.js.map

package/dist/strategies/fixed-strategy.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fixed-strategy.js","sourceRoot":"","sources":["../../src/strategies/fixed-strategy.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AAGnD,MAAM,kBAAkB,GAAG,GAAG,CAAC;AAC/B,MAAM,eAAe,GAAG,EAAE,CAAC;AAE3B,4DAA4D;AAC5D,MAAM,OAAO,qBAAqB;IAChC,IAAI,GAAG,OAAO,CAAC;IAEf,KAAK,CAAC,GAAY,EAAE,OAAwB;QAC1C,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,kBAAkB,CAAC;QAC1D,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,eAAe,CAAC;QAEnD,MAAM,IAAI,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC;QAC9B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAEhD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAElC,MAAM,MAAM,GAAoB,EAAE,CAAC;QACnC,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,IAAI,YAAY,GAAG,CAAC,CAAC;QAErB,OAAO,QAAQ,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;YAC/B,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,GAAG,SAAS,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;YAC5D,MAAM,UAAU,GAAG,KAAK,CAAC,KAAK,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;YACjD,MAAM,OAAO,GAAG,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACrC,MAAM,UAAU,GAAG,UAAU,CAAC,MAAM,CAAC;YAErC,MAAM,OAAO,GAAG,aAAa,CAAC,OAAO,CAAC,CAAC;YAEvC,MAAM,KAAK,GAAkB;gBAC3B,EAAE,EAAE,MAAM,CAAC,UAAU,EAAE;gBACvB,OAAO;gBACP,QAAQ,EAAE;oBACR,OAAO,EAAE,MAAM,CAAC,UAAU,EAAE;oBAC5B,WAAW,EAAE,EAAE;oBACf,UAAU;oBACV,UAAU,EAAE,OAAO,CAAC,UAAU;oBAC9B,UAAU,EAAE,YAAY;oBACxB,QAAQ,EAAE,YAAY,GAAG,UAAU;iBACpC;gBACD,GAAG,EAAE,OAAO;aACb,CAAC;YAEF,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAEnB,YAAY,IAAI,UAAU,CAAC;YAC3B,QAAQ,GAAG,MAAM,GAAG,OAAO,CAAC;YAE5B,IAAI,QAAQ,IAAI,MAAM;gBAAE,MAAM;YAC9B,IAAI,QAAQ,IAAI,KAAK,CAAC,MAAM;gBAAE,MAAM;YACpC,IAAI,MAAM,IAAI,KAAK,CAAC,MAAM;gBAAE,MAAM;QACpC,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;CACF;AAED,SAAS,aAAa,CAAC,IAAY;IACjC,OAAO;QACL,IAAI,EAAE,WAAW;QACjB,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAc,CAAC;KACrC,CAAC;AACrB,CAAC"}

package/dist/strategies/heading-strategy.d.ts

@@ -0,0 +1,8 @@
+import type { AnyNode, DocumentChunk, ChunkingOptions } from "@markitdownjs/shared";
+import type { ChunkingStrategy } from "../chunker.js";
+/** Splits a document by heading boundaries */
+export declare class HeadingChunkingStrategy implements ChunkingStrategy {
+    name: string;
+    chunk(ast: AnyNode, options: ChunkingOptions): DocumentChunk[];
+}
+//# sourceMappingURL=heading-strategy.d.ts.map

package/dist/strategies/heading-strategy.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"heading-strategy.d.ts","sourceRoot":"","sources":["../../src/strategies/heading-strategy.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,OAAO,EACP,aAAa,EACb,eAAe,EAEhB,MAAM,sBAAsB,CAAC;AAM9B,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAKtD,8CAA8C;AAC9C,qBAAa,uBAAwB,YAAW,gBAAgB;IAC9D,IAAI,SAAa;IAEjB,KAAK,CAAC,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,eAAe,GAAG,aAAa,EAAE;CA+D/D"}

package/dist/strategies/heading-strategy.js

@@ -0,0 +1,117 @@
+import { getNodeText, isNodeOfType, countTokens, } from "@markitdownjs/shared";
+import { createChunk } from "../chunker.js";
+const DEFAULT_MAX_TOKENS = 512;
+/** Splits a document by heading boundaries */
+export class HeadingChunkingStrategy {
+    name = "heading";
+    chunk(ast, options) {
+        const maxTokens = options.maxTokens ?? DEFAULT_MAX_TOKENS;
+        const headingPath = [];
+        const chunks = [];
+        let currentNodes = [];
+        let startIndex = 0;
+        let globalOffset = 0;
+        const flushChunk = () => {
+            if (currentNodes.length === 0)
+                return;
+            const chunk = createChunk([...currentNodes], [...headingPath], options, startIndex, globalOffset);
+            if (chunk) {
+                if (countTokens(chunk.ast) > maxTokens) {
+                    const split = splitLargeChunk(currentNodes, headingPath, options, maxTokens, startIndex);
+                    chunks.push(...split);
+                }
+                else {
+                    chunks.push(chunk);
+                }
+            }
+            currentNodes = [];
+            startIndex = globalOffset;
+        };
+        walk(ast, null);
+        flushChunk();
+        return chunks;
+        function walk(node, parent) {
+            if (isNodeOfType(node, "heading")) {
+                flushChunk();
+                headingPath.length = node.level - 1;
+                headingPath[node.level - 1] = getNodeText(node);
+            }
+            if ("children" in node && Array.isArray(node.children)) {
+                for (const child of node.children) {
+                    walk(child, node);
+                }
+            }
+            const nodeTokens = countTokens(node);
+            if (node.type !== "heading" &&
+                node.type !== "document" &&
+                !(parent && isNodeOfType(parent, "heading"))) {
+                currentNodes.push(node);
+                globalOffset += nodeTokens;
+            }
+            if (nodeTokens > 0 && countTokensArray(currentNodes) >= maxTokens) {
+                flushChunk();
+            }
+        }
+    }
+}
+function countTokensArray(nodes) {
+    return nodes.reduce((sum, n) => sum + countTokens(n), 0);
+}
+function splitLargeChunk(nodes, headingPath, options, maxTokens, startIndex) {
+    const chunks = [];
+    let buffer = [];
+    let offset = startIndex;
+    for (const node of nodes) {
+        const nodeTokens = countTokens(node);
+        const bufferTokens = countTokensArray(buffer);
+        if (bufferTokens + nodeTokens > maxTokens && buffer.length > 0) {
+            const chunk = createChunk(buffer, headingPath, options, offset, offset + bufferTokens);
+            if (chunk)
+                chunks.push(chunk);
+            offset += bufferTokens;
+            buffer = [];
+        }
+        if (nodeTokens > maxTokens && node.type === "paragraph" && "children" in node) {
+            const subChunks = splitParagraph(node, headingPath, options, maxTokens, offset);
+            chunks.push(...subChunks);
+            offset += nodeTokens;
+        }
+        else {
+            buffer.push(node);
+        }
+    }
+    if (buffer.length > 0) {
+        const bufferTokens = countTokensArray(buffer);
+        const chunk = createChunk(buffer, headingPath, options, offset, offset + bufferTokens);
+        if (chunk)
+            chunks.push(chunk);
+    }
+    return chunks;
+}
+function splitParagraph(node, headingPath, options, maxTokens, startIndex) {
+    const text = getNodeText(node);
+    const words = text.split(/\s+/).filter(Boolean);
+    const chunks = [];
+    let offset = startIndex;
+    for (let i = 0; i < words.length; i += maxTokens) {
+        const slice = words.slice(i, i + maxTokens);
+        const value = slice.join(" ");
+        const tokenCount = slice.length;
+        chunks.push({
+            id: crypto.randomUUID(),
+            content: value,
+            metadata: {
+                chunkId: crypto.randomUUID(),
+                headingPath: [...headingPath],
+                tokenCount,
+                sourceFile: options.sourceFile,
+                startIndex: offset,
+                endIndex: offset + tokenCount,
+            },
+            ast: { type: "paragraph", children: [{ type: "text", value }] },
+        });
+        offset += tokenCount;
+    }
+    return chunks;
+}
+//# sourceMappingURL=heading-strategy.js.map

package/dist/strategies/heading-strategy.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"heading-strategy.js","sourceRoot":"","sources":["../../src/strategies/heading-strategy.ts"],"names":[],"mappings":"AAMA,OAAO,EACL,WAAW,EACX,YAAY,EACZ,WAAW,GACZ,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAE5C,MAAM,kBAAkB,GAAG,GAAG,CAAC;AAE/B,8CAA8C;AAC9C,MAAM,OAAO,uBAAuB;IAClC,IAAI,GAAG,SAAS,CAAC;IAEjB,KAAK,CAAC,GAAY,EAAE,OAAwB;QAC1C,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,kBAAkB,CAAC;QAC1D,MAAM,WAAW,GAAa,EAAE,CAAC;QACjC,MAAM,MAAM,GAAoB,EAAE,CAAC;QACnC,IAAI,YAAY,GAAc,EAAE,CAAC;QACjC,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,IAAI,YAAY,GAAG,CAAC,CAAC;QAErB,MAAM,UAAU,GAAG,GAAS,EAAE;YAC5B,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO;YACtC,MAAM,KAAK,GAAG,WAAW,CACvB,CAAC,GAAG,YAAY,CAAC,EACjB,CAAC,GAAG,WAAW,CAAC,EAChB,OAAO,EACP,UAAU,EACV,YAAY,CACb,CAAC;YACF,IAAI,KAAK,EAAE,CAAC;gBACV,IAAI,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,SAAS,EAAE,CAAC;oBACvC,MAAM,KAAK,GAAG,eAAe,CAAC,YAAY,EAAE,WAAW,EAAE,OAAO,EAAE,SAAS,EAAE,UAAU,CAAC,CAAC;oBACzF,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC;gBACxB,CAAC;qBAAM,CAAC;oBACN,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACrB,CAAC;YACH,CAAC;YACD,YAAY,GAAG,EAAE,CAAC;YAClB,UAAU,GAAG,YAAY,CAAC;QAC5B,CAAC,CAAC;QAEF,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QAEhB,UAAU,EAAE,CAAC;QAEb,OAAO,MAAM,CAAC;QAEd,SAAS,IAAI,CAAC,IAAa,EAAE,MAAsB;YACjD,IAAI,YAAY,CAAc,IAAI,EAAE,SAAS,CAAC,EAAE,CAAC;gBAC/C,UAAU,EAAE,CAAC;gBACb,WAAW,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC;gBACpC,WAAW,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;YAClD,CAAC;YAED,IAAI,UAAU,IAAI,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACvD,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;oBAClC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;gBACpB,CAAC;YACH,CAAC;YAED,MAAM,UAAU,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;YACrC,IACE,IAAI,CAAC,IAAI,KAAK,SAAS;gBACvB,IAAI,CAAC,IAAI,KAAK,UAAU;gBACxB,CAAC,CAAC,MAAM,IAAI,YAAY,CAAc,MAAM,EAAE,SAAS,CAAC,CAAC,EACzD,CAAC;gBACD,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACxB,YAAY,IAAI,UAAU,CAAC;YAC7B,CAAC;YAED,IAAI,UAAU,GAAG,CAAC,IAAI,gBAAgB,CAAC,YAAY,CAAC,IAAI,SAAS,EAAE,CAAC;gBAClE,UAAU,EAAE,CAAC;YACf,CAAC;QACH,CAAC;IACH,CAAC;CACF;AAED,SAAS,gBAAgB,CAAC,KAAgB;IACxC,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,WAAW,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC;AAED,SAAS,eAAe,CACtB,KAAgB,EAChB,WAAqB,EACrB,OAAwB,EACxB,SAAiB,EACjB,UAAkB;IAElB,MAAM,MAAM,GAAoB,EAAE,CAAC;IACnC,IAAI,MAAM,GAAc,EAAE,CAAC;IAC3B,IAAI,MAAM,GAAG,UAAU,CAAC;IAExB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,UAAU,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;QACrC,MAAM,YAAY,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC;QAE9C,IAAI,YAAY,GAAG,UAAU,GAAG,SAAS,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC/D,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,EAAE,WAAW,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,GAAG,YAAY,CAAC,CAAC;YACvF,IAAI,KAAK;gBAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC9B,MAAM,IAAI,YAAY,CAAC;YACvB,MAAM,GAAG,EAAE,CAAC;QACd,CAAC;QAED,IAAI,UAAU,GAAG,SAAS,IAAI,IAAI,CAAC,IAAI,KAAK,WAAW,IAAI,UAAU,IAAI,IAAI,EAAE,CAAC;YAC9E,MAAM,SAAS,GAAG,cAAc,CAAC,IAAe,EAAE,WAAW,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;YAC3F,MAAM,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;YAC1B,MAAM,IAAI,UAAU,CAAC;QACvB,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpB,CAAC;IACH,CAAC;IAED,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtB,MAAM,YAAY,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC;QAC9C,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,EAAE,WAAW,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,GAAG,YAAY,CAAC,CAAC;QACvF,IAAI,KAAK;YAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAChC,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,cAAc,CACrB,IAAa,EACb,WAAqB,EACrB,OAAwB,EACxB,SAAiB,EACjB,UAAkB;IAElB,MAAM,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;IAC/B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAChD,MAAM,MAAM,GAAoB,EAAE,CAAC;IACnC,IAAI,MAAM,GAAG,UAAU,CAAC;IAExB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;QACjD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC;QAC5C,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC9B,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC;QAChC,MAAM,CAAC,IAAI,CAAC;YACV,EAAE,EAAE,MAAM,CAAC,UAAU,EAAE;YACvB,OAAO,EAAE,KAAK;YACd,QAAQ,EAAE;gBACR,OAAO,EAAE,MAAM,CAAC,UAAU,EAAE;gBAC5B,WAAW,EAAE,CAAC,GAAG,WAAW,CAAC;gBAC7B,UAAU;gBACV,UAAU,EAAE,OAAO,CAAC,UAAU;gBAC9B,UAAU,EAAE,MAAM;gBAClB,QAAQ,EAAE,MAAM,GAAG,UAAU;aAC9B;YACD,GAAG,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE;SAChE,CAAC,CAAC;QACH,MAAM,IAAI,UAAU,CAAC;IACvB,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/strategies/page-strategy.d.ts

@@ -0,0 +1,8 @@
+import type { AnyNode, DocumentChunk, ChunkingOptions } from "@markitdownjs/shared";
+import type { ChunkingStrategy } from "../chunker.js";
+/** Splits a document by page break boundaries */
+export declare class PageChunkingStrategy implements ChunkingStrategy {
+    name: string;
+    chunk(ast: AnyNode, options: ChunkingOptions): DocumentChunk[];
+}
+//# sourceMappingURL=page-strategy.d.ts.map

package/dist/strategies/page-strategy.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"page-strategy.d.ts","sourceRoot":"","sources":["../../src/strategies/page-strategy.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,OAAO,EACP,aAAa,EACb,eAAe,EAEhB,MAAM,sBAAsB,CAAC;AAM9B,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAKtD,iDAAiD;AACjD,qBAAa,oBAAqB,YAAW,gBAAgB;IAC3D,IAAI,SAAU;IAEd,KAAK,CAAC,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,eAAe,GAAG,aAAa,EAAE;CAsE/D"}

package/dist/strategies/page-strategy.js

@@ -0,0 +1,88 @@
+import { walkAst, isNodeOfType, countTokens, } from "@markitdownjs/shared";
+import { createChunk } from "../chunker.js";
+const DEFAULT_MAX_TOKENS = 512;
+/** Splits a document by page break boundaries */
+export class PageChunkingStrategy {
+    name = "page";
+    chunk(ast, options) {
+        const maxTokens = options.maxTokens ?? DEFAULT_MAX_TOKENS;
+        const chunks = [];
+        let currentNodes = [];
+        let currentPage;
+        let startIndex = 0;
+        let globalOffset = 0;
+        let hasPageBreaks = false;
+        const flushChunk = (page) => {
+            if (currentNodes.length === 0)
+                return;
+            const chunk = createChunk([...currentNodes], [], options, startIndex, globalOffset, page);
+            if (chunk) {
+                if (countTokens(chunk.ast) > maxTokens) {
+                    const split = splitLargeChunk(currentNodes, options, maxTokens, startIndex, page);
+                    chunks.push(...split);
+                }
+                else {
+                    chunks.push(chunk);
+                }
+            }
+            currentNodes = [];
+            startIndex = globalOffset;
+        };
+        walkAst(ast, (node, _parent) => {
+            if (isNodeOfType(node, "page-break")) {
+                hasPageBreaks = true;
+                flushChunk(currentPage);
+                currentPage = node.pageNumber;
+                return;
+            }
+            currentNodes.push(node);
+            const tokens = countTokens(node);
+            globalOffset += tokens;
+            if (tokens > 0 && countTokensArray(currentNodes) >= maxTokens) {
+                flushChunk(currentPage);
+            }
+        });
+        flushChunk(currentPage);
+        if (!hasPageBreaks) {
+            if (chunks.length === 0)
+                return [];
+            if (chunks.length === 1) {
+                const single = chunks[0];
+                const totalTokens = countTokens(single.ast);
+                if (totalTokens > maxTokens) {
+                    return splitLargeChunk([single.ast], options, maxTokens, single.metadata.startIndex, single.metadata.page);
+                }
+                return chunks;
+            }
+        }
+        return chunks;
+    }
+}
+function countTokensArray(nodes) {
+    return nodes.reduce((sum, n) => sum + countTokens(n), 0);
+}
+function splitLargeChunk(nodes, options, maxTokens, startIndex, page) {
+    const chunks = [];
+    let buffer = [];
+    let offset = startIndex;
+    for (const node of nodes) {
+        const nodeTokens = countTokens(node);
+        const bufferTokens = countTokensArray(buffer);
+        if (bufferTokens + nodeTokens > maxTokens && buffer.length > 0) {
+            const chunk = createChunk(buffer, [], options, offset, offset + bufferTokens, page);
+            if (chunk)
+                chunks.push(chunk);
+            offset += bufferTokens;
+            buffer = [];
+        }
+        buffer.push(node);
+    }
+    if (buffer.length > 0) {
+        const bufferTokens = countTokensArray(buffer);
+        const chunk = createChunk(buffer, [], options, offset, offset + bufferTokens, page);
+        if (chunk)
+            chunks.push(chunk);
+    }
+    return chunks;
+}
+//# sourceMappingURL=page-strategy.js.map

package/dist/strategies/page-strategy.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"page-strategy.js","sourceRoot":"","sources":["../../src/strategies/page-strategy.ts"],"names":[],"mappings":"AAMA,OAAO,EACL,OAAO,EACP,YAAY,EACZ,WAAW,GACZ,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAE5C,MAAM,kBAAkB,GAAG,GAAG,CAAC;AAE/B,iDAAiD;AACjD,MAAM,OAAO,oBAAoB;IAC/B,IAAI,GAAG,MAAM,CAAC;IAEd,KAAK,CAAC,GAAY,EAAE,OAAwB;QAC1C,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,kBAAkB,CAAC;QAC1D,MAAM,MAAM,GAAoB,EAAE,CAAC;QACnC,IAAI,YAAY,GAAc,EAAE,CAAC;QACjC,IAAI,WAA+B,CAAC;QACpC,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,IAAI,aAAa,GAAG,KAAK,CAAC;QAE1B,MAAM,UAAU,GAAG,CAAC,IAAa,EAAQ,EAAE;YACzC,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO;YACtC,MAAM,KAAK,GAAG,WAAW,CACvB,CAAC,GAAG,YAAY,CAAC,EACjB,EAAE,EACF,OAAO,EACP,UAAU,EACV,YAAY,EACZ,IAAI,CACL,CAAC;YACF,IAAI,KAAK,EAAE,CAAC;gBACV,IAAI,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,SAAS,EAAE,CAAC;oBACvC,MAAM,KAAK,GAAG,eAAe,CAAC,YAAY,EAAE,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,IAAI,CAAC,CAAC;oBAClF,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC;gBACxB,CAAC;qBAAM,CAAC;oBACN,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACrB,CAAC;YACH,CAAC;YACD,YAAY,GAAG,EAAE,CAAC;YAClB,UAAU,GAAG,YAAY,CAAC;QAC5B,CAAC,CAAC;QAEF,OAAO,CAAC,GAAG,EAAE,CAAC,IAAI,EAAE,OAAO,EAAE,EAAE;YAC7B,IAAI,YAAY,CAAgB,IAAI,EAAE,YAAY,CAAC,EAAE,CAAC;gBACpD,aAAa,GAAG,IAAI,CAAC;gBACrB,UAAU,CAAC,WAAW,CAAC,CAAC;gBACxB,WAAW,GAAG,IAAI,CAAC,UAAU,CAAC;gBAC9B,OAAO;YACT,CAAC;YAED,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACxB,MAAM,MAAM,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;YACjC,YAAY,IAAI,MAAM,CAAC;YAEvB,IAAI,MAAM,GAAG,CAAC,IAAI,gBAAgB,CAAC,YAAY,CAAC,IAAI,SAAS,EAAE,CAAC;gBAC9D,UAAU,CAAC,WAAW,CAAC,CAAC;YAC1B,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,UAAU,CAAC,WAAW,CAAC,CAAC;QAExB,IAAI,CAAC,aAAa,EAAE,CAAC;YACnB,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO,EAAE,CAAC;YACnC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACxB,MAAM,MAAM,GAAG,MAAM,CAAC,CAAC,CAAE,CAAC;gBAC1B,MAAM,WAAW,GAAG,WAAW,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBAC5C,IAAI,WAAW,GAAG,SAAS,EAAE,CAAC;oBAC5B,OAAO,eAAe,CACpB,CAAC,MAAM,CAAC,GAAG,CAAC,EACZ,OAAO,EACP,SAAS,EACT,MAAM,CAAC,QAAQ,CAAC,UAAU,EAC1B,MAAM,CAAC,QAAQ,CAAC,IAAI,CACrB,CAAC;gBACJ,CAAC;gBACD,OAAO,MAAM,CAAC;YAChB,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;CACF;AAED,SAAS,gBAAgB,CAAC,KAAgB;IACxC,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,WAAW,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC;AAED,SAAS,eAAe,CACtB,KAAgB,EAChB,OAAwB,EACxB,SAAiB,EACjB,UAAkB,EAClB,IAAa;IAEb,MAAM,MAAM,GAAoB,EAAE,CAAC;IACnC,IAAI,MAAM,GAAc,EAAE,CAAC;IAC3B,IAAI,MAAM,GAAG,UAAU,CAAC;IAExB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,UAAU,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;QACrC,MAAM,YAAY,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC;QAE9C,IAAI,YAAY,GAAG,UAAU,GAAG,SAAS,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC/D,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,EAAE,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,GAAG,YAAY,EAAE,IAAI,CAAC,CAAC;YACpF,IAAI,KAAK;gBAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC9B,MAAM,IAAI,YAAY,CAAC;YACvB,MAAM,GAAG,EAAE,CAAC;QACd,CAAC;QAED,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACpB,CAAC;IAED,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtB,MAAM,YAAY,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC;QAC9C,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,EAAE,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,GAAG,YAAY,EAAE,IAAI,CAAC,CAAC;QACpF,IAAI,KAAK;YAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAChC,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/strategies/semantic-strategy.d.ts

@@ -0,0 +1,8 @@
+import type { AnyNode, DocumentChunk, ChunkingOptions } from "@markitdownjs/shared";
+import type { ChunkingStrategy } from "../chunker.js";
+/** Splits a document by semantic boundaries with configurable overlap */
+export declare class SemanticChunkingStrategy implements ChunkingStrategy {
+    name: string;
+    chunk(ast: AnyNode, options: ChunkingOptions): DocumentChunk[];
+}
+//# sourceMappingURL=semantic-strategy.d.ts.map

package/dist/strategies/semantic-strategy.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"semantic-strategy.d.ts","sourceRoot":"","sources":["../../src/strategies/semantic-strategy.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,OAAO,EACP,aAAa,EACb,eAAe,EAGhB,MAAM,sBAAsB,CAAC;AAO9B,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAMtD,yEAAyE;AACzE,qBAAa,wBAAyB,YAAW,gBAAgB;IAC/D,IAAI,SAAc;IAElB,KAAK,CAAC,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,eAAe,GAAG,aAAa,EAAE;CA2D/D"}

package/dist/strategies/semantic-strategy.js

@@ -0,0 +1,132 @@
+import { walkAst, getNodeText, isNodeOfType, countTokens, } from "@markitdownjs/shared";
+import { createChunk } from "../chunker.js";
+const DEFAULT_MAX_TOKENS = 512;
+const DEFAULT_OVERLAP = 50;
+/** Splits a document by semantic boundaries with configurable overlap */
+export class SemanticChunkingStrategy {
+    name = "semantic";
+    chunk(ast, options) {
+        const maxTokens = options.maxTokens ?? DEFAULT_MAX_TOKENS;
+        const overlap = options.overlap ?? DEFAULT_OVERLAP;
+        const nodes = [];
+        walkAst(ast, (node, _parent) => {
+            if (node.type === "paragraph" ||
+                node.type === "heading" ||
+                node.type === "list" ||
+                node.type === "table" ||
+                node.type === "code" ||
+                node.type === "blockquote" ||
+                node.type === "thematic-break" ||
+                node.type === "horizontal-rule") {
+                nodes.push(node);
+            }
+        });
+        if (nodes.length === 0)
+            return [];
+        const semanticGroups = groupBySemanticBoundary(nodes);
+        const chunks = [];
+        let overlapText = "";
+        for (const group of semanticGroups) {
+            const groupTokens = group.reduce((sum, n) => sum + countTokens(n), 0);
+            if (overlapText && overlap > 0) {
+                const overlapWords = overlapText.split(/\s+/).filter(Boolean).slice(-overlap);
+                const overlapToken = overlapWords.join(" ");
+                if (overlapToken) {
+                    group.unshift({
+                        type: "paragraph",
+                        children: [{ type: "text", value: overlapToken }],
+                    });
+                }
+            }
+            if (groupTokens <= maxTokens) {
+                const chunk = createChunk(group, extractHeadingPath(group), options, 0, groupTokens);
+                if (chunk)
+                    chunks.push(chunk);
+                const lastText = getNodeText(group[group.length - 1]);
+                const words = lastText.split(/\s+/).filter(Boolean);
+                overlapText = words.slice(-overlap).join(" ");
+            }
+            else {
+                const subChunks = splitGroupByTokens(group, maxTokens, overlap, options);
+                chunks.push(...subChunks);
+                if (subChunks.length > 0) {
+                    const lastChunk = subChunks[subChunks.length - 1];
+                    const words = lastChunk.content.split(/\s+/).filter(Boolean);
+                    overlapText = words.slice(-overlap).join(" ");
+                }
+            }
+        }
+        return chunks;
+    }
+}
+function groupBySemanticBoundary(nodes) {
+    const groups = [];
+    let currentGroup = [];
+    for (const node of nodes) {
+        if (isNodeOfType(node, "heading")) {
+            if (currentGroup.length > 0) {
+                groups.push(currentGroup);
+                currentGroup = [];
+            }
+            currentGroup.push(node);
+        }
+        else if (node.type === "thematic-break" ||
+            node.type === "horizontal-rule") {
+            if (currentGroup.length > 0) {
+                groups.push(currentGroup);
+                currentGroup = [];
+            }
+        }
+        else {
+            currentGroup.push(node);
+        }
+    }
+    if (currentGroup.length > 0) {
+        groups.push(currentGroup);
+    }
+    return groups;
+}
+function extractHeadingPath(nodes) {
+    const path = [];
+    for (const node of nodes) {
+        if (isNodeOfType(node, "heading")) {
+            path.length = node.level - 1;
+            path[node.level - 1] = getNodeText(node);
+        }
+    }
+    return path;
+}
+function splitGroupByTokens(nodes, maxTokens, overlap, options) {
+    const chunks = [];
+    let buffer = [];
+    let bufferTokens = 0;
+    for (const node of nodes) {
+        const nodeTokens = countTokens(node);
+        if (bufferTokens + nodeTokens > maxTokens && buffer.length > 0) {
+            const chunk = createChunk(buffer, extractHeadingPath(buffer), options, 0, bufferTokens);
+            if (chunk)
+                chunks.push(chunk);
+            const lastText = getNodeText(buffer[buffer.length - 1]);
+            const overlapWords = lastText.split(/\s+/).filter(Boolean).slice(-overlap);
+            const overlapToken = overlapWords.join(" ");
+            buffer = [];
+            bufferTokens = 0;
+            if (overlapToken && overlap > 0) {
+                buffer.push({
+                    type: "paragraph",
+                    children: [{ type: "text", value: overlapToken }],
+                });
+                bufferTokens = overlap;
+            }
+        }
+        buffer.push(node);
+        bufferTokens += nodeTokens;
+    }
+    if (buffer.length > 0) {
+        const chunk = createChunk(buffer, extractHeadingPath(buffer), options, 0, bufferTokens);
+        if (chunk)
+            chunks.push(chunk);
+    }
+    return chunks;
+}
+//# sourceMappingURL=semantic-strategy.js.map

package/dist/strategies/semantic-strategy.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"semantic-strategy.js","sourceRoot":"","sources":["../../src/strategies/semantic-strategy.ts"],"names":[],"mappings":"AAOA,OAAO,EACL,OAAO,EACP,WAAW,EACX,YAAY,EACZ,WAAW,GACZ,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAE5C,MAAM,kBAAkB,GAAG,GAAG,CAAC;AAC/B,MAAM,eAAe,GAAG,EAAE,CAAC;AAE3B,yEAAyE;AACzE,MAAM,OAAO,wBAAwB;IACnC,IAAI,GAAG,UAAU,CAAC;IAElB,KAAK,CAAC,GAAY,EAAE,OAAwB;QAC1C,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,kBAAkB,CAAC;QAC1D,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,eAAe,CAAC;QACnD,MAAM,KAAK,GAAc,EAAE,CAAC;QAE5B,OAAO,CAAC,GAAG,EAAE,CAAC,IAAI,EAAE,OAAO,EAAE,EAAE;YAC7B,IACE,IAAI,CAAC,IAAI,KAAK,WAAW;gBACzB,IAAI,CAAC,IAAI,KAAK,SAAS;gBACvB,IAAI,CAAC,IAAI,KAAK,MAAM;gBACpB,IAAI,CAAC,IAAI,KAAK,OAAO;gBACrB,IAAI,CAAC,IAAI,KAAK,MAAM;gBACpB,IAAI,CAAC,IAAI,KAAK,YAAY;gBAC1B,IAAI,CAAC,IAAI,KAAK,gBAAgB;gBAC9B,IAAI,CAAC,IAAI,KAAK,iBAAiB,EAC/B,CAAC;gBACD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACnB,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAElC,MAAM,cAAc,GAAG,uBAAuB,CAAC,KAAK,CAAC,CAAC;QACtD,MAAM,MAAM,GAAoB,EAAE,CAAC;QACnC,IAAI,WAAW,GAAG,EAAE,CAAC;QAErB,KAAK,MAAM,KAAK,IAAI,cAAc,EAAE,CAAC;YACnC,MAAM,WAAW,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,WAAW,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YAEtE,IAAI,WAAW,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;gBAC/B,MAAM,YAAY,GAAG,WAAW,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC;gBAC9E,MAAM,YAAY,GAAG,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAC5C,IAAI,YAAY,EAAE,CAAC;oBACjB,KAAK,CAAC,OAAO,CAAC;wBACZ,IAAI,EAAE,WAAW;wBACjB,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,YAAY,EAAE,CAAC;qBACjC,CAAC,CAAC;gBACtB,CAAC;YACH,CAAC;YAED,IAAI,WAAW,IAAI,SAAS,EAAE,CAAC;gBAC7B,MAAM,KAAK,GAAG,WAAW,CAAC,KAAK,EAAE,kBAAkB,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,WAAW,CAAC,CAAC;gBACrF,IAAI,KAAK;oBAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAC9B,MAAM,QAAQ,GAAG,WAAW,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAE,CAAC,CAAC;gBACvD,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;gBACpD,WAAW,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAChD,CAAC;iBAAM,CAAC;gBACN,MAAM,SAAS,GAAG,kBAAkB,CAAC,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;gBACzE,MAAM,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;gBAC1B,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACzB,MAAM,SAAS,GAAG,SAAS,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAE,CAAC;oBACnD,MAAM,KAAK,GAAG,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;oBAC7D,WAAW,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAChD,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;CACF;AAED,SAAS,uBAAuB,CAAC,KAAgB;IAC/C,MAAM,MAAM,GAAgB,EAAE,CAAC;IAC/B,IAAI,YAAY,GAAc,EAAE,CAAC;IAEjC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,YAAY,CAAc,IAAI,EAAE,SAAS,CAAC,EAAE,CAAC;YAC/C,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC5B,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;gBAC1B,YAAY,GAAG,EAAE,CAAC;YACpB,CAAC;YACD,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1B,CAAC;aAAM,IACL,IAAI,CAAC,IAAI,KAAK,gBAAgB;YAC9B,IAAI,CAAC,IAAI,KAAK,iBAAiB,EAC/B,CAAC;YACD,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC5B,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;gBAC1B,YAAY,GAAG,EAAE,CAAC;YACpB,CAAC;QACH,CAAC;aAAM,CAAC;YACN,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5B,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IAC5B,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAgB;IAC1C,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,YAAY,CAAc,IAAI,EAAE,SAAS,CAAC,EAAE,CAAC;YAC/C,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC;YAC7B,IAAI,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;QAC3C,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,kBAAkB,CACzB,KAAgB,EAChB,SAAiB,EACjB,OAAe,EACf,OAAwB;IAExB,MAAM,MAAM,GAAoB,EAAE,CAAC;IACnC,IAAI,MAAM,GAAc,EAAE,CAAC;IAC3B,IAAI,YAAY,GAAG,CAAC,CAAC;IAErB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,UAAU,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;QAErC,IAAI,YAAY,GAAG,UAAU,GAAG,SAAS,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC/D,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,EAAE,kBAAkB,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,YAAY,CAAC,CAAC;YACxF,IAAI,KAAK;gBAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAE9B,MAAM,QAAQ,GAAG,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAE,CAAC,CAAC;YACzD,MAAM,YAAY,GAAG,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC;YAC3E,MAAM,YAAY,GAAG,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAE5C,MAAM,GAAG,EAAE,CAAC;YACZ,YAAY,GAAG,CAAC,CAAC;YAEjB,IAAI,YAAY,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;gBAChC,MAAM,CAAC,IAAI,CAAC;oBACV,IAAI,EAAE,WAAW;oBACjB,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,YAAY,EAAE,CAAC;iBACjC,CAAC,CAAC;gBACpB,YAAY,GAAG,OAAO,CAAC;YACzB,CAAC;QACH,CAAC;QAED,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAClB,YAAY,IAAI,UAAU,CAAC;IAC7B,CAAC;IAED,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtB,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,EAAE,kBAAkB,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,YAAY,CAAC,CAAC;QACxF,IAAI,KAAK;YAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAChC,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/package.json

@@ -0,0 +1,31 @@
+{
+  "name": "@markitdownjs/chunking",
+  "version": "0.1.0",
+  "description": "AI-ready document chunking strategies for MarkItDownJS",
+  "license": "MIT",
+  "type": "module",
+  "main": "./dist/index.js",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "import": "./dist/index.js",
+      "types": "./dist/index.d.ts"
+    }
+  },
+  "files": [
+    "dist"
+  ],
+  "dependencies": {
+    "@markitdownjs/shared": "0.1.0"
+  },
+  "devDependencies": {
+    "typescript": "^5.5.0"
+  },
+  "scripts": {
+    "build": "tsc --project tsconfig.json",
+    "dev": "tsc --watch --project tsconfig.json",
+    "typecheck": "tsc --noEmit",
+    "clean": "rm -rf dist tsconfig.tsbuildinfo"
+  }
+}