@markitdownjs/chunking 0.1.0 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +66 -0
- package/dist/__tests__/chunker.test.js +22 -20
- package/dist/__tests__/chunker.test.js.map +1 -1
- package/dist/chunker.d.ts.map +1 -1
- package/dist/chunker.js +1 -3
- package/dist/chunker.js.map +1 -1
- package/dist/strategies/heading-strategy.d.ts.map +1 -1
- package/dist/strategies/heading-strategy.js +1 -1
- package/dist/strategies/heading-strategy.js.map +1 -1
- package/dist/strategies/page-strategy.d.ts.map +1 -1
- package/dist/strategies/page-strategy.js +1 -1
- package/dist/strategies/page-strategy.js.map +1 -1
- package/dist/strategies/semantic-strategy.d.ts.map +1 -1
- package/dist/strategies/semantic-strategy.js +2 -3
- package/dist/strategies/semantic-strategy.js.map +1 -1
- package/package.json +2 -2
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2024 MarkItDownJS Contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
# @markitdownjs/chunking
|
|
2
|
+
|
|
3
|
+
[](https://www.npmjs.com/package/@markitdownjs/chunking)
|
|
4
|
+
[](https://opensource.org/licenses/MIT)
|
|
5
|
+
|
|
6
|
+
First-class document chunking for RAG pipelines. Splits a `DocumentNode` AST into semantic chunks with rich metadata — heading path, page number, token count, and chunk ID.
|
|
7
|
+
|
|
8
|
+
## Install
|
|
9
|
+
|
|
10
|
+
```bash
|
|
11
|
+
npm install @markitdownjs/chunking
|
|
12
|
+
```
|
|
13
|
+
|
|
14
|
+
## Usage
|
|
15
|
+
|
|
16
|
+
```ts
|
|
17
|
+
import { Chunker, HeadingChunkStrategy } from "@markitdownjs/chunking";
|
|
18
|
+
|
|
19
|
+
const chunker = new Chunker({ strategy: new HeadingChunkStrategy({ maxTokens: 512 }) });
|
|
20
|
+
const chunks = chunker.chunk(documentNode, { sourceFile: "report.pdf" });
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
## Chunk Schema
|
|
24
|
+
|
|
25
|
+
```ts
|
|
26
|
+
interface Chunk {
|
|
27
|
+
chunkId: string;
|
|
28
|
+
sourceFile: string;
|
|
29
|
+
pageNumber: number | null;
|
|
30
|
+
headingPath: string[];
|
|
31
|
+
tokenCount: number;
|
|
32
|
+
metadata: Record<string, unknown>;
|
|
33
|
+
content: string;
|
|
34
|
+
}
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
## Strategies
|
|
38
|
+
|
|
39
|
+
| Export | Splits on |
|
|
40
|
+
|---|---|
|
|
41
|
+
| `HeadingChunkStrategy` | Document headings (h1–h6), with `maxTokens` overflow splitting |
|
|
42
|
+
| `PageChunkStrategy` | Page boundaries extracted from the AST |
|
|
43
|
+
| `TokenChunkStrategy` | Fixed token window with configurable overlap |
|
|
44
|
+
| `SemanticChunkStrategy` | Sentence-boundary aware splits using embedding similarity |
|
|
45
|
+
|
|
46
|
+
```ts
|
|
47
|
+
import { Chunker, TokenChunkStrategy } from "@markitdownjs/chunking";
|
|
48
|
+
|
|
49
|
+
const chunker = new Chunker({
|
|
50
|
+
strategy: new TokenChunkStrategy({ maxTokens: 256, overlap: 32 }),
|
|
51
|
+
});
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
## Strategy Options
|
|
55
|
+
|
|
56
|
+
```ts
|
|
57
|
+
new HeadingChunkStrategy({
|
|
58
|
+
maxTokens: 512, // split oversized sections further
|
|
59
|
+
minTokens: 50, // merge sections below threshold
|
|
60
|
+
includeHeadingPath: true,
|
|
61
|
+
});
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
---
|
|
65
|
+
|
|
66
|
+
Part of the [MarkItDownJS](https://github.com/markitdownjs/markitdownjs) monorepo.
|
|
@@ -1,42 +1,44 @@
|
|
|
1
|
-
import { describe, it, expect } from
|
|
2
|
-
import { DocumentChunker } from
|
|
3
|
-
import { createNode } from
|
|
4
|
-
describe(
|
|
1
|
+
import { describe, it, expect } from "vitest";
|
|
2
|
+
import { DocumentChunker } from "../chunker.js";
|
|
3
|
+
import { createNode } from "@markitdownjs/shared";
|
|
4
|
+
describe("DocumentChunker", () => {
|
|
5
5
|
const chunker = new DocumentChunker();
|
|
6
6
|
const ast = createNode({
|
|
7
|
-
type:
|
|
7
|
+
type: "document",
|
|
8
8
|
children: [
|
|
9
9
|
createNode({
|
|
10
|
-
type:
|
|
10
|
+
type: "heading",
|
|
11
11
|
level: 1,
|
|
12
|
-
children: [createNode({ type:
|
|
12
|
+
children: [createNode({ type: "text", value: "Introduction" })],
|
|
13
13
|
}),
|
|
14
14
|
createNode({
|
|
15
|
-
type:
|
|
16
|
-
children: [
|
|
15
|
+
type: "paragraph",
|
|
16
|
+
children: [
|
|
17
|
+
createNode({ type: "text", value: "This is the introduction paragraph." }),
|
|
18
|
+
],
|
|
17
19
|
}),
|
|
18
20
|
createNode({
|
|
19
|
-
type:
|
|
21
|
+
type: "heading",
|
|
20
22
|
level: 1,
|
|
21
|
-
children: [createNode({ type:
|
|
23
|
+
children: [createNode({ type: "text", value: "Main Content" })],
|
|
22
24
|
}),
|
|
23
25
|
createNode({
|
|
24
|
-
type:
|
|
25
|
-
children: [createNode({ type:
|
|
26
|
+
type: "paragraph",
|
|
27
|
+
children: [createNode({ type: "text", value: "This is the main content." })],
|
|
26
28
|
}),
|
|
27
29
|
],
|
|
28
30
|
});
|
|
29
|
-
it(
|
|
30
|
-
const chunks = chunker.chunk(ast, { strategy:
|
|
31
|
+
it("should chunk by heading", () => {
|
|
32
|
+
const chunks = chunker.chunk(ast, { strategy: "heading" });
|
|
31
33
|
expect(chunks.length).toBeGreaterThanOrEqual(2);
|
|
32
|
-
expect(chunks[0].metadata.headingPath).toContain(
|
|
34
|
+
expect(chunks[0].metadata.headingPath).toContain("Introduction");
|
|
33
35
|
});
|
|
34
|
-
it(
|
|
35
|
-
const chunks = chunker.chunk(ast, { strategy:
|
|
36
|
+
it("should chunk by fixed size", () => {
|
|
37
|
+
const chunks = chunker.chunk(ast, { strategy: "fixed", maxTokens: 5, overlap: 2 });
|
|
36
38
|
expect(chunks.length).toBeGreaterThanOrEqual(1);
|
|
37
39
|
});
|
|
38
|
-
it(
|
|
39
|
-
const chunks = chunker.chunk(ast, { strategy:
|
|
40
|
+
it("should generate chunk IDs", () => {
|
|
41
|
+
const chunks = chunker.chunk(ast, { strategy: "heading" });
|
|
40
42
|
for (const chunk of chunks) {
|
|
41
43
|
expect(chunk.id).toBeDefined();
|
|
42
44
|
expect(chunk.id.length).toBeGreaterThan(0);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chunker.test.js","sourceRoot":"","sources":["../../src/__tests__/chunker.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAChD,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAGlD,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;IAC/B,MAAM,OAAO,GAAG,IAAI,eAAe,EAAE,CAAC;IAEtC,MAAM,GAAG,GAAG,UAAU,CAAe;QACnC,IAAI,EAAE,UAAU;QAChB,QAAQ,EAAE;YACR,UAAU,CAAc;gBACtB,IAAI,EAAE,SAAS;gBACf,KAAK,EAAE,CAAC;gBACR,QAAQ,EAAE,CAAC,UAAU,CAAW,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC,CAAC;aAC1E,CAAC;YACF,UAAU,CAAgB;gBACxB,IAAI,EAAE,WAAW;gBACjB,QAAQ,EAAE,
|
|
1
|
+
{"version":3,"file":"chunker.test.js","sourceRoot":"","sources":["../../src/__tests__/chunker.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAChD,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAGlD,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;IAC/B,MAAM,OAAO,GAAG,IAAI,eAAe,EAAE,CAAC;IAEtC,MAAM,GAAG,GAAG,UAAU,CAAe;QACnC,IAAI,EAAE,UAAU;QAChB,QAAQ,EAAE;YACR,UAAU,CAAc;gBACtB,IAAI,EAAE,SAAS;gBACf,KAAK,EAAE,CAAC;gBACR,QAAQ,EAAE,CAAC,UAAU,CAAW,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC,CAAC;aAC1E,CAAC;YACF,UAAU,CAAgB;gBACxB,IAAI,EAAE,WAAW;gBACjB,QAAQ,EAAE;oBACR,UAAU,CAAW,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,qCAAqC,EAAE,CAAC;iBACrF;aACF,CAAC;YACF,UAAU,CAAc;gBACtB,IAAI,EAAE,SAAS;gBACf,KAAK,EAAE,CAAC;gBACR,QAAQ,EAAE,CAAC,UAAU,CAAW,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC,CAAC;aAC1E,CAAC;YACF,UAAU,CAAgB;gBACxB,IAAI,EAAE,WAAW;gBACjB,QAAQ,EAAE,CAAC,UAAU,CAAW,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,2BAA2B,EAAE,CAAC,CAAC;aACvF,CAAC;SACH;KACF,CAAC,CAAC;IAEH,EAAE,CAAC,yBAAyB,EAAE,GAAG,EAAE;QACjC,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC,CAAC;QAC3D,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC;QAChD,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,SAAS,CAAC,cAAc,CAAC,CAAC;IACpE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4BAA4B,EAAE,GAAG,EAAE;QACpC,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC;QACnF,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC;IAClD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2BAA2B,EAAE,GAAG,EAAE;QACnC,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC,CAAC;QAC3D,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;YAC/B,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QAC7C,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
package/dist/chunker.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chunker.d.ts","sourceRoot":"","sources":["../src/chunker.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,
|
|
1
|
+
{"version":3,"file":"chunker.d.ts","sourceRoot":"","sources":["../src/chunker.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,aAAa,EAAiB,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAOnG,sDAAsD;AACtD,MAAM,WAAW,gBAAgB;IAC/B,+BAA+B;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,wCAAwC;IACxC,KAAK,CAAC,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,eAAe,GAAG,aAAa,EAAE,CAAC;CAChE;AAED,iEAAiE;AACjE,wBAAgB,WAAW,CACzB,KAAK,EAAE,OAAO,EAAE,EAChB,WAAW,EAAE,MAAM,EAAE,EACrB,OAAO,EAAE,eAAe,EACxB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,EAChB,IAAI,CAAC,EAAE,MAAM,GACZ,aAAa,GAAG,IAAI,CAmBtB;AAED,oEAAoE;AACpE,qBAAa,eAAe;IAC1B,OAAO,CAAC,UAAU,CAAgC;;IAUlD,0CAA0C;IAC1C,gBAAgB,CAAC,QAAQ,EAAE,gBAAgB,GAAG,IAAI;IAIlD,wCAAwC;IACxC,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,gBAAgB,GAAG,SAAS;IAIvD,yCAAyC;IACzC,cAAc,IAAI,MAAM,EAAE;IAI1B,gDAAgD;IAChD,KAAK,CAAC,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,eAAe,GAAG,aAAa,EAAE;CAQ/D"}
|
package/dist/chunker.js
CHANGED
|
@@ -9,9 +9,7 @@ export function createChunk(nodes, headingPath, options, startIndex, endIndex, p
|
|
|
9
9
|
return null;
|
|
10
10
|
const content = nodes.map((n) => getNodeText(n)).join("\n\n");
|
|
11
11
|
const tokenCount = content.split(/\s+/).filter(Boolean).length;
|
|
12
|
-
const root = nodes.length === 1
|
|
13
|
-
? nodes[0]
|
|
14
|
-
: { type: "document", children: nodes };
|
|
12
|
+
const root = nodes.length === 1 ? nodes[0] : { type: "document", children: nodes };
|
|
15
13
|
const metadata = {
|
|
16
14
|
chunkId: generateId(),
|
|
17
15
|
headingPath,
|
package/dist/chunker.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chunker.js","sourceRoot":"","sources":["../src/chunker.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"chunker.js","sourceRoot":"","sources":["../src/chunker.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAC/D,OAAO,EAAE,uBAAuB,EAAE,MAAM,kCAAkC,CAAC;AAC3E,OAAO,EAAE,oBAAoB,EAAE,MAAM,+BAA+B,CAAC;AACrE,OAAO,EAAE,wBAAwB,EAAE,MAAM,mCAAmC,CAAC;AAC7E,OAAO,EAAE,qBAAqB,EAAE,MAAM,gCAAgC,CAAC;AAUvE,iEAAiE;AACjE,MAAM,UAAU,WAAW,CACzB,KAAgB,EAChB,WAAqB,EACrB,OAAwB,EACxB,UAAkB,EAClB,QAAgB,EAChB,IAAa;IAEb,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAEpC,MAAM,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC9D,MAAM,UAAU,GAAG,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC;IAE/D,MAAM,IAAI,GAAY,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;IAE7F,MAAM,QAAQ,GAAkB;QAC9B,OAAO,EAAE,UAAU,EAAE;QACrB,WAAW;QACX,UAAU;QACV,UAAU,EAAE,OAAO,CAAC,UAAU;QAC9B,UAAU;QACV,QAAQ;QACR,IAAI;KACL,CAAC;IAEF,OAAO,EAAE,EAAE,EAAE,QAAQ,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC;AAChE,CAAC;AAED,oEAAoE;AACpE,MAAM,OAAO,eAAe;IAClB,UAAU,CAAgC;IAElD;QACE,IAAI,CAAC,UAAU,GAAG,IAAI,GAAG,EAAE,CAAC;QAC5B,IAAI,CAAC,gBAAgB,CAAC,IAAI,uBAAuB,EAAE,CAAC,CAAC;QACrD,IAAI,CAAC,gBAAgB,CAAC,IAAI,oBAAoB,EAAE,CAAC,CAAC;QAClD,IAAI,CAAC,gBAAgB,CAAC,IAAI,wBAAwB,EAAE,CAAC,CAAC;QACtD,IAAI,CAAC,gBAAgB,CAAC,IAAI,qBAAqB,EAAE,CAAC,CAAC;IACrD,CAAC;IAED,0CAA0C;IAC1C,gBAAgB,CAAC,QAA0B;QACzC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IAC/C,CAAC;IAED,wCAAwC;IACxC,WAAW,CAAC,IAAY;QACtB,OAAO,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACnC,CAAC;IAED,yCAAyC;IACzC,cAAc;QACZ,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC;IAC5C,CAAC;IAED,gDAAgD;IAChD,KAAK,CAAC,GAAY,EAAE,OAAwB;QAC1C,MAAM,YAAY,GAAG,OAAO,CAAC,QAAQ,IAAI,SAAS,CAAC;QACnD,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;QACnD,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,IAAI,KAAK,CAAC,8BAA8B,YAAY,EAAE,CAAC,CAAC;QAChE,CAAC;QACD,OAAO,QAAQ,CAAC,KAAK,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IACtC,CAAC;CACF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"heading-strategy.d.ts","sourceRoot":"","sources":["../../src/strategies/heading-strategy.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,
|
|
1
|
+
{"version":3,"file":"heading-strategy.d.ts","sourceRoot":"","sources":["../../src/strategies/heading-strategy.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,aAAa,EAAE,eAAe,EAAe,MAAM,sBAAsB,CAAC;AAEjG,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAKtD,8CAA8C;AAC9C,qBAAa,uBAAwB,YAAW,gBAAgB;IAC9D,IAAI,SAAa;IAEjB,KAAK,CAAC,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,eAAe,GAAG,aAAa,EAAE;CA+D/D"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { getNodeText, isNodeOfType, countTokens
|
|
1
|
+
import { getNodeText, isNodeOfType, countTokens } from "@markitdownjs/shared";
|
|
2
2
|
import { createChunk } from "../chunker.js";
|
|
3
3
|
const DEFAULT_MAX_TOKENS = 512;
|
|
4
4
|
/** Splits a document by heading boundaries */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"heading-strategy.js","sourceRoot":"","sources":["../../src/strategies/heading-strategy.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"heading-strategy.js","sourceRoot":"","sources":["../../src/strategies/heading-strategy.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AAE9E,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAE5C,MAAM,kBAAkB,GAAG,GAAG,CAAC;AAE/B,8CAA8C;AAC9C,MAAM,OAAO,uBAAuB;IAClC,IAAI,GAAG,SAAS,CAAC;IAEjB,KAAK,CAAC,GAAY,EAAE,OAAwB;QAC1C,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,kBAAkB,CAAC;QAC1D,MAAM,WAAW,GAAa,EAAE,CAAC;QACjC,MAAM,MAAM,GAAoB,EAAE,CAAC;QACnC,IAAI,YAAY,GAAc,EAAE,CAAC;QACjC,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,IAAI,YAAY,GAAG,CAAC,CAAC;QAErB,MAAM,UAAU,GAAG,GAAS,EAAE;YAC5B,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO;YACtC,MAAM,KAAK,GAAG,WAAW,CACvB,CAAC,GAAG,YAAY,CAAC,EACjB,CAAC,GAAG,WAAW,CAAC,EAChB,OAAO,EACP,UAAU,EACV,YAAY,CACb,CAAC;YACF,IAAI,KAAK,EAAE,CAAC;gBACV,IAAI,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,SAAS,EAAE,CAAC;oBACvC,MAAM,KAAK,GAAG,eAAe,CAAC,YAAY,EAAE,WAAW,EAAE,OAAO,EAAE,SAAS,EAAE,UAAU,CAAC,CAAC;oBACzF,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC;gBACxB,CAAC;qBAAM,CAAC;oBACN,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACrB,CAAC;YACH,CAAC;YACD,YAAY,GAAG,EAAE,CAAC;YAClB,UAAU,GAAG,YAAY,CAAC;QAC5B,CAAC,CAAC;QAEF,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QAEhB,UAAU,EAAE,CAAC;QAEb,OAAO,MAAM,CAAC;QAEd,SAAS,IAAI,CAAC,IAAa,EAAE,MAAsB;YACjD,IAAI,YAAY,CAAc,IAAI,EAAE,SAAS,CAAC,EAAE,CAAC;gBAC/C,UAAU,EAAE,CAAC;gBACb,WAAW,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC;gBACpC,WAAW,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;YAClD,CAAC;YAED,IAAI,UAAU,IAAI,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACvD,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;oBAClC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;gBACpB,CAAC;YACH,CAAC;YAED,MAAM,UAAU,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;YACrC,IACE,IAAI,CAAC,IAAI,KAAK,SAAS;gBACvB,IAAI,CAAC,IAAI,KAAK,UAAU;gBACxB,CAAC,CAAC,MAAM,IAAI,YAAY,CAAc,MAAM,EAAE,SAAS,CAAC,CAAC,EACzD,CAAC;gBACD,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACxB,YAAY,IAAI,UAAU,CAAC;YAC7B,CAAC;YAED,IAAI,UAAU,GAAG,CAAC,IAAI,gBAAgB,CAAC,YAAY,CAAC,IAAI,SAAS,EAAE,CAAC;gBAClE,UAAU,EAAE,CAAC;YACf,CAAC;QACH,CAAC;IACH,CAAC;CACF;AAED,SAAS,gBAAgB,CAAC,KAAgB;IACxC,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,WAAW,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC;AAED,SAAS,eAAe,CACtB,KAAgB,EAChB,WAAqB,EACrB,OAAwB,EACxB,SAAiB,EACjB,UAAkB;IAElB,MAAM,MAAM,GAAoB,EAAE,CAAC;IACnC,IAAI,MAAM,GAAc,EAAE,CAAC;IAC3B,IAAI,MAAM,GAAG,UAAU,CAAC;IAExB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,UAAU,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;QACrC,MAAM,YAAY,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC;QAE9C,IAAI,YAAY,GAAG,UAAU,GAAG,SAAS,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC/D,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,EAAE,WAAW,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,GAAG,YAAY,CAAC,CAAC;YACvF,IAAI,KAAK;gBAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC9B,MAAM,IAAI,YAAY,CAAC;YACvB,MAAM,GAAG,EAAE,CAAC;QACd,CAAC;QAED,IAAI,UAAU,GAAG,SAAS,IAAI,IAAI,CAAC,IAAI,KAAK,WAAW,IAAI,UAAU,IAAI,IAAI,EAAE,CAAC;YAC9E,MAAM,SAAS,GAAG,cAAc,CAAC,IAAe,EAAE,WAAW,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;YAC3F,MAAM,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;YAC1B,MAAM,IAAI,UAAU,CAAC;QACvB,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpB,CAAC;IACH,CAAC;IAED,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtB,MAAM,YAAY,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC;QAC9C,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,EAAE,WAAW,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,GAAG,YAAY,CAAC,CAAC;QACvF,IAAI,KAAK;YAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAChC,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,cAAc,CACrB,IAAa,EACb,WAAqB,EACrB,OAAwB,EACxB,SAAiB,EACjB,UAAkB;IAElB,MAAM,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;IAC/B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAChD,MAAM,MAAM,GAAoB,EAAE,CAAC;IACnC,IAAI,MAAM,GAAG,UAAU,CAAC;IAExB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;QACjD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC;QAC5C,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC9B,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC;QAChC,MAAM,CAAC,IAAI,CAAC;YACV,EAAE,EAAE,MAAM,CAAC,UAAU,EAAE;YACvB,OAAO,EAAE,KAAK;YACd,QAAQ,EAAE;gBACR,OAAO,EAAE,MAAM,CAAC,UAAU,EAAE;gBAC5B,WAAW,EAAE,CAAC,GAAG,WAAW,CAAC;gBAC7B,UAAU;gBACV,UAAU,EAAE,OAAO,CAAC,UAAU;gBAC9B,UAAU,EAAE,MAAM;gBAClB,QAAQ,EAAE,MAAM,GAAG,UAAU;aAC9B;YACD,GAAG,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE;SAChE,CAAC,CAAC;QACH,MAAM,IAAI,UAAU,CAAC;IACvB,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"page-strategy.d.ts","sourceRoot":"","sources":["../../src/strategies/page-strategy.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,
|
|
1
|
+
{"version":3,"file":"page-strategy.d.ts","sourceRoot":"","sources":["../../src/strategies/page-strategy.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,aAAa,EAAE,eAAe,EAAiB,MAAM,sBAAsB,CAAC;AAEnG,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAKtD,iDAAiD;AACjD,qBAAa,oBAAqB,YAAW,gBAAgB;IAC3D,IAAI,SAAU;IAEd,KAAK,CAAC,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,eAAe,GAAG,aAAa,EAAE;CA+D/D"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { walkAst, isNodeOfType, countTokens
|
|
1
|
+
import { walkAst, isNodeOfType, countTokens } from "@markitdownjs/shared";
|
|
2
2
|
import { createChunk } from "../chunker.js";
|
|
3
3
|
const DEFAULT_MAX_TOKENS = 512;
|
|
4
4
|
/** Splits a document by page break boundaries */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"page-strategy.js","sourceRoot":"","sources":["../../src/strategies/page-strategy.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"page-strategy.js","sourceRoot":"","sources":["../../src/strategies/page-strategy.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AAE1E,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAE5C,MAAM,kBAAkB,GAAG,GAAG,CAAC;AAE/B,iDAAiD;AACjD,MAAM,OAAO,oBAAoB;IAC/B,IAAI,GAAG,MAAM,CAAC;IAEd,KAAK,CAAC,GAAY,EAAE,OAAwB;QAC1C,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,kBAAkB,CAAC;QAC1D,MAAM,MAAM,GAAoB,EAAE,CAAC;QACnC,IAAI,YAAY,GAAc,EAAE,CAAC;QACjC,IAAI,WAA+B,CAAC;QACpC,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,IAAI,aAAa,GAAG,KAAK,CAAC;QAE1B,MAAM,UAAU,GAAG,CAAC,IAAa,EAAQ,EAAE;YACzC,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO;YACtC,MAAM,KAAK,GAAG,WAAW,CAAC,CAAC,GAAG,YAAY,CAAC,EAAE,EAAE,EAAE,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,IAAI,CAAC,CAAC;YAC1F,IAAI,KAAK,EAAE,CAAC;gBACV,IAAI,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,SAAS,EAAE,CAAC;oBACvC,MAAM,KAAK,GAAG,eAAe,CAAC,YAAY,EAAE,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,IAAI,CAAC,CAAC;oBAClF,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC;gBACxB,CAAC;qBAAM,CAAC;oBACN,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACrB,CAAC;YACH,CAAC;YACD,YAAY,GAAG,EAAE,CAAC;YAClB,UAAU,GAAG,YAAY,CAAC;QAC5B,CAAC,CAAC;QAEF,OAAO,CAAC,GAAG,EAAE,CAAC,IAAI,EAAE,OAAO,EAAE,EAAE;YAC7B,IAAI,YAAY,CAAgB,IAAI,EAAE,YAAY,CAAC,EAAE,CAAC;gBACpD,aAAa,GAAG,IAAI,CAAC;gBACrB,UAAU,CAAC,WAAW,CAAC,CAAC;gBACxB,WAAW,GAAG,IAAI,CAAC,UAAU,CAAC;gBAC9B,OAAO;YACT,CAAC;YAED,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACxB,MAAM,MAAM,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;YACjC,YAAY,IAAI,MAAM,CAAC;YAEvB,IAAI,MAAM,GAAG,CAAC,IAAI,gBAAgB,CAAC,YAAY,CAAC,IAAI,SAAS,EAAE,CAAC;gBAC9D,UAAU,CAAC,WAAW,CAAC,CAAC;YAC1B,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,UAAU,CAAC,WAAW,CAAC,CAAC;QAExB,IAAI,CAAC,aAAa,EAAE,CAAC;YACnB,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO,EAAE,CAAC;YACnC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACxB,MAAM,MAAM,GAAG,MAAM,CAAC,CAAC,CAAE,CAAC;gBAC1B,MAAM,WAAW,GAAG,WAAW,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBAC5C,IAAI,WAAW,GAAG,SAAS,EAAE,CAAC;oBAC5B,OAAO,eAAe,CACpB,CAAC,MAAM,CAAC,GAAG,CAAC,EACZ,OAAO,EACP,SAAS,EACT,MAAM,CAAC,QAAQ,CAAC,UAAU,EAC1B,MAAM,CAAC,QAAQ,CAAC,IAAI,CACrB,CAAC;gBACJ,CAAC;gBACD,OAAO,MAAM,CAAC;YAChB,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;CACF;AAED,SAAS,gBAAgB,CAAC,KAAgB;IACxC,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,WAAW,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC;AAED,SAAS,eAAe,CACtB,KAAgB,EAChB,OAAwB,EACxB,SAAiB,EACjB,UAAkB,EAClB,IAAa;IAEb,MAAM,MAAM,GAAoB,EAAE,CAAC;IACnC,IAAI,MAAM,GAAc,EAAE,CAAC;IAC3B,IAAI,MAAM,GAAG,UAAU,CAAC;IAExB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,UAAU,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;QACrC,MAAM,YAAY,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC;QAE9C,IAAI,YAAY,GAAG,UAAU,GAAG,SAAS,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC/D,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,EAAE,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,GAAG,YAAY,EAAE,IAAI,CAAC,CAAC;YACpF,IAAI,KAAK;gBAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC9B,MAAM,IAAI,YAAY,CAAC;YACvB,MAAM,GAAG,EAAE,CAAC;QACd,CAAC;QAED,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACpB,CAAC;IAED,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtB,MAAM,YAAY,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC;QAC9C,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,EAAE,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,GAAG,YAAY,EAAE,IAAI,CAAC,CAAC;QACpF,IAAI,KAAK;YAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAChC,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"semantic-strategy.d.ts","sourceRoot":"","sources":["../../src/strategies/semantic-strategy.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,OAAO,EACP,aAAa,EACb,eAAe,EAGhB,MAAM,sBAAsB,CAAC;
|
|
1
|
+
{"version":3,"file":"semantic-strategy.d.ts","sourceRoot":"","sources":["../../src/strategies/semantic-strategy.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,OAAO,EACP,aAAa,EACb,eAAe,EAGhB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAMtD,yEAAyE;AACzE,qBAAa,wBAAyB,YAAW,gBAAgB;IAC/D,IAAI,SAAc;IAElB,KAAK,CAAC,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,eAAe,GAAG,aAAa,EAAE;CA2D/D"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { walkAst, getNodeText, isNodeOfType, countTokens
|
|
1
|
+
import { walkAst, getNodeText, isNodeOfType, countTokens } from "@markitdownjs/shared";
|
|
2
2
|
import { createChunk } from "../chunker.js";
|
|
3
3
|
const DEFAULT_MAX_TOKENS = 512;
|
|
4
4
|
const DEFAULT_OVERLAP = 50;
|
|
@@ -70,8 +70,7 @@ function groupBySemanticBoundary(nodes) {
|
|
|
70
70
|
}
|
|
71
71
|
currentGroup.push(node);
|
|
72
72
|
}
|
|
73
|
-
else if (node.type === "thematic-break" ||
|
|
74
|
-
node.type === "horizontal-rule") {
|
|
73
|
+
else if (node.type === "thematic-break" || node.type === "horizontal-rule") {
|
|
75
74
|
if (currentGroup.length > 0) {
|
|
76
75
|
groups.push(currentGroup);
|
|
77
76
|
currentGroup = [];
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"semantic-strategy.js","sourceRoot":"","sources":["../../src/strategies/semantic-strategy.ts"],"names":[],"mappings":"AAOA,OAAO,
|
|
1
|
+
{"version":3,"file":"semantic-strategy.js","sourceRoot":"","sources":["../../src/strategies/semantic-strategy.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AAEvF,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAE5C,MAAM,kBAAkB,GAAG,GAAG,CAAC;AAC/B,MAAM,eAAe,GAAG,EAAE,CAAC;AAE3B,yEAAyE;AACzE,MAAM,OAAO,wBAAwB;IACnC,IAAI,GAAG,UAAU,CAAC;IAElB,KAAK,CAAC,GAAY,EAAE,OAAwB;QAC1C,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,kBAAkB,CAAC;QAC1D,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,eAAe,CAAC;QACnD,MAAM,KAAK,GAAc,EAAE,CAAC;QAE5B,OAAO,CAAC,GAAG,EAAE,CAAC,IAAI,EAAE,OAAO,EAAE,EAAE;YAC7B,IACE,IAAI,CAAC,IAAI,KAAK,WAAW;gBACzB,IAAI,CAAC,IAAI,KAAK,SAAS;gBACvB,IAAI,CAAC,IAAI,KAAK,MAAM;gBACpB,IAAI,CAAC,IAAI,KAAK,OAAO;gBACrB,IAAI,CAAC,IAAI,KAAK,MAAM;gBACpB,IAAI,CAAC,IAAI,KAAK,YAAY;gBAC1B,IAAI,CAAC,IAAI,KAAK,gBAAgB;gBAC9B,IAAI,CAAC,IAAI,KAAK,iBAAiB,EAC/B,CAAC;gBACD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACnB,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAElC,MAAM,cAAc,GAAG,uBAAuB,CAAC,KAAK,CAAC,CAAC;QACtD,MAAM,MAAM,GAAoB,EAAE,CAAC;QACnC,IAAI,WAAW,GAAG,EAAE,CAAC;QAErB,KAAK,MAAM,KAAK,IAAI,cAAc,EAAE,CAAC;YACnC,MAAM,WAAW,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,WAAW,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YAEtE,IAAI,WAAW,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;gBAC/B,MAAM,YAAY,GAAG,WAAW,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC;gBAC9E,MAAM,YAAY,GAAG,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAC5C,IAAI,YAAY,EAAE,CAAC;oBACjB,KAAK,CAAC,OAAO,CAAC;wBACZ,IAAI,EAAE,WAAW;wBACjB,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,YAAY,EAAE,CAAC;qBACjC,CAAC,CAAC;gBACtB,CAAC;YACH,CAAC;YAED,IAAI,WAAW,IAAI,SAAS,EAAE,CAAC;gBAC7B,MAAM,KAAK,GAAG,WAAW,CAAC,KAAK,EAAE,kBAAkB,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,WAAW,CAAC,CAAC;gBACrF,IAAI,KAAK;oBAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAC9B,MAAM,QAAQ,GAAG,WAAW,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAE,CAAC,CAAC;gBACvD,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;gBACpD,WAAW,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAChD,CAAC;iBAAM,CAAC;gBACN,MAAM,SAAS,GAAG,kBAAkB,CAAC,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;gBACzE,MAAM,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;gBAC1B,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACzB,MAAM,SAAS,GAAG,SAAS,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAE,CAAC;oBACnD,MAAM,KAAK,GAAG,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;oBAC7D,WAAW,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAChD,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;CACF;AAED,SAAS,uBAAuB,CAAC,KAAgB;IAC/C,MAAM,MAAM,GAAgB,EAAE,CAAC;IAC/B,IAAI,YAAY,GAAc,EAAE,CAAC;IAEjC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,YAAY,CAAc,IAAI,EAAE,SAAS,CAAC,EAAE,CAAC;YAC/C,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC5B,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;gBAC1B,YAAY,GAAG,EAAE,CAAC;YACpB,CAAC;YACD,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1B,CAAC;aAAM,IAAI,IAAI,CAAC,IAAI,KAAK,gBAAgB,IAAI,IAAI,CAAC,IAAI,KAAK,iBAAiB,EAAE,CAAC;YAC7E,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC5B,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;gBAC1B,YAAY,GAAG,EAAE,CAAC;YACpB,CAAC;QACH,CAAC;aAAM,CAAC;YACN,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5B,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IAC5B,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAgB;IAC1C,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,YAAY,CAAc,IAAI,EAAE,SAAS,CAAC,EAAE,CAAC;YAC/C,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC;YAC7B,IAAI,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;QAC3C,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,kBAAkB,CACzB,KAAgB,EAChB,SAAiB,EACjB,OAAe,EACf,OAAwB;IAExB,MAAM,MAAM,GAAoB,EAAE,CAAC;IACnC,IAAI,MAAM,GAAc,EAAE,CAAC;IAC3B,IAAI,YAAY,GAAG,CAAC,CAAC;IAErB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,UAAU,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;QAErC,IAAI,YAAY,GAAG,UAAU,GAAG,SAAS,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC/D,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,EAAE,kBAAkB,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,YAAY,CAAC,CAAC;YACxF,IAAI,KAAK;gBAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAE9B,MAAM,QAAQ,GAAG,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAE,CAAC,CAAC;YACzD,MAAM,YAAY,GAAG,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC;YAC3E,MAAM,YAAY,GAAG,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAE5C,MAAM,GAAG,EAAE,CAAC;YACZ,YAAY,GAAG,CAAC,CAAC;YAEjB,IAAI,YAAY,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;gBAChC,MAAM,CAAC,IAAI,CAAC;oBACV,IAAI,EAAE,WAAW;oBACjB,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,YAAY,EAAE,CAAC;iBACjC,CAAC,CAAC;gBACpB,YAAY,GAAG,OAAO,CAAC;YACzB,CAAC;QACH,CAAC;QAED,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAClB,YAAY,IAAI,UAAU,CAAC;IAC7B,CAAC;IAED,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtB,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,EAAE,kBAAkB,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,YAAY,CAAC,CAAC;QACxF,IAAI,KAAK;YAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAChC,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@markitdownjs/chunking",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.2",
|
|
4
4
|
"description": "AI-ready document chunking strategies for MarkItDownJS",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
"dist"
|
|
18
18
|
],
|
|
19
19
|
"dependencies": {
|
|
20
|
-
"@markitdownjs/shared": "0.
|
|
20
|
+
"@markitdownjs/shared": "0.2.0"
|
|
21
21
|
},
|
|
22
22
|
"devDependencies": {
|
|
23
23
|
"typescript": "^5.5.0"
|